Release 2024.6.0

cloudflared_udp_total_sessions was incorrectly a gauge when it
represents the total since the cloudflared process started and will
only ever increase.

Additionally adds new ICMP metrics for requests and replies.

Makefile

@@ -251,17 +251,11 @@ github-windows-upload:
 	python3 github_release.py --path built_artifacts/cloudflared-windows-386.exe --release-version $(VERSION) --name cloudflared-windows-386.exe
 	python3 github_release.py --path built_artifacts/cloudflared-windows-386.msi --release-version $(VERSION) --name cloudflared-windows-386.msi
 
-.PHONY: tunnelrpc-deps
-tunnelrpc-deps:
+.PHONY: capnp
+capnp:
 	which capnp  # https://capnproto.org/install.html
 	which capnpc-go  # go install zombiezen.com/go/capnproto2/capnpc-go@latest
-	capnp compile -ogo tunnelrpc/tunnelrpc.capnp
-
-.PHONY: quic-deps
-quic-deps:
-	which capnp
-	which capnpc-go
-	capnp compile -ogo quic/schema/quic_metadata_protocol.capnp
+	capnp compile -ogo tunnelrpc/proto/tunnelrpc.capnp tunnelrpc/proto/quic_metadata_protocol.capnp
 
 .PHONY: vet
 vet:
@@ -269,4 +263,4 @@ vet:
 
 .PHONY: fmt
 fmt:
-	goimports -l -w -local github.com/cloudflare/cloudflared $$(go list -mod=vendor -f '{{.Dir}}' -a ./... | fgrep -v tunnelrpc)
+	goimports -l -w -local github.com/cloudflare/cloudflared $$(go list -mod=vendor -f '{{.Dir}}' -a ./... | fgrep -v tunnelrpc/proto)

RELEASE_NOTES

@@ -1,3 +1,13 @@
+2024.6.0
+- 2024-05-30 TUN-8441: Correct UDP total sessions metric to a counter and add new ICMP metrics
+- 2024-05-28 TUN-8422: Add metrics for capnp method calls
+- 2024-05-24 TUN-8424: Refactor capnp registration server
+- 2024-05-23 TUN-8427: Fix BackoffHandler's internally shared clock structure
+- 2024-05-21 TUN-8425: Remove ICMP binding for quick tunnels
+- 2024-05-20 TUN-8423: Deprecate older legacy tunnel capnp interfaces
+- 2024-05-15 TUN-8419: Add capnp safe transport
+- 2024-05-13 TUN-8415: Refactor capnp rpc into a single module
+
 2024.5.0
 - 2024-05-07 TUN-8407: Upgrade go to version 1.22.2
 

cmd/cloudflared/tunnel/cmd.go

@@ -78,8 +78,8 @@ const (
 	// hostKeyPath is the path of the dir to save SSH host keys too
 	hostKeyPath = "host-key-path"
 
-	// udpUnregisterSessionTimeout is how long we wait before we stop trying to unregister a UDP session from the edge
-	udpUnregisterSessionTimeoutFlag = "udp-unregister-session-timeout"
+	// rpcTimeout is how long to wait for a Capnp RPC request to the edge
+	rpcTimeout = "rpc-timeout"
 
 	// writeStreamTimeout sets if we should have a timeout when writing data to a stream towards the destination (edge/origin).
 	writeStreamTimeout = "write-stream-timeout"
@@ -287,7 +287,7 @@ func routeFromFlag(c *cli.Context) (route cfapi.HostnameRoute, ok bool) {
 func StartServer(
 	c *cli.Context,
 	info *cliutil.BuildInfo,
-	namedTunnel *connection.NamedTunnelProperties,
+	namedTunnel *connection.TunnelProperties,
 	log *zerolog.Logger,
 ) error {
 	err := sentry.Init(sentry.ClientOptions{
@@ -409,6 +409,11 @@ func StartServer(
 		}
 	}
 
+	// Disable ICMP packet routing for quick tunnels
+	if quickTunnelURL != "" {
+		tunnelConfig.PacketConfig = nil
+	}
+
 	internalRules := []ingress.Rule{}
 	if features.Contains(features.FeatureManagementLogs) {
 		serviceIP := c.String("service-op-ip")
@@ -658,9 +663,9 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 		}),
 		altsrc.NewStringSliceFlag(&cli.StringSliceFlag{
 			Name:    "tag",
-			Usage:   "Custom tags used to identify this tunnel, in format `KEY=VALUE`. Multiple tags may be specified",
+			Usage:   "Custom tags used to identify this tunnel via added HTTP request headers to the origin, in format `KEY=VALUE`. Multiple tags may be specified.",
 			EnvVars: []string{"TUNNEL_TAG"},
-			Hidden:  shouldHide,
+			Hidden:  true,
 		}),
 		altsrc.NewDurationFlag(&cli.DurationFlag{
 			Name:   "heartbeat-interval",
@@ -695,7 +700,7 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			Hidden: true,
 		}),
 		altsrc.NewDurationFlag(&cli.DurationFlag{
-			Name:   udpUnregisterSessionTimeoutFlag,
+			Name:   rpcTimeout,
 			Value:  5 * time.Second,
 			Hidden: true,
 		}),

cmd/cloudflared/tunnel/configuration.go

@@ -27,7 +27,7 @@ import (
 	"github.com/cloudflare/cloudflared/orchestration"
 	"github.com/cloudflare/cloudflared/supervisor"
 	"github.com/cloudflare/cloudflared/tlsconfig"
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
 const (
@@ -108,7 +108,7 @@ func isSecretEnvVar(key string) bool {
 	return false
 }
 
-func dnsProxyStandAlone(c *cli.Context, namedTunnel *connection.NamedTunnelProperties) bool {
+func dnsProxyStandAlone(c *cli.Context, namedTunnel *connection.TunnelProperties) bool {
 	return c.IsSet("proxy-dns") &&
 		!(c.IsSet("name") || // adhoc-named tunnel
 			c.IsSet(ingress.HelloWorldFlag) || // quick or named tunnel
@@ -121,7 +121,7 @@ func prepareTunnelConfig(
 	info *cliutil.BuildInfo,
 	log, logTransport *zerolog.Logger,
 	observer *connection.Observer,
-	namedTunnel *connection.NamedTunnelProperties,
+	namedTunnel *connection.TunnelProperties,
 ) (*supervisor.TunnelConfig, *orchestration.Config, error) {
 	clientID, err := uuid.NewRandom()
 	if err != nil {
@@ -133,7 +133,7 @@ func prepareTunnelConfig(
 		log.Err(err).Msg("Tag parse failure")
 		return nil, nil, errors.Wrap(err, "Tag parse failure")
 	}
-	tags = append(tags, tunnelpogs.Tag{Name: "ID", Value: clientID.String()})
+	tags = append(tags, pogs.Tag{Name: "ID", Value: clientID.String()})
 
 	transportProtocol := c.String("protocol")
 
@@ -166,7 +166,7 @@ func prepareTunnelConfig(
 		)
 	}
 
-	namedTunnel.Client = tunnelpogs.ClientInfo{
+	namedTunnel.Client = pogs.ClientInfo{
 		ClientID: clientID[:],
 		Features: clientFeatures,
 		Version:  info.Version(),
@@ -246,7 +246,7 @@ func prepareTunnelConfig(
 		EdgeTLSConfigs:              edgeTLSConfigs,
 		FeatureSelector:             featureSelector,
 		MaxEdgeAddrRetries:          uint8(c.Int("max-edge-addr-retries")),
-		UDPUnregisterSessionTimeout: c.Duration(udpUnregisterSessionTimeoutFlag),
+		RPCTimeout:                  c.Duration(rpcTimeout),
 		WriteStreamTimeout:          c.Duration(writeStreamTimeout),
 		DisableQUICPathMTUDiscovery: c.Bool(quicDisablePathMTUDiscovery),
 	}

cmd/cloudflared/tunnel/quick_tunnel.go

@@ -79,7 +79,7 @@ func RunQuickTunnel(sc *subcommandContext) error {
 	return StartServer(
 		sc.c,
 		buildInfo,
-		&connection.NamedTunnelProperties{Credentials: credentials, QuickTunnelUrl: data.Result.Hostname},
+		&connection.TunnelProperties{Credentials: credentials, QuickTunnelUrl: data.Result.Hostname},
 		sc.log,
 	)
 }

cmd/cloudflared/tunnel/subcommand_context.go

@@ -261,7 +261,7 @@ func (sc *subcommandContext) runWithCredentials(credentials connection.Credentia
 	return StartServer(
 		sc.c,
 		buildInfo,
-		&connection.NamedTunnelProperties{Credentials: credentials},
+		&connection.TunnelProperties{Credentials: credentials},
 		sc.log,
 	)
 }

cmd/cloudflared/tunnel/tag.go

@@ -4,23 +4,23 @@ import (
 	"fmt"
 	"regexp"
 
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
 // Restrict key names to characters allowed in an HTTP header name.
 // Restrict key values to printable characters (what is recognised as data in an HTTP header value).
 var tagRegexp = regexp.MustCompile("^([a-zA-Z0-9!#$%&'*+\\-.^_`|~]+)=([[:print:]]+)$")
 
-func NewTagFromCLI(compoundTag string) (tunnelpogs.Tag, bool) {
+func NewTagFromCLI(compoundTag string) (pogs.Tag, bool) {
 	matches := tagRegexp.FindStringSubmatch(compoundTag)
 	if len(matches) == 0 {
-		return tunnelpogs.Tag{}, false
+		return pogs.Tag{}, false
 	}
-	return tunnelpogs.Tag{Name: matches[1], Value: matches[2]}, true
+	return pogs.Tag{Name: matches[1], Value: matches[2]}, true
 }
 
-func NewTagSliceFromCLI(tags []string) ([]tunnelpogs.Tag, error) {
-	var tagSlice []tunnelpogs.Tag
+func NewTagSliceFromCLI(tags []string) ([]pogs.Tag, error) {
+	var tagSlice []pogs.Tag
 	for _, compoundTag := range tags {
 		if tag, ok := NewTagFromCLI(compoundTag); ok {
 			tagSlice = append(tagSlice, tag)

cmd/cloudflared/tunnel/tag_test.go

@@ -3,7 +3,7 @@ package tunnel
 import (
 	"testing"
 
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 
 	"github.com/stretchr/testify/assert"
 )
@@ -11,12 +11,12 @@ import (
 func TestSingleTag(t *testing.T) {
 	testCases := []struct {
 		Input  string
-		Output tunnelpogs.Tag
+		Output pogs.Tag
 		Fail   bool
 	}{
-		{Input: "x=y", Output: tunnelpogs.Tag{Name: "x", Value: "y"}},
-		{Input: "More-Complex=Tag Values", Output: tunnelpogs.Tag{Name: "More-Complex", Value: "Tag Values"}},
-		{Input: "First=Equals=Wins", Output: tunnelpogs.Tag{Name: "First", Value: "Equals=Wins"}},
+		{Input: "x=y", Output: pogs.Tag{Name: "x", Value: "y"}},
+		{Input: "More-Complex=Tag Values", Output: pogs.Tag{Name: "More-Complex", Value: "Tag Values"}},
+		{Input: "First=Equals=Wins", Output: pogs.Tag{Name: "First", Value: "Equals=Wins"}},
 		{Input: "x=", Fail: true},
 		{Input: "=y", Fail: true},
 		{Input: "=", Fail: true},

connection/connection.go

@@ -42,7 +42,7 @@ type Orchestrator interface {
 	GetOriginProxy() (OriginProxy, error)
 }
 
-type NamedTunnelProperties struct {
+type TunnelProperties struct {
 	Credentials    Credentials
 	Client         pogs.ClientInfo
 	QuickTunnelUrl string

connection/control.go

@@ -6,25 +6,25 @@ import (
 	"net"
 	"time"
 
-	"github.com/rs/zerolog"
-
 	"github.com/cloudflare/cloudflared/management"
+	"github.com/cloudflare/cloudflared/tunnelrpc"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
-// RPCClientFunc derives a named tunnel rpc client that can then be used to register and unregister connections.
-type RPCClientFunc func(context.Context, io.ReadWriteCloser, *zerolog.Logger) NamedTunnelRPCClient
+// registerClient derives a named tunnel rpc client that can then be used to register and unregister connections.
+type registerClientFunc func(context.Context, io.ReadWriteCloser, time.Duration) tunnelrpc.RegistrationClient
 
 type controlStream struct {
 	observer *Observer
 
-	connectedFuse         ConnectedFuse
-	namedTunnelProperties *NamedTunnelProperties
-	connIndex             uint8
-	edgeAddress           net.IP
-	protocol              Protocol
+	connectedFuse    ConnectedFuse
+	tunnelProperties *TunnelProperties
+	connIndex        uint8
+	edgeAddress      net.IP
+	protocol         Protocol
 
-	newRPCClientFunc RPCClientFunc
+	registerClientFunc registerClientFunc
+	registerTimeout    time.Duration
 
 	gracefulShutdownC <-chan struct{}
 	gracePeriod       time.Duration
@@ -47,27 +47,29 @@ type TunnelConfigJSONGetter interface {
 func NewControlStream(
 	observer *Observer,
 	connectedFuse ConnectedFuse,
-	namedTunnelConfig *NamedTunnelProperties,
+	tunnelProperties *TunnelProperties,
 	connIndex uint8,
 	edgeAddress net.IP,
-	newRPCClientFunc RPCClientFunc,
+	registerClientFunc registerClientFunc,
+	registerTimeout time.Duration,
 	gracefulShutdownC <-chan struct{},
 	gracePeriod time.Duration,
 	protocol Protocol,
 ) ControlStreamHandler {
-	if newRPCClientFunc == nil {
-		newRPCClientFunc = newRegistrationRPCClient
+	if registerClientFunc == nil {
+		registerClientFunc = tunnelrpc.NewRegistrationClient
 	}
 	return &controlStream{
-		observer:              observer,
-		connectedFuse:         connectedFuse,
-		namedTunnelProperties: namedTunnelConfig,
-		newRPCClientFunc:      newRPCClientFunc,
-		connIndex:             connIndex,
-		edgeAddress:           edgeAddress,
-		gracefulShutdownC:     gracefulShutdownC,
-		gracePeriod:           gracePeriod,
-		protocol:              protocol,
+		observer:           observer,
+		connectedFuse:      connectedFuse,
+		tunnelProperties:   tunnelProperties,
+		registerClientFunc: registerClientFunc,
+		registerTimeout:    registerTimeout,
+		connIndex:          connIndex,
+		edgeAddress:        edgeAddress,
+		gracefulShutdownC:  gracefulShutdownC,
+		gracePeriod:        gracePeriod,
+		protocol:           protocol,
 	}
 }
 
@@ -77,13 +79,25 @@ func (c *controlStream) ServeControlStream(
 	connOptions *tunnelpogs.ConnectionOptions,
 	tunnelConfigGetter TunnelConfigJSONGetter,
 ) error {
-	rpcClient := c.newRPCClientFunc(ctx, rw, c.observer.log)
+	registrationClient := c.registerClientFunc(ctx, rw, c.registerTimeout)
 
-	registrationDetails, err := rpcClient.RegisterConnection(ctx, c.namedTunnelProperties, connOptions, c.connIndex, c.edgeAddress, c.observer)
+	registrationDetails, err := registrationClient.RegisterConnection(
+		ctx,
+		c.tunnelProperties.Credentials.Auth(),
+		c.tunnelProperties.Credentials.TunnelID,
+		connOptions,
+		c.connIndex,
+		c.edgeAddress)
 	if err != nil {
-		rpcClient.Close()
-		return err
+		defer registrationClient.Close()
+		if err.Error() == DuplicateConnectionError {
+			c.observer.metrics.regFail.WithLabelValues("dup_edge_conn", "registerConnection").Inc()
+			return errDuplicationConnection
+		}
+		c.observer.metrics.regFail.WithLabelValues("server_error", "registerConnection").Inc()
+		return serverRegistrationErrorFromRPC(err)
 	}
+	c.observer.metrics.regSuccess.WithLabelValues("registerConnection").Inc()
 
 	c.observer.logConnected(registrationDetails.UUID, c.connIndex, registrationDetails.Location, c.edgeAddress, c.protocol)
 	c.observer.sendConnectedEvent(c.connIndex, c.protocol, registrationDetails.Location)
@@ -92,21 +106,23 @@ func (c *controlStream) ServeControlStream(
 	// if conn index is 0 and tunnel is not remotely managed, then send local ingress rules configuration
 	if c.connIndex == 0 && !registrationDetails.TunnelIsRemotelyManaged {
 		if tunnelConfig, err := tunnelConfigGetter.GetConfigJSON(); err == nil {
-			if err := rpcClient.SendLocalConfiguration(ctx, tunnelConfig, c.observer); err != nil {
+			if err := registrationClient.SendLocalConfiguration(ctx, tunnelConfig); err != nil {
+				c.observer.metrics.localConfigMetrics.pushesErrors.Inc()
 				c.observer.log.Err(err).Msg("unable to send local configuration")
 			}
+			c.observer.metrics.localConfigMetrics.pushes.Inc()
 		} else {
 			c.observer.log.Err(err).Msg("failed to obtain current configuration")
 		}
 	}
 
-	c.waitForUnregister(ctx, rpcClient)
+	c.waitForUnregister(ctx, registrationClient)
 	return nil
 }
 
-func (c *controlStream) waitForUnregister(ctx context.Context, rpcClient NamedTunnelRPCClient) {
+func (c *controlStream) waitForUnregister(ctx context.Context, registrationClient tunnelrpc.RegistrationClient) {
 	// wait for connection termination or start of graceful shutdown
-	defer rpcClient.Close()
+	defer registrationClient.Close()
 	select {
 	case <-ctx.Done():
 		break
@@ -115,7 +131,7 @@ func (c *controlStream) waitForUnregister(ctx context.Context, rpcClient NamedTu
 	}
 
 	c.observer.sendUnregisteringEvent(c.connIndex)
-	rpcClient.GracefulShutdown(ctx, c.gracePeriod)
+	registrationClient.GracefulShutdown(ctx, c.gracePeriod)
 	c.observer.log.Info().
 		Int(management.EventTypeKey, int(management.Cloudflared)).
 		Uint8(LogFieldConnIndex, c.connIndex).

connection/http2.go

@@ -40,8 +40,6 @@ type HTTP2Connection struct {
 	connOptions  *tunnelpogs.ConnectionOptions
 	observer     *Observer
 	connIndex    uint8
-	// newRPCClientFunc allows us to mock RPCs during testing
-	newRPCClientFunc func(context.Context, io.ReadWriteCloser, *zerolog.Logger) NamedTunnelRPCClient
 
 	log                  *zerolog.Logger
 	activeRequestsWG     sync.WaitGroup
@@ -69,7 +67,6 @@ func NewHTTP2Connection(
 		connOptions:          connOptions,
 		observer:             observer,
 		connIndex:            connIndex,
-		newRPCClientFunc:     newRegistrationRPCClient,
 		controlStreamHandler: controlStreamHandler,
 		log:                  log,
 	}

connection/http2_test.go

@@ -20,8 +20,8 @@ import (
 	"github.com/stretchr/testify/require"
 	"golang.org/x/net/http2"
 
+	"github.com/cloudflare/cloudflared/tunnelrpc"
 	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
 var (
@@ -36,10 +36,11 @@ func newTestHTTP2Connection() (*HTTP2Connection, net.Conn) {
 	controlStream := NewControlStream(
 		obs,
 		mockConnectedFuse{},
-		&NamedTunnelProperties{},
+		&TunnelProperties{},
 		connIndex,
 		nil,
 		nil,
+		1*time.Second,
 		nil,
 		1*time.Second,
 		HTTP2,
@@ -168,23 +169,23 @@ type mockNamedTunnelRPCClient struct {
 	unregistered chan struct{}
 }
 
-func (mc mockNamedTunnelRPCClient) SendLocalConfiguration(c context.Context, config []byte, observer *Observer) error {
+func (mc mockNamedTunnelRPCClient) SendLocalConfiguration(c context.Context, config []byte) error {
 	return nil
 }
 
 func (mc mockNamedTunnelRPCClient) RegisterConnection(
-	c context.Context,
-	properties *NamedTunnelProperties,
-	options *tunnelpogs.ConnectionOptions,
+	ctx context.Context,
+	auth pogs.TunnelAuth,
+	tunnelID uuid.UUID,
+	options *pogs.ConnectionOptions,
 	connIndex uint8,
 	edgeAddress net.IP,
-	observer *Observer,
-) (*tunnelpogs.ConnectionDetails, error) {
+) (*pogs.ConnectionDetails, error) {
 	if mc.shouldFail != nil {
 		return nil, mc.shouldFail
 	}
 	close(mc.registered)
-	return &tunnelpogs.ConnectionDetails{
+	return &pogs.ConnectionDetails{
 		Location:                "LIS",
 		UUID:                    uuid.New(),
 		TunnelIsRemotelyManaged: false,
@@ -203,8 +204,8 @@ type mockRPCClientFactory struct {
 	unregistered chan struct{}
 }
 
-func (mf *mockRPCClientFactory) newMockRPCClient(context.Context, io.ReadWriteCloser, *zerolog.Logger) NamedTunnelRPCClient {
-	return mockNamedTunnelRPCClient{
+func (mf *mockRPCClientFactory) newMockRPCClient(context.Context, io.ReadWriteCloser, time.Duration) tunnelrpc.RegistrationClient {
+	return &mockNamedTunnelRPCClient{
 		shouldFail:   mf.shouldFail,
 		registered:   mf.registered,
 		unregistered: mf.unregistered,
@@ -360,10 +361,11 @@ func TestServeControlStream(t *testing.T) {
 	controlStream := NewControlStream(
 		obs,
 		mockConnectedFuse{},
-		&NamedTunnelProperties{},
+		&TunnelProperties{},
 		1,
 		nil,
 		rpcClientFactory.newMockRPCClient,
+		1*time.Second,
 		nil,
 		1*time.Second,
 		HTTP2,
@@ -412,10 +414,11 @@ func TestFailRegistration(t *testing.T) {
 	controlStream := NewControlStream(
 		obs,
 		mockConnectedFuse{},
-		&NamedTunnelProperties{},
+		&TunnelProperties{},
 		http2Conn.connIndex,
 		nil,
 		rpcClientFactory.newMockRPCClient,
+		1*time.Second,
 		nil,
 		1*time.Second,
 		HTTP2,
@@ -460,10 +463,11 @@ func TestGracefulShutdownHTTP2(t *testing.T) {
 	controlStream := NewControlStream(
 		obs,
 		mockConnectedFuse{},
-		&NamedTunnelProperties{},
+		&TunnelProperties{},
 		http2Conn.connIndex,
 		nil,
 		rpcClientFactory.newMockRPCClient,
+		1*time.Second,
 		shutdownC,
 		1*time.Second,
 		HTTP2,

connection/metrics.go

@@ -43,7 +43,6 @@ type localConfigMetrics struct {
 }
 
 type tunnelMetrics struct {
-	timerRetries    prometheus.Gauge
 	serverLocations *prometheus.GaugeVec
 	// locationLock is a mutex for oldServerLocations
 	locationLock sync.Mutex
@@ -351,15 +350,6 @@ func initTunnelMetrics() *tunnelMetrics {
 	)
 	prometheus.MustRegister(maxConcurrentRequestsPerTunnel)
 
-	timerRetries := prometheus.NewGauge(
-		prometheus.GaugeOpts{
-			Namespace: MetricsNamespace,
-			Subsystem: TunnelSubsystem,
-			Name:      "timer_retries",
-			Help:      "Unacknowledged heart beats count",
-		})
-	prometheus.MustRegister(timerRetries)
-
 	serverLocations := prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Namespace: MetricsNamespace,
@@ -416,7 +406,6 @@ func initTunnelMetrics() *tunnelMetrics {
 	prometheus.MustRegister(registerSuccess)
 
 	return &tunnelMetrics{
-		timerRetries:        timerRetries,
 		serverLocations:     serverLocations,
 		oldServerLocations:  make(map[string]string),
 		muxerMetrics:        newMuxerMetrics(),

connection/quic.go

@@ -28,9 +28,11 @@ import (
 	"github.com/cloudflare/cloudflared/ingress"
 	"github.com/cloudflare/cloudflared/management"
 	"github.com/cloudflare/cloudflared/packet"
-	quicpogs "github.com/cloudflare/cloudflared/quic"
+	cfdquic "github.com/cloudflare/cloudflared/quic"
 	"github.com/cloudflare/cloudflared/tracing"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	rpcquic "github.com/cloudflare/cloudflared/tunnelrpc/quic"
 )
 
 const (
@@ -59,14 +61,14 @@ type QUICConnection struct {
 	// sessionManager tracks active sessions. It receives datagrams from quic connection via datagramMuxer
 	sessionManager datagramsession.Manager
 	// datagramMuxer mux/demux datagrams from quic connection
-	datagramMuxer        *quicpogs.DatagramMuxerV2
+	datagramMuxer        *cfdquic.DatagramMuxerV2
 	packetRouter         *ingress.PacketRouter
 	controlStreamHandler ControlStreamHandler
 	connOptions          *tunnelpogs.ConnectionOptions
 	connIndex            uint8
 
-	udpUnregisterTimeout time.Duration
-	streamWriteTimeout   time.Duration
+	rpcTimeout         time.Duration
+	streamWriteTimeout time.Duration
 }
 
 // NewQUICConnection returns a new instance of QUICConnection.
@@ -82,7 +84,7 @@ func NewQUICConnection(
 	controlStreamHandler ControlStreamHandler,
 	logger *zerolog.Logger,
 	packetRouterConfig *ingress.GlobalRouterConfig,
-	udpUnregisterTimeout time.Duration,
+	rpcTimeout time.Duration,
 	streamWriteTimeout time.Duration,
 ) (*QUICConnection, error) {
 	udpConn, err := createUDPConnForConnIndex(connIndex, localAddr, logger)
@@ -104,7 +106,7 @@ func NewQUICConnection(
 	}
 
 	sessionDemuxChan := make(chan *packet.Session, demuxChanCapacity)
-	datagramMuxer := quicpogs.NewDatagramMuxerV2(session, logger, sessionDemuxChan)
+	datagramMuxer := cfdquic.NewDatagramMuxerV2(session, logger, sessionDemuxChan)
 	sessionManager := datagramsession.NewManager(logger, datagramMuxer.SendToSession, sessionDemuxChan)
 	packetRouter := ingress.NewPacketRouter(packetRouterConfig, datagramMuxer, logger)
 
@@ -118,7 +120,7 @@ func NewQUICConnection(
 		controlStreamHandler: controlStreamHandler,
 		connOptions:          connOptions,
 		connIndex:            connIndex,
-		udpUnregisterTimeout: udpUnregisterTimeout,
+		rpcTimeout:           rpcTimeout,
 		streamWriteTimeout:   streamWriteTimeout,
 	}, nil
 }
@@ -198,7 +200,7 @@ func (q *QUICConnection) acceptStream(ctx context.Context) error {
 
 func (q *QUICConnection) runStream(quicStream quic.Stream) {
 	ctx := quicStream.Context()
-	stream := quicpogs.NewSafeStreamCloser(quicStream, q.streamWriteTimeout, q.logger)
+	stream := cfdquic.NewSafeStreamCloser(quicStream, q.streamWriteTimeout, q.logger)
 	defer stream.Close()
 
 	// we are going to fuse readers/writers from stream <- cloudflared -> origin, and we want to guarantee that
@@ -206,7 +208,8 @@ func (q *QUICConnection) runStream(quicStream quic.Stream) {
 	// So, we wrap the stream with a no-op write closer and only this method can actually close write side of the stream.
 	// A call to close will simulate a close to the read-side, which will fail subsequent reads.
 	noCloseStream := &nopCloserReadWriter{ReadWriteCloser: stream}
-	if err := q.handleStream(ctx, noCloseStream); err != nil {
+	ss := rpcquic.NewCloudflaredServer(q.handleDataStream, q, q, q.rpcTimeout)
+	if err := ss.Serve(ctx, noCloseStream); err != nil {
 		q.logger.Debug().Err(err).Msg("Failed to handle QUIC stream")
 
 		// if we received an error at this level, then close write side of stream with an error, which will result in
@@ -215,30 +218,7 @@ func (q *QUICConnection) runStream(quicStream quic.Stream) {
 	}
 }
 
-func (q *QUICConnection) handleStream(ctx context.Context, stream io.ReadWriteCloser) error {
-	signature, err := quicpogs.DetermineProtocol(stream)
-	if err != nil {
-		return err
-	}
-	switch signature {
-	case quicpogs.DataStreamProtocolSignature:
-		reqServerStream, err := quicpogs.NewRequestServerStream(stream, signature)
-		if err != nil {
-			return err
-		}
-		return q.handleDataStream(ctx, reqServerStream)
-	case quicpogs.RPCStreamProtocolSignature:
-		rpcStream, err := quicpogs.NewRPCServerStream(stream, signature)
-		if err != nil {
-			return err
-		}
-		return q.handleRPCStream(rpcStream)
-	default:
-		return fmt.Errorf("unknown protocol %v", signature)
-	}
-}
-
-func (q *QUICConnection) handleDataStream(ctx context.Context, stream *quicpogs.RequestServerStream) error {
+func (q *QUICConnection) handleDataStream(ctx context.Context, stream *rpcquic.RequestServerStream) error {
 	request, err := stream.ReadConnectRequestData()
 	if err != nil {
 		return err
@@ -264,22 +244,22 @@ func (q *QUICConnection) handleDataStream(ctx context.Context, stream *quicpogs.
 // dispatchRequest will dispatch the request depending on the type and returns an error if it occurs.
 // More importantly, it also tells if the during processing of the request the ConnectResponse metadata was sent downstream.
 // This is important since it informs
-func (q *QUICConnection) dispatchRequest(ctx context.Context, stream *quicpogs.RequestServerStream, err error, request *quicpogs.ConnectRequest) (error, bool) {
+func (q *QUICConnection) dispatchRequest(ctx context.Context, stream *rpcquic.RequestServerStream, err error, request *pogs.ConnectRequest) (error, bool) {
 	originProxy, err := q.orchestrator.GetOriginProxy()
 	if err != nil {
 		return err, false
 	}
 
 	switch request.Type {
-	case quicpogs.ConnectionTypeHTTP, quicpogs.ConnectionTypeWebsocket:
+	case pogs.ConnectionTypeHTTP, pogs.ConnectionTypeWebsocket:
 		tracedReq, err := buildHTTPRequest(ctx, request, stream, q.connIndex, q.logger)
 		if err != nil {
 			return err, false
 		}
 		w := newHTTPResponseAdapter(stream)
-		return originProxy.ProxyHTTP(&w, tracedReq, request.Type == quicpogs.ConnectionTypeWebsocket), w.connectResponseSent
+		return originProxy.ProxyHTTP(&w, tracedReq, request.Type == pogs.ConnectionTypeWebsocket), w.connectResponseSent
 
-	case quicpogs.ConnectionTypeTCP:
+	case pogs.ConnectionTypeTCP:
 		rwa := &streamReadWriteAcker{RequestServerStream: stream}
 		metadata := request.MetadataMap()
 		return originProxy.ProxyTCP(ctx, rwa, &TCPRequest{
@@ -293,14 +273,6 @@ func (q *QUICConnection) dispatchRequest(ctx context.Context, stream *quicpogs.R
 	}
 }
 
-func (q *QUICConnection) handleRPCStream(rpcStream *quicpogs.RPCServerStream) error {
-	if err := rpcStream.Serve(q, q, q.logger); err != nil {
-		q.logger.Err(err).Msg("failed handling RPC stream")
-	}
-
-	return nil
-}
-
 // RegisterUdpSession is the RPC method invoked by edge to register and run a session
 func (q *QUICConnection) RegisterUdpSession(ctx context.Context, sessionID uuid.UUID, dstIP net.IP, dstPort uint16, closeAfterIdleHint time.Duration, traceContext string) (*tunnelpogs.RegisterUdpSessionResponse, error) {
 	traceCtx := tracing.NewTracedContext(ctx, traceContext, q.logger)
@@ -377,9 +349,9 @@ func (q *QUICConnection) closeUDPSession(ctx context.Context, sessionID uuid.UUI
 		return
 	}
 
-	stream := quicpogs.NewSafeStreamCloser(quicStream, q.streamWriteTimeout, q.logger)
+	stream := cfdquic.NewSafeStreamCloser(quicStream, q.streamWriteTimeout, q.logger)
 	defer stream.Close()
-	rpcClientStream, err := quicpogs.NewRPCClientStream(ctx, stream, q.udpUnregisterTimeout, q.logger)
+	rpcClientStream, err := rpcquic.NewSessionClient(ctx, stream, q.rpcTimeout)
 	if err != nil {
 		// Log this at debug because this is not an error if session was closed due to lost connection
 		// with edge
@@ -408,16 +380,16 @@ func (q *QUICConnection) UpdateConfiguration(ctx context.Context, version int32,
 // streamReadWriteAcker is a light wrapper over QUIC streams with a callback to send response back to
 // the client.
 type streamReadWriteAcker struct {
-	*quicpogs.RequestServerStream
+	*rpcquic.RequestServerStream
 	connectResponseSent bool
 }
 
 // AckConnection acks response back to the proxy.
 func (s *streamReadWriteAcker) AckConnection(tracePropagation string) error {
-	metadata := []quicpogs.Metadata{}
+	metadata := []pogs.Metadata{}
 	// Only add tracing if provided by origintunneld
 	if tracePropagation != "" {
-		metadata = append(metadata, quicpogs.Metadata{
+		metadata = append(metadata, pogs.Metadata{
 			Key: tracing.CanonicalCloudflaredTracingHeader,
 			Val: tracePropagation,
 		})
@@ -428,12 +400,12 @@ func (s *streamReadWriteAcker) AckConnection(tracePropagation string) error {
 
 // httpResponseAdapter translates responses written by the HTTP Proxy into ones that can be used in QUIC.
 type httpResponseAdapter struct {
-	*quicpogs.RequestServerStream
+	*rpcquic.RequestServerStream
 	headers             http.Header
 	connectResponseSent bool
 }
 
-func newHTTPResponseAdapter(s *quicpogs.RequestServerStream) httpResponseAdapter {
+func newHTTPResponseAdapter(s *rpcquic.RequestServerStream) httpResponseAdapter {
 	return httpResponseAdapter{RequestServerStream: s, headers: make(http.Header)}
 }
 
@@ -442,12 +414,12 @@ func (hrw *httpResponseAdapter) AddTrailer(trailerName, trailerValue string) {
 }
 
 func (hrw *httpResponseAdapter) WriteRespHeaders(status int, header http.Header) error {
-	metadata := make([]quicpogs.Metadata, 0)
-	metadata = append(metadata, quicpogs.Metadata{Key: "HttpStatus", Val: strconv.Itoa(status)})
+	metadata := make([]pogs.Metadata, 0)
+	metadata = append(metadata, pogs.Metadata{Key: "HttpStatus", Val: strconv.Itoa(status)})
 	for k, vv := range header {
 		for _, v := range vv {
 			httpHeaderKey := fmt.Sprintf("%s:%s", HTTPHeaderKey, k)
-			metadata = append(metadata, quicpogs.Metadata{Key: httpHeaderKey, Val: v})
+			metadata = append(metadata, pogs.Metadata{Key: httpHeaderKey, Val: v})
 		}
 	}
 
@@ -483,17 +455,17 @@ func (hrw *httpResponseAdapter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
 }
 
 func (hrw *httpResponseAdapter) WriteErrorResponse(err error) {
-	hrw.WriteConnectResponseData(err, quicpogs.Metadata{Key: "HttpStatus", Val: strconv.Itoa(http.StatusBadGateway)})
+	hrw.WriteConnectResponseData(err, pogs.Metadata{Key: "HttpStatus", Val: strconv.Itoa(http.StatusBadGateway)})
 }
 
-func (hrw *httpResponseAdapter) WriteConnectResponseData(respErr error, metadata ...quicpogs.Metadata) error {
+func (hrw *httpResponseAdapter) WriteConnectResponseData(respErr error, metadata ...pogs.Metadata) error {
 	hrw.connectResponseSent = true
 	return hrw.RequestServerStream.WriteConnectResponseData(respErr, metadata...)
 }
 
 func buildHTTPRequest(
 	ctx context.Context,
-	connectRequest *quicpogs.ConnectRequest,
+	connectRequest *pogs.ConnectRequest,
 	body io.ReadCloser,
 	connIndex uint8,
 	log *zerolog.Logger,
@@ -502,7 +474,7 @@ func buildHTTPRequest(
 	dest := connectRequest.Dest
 	method := metadata[HTTPMethodKey]
 	host := metadata[HTTPHostKey]
-	isWebsocket := connectRequest.Type == quicpogs.ConnectionTypeWebsocket
+	isWebsocket := connectRequest.Type == pogs.ConnectionTypeWebsocket
 
 	req, err := http.NewRequestWithContext(ctx, method, dest, body)
 	if err != nil {
@@ -597,11 +569,11 @@ func (np *nopCloserReadWriter) Close() error {
 
 // muxerWrapper wraps DatagramMuxerV2 to satisfy the packet.FunnelUniPipe interface
 type muxerWrapper struct {
-	muxer *quicpogs.DatagramMuxerV2
+	muxer *cfdquic.DatagramMuxerV2
 }
 
 func (rp *muxerWrapper) SendPacket(dst netip.Addr, pk packet.RawPacket) error {
-	return rp.muxer.SendPacket(quicpogs.RawPacket(pk))
+	return rp.muxer.SendPacket(cfdquic.RawPacket(pk))
 }
 
 func (rp *muxerWrapper) ReceivePacket(ctx context.Context) (packet.RawPacket, error) {
@@ -609,7 +581,7 @@ func (rp *muxerWrapper) ReceivePacket(ctx context.Context) (packet.RawPacket, er
 	if err != nil {
 		return packet.RawPacket{}, err
 	}
-	rawPacket, ok := pk.(quicpogs.RawPacket)
+	rawPacket, ok := pk.(cfdquic.RawPacket)
 	if ok {
 		return packet.RawPacket(rawPacket), nil
 	}

connection/quic_test.go

@@ -3,9 +3,14 @@ package connection
 import (
 	"bytes"
 	"context"
+	"crypto/rand"
+	"crypto/rsa"
 	"crypto/tls"
+	"crypto/x509"
+	"encoding/pem"
 	"fmt"
 	"io"
+	"math/big"
 	"net"
 	"net/http"
 	"net/url"
@@ -23,14 +28,15 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/cloudflare/cloudflared/datagramsession"
-	quicpogs "github.com/cloudflare/cloudflared/quic"
+	cfdquic "github.com/cloudflare/cloudflared/quic"
 	"github.com/cloudflare/cloudflared/tracing"
 	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	rpcquic "github.com/cloudflare/cloudflared/tunnelrpc/quic"
 )
 
 var (
-	testTLSServerConfig = quicpogs.GenerateTLSConfig()
+	testTLSServerConfig = GenerateTLSConfig()
 	testQUICConfig      = &quic.Config{
 		KeepAlivePeriod: 5 * time.Second,
 		EnableDatagrams: true,
@@ -50,16 +56,16 @@ func TestQUICServer(t *testing.T) {
 	var tests = []struct {
 		desc             string
 		dest             string
-		connectionType   quicpogs.ConnectionType
-		metadata         []quicpogs.Metadata
+		connectionType   pogs.ConnectionType
+		metadata         []pogs.Metadata
 		message          []byte
 		expectedResponse []byte
 	}{
 		{
 			desc:           "test http proxy",
 			dest:           "/ok",
-			connectionType: quicpogs.ConnectionTypeHTTP,
-			metadata: []quicpogs.Metadata{
+			connectionType: pogs.ConnectionTypeHTTP,
+			metadata: []pogs.Metadata{
 				{
 					Key: "HttpHeader:Cf-Ray",
 					Val: "123123123",
@@ -78,8 +84,8 @@ func TestQUICServer(t *testing.T) {
 		{
 			desc:           "test http body request streaming",
 			dest:           "/slow_echo_body",
-			connectionType: quicpogs.ConnectionTypeHTTP,
-			metadata: []quicpogs.Metadata{
+			connectionType: pogs.ConnectionTypeHTTP,
+			metadata: []pogs.Metadata{
 				{
 					Key: "HttpHeader:Cf-Ray",
 					Val: "123123123",
@@ -103,8 +109,8 @@ func TestQUICServer(t *testing.T) {
 		{
 			desc:           "test ws proxy",
 			dest:           "/ws/echo",
-			connectionType: quicpogs.ConnectionTypeWebsocket,
-			metadata: []quicpogs.Metadata{
+			connectionType: pogs.ConnectionTypeWebsocket,
+			metadata: []pogs.Metadata{
 				{
 					Key: "HttpHeader:Cf-Cloudflared-Proxy-Connection-Upgrade",
 					Val: "Websocket",
@@ -127,8 +133,8 @@ func TestQUICServer(t *testing.T) {
 		},
 		{
 			desc:             "test tcp proxy",
-			connectionType:   quicpogs.ConnectionTypeTCP,
-			metadata:         []quicpogs.Metadata{},
+			connectionType:   pogs.ConnectionTypeTCP,
+			metadata:         []pogs.Metadata{},
 			message:          []byte("Here is some tcp data"),
 			expectedResponse: []byte("Here is some tcp data"),
 		},
@@ -175,7 +181,7 @@ type fakeControlStream struct {
 	ControlStreamHandler
 }
 
-func (fakeControlStream) ServeControlStream(ctx context.Context, rw io.ReadWriteCloser, connOptions *tunnelpogs.ConnectionOptions, tunnelConfigGetter TunnelConfigJSONGetter) error {
+func (fakeControlStream) ServeControlStream(ctx context.Context, rw io.ReadWriteCloser, connOptions *pogs.ConnectionOptions, tunnelConfigGetter TunnelConfigJSONGetter) error {
 	<-ctx.Done()
 	return nil
 }
@@ -188,8 +194,8 @@ func quicServer(
 	t *testing.T,
 	listener *quic.Listener,
 	dest string,
-	connectionType quicpogs.ConnectionType,
-	metadata []quicpogs.Metadata,
+	connectionType pogs.ConnectionType,
+	metadata []pogs.Metadata,
 	message []byte,
 	expectedResponse []byte,
 ) {
@@ -198,9 +204,9 @@ func quicServer(
 
 	quicStream, err := session.OpenStreamSync(context.Background())
 	require.NoError(t, err)
-	stream := quicpogs.NewSafeStreamCloser(quicStream, defaultQUICTimeout, &log)
+	stream := cfdquic.NewSafeStreamCloser(quicStream, defaultQUICTimeout, &log)
 
-	reqClientStream := quicpogs.RequestClientStream{ReadWriteCloser: stream}
+	reqClientStream := rpcquic.RequestClientStream{ReadWriteCloser: stream}
 	err = reqClientStream.WriteConnectRequestData(dest, connectionType, metadata...)
 	require.NoError(t, err)
 
@@ -265,15 +271,15 @@ func (moc *mockOriginProxyWithRequest) ProxyHTTP(w ResponseWriter, tr *tracing.T
 func TestBuildHTTPRequest(t *testing.T) {
 	var tests = []struct {
 		name           string
-		connectRequest *quicpogs.ConnectRequest
+		connectRequest *pogs.ConnectRequest
 		body           io.ReadCloser
 		req            *http.Request
 	}{
 		{
 			name: "check if http.Request is built correctly with content length",
-			connectRequest: &quicpogs.ConnectRequest{
+			connectRequest: &pogs.ConnectRequest{
 				Dest: "http://test.com",
-				Metadata: []quicpogs.Metadata{
+				Metadata: []pogs.Metadata{
 					{
 						Key: "HttpHeader:Cf-Cloudflared-Proxy-Connection-Upgrade",
 						Val: "Websocket",
@@ -317,9 +323,9 @@ func TestBuildHTTPRequest(t *testing.T) {
 		},
 		{
 			name: "if content length isn't part of request headers, then it's not set",
-			connectRequest: &quicpogs.ConnectRequest{
+			connectRequest: &pogs.ConnectRequest{
 				Dest: "http://test.com",
-				Metadata: []quicpogs.Metadata{
+				Metadata: []pogs.Metadata{
 					{
 						Key: "HttpHeader:Cf-Cloudflared-Proxy-Connection-Upgrade",
 						Val: "Websocket",
@@ -358,9 +364,9 @@ func TestBuildHTTPRequest(t *testing.T) {
 		},
 		{
 			name: "if content length is 0, but transfer-encoding is chunked, body is not nil",
-			connectRequest: &quicpogs.ConnectRequest{
+			connectRequest: &pogs.ConnectRequest{
 				Dest: "http://test.com",
-				Metadata: []quicpogs.Metadata{
+				Metadata: []pogs.Metadata{
 					{
 						Key: "HttpHeader:Another-Header",
 						Val: "Misc",
@@ -400,9 +406,9 @@ func TestBuildHTTPRequest(t *testing.T) {
 		},
 		{
 			name: "if content length is 0, but transfer-encoding is gzip,chunked, body is not nil",
-			connectRequest: &quicpogs.ConnectRequest{
+			connectRequest: &pogs.ConnectRequest{
 				Dest: "http://test.com",
-				Metadata: []quicpogs.Metadata{
+				Metadata: []pogs.Metadata{
 					{
 						Key: "HttpHeader:Another-Header",
 						Val: "Misc",
@@ -442,10 +448,10 @@ func TestBuildHTTPRequest(t *testing.T) {
 		},
 		{
 			name: "if content length is 0, and connect request is a websocket, body is not nil",
-			connectRequest: &quicpogs.ConnectRequest{
-				Type: quicpogs.ConnectionTypeWebsocket,
+			connectRequest: &pogs.ConnectRequest{
+				Type: pogs.ConnectionTypeWebsocket,
 				Dest: "http://test.com",
-				Metadata: []quicpogs.Metadata{
+				Metadata: []pogs.Metadata{
 					{
 						Key: "HttpHeader:Another-Header",
 						Val: "Misc",
@@ -617,9 +623,9 @@ func serveSession(ctx context.Context, qc *QUICConnection, edgeQUICSession quic.
 	}()
 
 	// Send a message to the quic session on edge side, it should be deumx to this datagram v2 session
-	muxedPayload, err := quicpogs.SuffixSessionID(sessionID, payload)
+	muxedPayload, err := cfdquic.SuffixSessionID(sessionID, payload)
 	require.NoError(t, err)
-	muxedPayload, err = quicpogs.SuffixType(muxedPayload, quicpogs.DatagramTypeUDP)
+	muxedPayload, err = cfdquic.SuffixType(muxedPayload, cfdquic.DatagramTypeUDP)
 	require.NoError(t, err)
 
 	err = edgeQUICSession.SendDatagram(muxedPayload)
@@ -665,7 +671,7 @@ const (
 	closedByTimeout
 )
 
-func runRPCServer(ctx context.Context, session quic.Connection, sessionRPCServer tunnelpogs.SessionManager, configRPCServer tunnelpogs.ConfigurationManager, t *testing.T) {
+func runRPCServer(ctx context.Context, session quic.Connection, sessionRPCServer pogs.SessionManager, configRPCServer pogs.ConfigurationManager, t *testing.T) {
 	stream, err := session.AcceptStream(ctx)
 	require.NoError(t, err)
 
@@ -674,13 +680,15 @@ func runRPCServer(ctx context.Context, session quic.Connection, sessionRPCServer
 		stream, err = session.AcceptStream(ctx)
 		require.NoError(t, err)
 	}
-	protocol, err := quicpogs.DetermineProtocol(stream)
-	assert.NoError(t, err)
-	rpcServerStream, err := quicpogs.NewRPCServerStream(stream, protocol)
-	assert.NoError(t, err)
-
-	log := zerolog.New(os.Stdout)
-	err = rpcServerStream.Serve(sessionRPCServer, configRPCServer, &log)
+	ss := rpcquic.NewCloudflaredServer(
+		func(_ context.Context, _ *rpcquic.RequestServerStream) error {
+			return nil
+		},
+		sessionRPCServer,
+		configRPCServer,
+		10*time.Second,
+	)
+	err = ss.Serve(ctx, stream)
 	assert.NoError(t, err)
 }
 
@@ -726,7 +734,7 @@ func testQUICConnection(udpListenerAddr net.Addr, t *testing.T, index uint8) *QU
 		fakeControlStream{},
 		&log,
 		nil,
-		5*time.Second,
+		15*time.Second,
 		0*time.Second,
 	)
 	require.NoError(t, err)
@@ -744,3 +752,27 @@ func (m *mockReaderNoopWriter) Write(p []byte) (n int, err error) {
 func (m *mockReaderNoopWriter) Close() error {
 	return nil
 }
+
+// GenerateTLSConfig sets up a bare-bones TLS config for a QUIC server
+func GenerateTLSConfig() *tls.Config {
+	key, err := rsa.GenerateKey(rand.Reader, 1024)
+	if err != nil {
+		panic(err)
+	}
+	template := x509.Certificate{SerialNumber: big.NewInt(1)}
+	certDER, err := x509.CreateCertificate(rand.Reader, &template, &template, &key.PublicKey, key)
+	if err != nil {
+		panic(err)
+	}
+	keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)})
+	certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER})
+
+	tlsCert, err := tls.X509KeyPair(certPEM, keyPEM)
+	if err != nil {
+		panic(err)
+	}
+	return &tls.Config{
+		Certificates: []tls.Certificate{tlsCert},
+		NextProtos:   []string{"argotunnel"},
+	}
+}

connection/rpc.go

@@ -1,153 +0,0 @@
-package connection
-
-import (
-	"context"
-	"io"
-	"net"
-	"time"
-
-	"github.com/rs/zerolog"
-	"zombiezen.com/go/capnproto2/rpc"
-
-	"github.com/cloudflare/cloudflared/tunnelrpc"
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
-)
-
-type tunnelServerClient struct {
-	client    tunnelpogs.TunnelServer_PogsClient
-	transport rpc.Transport
-}
-
-// NewTunnelRPCClient creates and returns a new RPC client, which will communicate using a stream on the given muxer.
-// This method is exported for supervisor to call Authenticate RPC
-func NewTunnelServerClient(
-	ctx context.Context,
-	stream io.ReadWriteCloser,
-	log *zerolog.Logger,
-) *tunnelServerClient {
-	transport := tunnelrpc.NewTransportLogger(log, rpc.StreamTransport(stream))
-	conn := rpc.NewConn(
-		transport,
-		tunnelrpc.ConnLog(log),
-	)
-	registrationClient := tunnelpogs.RegistrationServer_PogsClient{Client: conn.Bootstrap(ctx), Conn: conn}
-	return &tunnelServerClient{
-		client:    tunnelpogs.TunnelServer_PogsClient{RegistrationServer_PogsClient: registrationClient, Client: conn.Bootstrap(ctx), Conn: conn},
-		transport: transport,
-	}
-}
-
-func (tsc *tunnelServerClient) Authenticate(ctx context.Context, classicTunnel *ClassicTunnelProperties, registrationOptions *tunnelpogs.RegistrationOptions) (tunnelpogs.AuthOutcome, error) {
-	authResp, err := tsc.client.Authenticate(ctx, classicTunnel.OriginCert, classicTunnel.Hostname, registrationOptions)
-	if err != nil {
-		return nil, err
-	}
-	return authResp.Outcome(), nil
-}
-
-func (tsc *tunnelServerClient) Close() {
-	// Closing the client will also close the connection
-	_ = tsc.client.Close()
-	_ = tsc.transport.Close()
-}
-
-type NamedTunnelRPCClient interface {
-	RegisterConnection(
-		c context.Context,
-		config *NamedTunnelProperties,
-		options *tunnelpogs.ConnectionOptions,
-		connIndex uint8,
-		edgeAddress net.IP,
-		observer *Observer,
-	) (*tunnelpogs.ConnectionDetails, error)
-	SendLocalConfiguration(
-		c context.Context,
-		config []byte,
-		observer *Observer,
-	) error
-	GracefulShutdown(ctx context.Context, gracePeriod time.Duration)
-	Close()
-}
-
-type registrationServerClient struct {
-	client    tunnelpogs.RegistrationServer_PogsClient
-	transport rpc.Transport
-}
-
-func newRegistrationRPCClient(
-	ctx context.Context,
-	stream io.ReadWriteCloser,
-	log *zerolog.Logger,
-) NamedTunnelRPCClient {
-	transport := tunnelrpc.NewTransportLogger(log, rpc.StreamTransport(stream))
-	conn := rpc.NewConn(
-		transport,
-		tunnelrpc.ConnLog(log),
-	)
-	return &registrationServerClient{
-		client:    tunnelpogs.RegistrationServer_PogsClient{Client: conn.Bootstrap(ctx), Conn: conn},
-		transport: transport,
-	}
-}
-
-func (rsc *registrationServerClient) RegisterConnection(
-	ctx context.Context,
-	properties *NamedTunnelProperties,
-	options *tunnelpogs.ConnectionOptions,
-	connIndex uint8,
-	edgeAddress net.IP,
-	observer *Observer,
-) (*tunnelpogs.ConnectionDetails, error) {
-	conn, err := rsc.client.RegisterConnection(
-		ctx,
-		properties.Credentials.Auth(),
-		properties.Credentials.TunnelID,
-		connIndex,
-		options,
-	)
-	if err != nil {
-		if err.Error() == DuplicateConnectionError {
-			observer.metrics.regFail.WithLabelValues("dup_edge_conn", "registerConnection").Inc()
-			return nil, errDuplicationConnection
-		}
-		observer.metrics.regFail.WithLabelValues("server_error", "registerConnection").Inc()
-		return nil, serverRegistrationErrorFromRPC(err)
-	}
-
-	observer.metrics.regSuccess.WithLabelValues("registerConnection").Inc()
-
-	return conn, nil
-}
-
-func (rsc *registrationServerClient) SendLocalConfiguration(ctx context.Context, config []byte, observer *Observer) (err error) {
-	observer.metrics.localConfigMetrics.pushes.Inc()
-	defer func() {
-		if err != nil {
-			observer.metrics.localConfigMetrics.pushesErrors.Inc()
-		}
-	}()
-
-	return rsc.client.SendLocalConfiguration(ctx, config)
-}
-
-func (rsc *registrationServerClient) GracefulShutdown(ctx context.Context, gracePeriod time.Duration) {
-	ctx, cancel := context.WithTimeout(ctx, gracePeriod)
-	defer cancel()
-	_ = rsc.client.UnregisterConnection(ctx)
-}
-
-func (rsc *registrationServerClient) Close() {
-	// Closing the client will also close the connection
-	_ = rsc.client.Close()
-	// Closing the transport also closes the stream
-	_ = rsc.transport.Close()
-}
-
-type rpcName string
-
-const (
-	register     rpcName = "register"
-	reconnect    rpcName = "reconnect"
-	unregister   rpcName = "unregister"
-	authenticate rpcName = " authenticate"
-)

datagramsession/metrics.go

@@ -15,7 +15,7 @@ var (
 		Name:      "active_sessions",
 		Help:      "Concurrent count of UDP sessions that are being proxied to any origin",
 	})
-	totalUDPSessions = prometheus.NewGauge(prometheus.GaugeOpts{
+	totalUDPSessions = prometheus.NewCounter(prometheus.CounterOpts{
 		Namespace: namespace,
 		Subsystem: "udp",
 		Name:      "total_sessions",

ingress/icmp_metrics.go

@@ -0,0 +1,39 @@
+package ingress
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const (
+	namespace = "cloudflared"
+)
+
+var (
+	icmpRequests = prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: namespace,
+		Subsystem: "icmp",
+		Name:      "total_requests",
+		Help:      "Total count of ICMP requests that have been proxied to any origin",
+	})
+	icmpReplies = prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: namespace,
+		Subsystem: "icmp",
+		Name:      "total_replies",
+		Help:      "Total count of ICMP replies that have been proxied from any origin",
+	})
+)
+
+func init() {
+	prometheus.MustRegister(
+		icmpRequests,
+		icmpReplies,
+	)
+}
+
+func incrementICMPRequest() {
+	icmpRequests.Inc()
+}
+
+func incrementICMPReply() {
+	icmpReplies.Inc()
+}

ingress/origin_icmp_proxy.go

@@ -105,6 +105,7 @@ func isEchoReply(msg *icmp.Message) bool {
 }
 
 func observeICMPRequest(logger *zerolog.Logger, span trace.Span, src string, dst string, echoID int, seq int) {
+	incrementICMPRequest()
 	logger.Debug().
 		Str("src", src).
 		Str("dst", dst).
@@ -118,6 +119,7 @@ func observeICMPRequest(logger *zerolog.Logger, span trace.Span, src string, dst
 }
 
 func observeICMPReply(logger *zerolog.Logger, span trace.Span, dst string, echoID int, seq int) {
+	incrementICMPReply()
 	logger.Debug().Str("dst", dst).Int("echoID", echoID).Int("seq", seq).Msg("Sent ICMP reply to edge")
 	span.SetAttributes(
 		attribute.String("dst", dst),

orchestration/orchestrator.go

@@ -14,7 +14,7 @@ import (
 	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/ingress"
 	"github.com/cloudflare/cloudflared/proxy"
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
 // Orchestrator manages configurations, so they can be updatable during runtime
@@ -32,7 +32,7 @@ type Orchestrator struct {
 	internalRules []ingress.Rule
 	// cloudflared Configuration
 	config *Config
-	tags   []tunnelpogs.Tag
+	tags   []pogs.Tag
 	log    *zerolog.Logger
 
 	// orchestrator must not handle any more updates after shutdownC is closed
@@ -43,7 +43,7 @@ type Orchestrator struct {
 
 func NewOrchestrator(ctx context.Context,
 	config *Config,
-	tags []tunnelpogs.Tag,
+	tags []pogs.Tag,
 	internalRules []ingress.Rule,
 	log *zerolog.Logger) (*Orchestrator, error) {
 	o := &Orchestrator{
@@ -65,7 +65,7 @@ func NewOrchestrator(ctx context.Context,
 }
 
 // UpdateConfig creates a new proxy with the new ingress rules
-func (o *Orchestrator) UpdateConfig(version int32, config []byte) *tunnelpogs.UpdateConfigurationResponse {
+func (o *Orchestrator) UpdateConfig(version int32, config []byte) *pogs.UpdateConfigurationResponse {
 	o.lock.Lock()
 	defer o.lock.Unlock()
 
@@ -74,7 +74,7 @@ func (o *Orchestrator) UpdateConfig(version int32, config []byte) *tunnelpogs.Up
 			Int32("current_version", o.currentVersion).
 			Int32("received_version", version).
 			Msg("Current version is equal or newer than received version")
-		return &tunnelpogs.UpdateConfigurationResponse{
+		return &pogs.UpdateConfigurationResponse{
 			LastAppliedVersion: o.currentVersion,
 		}
 	}
@@ -84,7 +84,7 @@ func (o *Orchestrator) UpdateConfig(version int32, config []byte) *tunnelpogs.Up
 			Int32("version", version).
 			Str("config", string(config)).
 			Msgf("Failed to deserialize new configuration")
-		return &tunnelpogs.UpdateConfigurationResponse{
+		return &pogs.UpdateConfigurationResponse{
 			LastAppliedVersion: o.currentVersion,
 			Err:                err,
 		}
@@ -95,7 +95,7 @@ func (o *Orchestrator) UpdateConfig(version int32, config []byte) *tunnelpogs.Up
 			Int32("version", version).
 			Str("config", string(config)).
 			Msgf("Failed to update ingress")
-		return &tunnelpogs.UpdateConfigurationResponse{
+		return &pogs.UpdateConfigurationResponse{
 			LastAppliedVersion: o.currentVersion,
 			Err:                err,
 		}
@@ -107,7 +107,7 @@ func (o *Orchestrator) UpdateConfig(version int32, config []byte) *tunnelpogs.Up
 		Str("config", string(config)).
 		Msg("Updated to new configuration")
 	configVersion.Set(float64(version))
-	return &tunnelpogs.UpdateConfigurationResponse{
+	return &pogs.UpdateConfigurationResponse{
 		LastAppliedVersion: o.currentVersion,
 	}
 }

orchestration/orchestrator_test.go

@@ -23,12 +23,12 @@ import (
 	"github.com/cloudflare/cloudflared/ingress"
 	"github.com/cloudflare/cloudflared/management"
 	"github.com/cloudflare/cloudflared/tracing"
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
 var (
 	testLogger = zerolog.Nop()
-	testTags   = []tunnelpogs.Tag{
+	testTags   = []pogs.Tag{
 		{
 			Name:  "package",
 			Value: "orchestration",

proxy/proxy.go

@@ -19,7 +19,7 @@ import (
 	"github.com/cloudflare/cloudflared/ingress"
 	"github.com/cloudflare/cloudflared/stream"
 	"github.com/cloudflare/cloudflared/tracing"
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
 const (
@@ -33,7 +33,7 @@ type Proxy struct {
 	ingressRules ingress.Ingress
 	warpRouting  *ingress.WarpRoutingService
 	management   *ingress.ManagementService
-	tags         []tunnelpogs.Tag
+	tags         []pogs.Tag
 	log          *zerolog.Logger
 }
 
@@ -41,7 +41,7 @@ type Proxy struct {
 func NewOriginProxy(
 	ingressRules ingress.Ingress,
 	warpRouting ingress.WarpRoutingConfig,
-	tags []tunnelpogs.Tag,
+	tags []pogs.Tag,
 	writeTimeout time.Duration,
 	log *zerolog.Logger,
 ) *Proxy {

proxy/proxy_test.go

@@ -30,11 +30,11 @@ import (
 	"github.com/cloudflare/cloudflared/ingress"
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/tracing"
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
 var (
-	testTags        = []tunnelpogs.Tag{{Name: "Name", Value: "value"}}
+	testTags        = []pogs.Tag{{Name: "Name", Value: "value"}}
 	noWarpRouting   = ingress.WarpRoutingConfig{}
 	testWarpRouting = ingress.WarpRoutingConfig{
 		ConnectTimeout: config.CustomDuration{Duration: time.Second},

quic/constants.go

@@ -0,0 +1,12 @@
+package quic
+
+import "time"
+
+const (
+	HandshakeIdleTimeout = 5 * time.Second
+	MaxIdleTimeout       = 5 * time.Second
+	MaxIdlePingPeriod    = 1 * time.Second
+
+	// MaxIncomingStreams is 2^60, which is the maximum supported value by Quic-Go
+	MaxIncomingStreams = 1 << 60
+)

quic/quic_protocol.go

@@ -1,274 +0,0 @@
-package quic
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"net"
-	"time"
-
-	capnp "zombiezen.com/go/capnproto2"
-	"zombiezen.com/go/capnproto2/rpc"
-
-	"github.com/google/uuid"
-	"github.com/rs/zerolog"
-
-	"github.com/cloudflare/cloudflared/tunnelrpc"
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
-)
-
-// ProtocolSignature defines the first 6 bytes of the stream, which is used to distinguish the type of stream. It
-// ensures whoever performs a handshake does not write data before writing the metadata.
-type ProtocolSignature [6]byte
-
-var (
-	// DataStreamProtocolSignature is a custom protocol signature for data stream
-	DataStreamProtocolSignature = ProtocolSignature{0x0A, 0x36, 0xCD, 0x12, 0xA1, 0x3E}
-
-	// RPCStreamProtocolSignature is a custom protocol signature for RPC stream
-	RPCStreamProtocolSignature = ProtocolSignature{0x52, 0xBB, 0x82, 0x5C, 0xDB, 0x65}
-)
-
-type protocolVersion string
-
-const (
-	protocolV1 protocolVersion = "01"
-
-	protocolVersionLength = 2
-
-	HandshakeIdleTimeout = 5 * time.Second
-	MaxIdleTimeout       = 5 * time.Second
-	MaxIdlePingPeriod    = 1 * time.Second
-
-	// MaxIncomingStreams is 2^60, which is the maximum supported value by Quic-Go
-	MaxIncomingStreams = 1 << 60
-)
-
-// RequestServerStream is a stream to serve requests
-type RequestServerStream struct {
-	io.ReadWriteCloser
-}
-
-func NewRequestServerStream(stream io.ReadWriteCloser, signature ProtocolSignature) (*RequestServerStream, error) {
-	if signature != DataStreamProtocolSignature {
-		return nil, fmt.Errorf("RequestClientStream can only be created from data stream")
-	}
-	return &RequestServerStream{stream}, nil
-}
-
-// ReadConnectRequestData reads the handshake data from a QUIC stream.
-func (rss *RequestServerStream) ReadConnectRequestData() (*ConnectRequest, error) {
-	// This is a NO-OP for now. We could cause a branching if we wanted to use multiple versions.
-	if _, err := readVersion(rss); err != nil {
-		return nil, err
-	}
-
-	msg, err := capnp.NewDecoder(rss).Decode()
-	if err != nil {
-		return nil, err
-	}
-
-	r := &ConnectRequest{}
-	if err := r.fromPogs(msg); err != nil {
-		return nil, err
-	}
-	return r, nil
-}
-
-// WriteConnectResponseData writes response to a QUIC stream.
-func (rss *RequestServerStream) WriteConnectResponseData(respErr error, metadata ...Metadata) error {
-	var connectResponse *ConnectResponse
-	if respErr != nil {
-		connectResponse = &ConnectResponse{
-			Error: respErr.Error(),
-		}
-	} else {
-		connectResponse = &ConnectResponse{
-			Metadata: metadata,
-		}
-	}
-
-	msg, err := connectResponse.toPogs()
-	if err != nil {
-		return err
-	}
-
-	if err := writeDataStreamPreamble(rss); err != nil {
-		return err
-	}
-	return capnp.NewEncoder(rss).Encode(msg)
-}
-
-type RequestClientStream struct {
-	io.ReadWriteCloser
-}
-
-// WriteConnectRequestData writes requestMeta to a stream.
-func (rcs *RequestClientStream) WriteConnectRequestData(dest string, connectionType ConnectionType, metadata ...Metadata) error {
-	connectRequest := &ConnectRequest{
-		Dest:     dest,
-		Type:     connectionType,
-		Metadata: metadata,
-	}
-
-	msg, err := connectRequest.toPogs()
-	if err != nil {
-		return err
-	}
-
-	if err := writeDataStreamPreamble(rcs); err != nil {
-		return err
-	}
-	return capnp.NewEncoder(rcs).Encode(msg)
-}
-
-// ReadConnectResponseData reads the response to a RequestMeta in a stream.
-func (rcs *RequestClientStream) ReadConnectResponseData() (*ConnectResponse, error) {
-	signature, err := DetermineProtocol(rcs)
-	if err != nil {
-		return nil, err
-	}
-	if signature != DataStreamProtocolSignature {
-		return nil, fmt.Errorf("wrong protocol signature %v", signature)
-	}
-
-	// This is a NO-OP for now. We could cause a branching if we wanted to use multiple versions.
-	if _, err := readVersion(rcs); err != nil {
-		return nil, err
-	}
-
-	msg, err := capnp.NewDecoder(rcs).Decode()
-	if err != nil {
-		return nil, err
-	}
-
-	r := &ConnectResponse{}
-	if err := r.fromPogs(msg); err != nil {
-		return nil, err
-	}
-	return r, nil
-}
-
-// RPCServerStream is a stream to serve RPCs. It is closed when the RPC client is done
-type RPCServerStream struct {
-	io.ReadWriteCloser
-}
-
-func NewRPCServerStream(stream io.ReadWriteCloser, protocol ProtocolSignature) (*RPCServerStream, error) {
-	if protocol != RPCStreamProtocolSignature {
-		return nil, fmt.Errorf("RPCStream can only be created from rpc stream")
-	}
-	return &RPCServerStream{stream}, nil
-}
-
-func (s *RPCServerStream) Serve(sessionManager tunnelpogs.SessionManager, configManager tunnelpogs.ConfigurationManager, logger *zerolog.Logger) error {
-	// RPC logs are very robust, create a new logger that only logs error to reduce noise
-	rpcLogger := logger.Level(zerolog.ErrorLevel)
-	rpcTransport := tunnelrpc.NewTransportLogger(&rpcLogger, rpc.StreamTransport(s))
-	defer rpcTransport.Close()
-
-	main := tunnelpogs.CloudflaredServer_ServerToClient(sessionManager, configManager)
-	rpcConn := rpc.NewConn(
-		rpcTransport,
-		rpc.MainInterface(main.Client),
-		tunnelrpc.ConnLog(&rpcLogger),
-	)
-	defer rpcConn.Close()
-
-	return rpcConn.Wait()
-}
-
-func DetermineProtocol(stream io.Reader) (ProtocolSignature, error) {
-	signature, err := readSignature(stream)
-	if err != nil {
-		return ProtocolSignature{}, err
-	}
-	switch signature {
-	case DataStreamProtocolSignature:
-		return DataStreamProtocolSignature, nil
-	case RPCStreamProtocolSignature:
-		return RPCStreamProtocolSignature, nil
-	default:
-		return ProtocolSignature{}, fmt.Errorf("unknown signature %v", signature)
-	}
-}
-
-func writeDataStreamPreamble(stream io.Writer) error {
-	if err := writeSignature(stream, DataStreamProtocolSignature); err != nil {
-		return err
-	}
-
-	return writeVersion(stream)
-}
-
-func writeVersion(stream io.Writer) error {
-	_, err := stream.Write([]byte(protocolV1)[:protocolVersionLength])
-	return err
-}
-
-func readVersion(stream io.Reader) (string, error) {
-	version := make([]byte, protocolVersionLength)
-	_, err := stream.Read(version)
-	return string(version), err
-}
-
-func readSignature(stream io.Reader) (ProtocolSignature, error) {
-	var signature ProtocolSignature
-	if _, err := io.ReadFull(stream, signature[:]); err != nil {
-		return ProtocolSignature{}, err
-	}
-	return signature, nil
-}
-
-func writeSignature(stream io.Writer, signature ProtocolSignature) error {
-	_, err := stream.Write(signature[:])
-	return err
-}
-
-// RPCClientStream is a stream to call methods of SessionManager
-type RPCClientStream struct {
-	client    tunnelpogs.CloudflaredServer_PogsClient
-	transport rpc.Transport
-
-	// Time we wait for the server to respond to a request before we close the connection.
-	rpcUnregisterUDPSessionDeadline time.Duration
-}
-
-func NewRPCClientStream(ctx context.Context, stream io.ReadWriteCloser, rpcUnregisterUDPSessionDeadline time.Duration, logger *zerolog.Logger) (*RPCClientStream, error) {
-	n, err := stream.Write(RPCStreamProtocolSignature[:])
-	if err != nil {
-		return nil, err
-	}
-	if n != len(RPCStreamProtocolSignature) {
-		return nil, fmt.Errorf("expect to write %d bytes for RPC stream protocol signature, wrote %d", len(RPCStreamProtocolSignature), n)
-	}
-	transport := tunnelrpc.NewTransportLogger(logger, rpc.StreamTransport(stream))
-	conn := rpc.NewConn(
-		transport,
-		tunnelrpc.ConnLog(logger),
-	)
-	return &RPCClientStream{
-		client:                          tunnelpogs.NewCloudflaredServer_PogsClient(conn.Bootstrap(ctx), conn),
-		transport:                       transport,
-		rpcUnregisterUDPSessionDeadline: rpcUnregisterUDPSessionDeadline,
-	}, nil
-}
-
-func (rcs *RPCClientStream) RegisterUdpSession(ctx context.Context, sessionID uuid.UUID, dstIP net.IP, dstPort uint16, closeIdleAfterHint time.Duration, traceContext string) (*tunnelpogs.RegisterUdpSessionResponse, error) {
-	return rcs.client.RegisterUdpSession(ctx, sessionID, dstIP, dstPort, closeIdleAfterHint, traceContext)
-}
-
-func (rcs *RPCClientStream) UnregisterUdpSession(ctx context.Context, sessionID uuid.UUID, message string) error {
-	ctx, cancel := context.WithTimeout(ctx, rcs.rpcUnregisterUDPSessionDeadline)
-	defer cancel()
-	return rcs.client.UnregisterUdpSession(ctx, sessionID, message)
-}
-
-func (rcs *RPCClientStream) UpdateConfiguration(ctx context.Context, version int32, config []byte) (*tunnelpogs.UpdateConfigurationResponse, error) {
-	return rcs.client.UpdateConfiguration(ctx, version, config)
-}
-
-func (rcs *RPCClientStream) Close() {
-	_ = rcs.client.Close()
-	_ = rcs.transport.Close()
-}

quic/safe_stream_test.go

@@ -2,8 +2,13 @@ package quic
 
 import (
 	"context"
+	"crypto/rand"
+	"crypto/rsa"
 	"crypto/tls"
+	"crypto/x509"
+	"encoding/pem"
 	"io"
+	"math/big"
 	"net"
 	"sync"
 	"testing"
@@ -147,3 +152,27 @@ func serverRoundTrip(t *testing.T, stream io.ReadWriteCloser, mustWork bool) {
 	}
 	require.NoError(t, err)
 }
+
+// GenerateTLSConfig sets up a bare-bones TLS config for a QUIC server
+func GenerateTLSConfig() *tls.Config {
+	key, err := rsa.GenerateKey(rand.Reader, 1024)
+	if err != nil {
+		panic(err)
+	}
+	template := x509.Certificate{SerialNumber: big.NewInt(1)}
+	certDER, err := x509.CreateCertificate(rand.Reader, &template, &template, &key.PublicKey, key)
+	if err != nil {
+		panic(err)
+	}
+	keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)})
+	certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER})
+
+	tlsCert, err := tls.X509KeyPair(certPEM, keyPEM)
+	if err != nil {
+		panic(err)
+	}
+	return &tls.Config{
+		Certificates: []tls.Certificate{tlsCert},
+		NextProtos:   []string{"argotunnel"},
+	}
+}

quic/schema/quic_metadata_protocol.capnp

@@ -1,28 +0,0 @@
-using Go = import "/go.capnp";
-@0xb29021ef7421cc32;
-
-$Go.package("schema");
-$Go.import("schema");
-
-
-struct ConnectRequest{
-	dest @0 :Text;
-	type @1 :ConnectionType;
-	metadata @2 :List(Metadata);
-}
-
-enum ConnectionType{
-	http @0;
-	websocket @1;
-	tcp @2;
-}
-
-struct Metadata {
-    key @0 :Text;
-	val @1 :Text;
-}
-
-struct ConnectResponse{
-	error @0 :Text;
-	metadata @1 :List(Metadata);
-}

quic/test_utils.go

@@ -1,34 +0,0 @@
-package quic
-
-import (
-	"crypto/rand"
-	"crypto/rsa"
-	"crypto/tls"
-	"crypto/x509"
-	"encoding/pem"
-	"math/big"
-)
-
-// GenerateTLSConfig sets up a bare-bones TLS config for a QUIC server
-func GenerateTLSConfig() *tls.Config {
-	key, err := rsa.GenerateKey(rand.Reader, 1024)
-	if err != nil {
-		panic(err)
-	}
-	template := x509.Certificate{SerialNumber: big.NewInt(1)}
-	certDER, err := x509.CreateCertificate(rand.Reader, &template, &template, &key.PublicKey, key)
-	if err != nil {
-		panic(err)
-	}
-	keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)})
-	certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER})
-
-	tlsCert, err := tls.X509KeyPair(certPEM, keyPEM)
-	if err != nil {
-		panic(err)
-	}
-	return &tls.Config{
-		Certificates: []tls.Certificate{tlsCert},
-		NextProtos:   []string{"argotunnel"},
-	}
-}

retry/backoffhandler.go

@@ -6,17 +6,16 @@ import (
 	"time"
 )
 
+const (
+	DefaultBaseTime time.Duration = time.Second
+)
+
 // Redeclare time functions so they can be overridden in tests.
-type clock struct {
+type Clock struct {
 	Now   func() time.Time
 	After func(d time.Duration) <-chan time.Time
 }
 
-var Clock = clock{
-	Now:   time.Now,
-	After: time.After,
-}
-
 // BackoffHandler manages exponential backoff and limits the maximum number of retries.
 // The base time period is 1 second, doubling with each retry.
 // After initial success, a grace period can be set to reset the backoff timer if
@@ -25,15 +24,26 @@ var Clock = clock{
 type BackoffHandler struct {
 	// MaxRetries sets the maximum number of retries to perform. The default value
 	// of 0 disables retry completely.
-	MaxRetries uint
+	maxRetries uint
 	// RetryForever caps the exponential backoff period according to MaxRetries
 	// but allows you to retry indefinitely.
-	RetryForever bool
+	retryForever bool
 	// BaseTime sets the initial backoff period.
-	BaseTime time.Duration
+	baseTime time.Duration
 
 	retries       uint
 	resetDeadline time.Time
+
+	Clock Clock
+}
+
+func NewBackoff(maxRetries uint, baseTime time.Duration, retryForever bool) BackoffHandler {
+	return BackoffHandler{
+		maxRetries:   maxRetries,
+		baseTime:     baseTime,
+		retryForever: retryForever,
+		Clock:        Clock{Now: time.Now, After: time.After},
+	}
 }
 
 func (b BackoffHandler) GetMaxBackoffDuration(ctx context.Context) (time.Duration, bool) {
@@ -44,11 +54,11 @@ func (b BackoffHandler) GetMaxBackoffDuration(ctx context.Context) (time.Duratio
 		return time.Duration(0), false
 	default:
 	}
-	if !b.resetDeadline.IsZero() && Clock.Now().After(b.resetDeadline) {
+	if !b.resetDeadline.IsZero() && b.Clock.Now().After(b.resetDeadline) {
 		// b.retries would be set to 0 at this point
 		return time.Second, true
 	}
-	if b.retries >= b.MaxRetries && !b.RetryForever {
+	if b.retries >= b.maxRetries && !b.retryForever {
 		return time.Duration(0), false
 	}
 	maxTimeToWait := b.GetBaseTime() * 1 << (b.retries + 1)
@@ -58,12 +68,12 @@ func (b BackoffHandler) GetMaxBackoffDuration(ctx context.Context) (time.Duratio
 // BackoffTimer returns a channel that sends the current time when the exponential backoff timeout expires.
 // Returns nil if the maximum number of retries have been used.
 func (b *BackoffHandler) BackoffTimer() <-chan time.Time {
-	if !b.resetDeadline.IsZero() && Clock.Now().After(b.resetDeadline) {
+	if !b.resetDeadline.IsZero() && b.Clock.Now().After(b.resetDeadline) {
 		b.retries = 0
 		b.resetDeadline = time.Time{}
 	}
-	if b.retries >= b.MaxRetries {
-		if !b.RetryForever {
+	if b.retries >= b.maxRetries {
+		if !b.retryForever {
 			return nil
 		}
 	} else {
@@ -71,7 +81,7 @@ func (b *BackoffHandler) BackoffTimer() <-chan time.Time {
 	}
 	maxTimeToWait := time.Duration(b.GetBaseTime() * 1 << (b.retries))
 	timeToWait := time.Duration(rand.Int63n(maxTimeToWait.Nanoseconds()))
-	return Clock.After(timeToWait)
+	return b.Clock.After(timeToWait)
 }
 
 // Backoff is used to wait according to exponential backoff. Returns false if the
@@ -94,16 +104,16 @@ func (b *BackoffHandler) Backoff(ctx context.Context) bool {
 func (b *BackoffHandler) SetGracePeriod() time.Duration {
 	maxTimeToWait := b.GetBaseTime() * 2 << (b.retries + 1)
 	timeToWait := time.Duration(rand.Int63n(maxTimeToWait.Nanoseconds()))
-	b.resetDeadline = Clock.Now().Add(timeToWait)
+	b.resetDeadline = b.Clock.Now().Add(timeToWait)
 
 	return timeToWait
 }
 
 func (b BackoffHandler) GetBaseTime() time.Duration {
-	if b.BaseTime == 0 {
-		return time.Second
+	if b.baseTime == 0 {
+		return DefaultBaseTime
 	}
-	return b.BaseTime
+	return b.baseTime
 }
 
 // Retries returns the number of retries consumed so far.
@@ -112,9 +122,10 @@ func (b *BackoffHandler) Retries() int {
 }
 
 func (b *BackoffHandler) ReachedMaxRetries() bool {
-	return b.retries == b.MaxRetries
+	return b.retries == b.maxRetries
 }
 
 func (b *BackoffHandler) ResetNow() {
-	b.resetDeadline = time.Now()
+	b.resetDeadline = b.Clock.Now()
+	b.retries = 0
 }

retry/backoffhandler_test.go

@@ -13,10 +13,9 @@ func immediateTimeAfter(time.Duration) <-chan time.Time {
 }
 
 func TestBackoffRetries(t *testing.T) {
-	// make backoff return immediately
-	Clock.After = immediateTimeAfter
 	ctx := context.Background()
-	backoff := BackoffHandler{MaxRetries: 3}
+	// make backoff return immediately
+	backoff := BackoffHandler{maxRetries: 3, Clock: Clock{time.Now, immediateTimeAfter}}
 	if !backoff.Backoff(ctx) {
 		t.Fatalf("backoff failed immediately")
 	}
@@ -32,10 +31,10 @@ func TestBackoffRetries(t *testing.T) {
 }
 
 func TestBackoffCancel(t *testing.T) {
-	// prevent backoff from returning normally
-	Clock.After = func(time.Duration) <-chan time.Time { return make(chan time.Time) }
 	ctx, cancelFunc := context.WithCancel(context.Background())
-	backoff := BackoffHandler{MaxRetries: 3}
+	// prevent backoff from returning normally
+	after := func(time.Duration) <-chan time.Time { return make(chan time.Time) }
+	backoff := BackoffHandler{maxRetries: 3, Clock: Clock{time.Now, after}}
 	cancelFunc()
 	if backoff.Backoff(ctx) {
 		t.Fatalf("backoff allowed after cancel")
@@ -46,13 +45,12 @@ func TestBackoffCancel(t *testing.T) {
 }
 
 func TestBackoffGracePeriod(t *testing.T) {
+	ctx := context.Background()
 	currentTime := time.Now()
 	// make Clock.Now return whatever we like
-	Clock.Now = func() time.Time { return currentTime }
+	now := func() time.Time { return currentTime }
 	// make backoff return immediately
-	Clock.After = immediateTimeAfter
-	ctx := context.Background()
-	backoff := BackoffHandler{MaxRetries: 1}
+	backoff := BackoffHandler{maxRetries: 1, Clock: Clock{now, immediateTimeAfter}}
 	if !backoff.Backoff(ctx) {
 		t.Fatalf("backoff failed immediately")
 	}
@@ -70,10 +68,9 @@ func TestBackoffGracePeriod(t *testing.T) {
 }
 
 func TestGetMaxBackoffDurationRetries(t *testing.T) {
-	// make backoff return immediately
-	Clock.After = immediateTimeAfter
 	ctx := context.Background()
-	backoff := BackoffHandler{MaxRetries: 3}
+	// make backoff return immediately
+	backoff := BackoffHandler{maxRetries: 3, Clock: Clock{time.Now, immediateTimeAfter}}
 	if _, ok := backoff.GetMaxBackoffDuration(ctx); !ok {
 		t.Fatalf("backoff failed immediately")
 	}
@@ -95,10 +92,9 @@ func TestGetMaxBackoffDurationRetries(t *testing.T) {
 }
 
 func TestGetMaxBackoffDuration(t *testing.T) {
-	// make backoff return immediately
-	Clock.After = immediateTimeAfter
 	ctx := context.Background()
-	backoff := BackoffHandler{MaxRetries: 3}
+	// make backoff return immediately
+	backoff := BackoffHandler{maxRetries: 3, Clock: Clock{time.Now, immediateTimeAfter}}
 	if duration, ok := backoff.GetMaxBackoffDuration(ctx); !ok || duration > time.Second*2 {
 		t.Fatalf("backoff (%s) didn't return < 2 seconds on first retry", duration)
 	}
@@ -117,10 +113,9 @@ func TestGetMaxBackoffDuration(t *testing.T) {
 }
 
 func TestBackoffRetryForever(t *testing.T) {
-	// make backoff return immediately
-	Clock.After = immediateTimeAfter
 	ctx := context.Background()
-	backoff := BackoffHandler{MaxRetries: 3, RetryForever: true}
+	// make backoff return immediately
+	backoff := BackoffHandler{maxRetries: 3, retryForever: true, Clock: Clock{time.Now, immediateTimeAfter}}
 	if duration, ok := backoff.GetMaxBackoffDuration(ctx); !ok || duration > time.Second*2 {
 		t.Fatalf("backoff (%s) didn't return < 2 seconds on first retry", duration)
 	}

supervisor/reconnect.go

@@ -1,138 +0,0 @@
-package supervisor
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"sync"
-	"time"
-
-	"github.com/prometheus/client_golang/prometheus"
-
-	"github.com/cloudflare/cloudflared/retry"
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
-)
-
-var (
-	errJWTUnset = errors.New("JWT unset")
-)
-
-// reconnectTunnelCredentialManager is invoked by functions in tunnel.go to
-// get/set parameters for ReconnectTunnel RPC calls.
-type reconnectCredentialManager struct {
-	mu          sync.RWMutex
-	jwt         []byte
-	eventDigest map[uint8][]byte
-	connDigest  map[uint8][]byte
-	authSuccess prometheus.Counter
-	authFail    *prometheus.CounterVec
-}
-
-func newReconnectCredentialManager(namespace, subsystem string, haConnections int) *reconnectCredentialManager {
-	authSuccess := prometheus.NewCounter(
-		prometheus.CounterOpts{
-			Namespace: namespace,
-			Subsystem: subsystem,
-			Name:      "tunnel_authenticate_success",
-			Help:      "Count of successful tunnel authenticate",
-		},
-	)
-	authFail := prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Namespace: namespace,
-			Subsystem: subsystem,
-			Name:      "tunnel_authenticate_fail",
-			Help:      "Count of tunnel authenticate errors by type",
-		},
-		[]string{"error"},
-	)
-	prometheus.MustRegister(authSuccess, authFail)
-	return &reconnectCredentialManager{
-		eventDigest: make(map[uint8][]byte, haConnections),
-		connDigest:  make(map[uint8][]byte, haConnections),
-		authSuccess: authSuccess,
-		authFail:    authFail,
-	}
-}
-
-func (cm *reconnectCredentialManager) ReconnectToken() ([]byte, error) {
-	cm.mu.RLock()
-	defer cm.mu.RUnlock()
-	if cm.jwt == nil {
-		return nil, errJWTUnset
-	}
-	return cm.jwt, nil
-}
-
-func (cm *reconnectCredentialManager) SetReconnectToken(jwt []byte) {
-	cm.mu.Lock()
-	defer cm.mu.Unlock()
-	cm.jwt = jwt
-}
-
-func (cm *reconnectCredentialManager) EventDigest(connID uint8) ([]byte, error) {
-	cm.mu.RLock()
-	defer cm.mu.RUnlock()
-	digest, ok := cm.eventDigest[connID]
-	if !ok {
-		return nil, fmt.Errorf("no event digest for connection %v", connID)
-	}
-	return digest, nil
-}
-
-func (cm *reconnectCredentialManager) SetEventDigest(connID uint8, digest []byte) {
-	cm.mu.Lock()
-	defer cm.mu.Unlock()
-	cm.eventDigest[connID] = digest
-}
-
-func (cm *reconnectCredentialManager) ConnDigest(connID uint8) ([]byte, error) {
-	cm.mu.RLock()
-	defer cm.mu.RUnlock()
-	digest, ok := cm.connDigest[connID]
-	if !ok {
-		return nil, fmt.Errorf("no connection digest for connection %v", connID)
-	}
-	return digest, nil
-}
-
-func (cm *reconnectCredentialManager) SetConnDigest(connID uint8, digest []byte) {
-	cm.mu.Lock()
-	defer cm.mu.Unlock()
-	cm.connDigest[connID] = digest
-}
-
-func (cm *reconnectCredentialManager) RefreshAuth(
-	ctx context.Context,
-	backoff *retry.BackoffHandler,
-	authenticate func(ctx context.Context, numPreviousAttempts int) (tunnelpogs.AuthOutcome, error),
-) (retryTimer <-chan time.Time, err error) {
-	authOutcome, err := authenticate(ctx, backoff.Retries())
-	if err != nil {
-		cm.authFail.WithLabelValues(err.Error()).Inc()
-		if _, ok := backoff.GetMaxBackoffDuration(ctx); ok {
-			return backoff.BackoffTimer(), nil
-		}
-		return nil, err
-	}
-	// clear backoff timer
-	backoff.SetGracePeriod()
-
-	switch outcome := authOutcome.(type) {
-	case tunnelpogs.AuthSuccess:
-		cm.SetReconnectToken(outcome.JWT())
-		cm.authSuccess.Inc()
-		return retry.Clock.After(outcome.RefreshAfter()), nil
-	case tunnelpogs.AuthUnknown:
-		duration := outcome.RefreshAfter()
-		cm.authFail.WithLabelValues(outcome.Error()).Inc()
-		return retry.Clock.After(duration), nil
-	case tunnelpogs.AuthFail:
-		cm.authFail.WithLabelValues(outcome.Error()).Inc()
-		return nil, outcome
-	default:
-		err := fmt.Errorf("refresh_auth: Unexpected outcome type %T", authOutcome)
-		cm.authFail.WithLabelValues(err.Error()).Inc()
-		return nil, err
-	}
-}

supervisor/reconnect_test.go

@@ -1,120 +0,0 @@
-package supervisor
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-
-	"github.com/cloudflare/cloudflared/retry"
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
-)
-
-func TestRefreshAuthBackoff(t *testing.T) {
-	rcm := newReconnectCredentialManager(t.Name(), t.Name(), 4)
-
-	var wait time.Duration
-	retry.Clock.After = func(d time.Duration) <-chan time.Time {
-		wait = d
-		return time.After(d)
-	}
-	backoff := &retry.BackoffHandler{MaxRetries: 3}
-	auth := func(ctx context.Context, n int) (tunnelpogs.AuthOutcome, error) {
-		return nil, fmt.Errorf("authentication failure")
-	}
-
-	// authentication failures should consume the backoff
-	for i := uint(0); i < backoff.MaxRetries; i++ {
-		retryChan, err := rcm.RefreshAuth(context.Background(), backoff, auth)
-		require.NoError(t, err)
-		require.NotNil(t, retryChan)
-		require.Greater(t, wait.Seconds(), 0.0)
-		require.Less(t, wait.Seconds(), float64((1<<(i+1))*time.Second))
-	}
-	retryChan, err := rcm.RefreshAuth(context.Background(), backoff, auth)
-	require.Error(t, err)
-	require.Nil(t, retryChan)
-
-	// now we actually make contact with the remote server
-	_, _ = rcm.RefreshAuth(context.Background(), backoff, func(ctx context.Context, n int) (tunnelpogs.AuthOutcome, error) {
-		return tunnelpogs.NewAuthUnknown(errors.New("auth unknown"), 19), nil
-	})
-
-	// The backoff timer should have been reset. To confirm this, make timeNow
-	// return a value after the backoff timer's grace period
-	retry.Clock.Now = func() time.Time {
-		expectedGracePeriod := time.Duration(time.Second * 2 << backoff.MaxRetries)
-		return time.Now().Add(expectedGracePeriod * 2)
-	}
-	_, ok := backoff.GetMaxBackoffDuration(context.Background())
-	require.True(t, ok)
-}
-
-func TestRefreshAuthSuccess(t *testing.T) {
-	rcm := newReconnectCredentialManager(t.Name(), t.Name(), 4)
-
-	var wait time.Duration
-	retry.Clock.After = func(d time.Duration) <-chan time.Time {
-		wait = d
-		return time.After(d)
-	}
-
-	backoff := &retry.BackoffHandler{MaxRetries: 3}
-	auth := func(ctx context.Context, n int) (tunnelpogs.AuthOutcome, error) {
-		return tunnelpogs.NewAuthSuccess([]byte("jwt"), 19), nil
-	}
-
-	retryChan, err := rcm.RefreshAuth(context.Background(), backoff, auth)
-	assert.NoError(t, err)
-	assert.NotNil(t, retryChan)
-	assert.Equal(t, 19*time.Hour, wait)
-
-	token, err := rcm.ReconnectToken()
-	assert.NoError(t, err)
-	assert.Equal(t, []byte("jwt"), token)
-}
-
-func TestRefreshAuthUnknown(t *testing.T) {
-	rcm := newReconnectCredentialManager(t.Name(), t.Name(), 4)
-
-	var wait time.Duration
-	retry.Clock.After = func(d time.Duration) <-chan time.Time {
-		wait = d
-		return time.After(d)
-	}
-
-	backoff := &retry.BackoffHandler{MaxRetries: 3}
-	auth := func(ctx context.Context, n int) (tunnelpogs.AuthOutcome, error) {
-		return tunnelpogs.NewAuthUnknown(errors.New("auth unknown"), 19), nil
-	}
-
-	retryChan, err := rcm.RefreshAuth(context.Background(), backoff, auth)
-	assert.NoError(t, err)
-	assert.NotNil(t, retryChan)
-	assert.Equal(t, 19*time.Hour, wait)
-
-	token, err := rcm.ReconnectToken()
-	assert.Equal(t, errJWTUnset, err)
-	assert.Nil(t, token)
-}
-
-func TestRefreshAuthFail(t *testing.T) {
-	rcm := newReconnectCredentialManager(t.Name(), t.Name(), 4)
-
-	backoff := &retry.BackoffHandler{MaxRetries: 3}
-	auth := func(ctx context.Context, n int) (tunnelpogs.AuthOutcome, error) {
-		return tunnelpogs.NewAuthFail(errors.New("auth fail")), nil
-	}
-
-	retryChan, err := rcm.RefreshAuth(context.Background(), backoff, auth)
-	assert.Error(t, err)
-	assert.Nil(t, retryChan)
-
-	token, err := rcm.ReconnectToken()
-	assert.Equal(t, errJWTUnset, err)
-	assert.Nil(t, token)
-}

supervisor/supervisor.go

@@ -49,8 +49,6 @@ type Supervisor struct {
 	log          *ConnAwareLogger
 	logTransport *zerolog.Logger
 
-	reconnectCredentialManager *reconnectCredentialManager
-
 	reconnectCh       chan ReconnectSignal
 	gracefulShutdownC <-chan struct{}
 }
@@ -76,8 +74,6 @@ func NewSupervisor(config *TunnelConfig, orchestrator *orchestration.Orchestrato
 		return nil, err
 	}
 
-	reconnectCredentialManager := newReconnectCredentialManager(connection.MetricsNamespace, connection.TunnelSubsystem, config.HAConnections)
-
 	tracker := tunnelstate.NewConnTracker(config.Log)
 	log := NewConnAwareLogger(config.Log, tracker, config.Observer)
 
@@ -87,7 +83,6 @@ func NewSupervisor(config *TunnelConfig, orchestrator *orchestration.Orchestrato
 	edgeTunnelServer := EdgeTunnelServer{
 		config:            config,
 		orchestrator:      orchestrator,
-		credentialManager: reconnectCredentialManager,
 		edgeAddrs:         edgeIPs,
 		edgeAddrHandler:   edgeAddrHandler,
 		edgeBindAddr:      edgeBindAddr,
@@ -98,18 +93,17 @@ func NewSupervisor(config *TunnelConfig, orchestrator *orchestration.Orchestrato
 	}
 
 	return &Supervisor{
-		config:                     config,
-		orchestrator:               orchestrator,
-		edgeIPs:                    edgeIPs,
-		edgeTunnelServer:           &edgeTunnelServer,
-		tunnelErrors:               make(chan tunnelError),
-		tunnelsConnecting:          map[int]chan struct{}{},
-		tunnelsProtocolFallback:    map[int]*protocolFallback{},
-		log:                        log,
-		logTransport:               config.LogTransport,
-		reconnectCredentialManager: reconnectCredentialManager,
-		reconnectCh:                reconnectCh,
-		gracefulShutdownC:          gracefulShutdownC,
+		config:                  config,
+		orchestrator:            orchestrator,
+		edgeIPs:                 edgeIPs,
+		edgeTunnelServer:        &edgeTunnelServer,
+		tunnelErrors:            make(chan tunnelError),
+		tunnelsConnecting:       map[int]chan struct{}{},
+		tunnelsProtocolFallback: map[int]*protocolFallback{},
+		log:                     log,
+		logTransport:            config.LogTransport,
+		reconnectCh:             reconnectCh,
+		gracefulShutdownC:       gracefulShutdownC,
 	}, nil
 }
 
@@ -138,7 +132,7 @@ func (s *Supervisor) Run(
 	var tunnelsWaiting []int
 	tunnelsActive := s.config.HAConnections
 
-	backoff := retry.BackoffHandler{MaxRetries: s.config.Retries, BaseTime: tunnelRetryDuration, RetryForever: true}
+	backoff := retry.NewBackoff(s.config.Retries, tunnelRetryDuration, true)
 	var backoffTimer <-chan time.Time
 
 	shuttingDown := false
@@ -212,7 +206,7 @@ func (s *Supervisor) initialize(
 		s.config.HAConnections = availableAddrs
 	}
 	s.tunnelsProtocolFallback[0] = &protocolFallback{
-		retry.BackoffHandler{MaxRetries: s.config.Retries, RetryForever: true},
+		retry.NewBackoff(s.config.Retries, retry.DefaultBaseTime, true),
 		s.config.ProtocolSelector.Current(),
 		false,
 	}
@@ -234,7 +228,7 @@ func (s *Supervisor) initialize(
 	// At least one successful connection, so start the rest
 	for i := 1; i < s.config.HAConnections; i++ {
 		s.tunnelsProtocolFallback[i] = &protocolFallback{
-			retry.BackoffHandler{MaxRetries: s.config.Retries, RetryForever: true},
+			retry.NewBackoff(s.config.Retries, retry.DefaultBaseTime, true),
 			// Set the protocol we know the first tunnel connected with.
 			s.tunnelsProtocolFallback[0].protocol,
 			false,

supervisor/tunnel.go

@@ -10,7 +10,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/google/uuid"
 	"github.com/pkg/errors"
 	"github.com/quic-go/quic-go"
 	"github.com/rs/zerolog"
@@ -27,8 +26,7 @@ import (
 	quicpogs "github.com/cloudflare/cloudflared/quic"
 	"github.com/cloudflare/cloudflared/retry"
 	"github.com/cloudflare/cloudflared/signal"
-	"github.com/cloudflare/cloudflared/tunnelrpc"
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 	"github.com/cloudflare/cloudflared/tunnelstate"
 )
 
@@ -49,7 +47,7 @@ type TunnelConfig struct {
 	HAConnections      int
 	IsAutoupdated      bool
 	LBPool             string
-	Tags               []tunnelpogs.Tag
+	Tags               []pogs.Tag
 	Log                *zerolog.Logger
 	LogTransport       *zerolog.Logger
 	Observer           *connection.Observer
@@ -60,47 +58,25 @@ type TunnelConfig struct {
 
 	NeedPQ bool
 
-	NamedTunnel      *connection.NamedTunnelProperties
+	NamedTunnel      *connection.TunnelProperties
 	ProtocolSelector connection.ProtocolSelector
 	EdgeTLSConfigs   map[connection.Protocol]*tls.Config
 	PacketConfig     *ingress.GlobalRouterConfig
 
-	UDPUnregisterSessionTimeout time.Duration
-	WriteStreamTimeout          time.Duration
+	RPCTimeout         time.Duration
+	WriteStreamTimeout time.Duration
 
 	DisableQUICPathMTUDiscovery bool
 
 	FeatureSelector *features.FeatureSelector
 }
 
-func (c *TunnelConfig) registrationOptions(connectionID uint8, OriginLocalIP string, uuid uuid.UUID) *tunnelpogs.RegistrationOptions {
-	policy := tunnelrpc.ExistingTunnelPolicy_balance
-	if c.HAConnections <= 1 && c.LBPool == "" {
-		policy = tunnelrpc.ExistingTunnelPolicy_disconnect
-	}
-	return &tunnelpogs.RegistrationOptions{
-		ClientID:             c.ClientID,
-		Version:              c.ReportedVersion,
-		OS:                   c.OSArch,
-		ExistingTunnelPolicy: policy,
-		PoolName:             c.LBPool,
-		Tags:                 c.Tags,
-		ConnectionID:         connectionID,
-		OriginLocalIP:        OriginLocalIP,
-		IsAutoupdated:        c.IsAutoupdated,
-		RunFromTerminal:      c.RunFromTerminal,
-		CompressionQuality:   0,
-		UUID:                 uuid.String(),
-		Features:             c.SupportedFeatures(),
-	}
-}
-
-func (c *TunnelConfig) connectionOptions(originLocalAddr string, numPreviousAttempts uint8) *tunnelpogs.ConnectionOptions {
+func (c *TunnelConfig) connectionOptions(originLocalAddr string, numPreviousAttempts uint8) *pogs.ConnectionOptions {
 	// attempt to parse out origin IP, but don't fail since it's informational field
 	host, _, _ := net.SplitHostPort(originLocalAddr)
 	originIP := net.ParseIP(host)
 
-	return &tunnelpogs.ConnectionOptions{
+	return &pogs.ConnectionOptions{
 		Client:              c.NamedTunnel.Client,
 		OriginLocalIP:       originIP,
 		ReplaceExisting:     c.ReplaceExisting,
@@ -203,7 +179,6 @@ func (f *ipAddrFallback) ShouldGetNewAddress(connIndex uint8, err error) (needsN
 type EdgeTunnelServer struct {
 	config            *TunnelConfig
 	orchestrator      *orchestration.Orchestrator
-	credentialManager *reconnectCredentialManager
 	edgeAddrHandler   EdgeAddrHandler
 	edgeAddrs         *edgediscovery.Edge
 	edgeBindAddr      net.IP
@@ -479,6 +454,7 @@ func (e *EdgeTunnelServer) serveConnection(
 		connIndex,
 		addr.UDP.IP,
 		nil,
+		e.config.RPCTimeout,
 		e.gracefulShutdownC,
 		e.config.GracePeriod,
 		protocol,
@@ -531,7 +507,7 @@ func (e *EdgeTunnelServer) serveHTTP2(
 	ctx context.Context,
 	connLog *ConnAwareLogger,
 	tlsServerConn net.Conn,
-	connOptions *tunnelpogs.ConnectionOptions,
+	connOptions *pogs.ConnectionOptions,
 	controlStreamHandler connection.ControlStreamHandler,
 	connIndex uint8,
 ) error {
@@ -573,7 +549,7 @@ func (e *EdgeTunnelServer) serveQUIC(
 	ctx context.Context,
 	edgeAddr *net.UDPAddr,
 	connLogger *ConnAwareLogger,
-	connOptions *tunnelpogs.ConnectionOptions,
+	connOptions *pogs.ConnectionOptions,
 	controlStreamHandler connection.ControlStreamHandler,
 	connIndex uint8,
 ) (err error, recoverable bool) {
@@ -614,7 +590,7 @@ func (e *EdgeTunnelServer) serveQUIC(
 		controlStreamHandler,
 		connLogger.Logger(),
 		e.config.PacketConfig,
-		e.config.UDPUnregisterSessionTimeout,
+		e.config.RPCTimeout,
 		e.config.WriteStreamTimeout,
 	)
 	if err != nil {

supervisor/tunnel_test.go

@@ -24,14 +24,18 @@ func (dmf *dynamicMockFetcher) fetch() edgediscovery.PercentageFetcher {
 	}
 }
 
+func immediateTimeAfter(time.Duration) <-chan time.Time {
+	c := make(chan time.Time, 1)
+	c <- time.Now()
+	return c
+}
+
 func TestWaitForBackoffFallback(t *testing.T) {
 	maxRetries := uint(3)
-	backoff := retry.BackoffHandler{
-		MaxRetries: maxRetries,
-		BaseTime:   time.Millisecond * 10,
-	}
+	backoff := retry.NewBackoff(maxRetries, 40*time.Millisecond, false)
+	backoff.Clock.After = immediateTimeAfter
 	log := zerolog.Nop()
-	resolveTTL := time.Duration(0)
+	resolveTTL := 10 * time.Second
 	mockFetcher := dynamicMockFetcher{
 		protocolPercents: edgediscovery.ProtocolPercents{edgediscovery.ProtocolPercent{Protocol: "quic", Percentage: 100}},
 	}
@@ -64,21 +68,23 @@ func TestWaitForBackoffFallback(t *testing.T) {
 	}
 
 	// Retry fallback protocol
-	for i := 0; i < int(maxRetries); i++ {
-		protoFallback.BackoffTimer() // simulate retry
-		ok := selectNextProtocol(&log, protoFallback, protocolSelector, nil)
-		assert.True(t, ok)
-		fallback, ok := protocolSelector.Fallback()
-		assert.True(t, ok)
-		assert.Equal(t, fallback, protoFallback.protocol)
-	}
+	protoFallback.BackoffTimer() // simulate retry
+	ok := selectNextProtocol(&log, protoFallback, protocolSelector, nil)
+	assert.True(t, ok)
+	fallback, ok := protocolSelector.Fallback()
+	assert.True(t, ok)
+	assert.Equal(t, fallback, protoFallback.protocol)
+	assert.Equal(t, connection.HTTP2, protoFallback.protocol)
 
 	currentGlobalProtocol := protocolSelector.Current()
 	assert.Equal(t, initProtocol, currentGlobalProtocol)
 
+	// Simulate max retries again (retries reset after protocol switch)
+	for i := 0; i < int(maxRetries); i++ {
+		protoFallback.BackoffTimer()
+	}
 	// No protocol to fallback, return error
-	protoFallback.BackoffTimer() // simulate retry
-	ok := selectNextProtocol(&log, protoFallback, protocolSelector, nil)
+	ok = selectNextProtocol(&log, protoFallback, protocolSelector, nil)
 	assert.False(t, ok)
 
 	protoFallback.reset()

token/token.go

@@ -94,9 +94,10 @@ func errDeleteTokenFailed(lockFilePath string) error {
 // newLock will get a new file lock
 func newLock(path string) *lock {
 	lockPath := path + ".lock"
+	backoff := retry.NewBackoff(uint(7), retry.DefaultBaseTime, false)
 	return &lock{
 		lockFilePath: lockPath,
-		backoff:      &retry.BackoffHandler{MaxRetries: 7},
+		backoff:      &backoff,
 		sigHandler: &signalHandler{
 			signals: []os.Signal{syscall.SIGINT, syscall.SIGTERM},
 		},

tunnelrpc/go.capnp

@@ -1,15 +0,0 @@
-# Generate go.capnp.out with:
-# capnp compile -o- go.capnp > go.capnp.out
-# Must run inside this directory to preserve paths.
-
-@0xd12a1c51fedd6c88;
-
-annotation package(file) :Text;
-annotation import(file) :Text;
-annotation doc(struct, field, enum) :Text;
-annotation tag(enumerant) :Text;
-annotation notag(enumerant) :Void;
-annotation customtype(field) :Text;
-annotation name(struct, field, union, enum, enumerant, interface, method, param, annotation, const, group) :Text;
-
-$package("capnp");

tunnelrpc/log.go

@@ -1,43 +0,0 @@
-package tunnelrpc
-
-import (
-	"context"
-
-	"github.com/rs/zerolog"
-	"golang.org/x/net/trace"
-	"zombiezen.com/go/capnproto2/rpc"
-)
-
-// ConnLogger wraps a Zerolog Logger for a connection.
-type ConnLogger struct {
-	Log *zerolog.Logger
-}
-
-func (c ConnLogger) Infof(ctx context.Context, format string, args ...interface{}) {
-	c.Log.Info().Msgf(format, args...)
-}
-
-func (c ConnLogger) Errorf(ctx context.Context, format string, args ...interface{}) {
-	c.Log.Error().Msgf(format, args...)
-}
-
-func ConnLog(log *zerolog.Logger) rpc.ConnOption {
-	return rpc.ConnLog(ConnLogger{log})
-}
-
-// ConnTracer wraps a trace.EventLog for a connection.
-type ConnTracer struct {
-	Events trace.EventLog
-}
-
-func (c ConnTracer) Infof(ctx context.Context, format string, args ...interface{}) {
-	c.Events.Printf(format, args...)
-}
-
-func (c ConnTracer) Errorf(ctx context.Context, format string, args ...interface{}) {
-	c.Events.Errorf(format, args...)
-}
-
-func ConnTrace(events trace.EventLog) rpc.ConnOption {
-	return rpc.ConnLog(ConnTracer{events})
-}

tunnelrpc/logtransport.go

@@ -1,45 +0,0 @@
-// Package logtransport provides a transport that logs all of its messages.
-package tunnelrpc
-
-import (
-	"bytes"
-	"context"
-
-	"github.com/rs/zerolog"
-	"zombiezen.com/go/capnproto2/encoding/text"
-	"zombiezen.com/go/capnproto2/rpc"
-	rpccapnp "zombiezen.com/go/capnproto2/std/capnp/rpc"
-)
-
-type transport struct {
-	rpc.Transport
-	log *zerolog.Logger
-}
-
-// NewTransportLogger creates a new logger that proxies messages to and from t and
-// logs them to log.  If log is nil, then the log package's default
-// logger is used.
-func NewTransportLogger(log *zerolog.Logger, t rpc.Transport) rpc.Transport {
-	return &transport{Transport: t, log: log}
-}
-
-func (t *transport) SendMessage(ctx context.Context, msg rpccapnp.Message) error {
-	t.log.Trace().Msgf("rpc tx: %s", formatMsg(msg))
-	return t.Transport.SendMessage(ctx, msg)
-}
-
-func (t *transport) RecvMessage(ctx context.Context) (rpccapnp.Message, error) {
-	msg, err := t.Transport.RecvMessage(ctx)
-	if err != nil {
-		t.log.Debug().Msgf("rpc rx error: %s", err)
-		return msg, err
-	}
-	t.log.Trace().Msgf("rpc rx: %s", formatMsg(msg))
-	return msg, nil
-}
-
-func formatMsg(m rpccapnp.Message) string {
-	var buf bytes.Buffer
-	_ = text.NewEncoder(&buf).Encode(0x91b79f1f808db032, m.Struct)
-	return buf.String()
-}

tunnelrpc/metrics/metrics.go

@@ -0,0 +1,143 @@
+package metrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const (
+	metricsNamespace = "cloudflared"
+	rpcSubsystem     = "rpc"
+)
+
+// CloudflaredServer operation labels
+// CloudflaredServer is an extension of SessionManager with additional methods, but it's helpful
+// to visualize it separately in the metrics since they are technically different client/servers.
+const (
+	Cloudflared = "cloudflared"
+)
+
+// ConfigurationManager operation labels
+const (
+	ConfigurationManager = "config"
+
+	OperationUpdateConfiguration = "update_configuration"
+)
+
+// SessionManager operation labels
+const (
+	SessionManager = "session"
+
+	OperationRegisterUdpSession   = "register_udp_session"
+	OperationUnregisterUdpSession = "unregister_udp_session"
+)
+
+// RegistrationServer operation labels
+const (
+	Registration = "registration"
+
+	OperationRegisterConnection       = "register_connection"
+	OperationUnregisterConnection     = "unregister_connection"
+	OperationUpdateLocalConfiguration = "update_local_configuration"
+)
+
+type rpcMetrics struct {
+	serverOperations        *prometheus.CounterVec
+	serverFailures          *prometheus.CounterVec
+	serverOperationsLatency *prometheus.HistogramVec
+
+	ClientOperations        *prometheus.CounterVec
+	ClientFailures          *prometheus.CounterVec
+	ClientOperationsLatency *prometheus.HistogramVec
+}
+
+var CapnpMetrics *rpcMetrics = &rpcMetrics{
+	serverOperations: prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: metricsNamespace,
+			Subsystem: rpcSubsystem,
+			Name:      "server_operations",
+			Help:      "Number of rpc methods by handler served",
+		},
+		[]string{"handler", "method"},
+	),
+	serverFailures: prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: metricsNamespace,
+			Subsystem: rpcSubsystem,
+			Name:      "server_failures",
+			Help:      "Number of rpc methods failures by handler served",
+		},
+		[]string{"handler", "method"},
+	),
+	serverOperationsLatency: prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Namespace: metricsNamespace,
+			Subsystem: rpcSubsystem,
+			Name:      "server_latency_secs",
+			Help:      "Latency of rpc methods by handler served",
+			// Bucket starts at 50ms, each bucket grows by a factor of 3, up to 5 buckets and is expressed as seconds:
+			// 50ms, 150ms, 450ms, 1350ms, 4050ms
+			Buckets: prometheus.ExponentialBuckets(0.05, 3, 5),
+		},
+		[]string{"handler", "method"},
+	),
+	ClientOperations: prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: metricsNamespace,
+			Subsystem: rpcSubsystem,
+			Name:      "client_operations",
+			Help:      "Number of rpc methods by handler requested",
+		},
+		[]string{"handler", "method"},
+	),
+	ClientFailures: prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: metricsNamespace,
+			Subsystem: rpcSubsystem,
+			Name:      "client_failures",
+			Help:      "Number of rpc method failures by handler requested",
+		},
+		[]string{"handler", "method"},
+	),
+	ClientOperationsLatency: prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Namespace: metricsNamespace,
+			Subsystem: rpcSubsystem,
+			Name:      "client_latency_secs",
+			Help:      "Latency of rpc methods by handler requested",
+			// Bucket starts at 50ms, each bucket grows by a factor of 3, up to 5 buckets and is expressed as seconds:
+			// 50ms, 150ms, 450ms, 1350ms, 4050ms
+			Buckets: prometheus.ExponentialBuckets(0.05, 3, 5),
+		},
+		[]string{"handler", "method"},
+	),
+}
+
+func ObserveServerHandler(inner func() error, handler, method string) error {
+	defer CapnpMetrics.serverOperations.WithLabelValues(handler, method).Inc()
+	timer := prometheus.NewTimer(prometheus.ObserverFunc(func(s float64) {
+		CapnpMetrics.serverOperationsLatency.WithLabelValues(handler, method).Observe(s)
+	}))
+	defer timer.ObserveDuration()
+
+	err := inner()
+	if err != nil {
+		CapnpMetrics.serverFailures.WithLabelValues(handler, method).Inc()
+	}
+	return err
+}
+
+func NewClientOperationLatencyObserver(server string, method string) *prometheus.Timer {
+	return prometheus.NewTimer(prometheus.ObserverFunc(func(s float64) {
+		CapnpMetrics.ClientOperationsLatency.WithLabelValues(server, method).Observe(s)
+	}))
+}
+
+func init() {
+	prometheus.MustRegister(CapnpMetrics.serverOperations)
+	prometheus.MustRegister(CapnpMetrics.serverFailures)
+	prometheus.MustRegister(CapnpMetrics.serverOperationsLatency)
+	prometheus.MustRegister(CapnpMetrics.ClientOperations)
+	prometheus.MustRegister(CapnpMetrics.ClientFailures)
+	prometheus.MustRegister(CapnpMetrics.ClientOperationsLatency)
+}

tunnelrpc/pogs/auth_outcome.go

@@ -1,131 +0,0 @@
-package pogs
-
-import (
-	"errors"
-	"time"
-)
-
-// AuthenticateResponse is the serialized response from the Authenticate RPC.
-// It's a 1:1 representation of the capnp message, so it's not very useful for programmers.
-// Instead, you should call the `Outcome()` method to get a programmer-friendly sum type, with one
-// case for each possible outcome.
-type AuthenticateResponse struct {
-	PermanentErr      string
-	RetryableErr      string
-	Jwt               []byte
-	HoursUntilRefresh uint8
-}
-
-// Outcome turns the deserialized response of Authenticate into a programmer-friendly sum type.
-func (ar AuthenticateResponse) Outcome() AuthOutcome {
-	// If the user's authentication was unsuccessful, the server will return an error explaining why.
-	// cloudflared should fatal with this error.
-	if ar.PermanentErr != "" {
-		return NewAuthFail(errors.New(ar.PermanentErr))
-	}
-
-	// If there was a network error, then cloudflared should retry later,
-	// because origintunneld couldn't prove whether auth was correct or not.
-	if ar.RetryableErr != "" {
-		return NewAuthUnknown(errors.New(ar.RetryableErr), ar.HoursUntilRefresh)
-	}
-
-	// If auth succeeded, return the token and refresh it when instructed.
-	if len(ar.Jwt) > 0 {
-		return NewAuthSuccess(ar.Jwt, ar.HoursUntilRefresh)
-	}
-
-	// Otherwise the state got messed up.
-	return nil
-}
-
-// AuthOutcome is a programmer-friendly sum type denoting the possible outcomes of Authenticate.
-type AuthOutcome interface {
-	isAuthOutcome()
-	// Serialize into an AuthenticateResponse which can be sent via Capnp
-	Serialize() AuthenticateResponse
-}
-
-// AuthSuccess means the backend successfully authenticated this cloudflared.
-type AuthSuccess struct {
-	jwt               []byte
-	hoursUntilRefresh uint8
-}
-
-func NewAuthSuccess(jwt []byte, hoursUntilRefresh uint8) AuthSuccess {
-	return AuthSuccess{jwt: jwt, hoursUntilRefresh: hoursUntilRefresh}
-}
-
-func (ao AuthSuccess) JWT() []byte {
-	return ao.jwt
-}
-
-// RefreshAfter is how long cloudflared should wait before rerunning Authenticate.
-func (ao AuthSuccess) RefreshAfter() time.Duration {
-	return hoursToTime(ao.hoursUntilRefresh)
-}
-
-// Serialize into an AuthenticateResponse which can be sent via Capnp
-func (ao AuthSuccess) Serialize() AuthenticateResponse {
-	return AuthenticateResponse{
-		Jwt:               ao.jwt,
-		HoursUntilRefresh: ao.hoursUntilRefresh,
-	}
-}
-
-func (ao AuthSuccess) isAuthOutcome() {}
-
-// AuthFail means this cloudflared has the wrong auth and should exit.
-type AuthFail struct {
-	err error
-}
-
-func NewAuthFail(err error) AuthFail {
-	return AuthFail{err: err}
-}
-
-func (ao AuthFail) Error() string {
-	return ao.err.Error()
-}
-
-// Serialize into an AuthenticateResponse which can be sent via Capnp
-func (ao AuthFail) Serialize() AuthenticateResponse {
-	return AuthenticateResponse{
-		PermanentErr: ao.err.Error(),
-	}
-}
-
-func (ao AuthFail) isAuthOutcome() {}
-
-// AuthUnknown means the backend couldn't finish checking authentication. Try again later.
-type AuthUnknown struct {
-	err               error
-	hoursUntilRefresh uint8
-}
-
-func NewAuthUnknown(err error, hoursUntilRefresh uint8) AuthUnknown {
-	return AuthUnknown{err: err, hoursUntilRefresh: hoursUntilRefresh}
-}
-
-func (ao AuthUnknown) Error() string {
-	return ao.err.Error()
-}
-
-// RefreshAfter is how long cloudflared should wait before rerunning Authenticate.
-func (ao AuthUnknown) RefreshAfter() time.Duration {
-	return hoursToTime(ao.hoursUntilRefresh)
-}
-
-// Serialize into an AuthenticateResponse which can be sent via Capnp
-func (ao AuthUnknown) Serialize() AuthenticateResponse {
-	return AuthenticateResponse{
-		RetryableErr:      ao.err.Error(),
-		HoursUntilRefresh: ao.hoursUntilRefresh,
-	}
-}
-
-func (ao AuthUnknown) isAuthOutcome() {}
-
-func hoursToTime(hours uint8) time.Duration {
-	return time.Duration(hours) * time.Hour
-}

tunnelrpc/pogs/auth_serialize.go

@@ -1,78 +0,0 @@
-package pogs
-
-import (
-	"context"
-
-	"zombiezen.com/go/capnproto2/pogs"
-	"zombiezen.com/go/capnproto2/server"
-
-	"github.com/cloudflare/cloudflared/tunnelrpc"
-)
-
-func (i TunnelServer_PogsImpl) Authenticate(p tunnelrpc.TunnelServer_authenticate) error {
-	originCert, err := p.Params.OriginCert()
-	if err != nil {
-		return err
-	}
-	hostname, err := p.Params.Hostname()
-	if err != nil {
-		return err
-	}
-	options, err := p.Params.Options()
-	if err != nil {
-		return err
-	}
-	pogsOptions, err := UnmarshalRegistrationOptions(options)
-	if err != nil {
-		return err
-	}
-
-	server.Ack(p.Options)
-	resp, err := i.impl.Authenticate(p.Ctx, originCert, hostname, pogsOptions)
-	if err != nil {
-		return err
-	}
-	result, err := p.Results.NewResult()
-	if err != nil {
-		return err
-	}
-	return MarshalAuthenticateResponse(result, resp)
-}
-
-func MarshalAuthenticateResponse(s tunnelrpc.AuthenticateResponse, p *AuthenticateResponse) error {
-	return pogs.Insert(tunnelrpc.AuthenticateResponse_TypeID, s.Struct, p)
-}
-
-func (c TunnelServer_PogsClient) Authenticate(ctx context.Context, originCert []byte, hostname string, options *RegistrationOptions) (*AuthenticateResponse, error) {
-	client := tunnelrpc.TunnelServer{Client: c.Client}
-	promise := client.Authenticate(ctx, func(p tunnelrpc.TunnelServer_authenticate_Params) error {
-		err := p.SetOriginCert(originCert)
-		if err != nil {
-			return err
-		}
-		err = p.SetHostname(hostname)
-		if err != nil {
-			return err
-		}
-		registrationOptions, err := p.NewOptions()
-		if err != nil {
-			return err
-		}
-		err = MarshalRegistrationOptions(registrationOptions, options)
-		if err != nil {
-			return err
-		}
-		return nil
-	})
-	retval, err := promise.Result().Struct()
-	if err != nil {
-		return nil, err
-	}
-	return UnmarshalAuthenticateResponse(retval)
-}
-
-func UnmarshalAuthenticateResponse(s tunnelrpc.AuthenticateResponse) (*AuthenticateResponse, error) {
-	p := new(AuthenticateResponse)
-	err := pogs.Extract(p, tunnelrpc.AuthenticateResponse_TypeID, s.Struct)
-	return p, err
-}

tunnelrpc/pogs/auth_test.go

@@ -1,136 +0,0 @@
-package pogs
-
-import (
-	"fmt"
-	"reflect"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-	capnp "zombiezen.com/go/capnproto2"
-
-	"github.com/cloudflare/cloudflared/tunnelrpc"
-)
-
-// Ensure the AuthOutcome sum is correct
-var _ AuthOutcome = &AuthSuccess{}
-var _ AuthOutcome = &AuthFail{}
-var _ AuthOutcome = &AuthUnknown{}
-
-// Unit tests for AuthenticateResponse.Outcome()
-func TestAuthenticateResponseOutcome(t *testing.T) {
-	type fields struct {
-		PermanentErr      string
-		RetryableErr      string
-		Jwt               []byte
-		HoursUntilRefresh uint8
-	}
-	tests := []struct {
-		name   string
-		fields fields
-		want   AuthOutcome
-	}{
-		{"success",
-			fields{Jwt: []byte("asdf"), HoursUntilRefresh: 6},
-			AuthSuccess{jwt: []byte("asdf"), hoursUntilRefresh: 6},
-		},
-		{"fail",
-			fields{PermanentErr: "bad creds"},
-			AuthFail{err: fmt.Errorf("bad creds")},
-		},
-		{"error",
-			fields{RetryableErr: "bad conn", HoursUntilRefresh: 6},
-			AuthUnknown{err: fmt.Errorf("bad conn"), hoursUntilRefresh: 6},
-		},
-		{"nil (no fields are set)",
-			fields{},
-			nil,
-		},
-		{"nil (too few fields are set)",
-			fields{HoursUntilRefresh: 6},
-			nil,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			ar := AuthenticateResponse{
-				PermanentErr:      tt.fields.PermanentErr,
-				RetryableErr:      tt.fields.RetryableErr,
-				Jwt:               tt.fields.Jwt,
-				HoursUntilRefresh: tt.fields.HoursUntilRefresh,
-			}
-			got := ar.Outcome()
-			if !reflect.DeepEqual(got, tt.want) {
-				t.Errorf("AuthenticateResponse.Outcome() = %T, want %v", got, tt.want)
-			}
-			if got != nil && !reflect.DeepEqual(got.Serialize(), ar) {
-				t.Errorf(".Outcome() and .Serialize() should be inverses but weren't. Expected %v, got %v", ar, got.Serialize())
-			}
-		})
-	}
-}
-
-func TestAuthSuccess(t *testing.T) {
-	input := NewAuthSuccess([]byte("asdf"), 6)
-	output, ok := input.Serialize().Outcome().(AuthSuccess)
-	assert.True(t, ok)
-	assert.Equal(t, input, output)
-}
-
-func TestAuthUnknown(t *testing.T) {
-	input := NewAuthUnknown(fmt.Errorf("pdx unreachable"), 6)
-	output, ok := input.Serialize().Outcome().(AuthUnknown)
-	assert.True(t, ok)
-	assert.Equal(t, input, output)
-}
-
-func TestAuthFail(t *testing.T) {
-	input := NewAuthFail(fmt.Errorf("wrong creds"))
-	output, ok := input.Serialize().Outcome().(AuthFail)
-	assert.True(t, ok)
-	assert.Equal(t, input, output)
-}
-
-func TestWhenToRefresh(t *testing.T) {
-	expected := 4 * time.Hour
-	actual := hoursToTime(4)
-	if expected != actual {
-		t.Fatalf("expected %v hours, got %v", expected, actual)
-	}
-}
-
-// Test that serializing and deserializing AuthenticationResponse undo each other.
-func TestSerializeAuthenticationResponse(t *testing.T) {
-
-	tests := []*AuthenticateResponse{
-		{
-			Jwt:               []byte("\xbd\xb2\x3d\xbc\x20\xe2\x8c\x98"),
-			HoursUntilRefresh: 24,
-		},
-		{
-			PermanentErr: "bad auth",
-		},
-		{
-			RetryableErr:      "bad connection",
-			HoursUntilRefresh: 24,
-		},
-	}
-
-	for i, testCase := range tests {
-		_, seg, err := capnp.NewMessage(capnp.SingleSegment(nil))
-		assert.NoError(t, err)
-		capnpEntity, err := tunnelrpc.NewAuthenticateResponse(seg)
-		if !assert.NoError(t, err) {
-			t.Fatal("Couldn't initialize a new message")
-		}
-		err = MarshalAuthenticateResponse(capnpEntity, testCase)
-		if !assert.NoError(t, err, "testCase index %v failed to marshal", i) {
-			continue
-		}
-		result, err := UnmarshalAuthenticateResponse(capnpEntity)
-		if !assert.NoError(t, err, "testCase index %v failed to unmarshal", i) {
-			continue
-		}
-		assert.Equal(t, testCase, result, "testCase index %v didn't preserve struct through marshalling and unmarshalling", i)
-	}
-}

tunnelrpc/pogs/cloudflared_server.go

@@ -1,9 +1,10 @@
 package pogs
 
 import (
-	"github.com/cloudflare/cloudflared/tunnelrpc"
 	capnp "zombiezen.com/go/capnproto2"
 	"zombiezen.com/go/capnproto2/rpc"
+
+	"github.com/cloudflare/cloudflared/tunnelrpc/proto"
 )
 
 type CloudflaredServer interface {
@@ -16,8 +17,8 @@ type CloudflaredServer_PogsImpl struct {
 	ConfigurationManager_PogsImpl
 }
 
-func CloudflaredServer_ServerToClient(s SessionManager, c ConfigurationManager) tunnelrpc.CloudflaredServer {
-	return tunnelrpc.CloudflaredServer_ServerToClient(CloudflaredServer_PogsImpl{
+func CloudflaredServer_ServerToClient(s SessionManager, c ConfigurationManager) proto.CloudflaredServer {
+	return proto.CloudflaredServer_ServerToClient(CloudflaredServer_PogsImpl{
 		SessionManager_PogsImpl:       SessionManager_PogsImpl{s},
 		ConfigurationManager_PogsImpl: ConfigurationManager_PogsImpl{c},
 	})

tunnelrpc/pogs/configuration_manager.go

@@ -4,13 +4,16 @@ import (
 	"context"
 	"fmt"
 
-	"github.com/cloudflare/cloudflared/tunnelrpc"
 	capnp "zombiezen.com/go/capnproto2"
 	"zombiezen.com/go/capnproto2/rpc"
 	"zombiezen.com/go/capnproto2/server"
+
+	"github.com/cloudflare/cloudflared/tunnelrpc/metrics"
+	"github.com/cloudflare/cloudflared/tunnelrpc/proto"
 )
 
 type ConfigurationManager interface {
+	// UpdateConfiguration is the call provided to cloudflared to load the latest remote configuration.
 	UpdateConfiguration(ctx context.Context, version int32, config []byte) *UpdateConfigurationResponse
 }
 
@@ -18,11 +21,15 @@ type ConfigurationManager_PogsImpl struct {
 	impl ConfigurationManager
 }
 
-func ConfigurationManager_ServerToClient(c ConfigurationManager) tunnelrpc.ConfigurationManager {
-	return tunnelrpc.ConfigurationManager_ServerToClient(ConfigurationManager_PogsImpl{c})
+func ConfigurationManager_ServerToClient(c ConfigurationManager) proto.ConfigurationManager {
+	return proto.ConfigurationManager_ServerToClient(ConfigurationManager_PogsImpl{c})
 }
 
-func (i ConfigurationManager_PogsImpl) UpdateConfiguration(p tunnelrpc.ConfigurationManager_updateConfiguration) error {
+func (i ConfigurationManager_PogsImpl) UpdateConfiguration(p proto.ConfigurationManager_updateConfiguration) error {
+	return metrics.ObserveServerHandler(func() error { return i.updateConfiguration(p) }, metrics.ConfigurationManager, metrics.OperationUpdateConfiguration)
+}
+
+func (i ConfigurationManager_PogsImpl) updateConfiguration(p proto.ConfigurationManager_updateConfiguration) error {
 	server.Ack(p.Options)
 
 	version := p.Params.Version()
@@ -51,8 +58,8 @@ func (c ConfigurationManager_PogsClient) Close() error {
 }
 
 func (c ConfigurationManager_PogsClient) UpdateConfiguration(ctx context.Context, version int32, config []byte) (*UpdateConfigurationResponse, error) {
-	client := tunnelrpc.ConfigurationManager{Client: c.Client}
-	promise := client.UpdateConfiguration(ctx, func(p tunnelrpc.ConfigurationManager_updateConfiguration_Params) error {
+	client := proto.ConfigurationManager{Client: c.Client}
+	promise := client.UpdateConfiguration(ctx, func(p proto.ConfigurationManager_updateConfiguration_Params) error {
 		p.SetVersion(version)
 		return p.SetConfig(config)
 	})
@@ -74,7 +81,7 @@ type UpdateConfigurationResponse struct {
 	Err                error `json:"err"`
 }
 
-func (p *UpdateConfigurationResponse) Marshal(s tunnelrpc.UpdateConfigurationResponse) error {
+func (p *UpdateConfigurationResponse) Marshal(s proto.UpdateConfigurationResponse) error {
 	s.SetLatestAppliedVersion(p.LastAppliedVersion)
 	if p.Err != nil {
 		return s.SetErr(p.Err.Error())
@@ -82,7 +89,7 @@ func (p *UpdateConfigurationResponse) Marshal(s tunnelrpc.UpdateConfigurationRes
 	return nil
 }
 
-func (p *UpdateConfigurationResponse) Unmarshal(s tunnelrpc.UpdateConfigurationResponse) error {
+func (p *UpdateConfigurationResponse) Unmarshal(s proto.UpdateConfigurationResponse) error {
 	p.LastAppliedVersion = s.LatestAppliedVersion()
 	respErr, err := s.Err()
 	if err != nil {

tunnelrpc/pogs/quic_metadata_protocol.go

@@ -1,4 +1,4 @@
-package quic
+package pogs
 
 import (
 	"fmt"
@@ -6,7 +6,7 @@ import (
 	capnp "zombiezen.com/go/capnproto2"
 	"zombiezen.com/go/capnproto2/pogs"
 
-	"github.com/cloudflare/cloudflared/quic/schema"
+	"github.com/cloudflare/cloudflared/tunnelrpc/proto"
 )
 
 // ConnectionType indicates the type of underlying connection proxied within the QUIC stream.
@@ -52,26 +52,26 @@ func (r *ConnectRequest) MetadataMap() map[string]string {
 	return metadataMap
 }
 
-func (r *ConnectRequest) fromPogs(msg *capnp.Message) error {
-	metadata, err := schema.ReadRootConnectRequest(msg)
+func (r *ConnectRequest) FromPogs(msg *capnp.Message) error {
+	metadata, err := proto.ReadRootConnectRequest(msg)
 	if err != nil {
 		return err
 	}
-	return pogs.Extract(r, schema.ConnectRequest_TypeID, metadata.Struct)
+	return pogs.Extract(r, proto.ConnectRequest_TypeID, metadata.Struct)
 }
 
-func (r *ConnectRequest) toPogs() (*capnp.Message, error) {
+func (r *ConnectRequest) ToPogs() (*capnp.Message, error) {
 	msg, seg, err := capnp.NewMessage(capnp.SingleSegment(nil))
 	if err != nil {
 		return nil, err
 	}
 
-	root, err := schema.NewRootConnectRequest(seg)
+	root, err := proto.NewRootConnectRequest(seg)
 	if err != nil {
 		return nil, err
 	}
 
-	if err := pogs.Insert(schema.ConnectRequest_TypeID, root.Struct, r); err != nil {
+	if err := pogs.Insert(proto.ConnectRequest_TypeID, root.Struct, r); err != nil {
 		return nil, err
 	}
 
@@ -84,26 +84,26 @@ type ConnectResponse struct {
 	Metadata []Metadata `capnp:"metadata"`
 }
 
-func (r *ConnectResponse) fromPogs(msg *capnp.Message) error {
-	metadata, err := schema.ReadRootConnectResponse(msg)
+func (r *ConnectResponse) FromPogs(msg *capnp.Message) error {
+	metadata, err := proto.ReadRootConnectResponse(msg)
 	if err != nil {
 		return err
 	}
-	return pogs.Extract(r, schema.ConnectResponse_TypeID, metadata.Struct)
+	return pogs.Extract(r, proto.ConnectResponse_TypeID, metadata.Struct)
 }
 
-func (r *ConnectResponse) toPogs() (*capnp.Message, error) {
+func (r *ConnectResponse) ToPogs() (*capnp.Message, error) {
 	msg, seg, err := capnp.NewMessage(capnp.SingleSegment(nil))
 	if err != nil {
 		return nil, err
 	}
 
-	root, err := schema.NewRootConnectResponse(seg)
+	root, err := proto.NewRootConnectResponse(seg)
 	if err != nil {
 		return nil, err
 	}
 
-	if err := pogs.Insert(schema.ConnectResponse_TypeID, root.Struct, r); err != nil {
+	if err := pogs.Insert(proto.ConnectResponse_TypeID, root.Struct, r); err != nil {
 		return nil, err
 	}
 

tunnelrpc/pogs/reconnect_tunnel.go

@@ -1,93 +0,0 @@
-package pogs
-
-import (
-	"context"
-
-	"zombiezen.com/go/capnproto2/server"
-
-	"github.com/cloudflare/cloudflared/tunnelrpc"
-)
-
-func (i TunnelServer_PogsImpl) ReconnectTunnel(p tunnelrpc.TunnelServer_reconnectTunnel) error {
-	jwt, err := p.Params.Jwt()
-	if err != nil {
-		return err
-	}
-	eventDigest, err := p.Params.EventDigest()
-	if err != nil {
-		return err
-	}
-	connDigest, err := p.Params.ConnDigest()
-	if err != nil {
-		return err
-	}
-	hostname, err := p.Params.Hostname()
-	if err != nil {
-		return err
-	}
-	options, err := p.Params.Options()
-	if err != nil {
-		return err
-	}
-	pogsOptions, err := UnmarshalRegistrationOptions(options)
-	if err != nil {
-		return err
-	}
-	server.Ack(p.Options)
-	registration, err := i.impl.ReconnectTunnel(p.Ctx, jwt, eventDigest, connDigest, hostname, pogsOptions)
-	if err != nil {
-		return err
-	}
-	result, err := p.Results.NewResult()
-	if err != nil {
-		return err
-	}
-	return MarshalTunnelRegistration(result, registration)
-}
-
-func (c TunnelServer_PogsClient) ReconnectTunnel(
-	ctx context.Context,
-	jwt,
-	eventDigest []byte,
-	connDigest []byte,
-	hostname string,
-	options *RegistrationOptions,
-) *TunnelRegistration {
-	client := tunnelrpc.TunnelServer{Client: c.Client}
-	promise := client.ReconnectTunnel(ctx, func(p tunnelrpc.TunnelServer_reconnectTunnel_Params) error {
-		err := p.SetJwt(jwt)
-		if err != nil {
-			return err
-		}
-		err = p.SetEventDigest(eventDigest)
-		if err != nil {
-			return err
-		}
-		err = p.SetConnDigest(connDigest)
-		if err != nil {
-			return err
-		}
-		err = p.SetHostname(hostname)
-		if err != nil {
-			return err
-		}
-		registrationOptions, err := p.NewOptions()
-		if err != nil {
-			return err
-		}
-		err = MarshalRegistrationOptions(registrationOptions, options)
-		if err != nil {
-			return err
-		}
-		return nil
-	})
-	retval, err := promise.Result().Struct()
-	if err != nil {
-		return NewRetryableRegistrationError(err, defaultRetryAfterSeconds).Serialize()
-	}
-	registration, err := UnmarshalTunnelRegistration(retval)
-	if err != nil {
-		return NewRetryableRegistrationError(err, defaultRetryAfterSeconds).Serialize()
-	}
-	return registration
-}

tunnelrpc/pogs/registration_server.go

@@ -12,12 +12,18 @@ import (
 	"zombiezen.com/go/capnproto2/rpc"
 	"zombiezen.com/go/capnproto2/server"
 
-	"github.com/cloudflare/cloudflared/tunnelrpc"
+	"github.com/cloudflare/cloudflared/tunnelrpc/metrics"
+	"github.com/cloudflare/cloudflared/tunnelrpc/proto"
 )
 
 type RegistrationServer interface {
+	// RegisterConnection is the call typically handled by the edge to initiate and authenticate a new connection
+	// for cloudflared.
 	RegisterConnection(ctx context.Context, auth TunnelAuth, tunnelID uuid.UUID, connIndex byte, options *ConnectionOptions) (*ConnectionDetails, error)
+	// UnregisterConnection is the call typically handled by the edge to close an existing connection for cloudflared.
 	UnregisterConnection(ctx context.Context)
+	// UpdateLocalConfiguration is the call typically handled by the edge for cloudflared to provide the current
+	// configuration it is operating with.
 	UpdateLocalConfiguration(ctx context.Context, config []byte) error
 }
 
@@ -25,11 +31,15 @@ type RegistrationServer_PogsImpl struct {
 	impl RegistrationServer
 }
 
-func RegistrationServer_ServerToClient(s RegistrationServer) tunnelrpc.RegistrationServer {
-	return tunnelrpc.RegistrationServer_ServerToClient(RegistrationServer_PogsImpl{s})
+func RegistrationServer_ServerToClient(s RegistrationServer) proto.RegistrationServer {
+	return proto.RegistrationServer_ServerToClient(RegistrationServer_PogsImpl{s})
 }
 
-func (i RegistrationServer_PogsImpl) RegisterConnection(p tunnelrpc.RegistrationServer_registerConnection) error {
+func (i RegistrationServer_PogsImpl) RegisterConnection(p proto.RegistrationServer_registerConnection) error {
+	return metrics.ObserveServerHandler(func() error { return i.registerConnection(p) }, metrics.Registration, metrics.OperationRegisterConnection)
+}
+
+func (i RegistrationServer_PogsImpl) registerConnection(p proto.RegistrationServer_registerConnection) error {
 	server.Ack(p.Options)
 
 	auth, err := p.Params.Auth()
@@ -82,14 +92,19 @@ func (i RegistrationServer_PogsImpl) RegisterConnection(p tunnelrpc.Registration
 	}
 }
 
-func (i RegistrationServer_PogsImpl) UnregisterConnection(p tunnelrpc.RegistrationServer_unregisterConnection) error {
-	server.Ack(p.Options)
-
-	i.impl.UnregisterConnection(p.Ctx)
-	return nil
+func (i RegistrationServer_PogsImpl) UnregisterConnection(p proto.RegistrationServer_unregisterConnection) error {
+	return metrics.ObserveServerHandler(func() error {
+		server.Ack(p.Options)
+		i.impl.UnregisterConnection(p.Ctx)
+		return nil // No metrics will be reported for failure as this method has no return value
+	}, metrics.Registration, metrics.OperationUnregisterConnection)
 }
 
-func (i RegistrationServer_PogsImpl) UpdateLocalConfiguration(c tunnelrpc.RegistrationServer_updateLocalConfiguration) error {
+func (i RegistrationServer_PogsImpl) UpdateLocalConfiguration(p proto.RegistrationServer_updateLocalConfiguration) error {
+	return metrics.ObserveServerHandler(func() error { return i.updateLocalConfiguration(p) }, metrics.Registration, metrics.OperationUpdateLocalConfiguration)
+}
+
+func (i RegistrationServer_PogsImpl) updateLocalConfiguration(c proto.RegistrationServer_updateLocalConfiguration) error {
 	server.Ack(c.Options)
 
 	configBytes, err := c.Params.Config()
@@ -105,14 +120,21 @@ type RegistrationServer_PogsClient struct {
 	Conn   *rpc.Conn
 }
 
+func NewRegistrationServer_PogsClient(client capnp.Client, conn *rpc.Conn) RegistrationServer_PogsClient {
+	return RegistrationServer_PogsClient{
+		Client: client,
+		Conn:   conn,
+	}
+}
+
 func (c RegistrationServer_PogsClient) Close() error {
 	c.Client.Close()
 	return c.Conn.Close()
 }
 
 func (c RegistrationServer_PogsClient) RegisterConnection(ctx context.Context, auth TunnelAuth, tunnelID uuid.UUID, connIndex byte, options *ConnectionOptions) (*ConnectionDetails, error) {
-	client := tunnelrpc.TunnelServer{Client: c.Client}
-	promise := client.RegisterConnection(ctx, func(p tunnelrpc.RegistrationServer_registerConnection_Params) error {
+	client := proto.TunnelServer{Client: c.Client}
+	promise := client.RegisterConnection(ctx, func(p proto.RegistrationServer_registerConnection_Params) error {
 		tunnelAuth, err := p.NewAuth()
 		if err != nil {
 			return err
@@ -145,7 +167,7 @@ func (c RegistrationServer_PogsClient) RegisterConnection(ctx context.Context, a
 	}
 	result := response.Result()
 	switch result.Which() {
-	case tunnelrpc.ConnectionResponse_result_Which_error:
+	case proto.ConnectionResponse_result_Which_error:
 		resultError, err := result.Error()
 		if err != nil {
 			return nil, wrapRPCError(err)
@@ -160,7 +182,7 @@ func (c RegistrationServer_PogsClient) RegisterConnection(ctx context.Context, a
 		}
 		return nil, err
 
-	case tunnelrpc.ConnectionResponse_result_Which_connectionDetails:
+	case proto.ConnectionResponse_result_Which_connectionDetails:
 		connDetails, err := result.ConnectionDetails()
 		if err != nil {
 			return nil, wrapRPCError(err)
@@ -176,8 +198,8 @@ func (c RegistrationServer_PogsClient) RegisterConnection(ctx context.Context, a
 }
 
 func (c RegistrationServer_PogsClient) SendLocalConfiguration(ctx context.Context, config []byte) error {
-	client := tunnelrpc.TunnelServer{Client: c.Client}
-	promise := client.UpdateLocalConfiguration(ctx, func(p tunnelrpc.RegistrationServer_updateLocalConfiguration_Params) error {
+	client := proto.TunnelServer{Client: c.Client}
+	promise := client.UpdateLocalConfiguration(ctx, func(p proto.RegistrationServer_updateLocalConfiguration_Params) error {
 		if err := p.SetConfig(config); err != nil {
 			return err
 		}
@@ -194,8 +216,8 @@ func (c RegistrationServer_PogsClient) SendLocalConfiguration(ctx context.Contex
 }
 
 func (c RegistrationServer_PogsClient) UnregisterConnection(ctx context.Context) error {
-	client := tunnelrpc.TunnelServer{Client: c.Client}
-	promise := client.UnregisterConnection(ctx, func(p tunnelrpc.RegistrationServer_unregisterConnection_Params) error {
+	client := proto.TunnelServer{Client: c.Client}
+	promise := client.UnregisterConnection(ctx, func(p proto.RegistrationServer_unregisterConnection_Params) error {
 		return nil
 	})
 	_, err := promise.Struct()
@@ -225,20 +247,20 @@ type TunnelAuth struct {
 	TunnelSecret []byte
 }
 
-func (p *ConnectionOptions) MarshalCapnproto(s tunnelrpc.ConnectionOptions) error {
-	return pogs.Insert(tunnelrpc.ConnectionOptions_TypeID, s.Struct, p)
+func (p *ConnectionOptions) MarshalCapnproto(s proto.ConnectionOptions) error {
+	return pogs.Insert(proto.ConnectionOptions_TypeID, s.Struct, p)
 }
 
-func (p *ConnectionOptions) UnmarshalCapnproto(s tunnelrpc.ConnectionOptions) error {
-	return pogs.Extract(p, tunnelrpc.ConnectionOptions_TypeID, s.Struct)
+func (p *ConnectionOptions) UnmarshalCapnproto(s proto.ConnectionOptions) error {
+	return pogs.Extract(p, proto.ConnectionOptions_TypeID, s.Struct)
 }
 
-func (a *TunnelAuth) MarshalCapnproto(s tunnelrpc.TunnelAuth) error {
-	return pogs.Insert(tunnelrpc.TunnelAuth_TypeID, s.Struct, a)
+func (a *TunnelAuth) MarshalCapnproto(s proto.TunnelAuth) error {
+	return pogs.Insert(proto.TunnelAuth_TypeID, s.Struct, a)
 }
 
-func (a *TunnelAuth) UnmarshalCapnproto(s tunnelrpc.TunnelAuth) error {
-	return pogs.Extract(a, tunnelrpc.TunnelAuth_TypeID, s.Struct)
+func (a *TunnelAuth) UnmarshalCapnproto(s proto.TunnelAuth) error {
+	return pogs.Extract(a, proto.TunnelAuth_TypeID, s.Struct)
 }
 
 type ConnectionDetails struct {
@@ -247,7 +269,7 @@ type ConnectionDetails struct {
 	TunnelIsRemotelyManaged bool
 }
 
-func (details *ConnectionDetails) MarshalCapnproto(s tunnelrpc.ConnectionDetails) error {
+func (details *ConnectionDetails) MarshalCapnproto(s proto.ConnectionDetails) error {
 	if err := s.SetUuid(details.UUID[:]); err != nil {
 		return err
 	}
@@ -259,7 +281,7 @@ func (details *ConnectionDetails) MarshalCapnproto(s tunnelrpc.ConnectionDetails
 	return nil
 }
 
-func (details *ConnectionDetails) UnmarshalCapnproto(s tunnelrpc.ConnectionDetails) error {
+func (details *ConnectionDetails) UnmarshalCapnproto(s proto.ConnectionDetails) error {
 	uuidBytes, err := s.Uuid()
 	if err != nil {
 		return err
@@ -277,7 +299,7 @@ func (details *ConnectionDetails) UnmarshalCapnproto(s tunnelrpc.ConnectionDetai
 	return err
 }
 
-func MarshalError(s tunnelrpc.ConnectionError, err error) error {
+func MarshalError(s proto.ConnectionError, err error) error {
 	if err := s.SetCause(err.Error()); err != nil {
 		return err
 	}

tunnelrpc/pogs/registration_server_test.go

@@ -13,7 +13,7 @@ import (
 	capnp "zombiezen.com/go/capnproto2"
 	"zombiezen.com/go/capnproto2/rpc"
 
-	"github.com/cloudflare/cloudflared/tunnelrpc"
+	"github.com/cloudflare/cloudflared/tunnelrpc/proto"
 )
 
 const testAccountTag = "abc123"
@@ -34,7 +34,7 @@ func TestMarshalConnectionOptions(t *testing.T) {
 
 	_, seg, err := capnp.NewMessage(capnp.SingleSegment(nil))
 	require.NoError(t, err)
-	capnpOpts, err := tunnelrpc.NewConnectionOptions(seg)
+	capnpOpts, err := proto.NewConnectionOptions(seg)
 	require.NoError(t, err)
 
 	err = orig.MarshalCapnproto(capnpOpts)
@@ -54,18 +54,14 @@ func TestConnectionRegistrationRPC(t *testing.T) {
 
 	// Server-side
 	testImpl := testConnectionRegistrationServer{}
-	srv := TunnelServer_ServerToClient(&testImpl)
+	srv := RegistrationServer_ServerToClient(&testImpl)
 	serverConn := rpc.NewConn(t1, rpc.MainInterface(srv.Client))
 	defer serverConn.Wait()
 
 	ctx := context.Background()
 	clientConn := rpc.NewConn(t2)
 	defer clientConn.Close()
-	client := TunnelServer_PogsClient{
-		RegistrationServer_PogsClient: RegistrationServer_PogsClient{
-			Client: clientConn.Bootstrap(ctx),
-			Conn:   clientConn,
-		},
+	client := RegistrationServer_PogsClient{
 		Client: clientConn.Bootstrap(ctx),
 		Conn:   clientConn,
 	}
@@ -123,8 +119,6 @@ func TestConnectionRegistrationRPC(t *testing.T) {
 }
 
 type testConnectionRegistrationServer struct {
-	mockTunnelServerBase
-
 	details *ConnectionDetails
 	err     error
 }
@@ -147,3 +141,7 @@ func (t *testConnectionRegistrationServer) RegisterConnection(ctx context.Contex
 
 	panic("either details or err mush be set")
 }
+
+func (t *testConnectionRegistrationServer) UnregisterConnection(ctx context.Context) {
+	panic("unimplemented: UnregisterConnection")
+}

tunnelrpc/pogs/session_manager.go

@@ -6,11 +6,13 @@ import (
 	"net"
 	"time"
 
-	"github.com/cloudflare/cloudflared/tunnelrpc"
 	"github.com/google/uuid"
 	capnp "zombiezen.com/go/capnproto2"
 	"zombiezen.com/go/capnproto2/rpc"
 	"zombiezen.com/go/capnproto2/server"
+
+	"github.com/cloudflare/cloudflared/tunnelrpc/metrics"
+	"github.com/cloudflare/cloudflared/tunnelrpc/proto"
 )
 
 type SessionManager interface {
@@ -26,11 +28,15 @@ type SessionManager_PogsImpl struct {
 	impl SessionManager
 }
 
-func SessionManager_ServerToClient(s SessionManager) tunnelrpc.SessionManager {
-	return tunnelrpc.SessionManager_ServerToClient(SessionManager_PogsImpl{s})
+func SessionManager_ServerToClient(s SessionManager) proto.SessionManager {
+	return proto.SessionManager_ServerToClient(SessionManager_PogsImpl{s})
 }
 
-func (i SessionManager_PogsImpl) RegisterUdpSession(p tunnelrpc.SessionManager_registerUdpSession) error {
+func (i SessionManager_PogsImpl) RegisterUdpSession(p proto.SessionManager_registerUdpSession) error {
+	return metrics.ObserveServerHandler(func() error { return i.registerUdpSession(p) }, metrics.SessionManager, metrics.OperationRegisterUdpSession)
+}
+
+func (i SessionManager_PogsImpl) registerUdpSession(p proto.SessionManager_registerUdpSession) error {
 	server.Ack(p.Options)
 
 	sessionIDRaw, err := p.Params.SessionId()
@@ -76,7 +82,11 @@ func (i SessionManager_PogsImpl) RegisterUdpSession(p tunnelrpc.SessionManager_r
 	return resp.Marshal(result)
 }
 
-func (i SessionManager_PogsImpl) UnregisterUdpSession(p tunnelrpc.SessionManager_unregisterUdpSession) error {
+func (i SessionManager_PogsImpl) UnregisterUdpSession(p proto.SessionManager_unregisterUdpSession) error {
+	return metrics.ObserveServerHandler(func() error { return i.unregisterUdpSession(p) }, metrics.SessionManager, metrics.OperationUnregisterUdpSession)
+}
+
+func (i SessionManager_PogsImpl) unregisterUdpSession(p proto.SessionManager_unregisterUdpSession) error {
 	server.Ack(p.Options)
 
 	sessionIDRaw, err := p.Params.SessionId()
@@ -101,7 +111,7 @@ type RegisterUdpSessionResponse struct {
 	Spans []byte // Spans in protobuf format
 }
 
-func (p *RegisterUdpSessionResponse) Marshal(s tunnelrpc.RegisterUdpSessionResponse) error {
+func (p *RegisterUdpSessionResponse) Marshal(s proto.RegisterUdpSessionResponse) error {
 	if p.Err != nil {
 		return s.SetErr(p.Err.Error())
 	}
@@ -111,7 +121,7 @@ func (p *RegisterUdpSessionResponse) Marshal(s tunnelrpc.RegisterUdpSessionRespo
 	return nil
 }
 
-func (p *RegisterUdpSessionResponse) Unmarshal(s tunnelrpc.RegisterUdpSessionResponse) error {
+func (p *RegisterUdpSessionResponse) Unmarshal(s proto.RegisterUdpSessionResponse) error {
 	respErr, err := s.Err()
 	if err != nil {
 		return err
@@ -131,14 +141,21 @@ type SessionManager_PogsClient struct {
 	Conn   *rpc.Conn
 }
 
+func NewSessionManager_PogsClient(client capnp.Client, conn *rpc.Conn) SessionManager_PogsClient {
+	return SessionManager_PogsClient{
+		Client: client,
+		Conn:   conn,
+	}
+}
+
 func (c SessionManager_PogsClient) Close() error {
 	c.Client.Close()
 	return c.Conn.Close()
 }
 
 func (c SessionManager_PogsClient) RegisterUdpSession(ctx context.Context, sessionID uuid.UUID, dstIP net.IP, dstPort uint16, closeAfterIdleHint time.Duration, traceContext string) (*RegisterUdpSessionResponse, error) {
-	client := tunnelrpc.SessionManager{Client: c.Client}
-	promise := client.RegisterUdpSession(ctx, func(p tunnelrpc.SessionManager_registerUdpSession_Params) error {
+	client := proto.SessionManager{Client: c.Client}
+	promise := client.RegisterUdpSession(ctx, func(p proto.SessionManager_registerUdpSession_Params) error {
 		if err := p.SetSessionId(sessionID[:]); err != nil {
 			return err
 		}
@@ -164,8 +181,8 @@ func (c SessionManager_PogsClient) RegisterUdpSession(ctx context.Context, sessi
 }
 
 func (c SessionManager_PogsClient) UnregisterUdpSession(ctx context.Context, sessionID uuid.UUID, message string) error {
-	client := tunnelrpc.SessionManager{Client: c.Client}
-	promise := client.UnregisterUdpSession(ctx, func(p tunnelrpc.SessionManager_unregisterUdpSession_Params) error {
+	client := proto.SessionManager{Client: c.Client}
+	promise := client.UnregisterUdpSession(ctx, func(p proto.SessionManager_unregisterUdpSession_Params) error {
 		if err := p.SetSessionId(sessionID[:]); err != nil {
 			return err
 		}

tunnelrpc/pogs/support_test.go

@@ -1,40 +0,0 @@
-package pogs
-
-import (
-	"context"
-
-	"github.com/google/uuid"
-)
-
-// mockTunnelServerBase provides a placeholder implementation
-// for TunnelServer interface that can be used to build
-// mocks for specific unit tests without having to implement every method
-type mockTunnelServerBase struct{}
-
-func (mockTunnelServerBase) RegisterConnection(ctx context.Context, auth TunnelAuth, tunnelID uuid.UUID, connIndex byte, options *ConnectionOptions) (*ConnectionDetails, error) {
-	panic("unexpected call to RegisterConnection")
-}
-
-func (mockTunnelServerBase) UnregisterConnection(ctx context.Context) {
-	panic("unexpected call to UnregisterConnection")
-}
-
-func (mockTunnelServerBase) RegisterTunnel(ctx context.Context, originCert []byte, hostname string, options *RegistrationOptions) *TunnelRegistration {
-	panic("unexpected call to RegisterTunnel")
-}
-
-func (mockTunnelServerBase) GetServerInfo(ctx context.Context) (*ServerInfo, error) {
-	panic("unexpected call to GetServerInfo")
-}
-
-func (mockTunnelServerBase) UnregisterTunnel(ctx context.Context, gracePeriodNanoSec int64) error {
-	panic("unexpected call to UnregisterTunnel")
-}
-
-func (mockTunnelServerBase) Authenticate(ctx context.Context, originCert []byte, hostname string, options *RegistrationOptions) (*AuthenticateResponse, error) {
-	panic("unexpected call to Authenticate")
-}
-
-func (mockTunnelServerBase) ReconnectTunnel(ctx context.Context, jwt, eventDigest, connDigest []byte, hostname string, options *RegistrationOptions) (*TunnelRegistration, error) {
-	panic("unexpected call to ReconnectTunnel")
-}

tunnelrpc/pogs/tag.go

@@ -0,0 +1,8 @@
+package pogs
+
+// Tag previously was a legacy tunnel capnp struct but was deprecated. To help reduce the amount of changes imposed
+// by removing this simple struct, it was copied out of the capnp and provided here instead.
+type Tag struct {
+	Name  string
+	Value string
+}

tunnelrpc/pogs/tunnelrpc.go

@@ -1,334 +0,0 @@
-package pogs
-
-import (
-	"context"
-	"fmt"
-
-	capnp "zombiezen.com/go/capnproto2"
-	"zombiezen.com/go/capnproto2/pogs"
-	"zombiezen.com/go/capnproto2/rpc"
-	"zombiezen.com/go/capnproto2/server"
-
-	"github.com/cloudflare/cloudflared/tunnelrpc"
-)
-
-const (
-	defaultRetryAfterSeconds = 15
-)
-
-type Authentication struct {
-	Key         string
-	Email       string
-	OriginCAKey string
-}
-
-func MarshalAuthentication(s tunnelrpc.Authentication, p *Authentication) error {
-	return pogs.Insert(tunnelrpc.Authentication_TypeID, s.Struct, p)
-}
-
-func UnmarshalAuthentication(s tunnelrpc.Authentication) (*Authentication, error) {
-	p := new(Authentication)
-	err := pogs.Extract(p, tunnelrpc.Authentication_TypeID, s.Struct)
-	return p, err
-}
-
-type TunnelRegistration struct {
-	SuccessfulTunnelRegistration
-	Err               string
-	PermanentFailure  bool
-	RetryAfterSeconds uint16
-}
-
-type SuccessfulTunnelRegistration struct {
-	Url         string
-	LogLines    []string
-	TunnelID    string `capnp:"tunnelID"`
-	EventDigest []byte
-	ConnDigest  []byte
-}
-
-func NewSuccessfulTunnelRegistration(
-	url string,
-	logLines []string,
-	tunnelID string,
-	eventDigest []byte,
-	connDigest []byte,
-) *TunnelRegistration {
-	// Marshal nil will result in an error
-	if logLines == nil {
-		logLines = []string{}
-	}
-	return &TunnelRegistration{
-		SuccessfulTunnelRegistration: SuccessfulTunnelRegistration{
-			Url:         url,
-			LogLines:    logLines,
-			TunnelID:    tunnelID,
-			EventDigest: eventDigest,
-			ConnDigest:  connDigest,
-		},
-	}
-}
-
-// Not calling this function Error() to avoid confusion with implementing error interface
-func (tr TunnelRegistration) DeserializeError() TunnelRegistrationError {
-	if tr.Err != "" {
-		err := fmt.Errorf(tr.Err)
-		if tr.PermanentFailure {
-			return NewPermanentRegistrationError(err)
-		}
-		retryAfterSeconds := tr.RetryAfterSeconds
-		if retryAfterSeconds < defaultRetryAfterSeconds {
-			retryAfterSeconds = defaultRetryAfterSeconds
-		}
-		return NewRetryableRegistrationError(err, retryAfterSeconds)
-	}
-	return nil
-}
-
-type TunnelRegistrationError interface {
-	error
-	Serialize() *TunnelRegistration
-	IsPermanent() bool
-}
-
-type PermanentRegistrationError struct {
-	err string
-}
-
-func NewPermanentRegistrationError(err error) TunnelRegistrationError {
-	return &PermanentRegistrationError{
-		err: err.Error(),
-	}
-}
-
-func (pre *PermanentRegistrationError) Error() string {
-	return pre.err
-}
-
-func (pre *PermanentRegistrationError) Serialize() *TunnelRegistration {
-	return &TunnelRegistration{
-		Err:              pre.err,
-		PermanentFailure: true,
-	}
-}
-
-func (*PermanentRegistrationError) IsPermanent() bool {
-	return true
-}
-
-type RetryableRegistrationError struct {
-	err               string
-	retryAfterSeconds uint16
-}
-
-func NewRetryableRegistrationError(err error, retryAfterSeconds uint16) TunnelRegistrationError {
-	return &RetryableRegistrationError{
-		err:               err.Error(),
-		retryAfterSeconds: retryAfterSeconds,
-	}
-}
-
-func (rre *RetryableRegistrationError) Error() string {
-	return rre.err
-}
-
-func (rre *RetryableRegistrationError) Serialize() *TunnelRegistration {
-	return &TunnelRegistration{
-		Err:               rre.err,
-		PermanentFailure:  false,
-		RetryAfterSeconds: rre.retryAfterSeconds,
-	}
-}
-
-func (*RetryableRegistrationError) IsPermanent() bool {
-	return false
-}
-
-func MarshalTunnelRegistration(s tunnelrpc.TunnelRegistration, p *TunnelRegistration) error {
-	return pogs.Insert(tunnelrpc.TunnelRegistration_TypeID, s.Struct, p)
-}
-
-func UnmarshalTunnelRegistration(s tunnelrpc.TunnelRegistration) (*TunnelRegistration, error) {
-	p := new(TunnelRegistration)
-	err := pogs.Extract(p, tunnelrpc.TunnelRegistration_TypeID, s.Struct)
-	return p, err
-}
-
-type RegistrationOptions struct {
-	ClientID             string `capnp:"clientId"`
-	Version              string
-	OS                   string `capnp:"os"`
-	ExistingTunnelPolicy tunnelrpc.ExistingTunnelPolicy
-	PoolName             string `capnp:"poolName"`
-	Tags                 []Tag
-	ConnectionID         uint8  `capnp:"connectionId"`
-	OriginLocalIP        string `capnp:"originLocalIp"`
-	IsAutoupdated        bool   `capnp:"isAutoupdated"`
-	RunFromTerminal      bool   `capnp:"runFromTerminal"`
-	CompressionQuality   uint64 `capnp:"compressionQuality"`
-	UUID                 string `capnp:"uuid"`
-	NumPreviousAttempts  uint8
-	Features             []string
-}
-
-func MarshalRegistrationOptions(s tunnelrpc.RegistrationOptions, p *RegistrationOptions) error {
-	return pogs.Insert(tunnelrpc.RegistrationOptions_TypeID, s.Struct, p)
-}
-
-func UnmarshalRegistrationOptions(s tunnelrpc.RegistrationOptions) (*RegistrationOptions, error) {
-	p := new(RegistrationOptions)
-	err := pogs.Extract(p, tunnelrpc.RegistrationOptions_TypeID, s.Struct)
-	return p, err
-}
-
-type Tag struct {
-	Name  string `json:"name"`
-	Value string `json:"value"`
-}
-
-type ServerInfo struct {
-	LocationName string
-}
-
-func MarshalServerInfo(s tunnelrpc.ServerInfo, p *ServerInfo) error {
-	return pogs.Insert(tunnelrpc.ServerInfo_TypeID, s.Struct, p)
-}
-
-func UnmarshalServerInfo(s tunnelrpc.ServerInfo) (*ServerInfo, error) {
-	p := new(ServerInfo)
-	err := pogs.Extract(p, tunnelrpc.ServerInfo_TypeID, s.Struct)
-	return p, err
-}
-
-type TunnelServer interface {
-	RegistrationServer
-	RegisterTunnel(ctx context.Context, originCert []byte, hostname string, options *RegistrationOptions) *TunnelRegistration
-	GetServerInfo(ctx context.Context) (*ServerInfo, error)
-	UnregisterTunnel(ctx context.Context, gracePeriodNanoSec int64) error
-	Authenticate(ctx context.Context, originCert []byte, hostname string, options *RegistrationOptions) (*AuthenticateResponse, error)
-	ReconnectTunnel(ctx context.Context, jwt, eventDigest, connDigest []byte, hostname string, options *RegistrationOptions) (*TunnelRegistration, error)
-}
-
-func TunnelServer_ServerToClient(s TunnelServer) tunnelrpc.TunnelServer {
-	return tunnelrpc.TunnelServer_ServerToClient(TunnelServer_PogsImpl{RegistrationServer_PogsImpl{s}, s})
-}
-
-type TunnelServer_PogsImpl struct {
-	RegistrationServer_PogsImpl
-	impl TunnelServer
-}
-
-func (i TunnelServer_PogsImpl) RegisterTunnel(p tunnelrpc.TunnelServer_registerTunnel) error {
-	originCert, err := p.Params.OriginCert()
-	if err != nil {
-		return err
-	}
-	hostname, err := p.Params.Hostname()
-	if err != nil {
-		return err
-	}
-	options, err := p.Params.Options()
-	if err != nil {
-		return err
-	}
-	pogsOptions, err := UnmarshalRegistrationOptions(options)
-	if err != nil {
-		return err
-	}
-	server.Ack(p.Options)
-	registration := i.impl.RegisterTunnel(p.Ctx, originCert, hostname, pogsOptions)
-
-	result, err := p.Results.NewResult()
-	if err != nil {
-		return err
-	}
-	return MarshalTunnelRegistration(result, registration)
-}
-
-func (i TunnelServer_PogsImpl) GetServerInfo(p tunnelrpc.TunnelServer_getServerInfo) error {
-	server.Ack(p.Options)
-	serverInfo, err := i.impl.GetServerInfo(p.Ctx)
-	if err != nil {
-		return err
-	}
-	result, err := p.Results.NewResult()
-	if err != nil {
-		return err
-	}
-	return MarshalServerInfo(result, serverInfo)
-}
-
-func (i TunnelServer_PogsImpl) UnregisterTunnel(p tunnelrpc.TunnelServer_unregisterTunnel) error {
-	gracePeriodNanoSec := p.Params.GracePeriodNanoSec()
-	server.Ack(p.Options)
-	return i.impl.UnregisterTunnel(p.Ctx, gracePeriodNanoSec)
-}
-
-func (i TunnelServer_PogsImpl) ObsoleteDeclarativeTunnelConnect(p tunnelrpc.TunnelServer_obsoleteDeclarativeTunnelConnect) error {
-	return fmt.Errorf("RPC to create declarative tunnel connection has been deprecated")
-}
-
-type TunnelServer_PogsClient struct {
-	RegistrationServer_PogsClient
-	Client capnp.Client
-	Conn   *rpc.Conn
-}
-
-func (c TunnelServer_PogsClient) Close() error {
-	c.Client.Close()
-	return c.Conn.Close()
-}
-
-func (c TunnelServer_PogsClient) RegisterTunnel(ctx context.Context, originCert []byte, hostname string, options *RegistrationOptions) *TunnelRegistration {
-	client := tunnelrpc.TunnelServer{Client: c.Client}
-	promise := client.RegisterTunnel(ctx, func(p tunnelrpc.TunnelServer_registerTunnel_Params) error {
-		err := p.SetOriginCert(originCert)
-		if err != nil {
-			return err
-		}
-		err = p.SetHostname(hostname)
-		if err != nil {
-			return err
-		}
-		registrationOptions, err := p.NewOptions()
-		if err != nil {
-			return err
-		}
-		err = MarshalRegistrationOptions(registrationOptions, options)
-		if err != nil {
-			return err
-		}
-		return nil
-	})
-	retval, err := promise.Result().Struct()
-	if err != nil {
-		return NewRetryableRegistrationError(err, defaultRetryAfterSeconds).Serialize()
-	}
-	registration, err := UnmarshalTunnelRegistration(retval)
-	if err != nil {
-		return NewRetryableRegistrationError(err, defaultRetryAfterSeconds).Serialize()
-	}
-	return registration
-}
-
-func (c TunnelServer_PogsClient) GetServerInfo(ctx context.Context) (*ServerInfo, error) {
-	client := tunnelrpc.TunnelServer{Client: c.Client}
-	promise := client.GetServerInfo(ctx, func(p tunnelrpc.TunnelServer_getServerInfo_Params) error {
-		return nil
-	})
-	retval, err := promise.Result().Struct()
-	if err != nil {
-		return nil, err
-	}
-	return UnmarshalServerInfo(retval)
-}
-
-func (c TunnelServer_PogsClient) UnregisterTunnel(ctx context.Context, gracePeriodNanoSec int64) error {
-	client := tunnelrpc.TunnelServer{Client: c.Client}
-	promise := client.UnregisterTunnel(ctx, func(p tunnelrpc.TunnelServer_unregisterTunnel_Params) error {
-		p.SetGracePeriodNanoSec(gracePeriodNanoSec)
-		return nil
-	})
-	_, err := promise.Struct()
-	return err
-}

tunnelrpc/pogs/tunnelrpc_test.go

@@ -1,57 +0,0 @@
-package pogs
-
-import (
-	"fmt"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	capnp "zombiezen.com/go/capnproto2"
-
-	"github.com/cloudflare/cloudflared/tunnelrpc"
-)
-
-const (
-	testURL               = "tunnel.example.com"
-	testTunnelID          = "asdfghjkl;"
-	testRetryAfterSeconds = 19
-)
-
-var (
-	testErr         = fmt.Errorf("Invalid credential")
-	testLogLines    = []string{"all", "working"}
-	testEventDigest = []byte("asdf")
-	testConnDigest  = []byte("lkjh")
-)
-
-// *PermanentRegistrationError implements TunnelRegistrationError
-var _ TunnelRegistrationError = (*PermanentRegistrationError)(nil)
-
-// *RetryableRegistrationError implements TunnelRegistrationError
-var _ TunnelRegistrationError = (*RetryableRegistrationError)(nil)
-
-func TestTunnelRegistration(t *testing.T) {
-	testCases := []*TunnelRegistration{
-		NewSuccessfulTunnelRegistration(testURL, testLogLines, testTunnelID, testEventDigest, testConnDigest),
-		NewSuccessfulTunnelRegistration(testURL, nil, testTunnelID, testEventDigest, testConnDigest),
-		NewPermanentRegistrationError(testErr).Serialize(),
-		NewRetryableRegistrationError(testErr, testRetryAfterSeconds).Serialize(),
-	}
-	for i, testCase := range testCases {
-		_, seg, err := capnp.NewMessage(capnp.SingleSegment(nil))
-		assert.NoError(t, err)
-		capnpEntity, err := tunnelrpc.NewTunnelRegistration(seg)
-		if !assert.NoError(t, err) {
-			t.Fatal("Couldn't initialize a new message")
-		}
-		err = MarshalTunnelRegistration(capnpEntity, testCase)
-		if !assert.NoError(t, err, "testCase #%v failed to marshal", i) {
-			continue
-		}
-		result, err := UnmarshalTunnelRegistration(capnpEntity)
-		if !assert.NoError(t, err, "testCase #%v failed to unmarshal", i) {
-			continue
-		}
-		assert.Equal(t, testCase, result, "testCase index %v didn't preserve struct through marshalling and unmarshalling", i)
-	}
-
-}

tunnelrpc/proto/go.capnp

@@ -0,0 +1,31 @@
+# Generate go.capnp.out with:
+# capnp compile -o- go.capnp > go.capnp.out
+# Must run inside this directory to preserve paths.
+
+@0xd12a1c51fedd6c88;
+
+annotation package(file) :Text;
+# The Go package name for the generated file.
+
+annotation import(file) :Text;
+# The Go import path that the generated file is accessible from.
+# Used to generate import statements and check if two types are in the
+# same package.
+
+annotation doc(struct, field, enum) :Text;
+# Adds a doc comment to the generated code.
+
+annotation tag(enumerant) :Text;
+# Changes the string representation of the enum in the generated code.
+
+annotation notag(enumerant) :Void;
+# Removes the string representation of the enum in the generated code.
+
+annotation customtype(field) :Text;
+# OBSOLETE, not used by code generator.
+
+annotation name(struct, field, union, enum, enumerant, interface, method, param, annotation, const, group) :Text;
+# Used to rename the element in the generated code.
+
+$package("capnp");
+$import("zombiezen.com/go/capnproto2");

tunnelrpc/proto/quic_metadata_protocol.capnp

@@ -0,0 +1,28 @@
+using Go = import "go.capnp";
+@0xb29021ef7421cc32;
+
+$Go.package("proto");
+$Go.import("github.com/cloudflare/cloudflared/tunnelrpc");
+
+
+struct ConnectRequest @0xc47116a1045e4061 {
+	dest @0 :Text;
+	type @1 :ConnectionType;
+	metadata @2 :List(Metadata);
+}
+
+enum ConnectionType @0xc52e1bac26d379c8 {
+	http @0;
+	websocket @1;
+	tcp @2;
+}
+
+struct Metadata @0xe1446b97bfd1cd37 {
+    key @0 :Text;
+	val @1 :Text;
+}
+
+struct ConnectResponse @0xb1032ec91cef8727 {
+	error @0 :Text;
+	metadata @1 :List(Metadata);
+}

tunnelrpc/proto/quic_metadata_protocol.capnp.go

@@ -1,6 +1,6 @@
 // Code generated by capnpc-go. DO NOT EDIT.
 
-package schema
+package proto
 
 import (
 	capnp "zombiezen.com/go/capnproto2"
@@ -357,34 +357,34 @@ func (p ConnectResponse_Promise) Struct() (ConnectResponse, error) {
 	return ConnectResponse{s}, err
 }
 
-const schema_b29021ef7421cc32 = "x\xda\xb4\x91\xcfk\x13A\x1c\xc5\xdf\x9bI\xba\x1e\xa2" +
-	"\x9b!\xd5\x8b\x8a\xa4\xf8+ES\xdb(\xa2\xa7\x80\x15" +
-	"TZ\xcc\x14\xcf\x96u;\x98\x92vw\x92\x9dZ\xf2" +
-	"\x17x\x15/\xe2\xd1\xbb \x15<\x0b\xa2\xa0\xa2\x07\x11" +
-	"\xff\x80\xfe\x05=y\xf0\xb42)\xdb@)\x08Bo" +
-	"\xdfy<\xe6}\xbe\xdfW\xfd\xd5\x16\xb3\xe5\xc7\x04t" +
-	"\xb5<\x91_x\xbas\xeaKSnA5\x98\xcf}" +
-	"\xab\xbb\x9d\xfa\xb3\xb7(\x8b\x00\x98}\xf9\x95\xea]\x00" +
-	"\xa8\xadM0\x8f\xda\x0fK\xafN\xf4?B7\xb8\xdf" +
-	"\xda\xaa\xf3\x03k7\x18\x00\xb5k|\x03\xe6\x9f\x87?" +
-	"\xcf\xbf>\xd9\xfc\x04\xd5\x10c3\xd8\xda\xf6\xce?#" +
-	"\xe7o\xde\x07\xf3\xeb\xdf\x7f\xbc\x7f\xd1\x9b\xdf>\x80\xa0" +
-	"uT<g\xed\x9c\x1fku\xe1!\xfa\x1b\xab\xf1L" +
-	"\x16w'\xccz4\xe3\x1f\xcb\xeb\xc6E+\x91\x8b\x96" +
-	"\xed ui\x9c\xae5\xe3\xc8&\xf6\xe6\xad4IL" +
-	"\xec\x96Lf\xd3$3@\x87\xd4Gd\x09(\x11P" +
-	"\x8d9@\x9f\x95\xd4W\x04\x159I/^\xbe\x07\xe8" +
-	"K\x92\xfa\x8e\xe0\x193\x18\xa4\x03V X\x01\xf3\"" +
-	"\x06\x00\x8f\x81\x1dIV\xc7\xe8\xa0\x17\xff\x87\xae\xbfa" +
-	"27b\xab\xec\xb1\xdd\x9e\x06t[R/\x08\x16h" +
-	"w\xbd6/\xa9;\x82Jp\x92\x02P\x8b\x9ewA" +
-	"Rw\x05\xc3\x15\x93\xb9\x027tCk\x18\x8e[\x00" +
-	"\x19\x1e\xda\x16\xabi\xf2`h\xcd\xee\x16#\xb0\xd3\xd3" +
-	"\xfe3u|\x09\xa0Pj\x0a\x08\xbb\xce\xd9|\xd3<" +
-	"\xca\xd2\xb8g@\x17\xb8\xd8\xeeE\x95\xff\x19\xb5\xb8\xab" +
-	"3\xdaW\xe3\xd4A5z\xf1\xa2\xa4\xbe*\x18\xf4\xcc" +
-	"\xb0\xb8J\xf0$Z+\xe6\xbf\x01\x00\x00\xff\xff\xf5\xed" +
-	"\xc9\xfe"
+const schema_b29021ef7421cc32 = "x\xda\xb4\x911k\x14A\x1c\xc5\xdf\x9b\xcde-\x0e" +
+	"\xf7\x86KlT\xc2\x05Q\x13\xdc\x8b\xc9\x09\xa2 \x1c" +
+	"\x18A%\xc1\x9b`mX7\x83\x09w\xee\xce\xed\xce" +
+	"\x19\xee\x13\xd8\xda\x89\xa5\xbd \x09X\xdb((h!" +
+	"\x16\xd6\x0a66\xf9\x04\xb22\x0b\x9b\x83\x90B\x04\xbb" +
+	"\xe1\xcd\x9by\xbf\xff\xff5\xbeu\xc5r\xed\x11\x01\xd5" +
+	"\xa8M\x17\x17\x9e\x1e\x9c\xf9\xd8\xf6\xf6 C\x16+\x9f" +
+	"Z\xf6\xa0\xf5l\x1f5\xe1\x03\xcb/~Q\xbe\xf1\x01" +
+	"\xb9\xb7\x0b\x16Q\xf7\xc1\xd4\xcbS\xc3wP!\x8fZ" +
+	";-\xfe`\xf3\x06}\xa0y\x8d\xaf\xc1\xe2\xc3\xf8\xeb" +
+	"\xf9W\xa7\xdb\xef!C11\x83\x9d\x9f\xceI\xf7\xa8" +
+	"\xf9\x9b\xf7\xc0\xe2\xea\xe7/o\x9f\xf7W\xbf\x1fC\xd0" +
+	"\x99\x15\xfbl\x86\xa5yA8\x08;J\x12=\xc8\xcc" +
+	"t\xbcd\xb2\xd4\xa6K\xc3\xd1N\xbc\xf9X\xdbh+" +
+	"\xb2\xd1f\xa9\xc5\xe9\xa0\x1dG&1\xd7o\xa6I\xa2" +
+	"c\xbb\xa1s\x13\xa4I\xae{\xa4:\xe1M\x01S\x04" +
+	"\xe4\xc2\x0a\xa0\xceyT\x97\x05%9C'\x86w\x01" +
+	"u\xc9\xa3\xba-8\xa7\xb3,\xcdX\x87`\x1d,\xaa" +
+	"\x14\x00<\x09\xf6<\xb21\xa1\x07\x9d\xf8\xaf\x80\xc3\x91" +
+	"\xafs\xeb\xf8\xea\x87|\xb7\x16\x01\xd5\xf5\xa8\xd6\x04+" +
+	"\xbc;N[\xf5\xa8z\x82Rp\x86\x02\x90\xeb\x8ey" +
+	"\xcd\xa3\xda\x16\x0c\xb6tn+\xe4\xc0\x8e\x8df0)" +
+	"\x03d\xf0_'\xd9I\x93\xfb\xfe\xd8\x94\x9b\xae\x97p" +
+	"g\x17\xdd\x87rv\x03\xa0\x90r\x1e\x08\xb6\xad5\xc5" +
+	"\xae~\x98\xa7q_\x83\xd6\xb7\xb19\x8c\xab\xfdU\xdc" +
+	"\xba\xb6s\xe5\xc5\x91J\xe7\x8f\xab\xd4\x89\x17=\xaa+" +
+	"\x82~_\x8f\xab\xed\xf8O\xa2Au\xfe\x13\x00\x00\xff" +
+	"\xff\x1d\xce\xd1\xb0"
 
 func init() {
 	schemas.Register(schema_b29021ef7421cc32,

tunnelrpc/proto/tunnelrpc.capnp

@@ -1,15 +1,23 @@
 using Go = import "go.capnp";
 @0xdb8274f9144abc7e;
-$Go.package("tunnelrpc");
+$Go.package("proto");
 $Go.import("github.com/cloudflare/cloudflared/tunnelrpc");
 
-struct Authentication {
+# === DEPRECATED Legacy Tunnel Authentication and Registration methods/servers ===
+# 
+# These structs and interfaces are no longer used but it is important to keep
+# them around to make sure backwards compatibility within the rpc protocol is
+# maintained.
+
+struct Authentication @0xc082ef6e0d42ed1d {
+    # DEPRECATED: Legacy tunnel authentication mechanism
     key @0 :Text;
     email @1 :Text;
     originCAKey @2 :Text;
 }
 
-struct TunnelRegistration {
+struct TunnelRegistration @0xf41a0f001ad49e46 {
+    # DEPRECATED: Legacy tunnel authentication mechanism
     err @0 :Text;
     # the url to access the tunnel
     url @1 :Text;
@@ -27,7 +35,9 @@ struct TunnelRegistration {
     connDigest @7 :Data;
 }
 
-struct RegistrationOptions {
+struct RegistrationOptions @0xc793e50592935b4a {
+    # DEPRECATED: Legacy tunnel authentication mechanism
+    
     # The tunnel client's unique identifier, used to verify a reconnection.
     clientId @0 :Text;
     # Information about the running binary.
@@ -56,29 +66,51 @@ struct RegistrationOptions {
     features @13 :List(Text);
 }
 
-struct Tag {
-    name @0 :Text;
-    value @1 :Text;
-}
+enum ExistingTunnelPolicy @0x84cb9536a2cf6d3c {
+    # DEPRECATED: Legacy tunnel registration mechanism
 
-enum ExistingTunnelPolicy {
     ignore @0;
     disconnect @1;
     balance @2;
 }
 
-struct ServerInfo {
+struct ServerInfo @0xf2c68e2547ec3866 {
+    # DEPRECATED: Legacy tunnel registration mechanism
+
     locationName @0 :Text;
 }
 
-struct AuthenticateResponse {
+struct AuthenticateResponse @0x82c325a07ad22a65 {
+    # DEPRECATED: Legacy tunnel registration mechanism
+
     permanentErr @0 :Text;
     retryableErr @1 :Text;
     jwt @2 :Data;
     hoursUntilRefresh @3 :UInt8;
 }
 
-struct ClientInfo {
+interface TunnelServer @0xea58385c65416035 extends (RegistrationServer) {
+    # DEPRECATED: Legacy tunnel authentication server
+
+    registerTunnel @0 (originCert :Data, hostname :Text, options :RegistrationOptions) -> (result :TunnelRegistration);
+    getServerInfo @1 () -> (result :ServerInfo);
+    unregisterTunnel @2 (gracePeriodNanoSec :Int64) -> ();
+    # obsoleteDeclarativeTunnelConnect RPC deprecated in TUN-3019
+    obsoleteDeclarativeTunnelConnect @3 () -> ();
+    authenticate @4 (originCert :Data, hostname :Text, options :RegistrationOptions) -> (result :AuthenticateResponse);
+    reconnectTunnel @5 (jwt :Data, eventDigest :Data, connDigest :Data, hostname :Text, options :RegistrationOptions) -> (result :TunnelRegistration);
+}
+
+struct Tag @0xcbd96442ae3bb01a {
+    # DEPRECATED: Legacy tunnel additional HTTP header mechanism 
+
+    name @0 :Text;
+    value @1 :Text;
+}
+
+# === End DEPRECATED Objects ===
+
+struct ClientInfo @0x83ced0145b2f114b {
     # The tunnel client's unique identifier, used to verify a reconnection.
     clientId @0 :Data;
     # Set of features this cloudflared knows it supports
@@ -89,7 +121,7 @@ struct ClientInfo {
     arch @3 :Text;
 }
 
-struct ConnectionOptions {
+struct ConnectionOptions @0xb4bf9861fe035d04 {
     # client details
     client @0 :ClientInfo;
     # origin LAN IP
@@ -102,21 +134,21 @@ struct ConnectionOptions {
     numPreviousAttempts @4 :UInt8;
 }
 
-struct ConnectionResponse {
+struct ConnectionResponse @0xdbaa9d03d52b62dc {
     result :union {
         error @0 :ConnectionError;
         connectionDetails @1 :ConnectionDetails;
     }
 }
 
-struct ConnectionError {
+struct ConnectionError @0xf5f383d2785edb86 {
     cause @0 :Text;
     # How long should this connection wait to retry in ns
     retryAfter @1 :Int64;
     shouldRetry @2 :Bool;
 }
 
-struct ConnectionDetails {
+struct ConnectionDetails @0xb5f39f082b9ac18a {
     # identifier of this connection
     uuid @0 :Data;
     # airport code of the colo where this connection landed
@@ -125,39 +157,29 @@ struct ConnectionDetails {
     tunnelIsRemotelyManaged @2: Bool;
 }
 
-struct TunnelAuth {
+struct TunnelAuth @0x9496331ab9cd463f {
   accountTag @0 :Text;
   tunnelSecret @1 :Data;
 }
 
-interface RegistrationServer {
+interface RegistrationServer @0xf71695ec7fe85497 {
     registerConnection @0 (auth :TunnelAuth, tunnelId :Data, connIndex :UInt8, options :ConnectionOptions) -> (result :ConnectionResponse);
     unregisterConnection @1 () -> ();
     updateLocalConfiguration @2 (config :Data) -> ();
 }
 
-interface TunnelServer extends (RegistrationServer) {
-    registerTunnel @0 (originCert :Data, hostname :Text, options :RegistrationOptions) -> (result :TunnelRegistration);
-    getServerInfo @1 () -> (result :ServerInfo);
-    unregisterTunnel @2 (gracePeriodNanoSec :Int64) -> ();
-    # obsoleteDeclarativeTunnelConnect RPC deprecated in TUN-3019
-    obsoleteDeclarativeTunnelConnect @3 () -> ();
-    authenticate @4 (originCert :Data, hostname :Text, options :RegistrationOptions) -> (result :AuthenticateResponse);
-    reconnectTunnel @5 (jwt :Data, eventDigest :Data, connDigest :Data, hostname :Text, options :RegistrationOptions) -> (result :TunnelRegistration);
-}
-
-struct RegisterUdpSessionResponse {
+struct RegisterUdpSessionResponse @0xab6d5210c1f26687 {
     err @0 :Text;
     spans @1 :Data;
 }
 
-interface SessionManager {
+interface SessionManager @0x839445a59fb01686 {
     # Let the edge decide closeAfterIdle to make sure cloudflared doesn't close session before the edge closes its side
     registerUdpSession @0 (sessionId :Data, dstIp :Data, dstPort :UInt16, closeAfterIdleHint :Int64, traceContext :Text = "") -> (result :RegisterUdpSessionResponse);
     unregisterUdpSession @1 (sessionId :Data, message :Text) -> ();
 }
 
-struct UpdateConfigurationResponse {
+struct UpdateConfigurationResponse @0xdb58ff694ba05cf9 {
     # Latest configuration that was applied successfully. The err field might be populated at the same time to indicate
     # that cloudflared is using an older configuration because the latest cannot be applied
     latestAppliedVersion @0 :Int32;
@@ -166,8 +188,8 @@ struct UpdateConfigurationResponse {
 }
 
 # ConfigurationManager defines RPC to manage cloudflared configuration remotely
-interface ConfigurationManager {
+interface ConfigurationManager @0xb48edfbdaa25db04 {
     updateConfiguration @0 (version :Int32, config :Data) -> (result: UpdateConfigurationResponse);
 }
 
-interface CloudflaredServer extends(SessionManager, ConfigurationManager) {}
+interface CloudflaredServer @0xf548cef9dea2a4a1 extends(SessionManager, ConfigurationManager) {}

tunnelrpc/quic/cloudflared_client.go

@@ -0,0 +1,89 @@
+package quic
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net"
+	"time"
+
+	"zombiezen.com/go/capnproto2/rpc"
+
+	"github.com/google/uuid"
+
+	"github.com/cloudflare/cloudflared/tunnelrpc"
+	"github.com/cloudflare/cloudflared/tunnelrpc/metrics"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+)
+
+// CloudflaredClient calls capnp rpc methods of SessionManager and ConfigurationManager.
+type CloudflaredClient struct {
+	client         pogs.CloudflaredServer_PogsClient
+	transport      rpc.Transport
+	requestTimeout time.Duration
+}
+
+func NewCloudflaredClient(ctx context.Context, stream io.ReadWriteCloser, requestTimeout time.Duration) (*CloudflaredClient, error) {
+	n, err := stream.Write(rpcStreamProtocolSignature[:])
+	if err != nil {
+		return nil, err
+	}
+	if n != len(rpcStreamProtocolSignature) {
+		return nil, fmt.Errorf("expect to write %d bytes for RPC stream protocol signature, wrote %d", len(rpcStreamProtocolSignature), n)
+	}
+	transport := tunnelrpc.SafeTransport(stream)
+	conn := rpc.NewConn(transport)
+	client := pogs.NewCloudflaredServer_PogsClient(conn.Bootstrap(ctx), conn)
+	return &CloudflaredClient{
+		client:         client,
+		transport:      transport,
+		requestTimeout: requestTimeout,
+	}, nil
+}
+
+func (c *CloudflaredClient) RegisterUdpSession(ctx context.Context, sessionID uuid.UUID, dstIP net.IP, dstPort uint16, closeIdleAfterHint time.Duration, traceContext string) (*pogs.RegisterUdpSessionResponse, error) {
+	ctx, cancel := context.WithTimeout(ctx, c.requestTimeout)
+	defer cancel()
+	defer metrics.CapnpMetrics.ClientOperations.WithLabelValues(metrics.Cloudflared, metrics.OperationRegisterUdpSession).Inc()
+	timer := metrics.NewClientOperationLatencyObserver(metrics.Cloudflared, metrics.OperationRegisterUdpSession)
+	defer timer.ObserveDuration()
+
+	resp, err := c.client.RegisterUdpSession(ctx, sessionID, dstIP, dstPort, closeIdleAfterHint, traceContext)
+	if err != nil {
+		metrics.CapnpMetrics.ClientFailures.WithLabelValues(metrics.Cloudflared, metrics.OperationRegisterUdpSession).Inc()
+	}
+	return resp, err
+}
+
+func (c *CloudflaredClient) UnregisterUdpSession(ctx context.Context, sessionID uuid.UUID, message string) error {
+	ctx, cancel := context.WithTimeout(ctx, c.requestTimeout)
+	defer cancel()
+	defer metrics.CapnpMetrics.ClientOperations.WithLabelValues(metrics.Cloudflared, metrics.OperationUnregisterUdpSession).Inc()
+	timer := metrics.NewClientOperationLatencyObserver(metrics.Cloudflared, metrics.OperationUnregisterUdpSession)
+	defer timer.ObserveDuration()
+
+	err := c.client.UnregisterUdpSession(ctx, sessionID, message)
+	if err != nil {
+		metrics.CapnpMetrics.ClientFailures.WithLabelValues(metrics.Cloudflared, metrics.OperationUnregisterUdpSession).Inc()
+	}
+	return err
+}
+
+func (c *CloudflaredClient) UpdateConfiguration(ctx context.Context, version int32, config []byte) (*pogs.UpdateConfigurationResponse, error) {
+	ctx, cancel := context.WithTimeout(ctx, c.requestTimeout)
+	defer cancel()
+	defer metrics.CapnpMetrics.ClientOperations.WithLabelValues(metrics.Cloudflared, metrics.OperationUpdateConfiguration).Inc()
+	timer := metrics.NewClientOperationLatencyObserver(metrics.Cloudflared, metrics.OperationUpdateConfiguration)
+	defer timer.ObserveDuration()
+
+	resp, err := c.client.UpdateConfiguration(ctx, version, config)
+	if err != nil {
+		metrics.CapnpMetrics.ClientFailures.WithLabelValues(metrics.Cloudflared, metrics.OperationUpdateConfiguration).Inc()
+	}
+	return resp, err
+}
+
+func (c *CloudflaredClient) Close() {
+	_ = c.client.Close()
+	_ = c.transport.Close()
+}

tunnelrpc/quic/cloudflared_server.go

@@ -0,0 +1,70 @@
+package quic
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"time"
+
+	"zombiezen.com/go/capnproto2/rpc"
+
+	"github.com/cloudflare/cloudflared/tunnelrpc"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+)
+
+// HandleRequestFunc wraps the proxied request from the upstream and also provides methods on the stream to
+// handle the response back.
+type HandleRequestFunc = func(ctx context.Context, stream *RequestServerStream) error
+
+// CloudflaredServer provides a handler interface for a client to provide methods to handle the different types of
+// requests that can be communicated by the stream.
+type CloudflaredServer struct {
+	handleRequest   HandleRequestFunc
+	sessionManager  pogs.SessionManager
+	configManager   pogs.ConfigurationManager
+	responseTimeout time.Duration
+}
+
+func NewCloudflaredServer(handleRequest HandleRequestFunc, sessionManager pogs.SessionManager, configManager pogs.ConfigurationManager, responseTimeout time.Duration) *CloudflaredServer {
+	return &CloudflaredServer{
+		handleRequest:   handleRequest,
+		sessionManager:  sessionManager,
+		configManager:   configManager,
+		responseTimeout: responseTimeout,
+	}
+}
+
+// Serve executes the defined handlers in ServerStream on the provided stream if it is a proper RPC stream with the
+// correct preamble protocol signature.
+func (s *CloudflaredServer) Serve(ctx context.Context, stream io.ReadWriteCloser) error {
+	signature, err := determineProtocol(stream)
+	if err != nil {
+		return err
+	}
+	switch signature {
+	case dataStreamProtocolSignature:
+		return s.handleRequest(ctx, &RequestServerStream{stream})
+	case rpcStreamProtocolSignature:
+		return s.handleRPC(ctx, stream)
+	default:
+		return fmt.Errorf("unknown protocol %v", signature)
+	}
+}
+
+func (s *CloudflaredServer) handleRPC(ctx context.Context, stream io.ReadWriteCloser) error {
+	ctx, cancel := context.WithTimeout(ctx, s.responseTimeout)
+	defer cancel()
+	transport := tunnelrpc.SafeTransport(stream)
+	defer transport.Close()
+
+	main := pogs.CloudflaredServer_ServerToClient(s.sessionManager, s.configManager)
+	rpcConn := rpc.NewConn(transport, rpc.MainInterface(main.Client))
+	defer rpcConn.Close()
+
+	// We ignore the errors here because if cloudflared fails to handle a request, we will just move on.
+	select {
+	case <-rpcConn.Done():
+	case <-ctx.Done():
+	}
+	return nil
+}

tunnelrpc/quic/protocol.go

@@ -0,0 +1,78 @@
+package quic
+
+import (
+	"fmt"
+	"io"
+)
+
+// protocolSignature defines the first 6 bytes of the stream, which is used to distinguish the type of stream. It
+// ensures whoever performs a handshake does not write data before writing the metadata.
+type protocolSignature [6]byte
+
+var (
+	// dataStreamProtocolSignature is a custom protocol signature for data stream
+	dataStreamProtocolSignature = protocolSignature{0x0A, 0x36, 0xCD, 0x12, 0xA1, 0x3E}
+
+	// rpcStreamProtocolSignature is a custom protocol signature for RPC stream
+	rpcStreamProtocolSignature = protocolSignature{0x52, 0xBB, 0x82, 0x5C, 0xDB, 0x65}
+
+	errDataStreamNotSupported = fmt.Errorf("data protocol not supported")
+	errRPCStreamNotSupported  = fmt.Errorf("rpc protocol not supported")
+)
+
+type protocolVersion string
+
+const (
+	protocolV1 protocolVersion = "01"
+
+	protocolVersionLength = 2
+)
+
+// determineProtocol reads the first 6 bytes from the stream to determine which protocol is spoken by the client.
+// The protocols are magic byte arrays understood by both sides of the stream.
+func determineProtocol(stream io.Reader) (protocolSignature, error) {
+	signature, err := readSignature(stream)
+	if err != nil {
+		return protocolSignature{}, err
+	}
+	switch signature {
+	case dataStreamProtocolSignature:
+		return dataStreamProtocolSignature, nil
+	case rpcStreamProtocolSignature:
+		return rpcStreamProtocolSignature, nil
+	default:
+		return protocolSignature{}, fmt.Errorf("unknown signature %v", signature)
+	}
+}
+
+func writeDataStreamPreamble(stream io.Writer) error {
+	if err := writeSignature(stream, dataStreamProtocolSignature); err != nil {
+		return err
+	}
+
+	return writeVersion(stream)
+}
+
+func writeVersion(stream io.Writer) error {
+	_, err := stream.Write([]byte(protocolV1)[:protocolVersionLength])
+	return err
+}
+
+func readVersion(stream io.Reader) (string, error) {
+	version := make([]byte, protocolVersionLength)
+	_, err := stream.Read(version)
+	return string(version), err
+}
+
+func readSignature(stream io.Reader) (protocolSignature, error) {
+	var signature protocolSignature
+	if _, err := io.ReadFull(stream, signature[:]); err != nil {
+		return protocolSignature{}, err
+	}
+	return signature, nil
+}
+
+func writeSignature(stream io.Writer, signature protocolSignature) error {
+	_, err := stream.Write(signature[:])
+	return err
+}

tunnelrpc/quic/request_client_stream.go

@@ -0,0 +1,61 @@
+package quic
+
+import (
+	"fmt"
+	"io"
+
+	capnp "zombiezen.com/go/capnproto2"
+
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+)
+
+// RequestClientStream is a stream to provide requests to the server. This operation is typically driven by the edge service.
+type RequestClientStream struct {
+	io.ReadWriteCloser
+}
+
+// WriteConnectRequestData writes requestMeta to a stream.
+func (rcs *RequestClientStream) WriteConnectRequestData(dest string, connectionType pogs.ConnectionType, metadata ...pogs.Metadata) error {
+	connectRequest := &pogs.ConnectRequest{
+		Dest:     dest,
+		Type:     connectionType,
+		Metadata: metadata,
+	}
+
+	msg, err := connectRequest.ToPogs()
+	if err != nil {
+		return err
+	}
+
+	if err := writeDataStreamPreamble(rcs); err != nil {
+		return err
+	}
+	return capnp.NewEncoder(rcs).Encode(msg)
+}
+
+// ReadConnectResponseData reads the response from the rpc stream to a ConnectResponse.
+func (rcs *RequestClientStream) ReadConnectResponseData() (*pogs.ConnectResponse, error) {
+	signature, err := determineProtocol(rcs)
+	if err != nil {
+		return nil, err
+	}
+	if signature != dataStreamProtocolSignature {
+		return nil, fmt.Errorf("wrong protocol signature %v", signature)
+	}
+
+	// This is a NO-OP for now. We could cause a branching if we wanted to use multiple versions.
+	if _, err := readVersion(rcs); err != nil {
+		return nil, err
+	}
+
+	msg, err := capnp.NewDecoder(rcs).Decode()
+	if err != nil {
+		return nil, err
+	}
+
+	r := &pogs.ConnectResponse{}
+	if err := r.FromPogs(msg); err != nil {
+		return nil, err
+	}
+	return r, nil
+}

tunnelrpc/quic/request_server_stream.go

@@ -0,0 +1,57 @@
+package quic
+
+import (
+	"io"
+
+	capnp "zombiezen.com/go/capnproto2"
+
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+)
+
+// RequestServerStream is a stream to serve requests
+type RequestServerStream struct {
+	io.ReadWriteCloser
+}
+
+// ReadConnectRequestData reads the handshake data from a QUIC stream.
+func (rss *RequestServerStream) ReadConnectRequestData() (*pogs.ConnectRequest, error) {
+	// This is a NO-OP for now. We could cause a branching if we wanted to use multiple versions.
+	if _, err := readVersion(rss); err != nil {
+		return nil, err
+	}
+
+	msg, err := capnp.NewDecoder(rss).Decode()
+	if err != nil {
+		return nil, err
+	}
+
+	r := &pogs.ConnectRequest{}
+	if err := r.FromPogs(msg); err != nil {
+		return nil, err
+	}
+	return r, nil
+}
+
+// WriteConnectResponseData writes response to a QUIC stream.
+func (rss *RequestServerStream) WriteConnectResponseData(respErr error, metadata ...pogs.Metadata) error {
+	var connectResponse *pogs.ConnectResponse
+	if respErr != nil {
+		connectResponse = &pogs.ConnectResponse{
+			Error: respErr.Error(),
+		}
+	} else {
+		connectResponse = &pogs.ConnectResponse{
+			Metadata: metadata,
+		}
+	}
+
+	msg, err := connectResponse.ToPogs()
+	if err != nil {
+		return err
+	}
+
+	if err := writeDataStreamPreamble(rss); err != nil {
+		return err
+	}
+	return capnp.NewEncoder(rss).Encode(msg)
+}

tunnelrpc/quic/request_server_stream_test.go

@@ -11,11 +11,10 @@ import (
 	"time"
 
 	"github.com/google/uuid"
-	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
 const (
@@ -26,14 +25,14 @@ func TestConnectRequestData(t *testing.T) {
 	var tests = []struct {
 		name           string
 		hostname       string
-		connectionType ConnectionType
-		metadata       []Metadata
+		connectionType pogs.ConnectionType
+		metadata       []pogs.Metadata
 	}{
 		{
 			name:           "Signature verified and request metadata is unmarshaled and read correctly",
 			hostname:       "tunnel.com",
-			connectionType: ConnectionTypeHTTP,
-			metadata: []Metadata{
+			connectionType: pogs.ConnectionTypeHTTP,
+			metadata: []pogs.Metadata{
 				{
 					Key: "key",
 					Val: "1234",
@@ -47,10 +46,10 @@ func TestConnectRequestData(t *testing.T) {
 			reqClientStream := RequestClientStream{noopCloser{b}}
 			err := reqClientStream.WriteConnectRequestData(test.hostname, test.connectionType, test.metadata...)
 			require.NoError(t, err)
-			protocol, err := DetermineProtocol(b)
-			require.NoError(t, err)
-			reqServerStream, err := NewRequestServerStream(noopCloser{b}, protocol)
+			protocol, err := determineProtocol(b)
 			require.NoError(t, err)
+			require.Equal(t, dataStreamProtocolSignature, protocol)
+			reqServerStream := RequestServerStream{&noopCloser{b}}
 
 			reqMeta, err := reqServerStream.ReadConnectRequestData()
 			require.NoError(t, err)
@@ -66,11 +65,11 @@ func TestConnectResponseMeta(t *testing.T) {
 	var tests = []struct {
 		name     string
 		err      error
-		metadata []Metadata
+		metadata []pogs.Metadata
 	}{
 		{
 			name: "Signature verified and response metadata is unmarshaled and read correctly",
-			metadata: []Metadata{
+			metadata: []pogs.Metadata{
 				{
 					Key: "key",
 					Val: "1234",
@@ -80,7 +79,7 @@ func TestConnectResponseMeta(t *testing.T) {
 		{
 			name: "If error is not empty, other fields should be blank",
 			err:  errors.New("something happened"),
-			metadata: []Metadata{
+			metadata: []pogs.Metadata{
 				{
 					Key: "key",
 					Val: "1234",
@@ -142,22 +141,18 @@ func TestRegisterUdpSession(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			logger := zerolog.Nop()
 			clientStream, serverStream := newMockRPCStreams()
 			sessionRegisteredChan := make(chan struct{})
 			go func() {
-				protocol, err := DetermineProtocol(serverStream)
-				assert.NoError(t, err)
-				rpcServerStream, err := NewRPCServerStream(serverStream, protocol)
-				assert.NoError(t, err)
-				err = rpcServerStream.Serve(test.sessionRPCServer, nil, &logger)
+				ss := NewCloudflaredServer(nil, test.sessionRPCServer, nil, 10*time.Second)
+				err := ss.Serve(context.Background(), serverStream)
 				assert.NoError(t, err)
 
 				serverStream.Close()
 				close(sessionRegisteredChan)
 			}()
 
-			rpcClientStream, err := NewRPCClientStream(context.Background(), clientStream, 5*time.Second, &logger)
+			rpcClientStream, err := NewCloudflaredClient(context.Background(), clientStream, 5*time.Second)
 			assert.NoError(t, err)
 
 			reg, err := rpcClientStream.RegisterUdpSession(context.Background(), test.sessionRPCServer.sessionID, test.sessionRPCServer.dstIP, test.sessionRPCServer.dstPort, testCloseIdleAfterHint, test.sessionRPCServer.traceContext)
@@ -192,14 +187,10 @@ func TestManageConfiguration(t *testing.T) {
 		config:  config,
 	}
 
-	logger := zerolog.Nop()
 	updatedChan := make(chan struct{})
 	go func() {
-		protocol, err := DetermineProtocol(serverStream)
-		assert.NoError(t, err)
-		rpcServerStream, err := NewRPCServerStream(serverStream, protocol)
-		assert.NoError(t, err)
-		err = rpcServerStream.Serve(nil, configRPCServer, &logger)
+		server := NewCloudflaredServer(nil, nil, configRPCServer, 10*time.Second)
+		err := server.Serve(context.Background(), serverStream)
 		assert.NoError(t, err)
 
 		serverStream.Close()
@@ -208,7 +199,7 @@ func TestManageConfiguration(t *testing.T) {
 
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
 	defer cancel()
-	rpcClientStream, err := NewRPCClientStream(ctx, clientStream, 5*time.Second, &logger)
+	rpcClientStream, err := NewCloudflaredClient(ctx, clientStream, 5*time.Second)
 	assert.NoError(t, err)
 
 	result, err := rpcClientStream.UpdateConfiguration(ctx, version, config)
@@ -230,7 +221,7 @@ type mockSessionRPCServer struct {
 	traceContext      string
 }
 
-func (s mockSessionRPCServer) RegisterUdpSession(_ context.Context, sessionID uuid.UUID, dstIP net.IP, dstPort uint16, closeIdleAfter time.Duration, traceContext string) (*tunnelpogs.RegisterUdpSessionResponse, error) {
+func (s mockSessionRPCServer) RegisterUdpSession(_ context.Context, sessionID uuid.UUID, dstIP net.IP, dstPort uint16, closeIdleAfter time.Duration, traceContext string) (*pogs.RegisterUdpSessionResponse, error) {
 	if s.sessionID != sessionID {
 		return nil, fmt.Errorf("expect session ID %s, got %s", s.sessionID, sessionID)
 	}
@@ -246,7 +237,7 @@ func (s mockSessionRPCServer) RegisterUdpSession(_ context.Context, sessionID uu
 	if s.traceContext != traceContext {
 		return nil, fmt.Errorf("expect traceContext %s, got %s", s.traceContext, traceContext)
 	}
-	return &tunnelpogs.RegisterUdpSessionResponse{}, nil
+	return &pogs.RegisterUdpSessionResponse{}, nil
 }
 
 func (s mockSessionRPCServer) UnregisterUdpSession(_ context.Context, sessionID uuid.UUID, message string) error {
@@ -264,18 +255,18 @@ type mockConfigRPCServer struct {
 	config  []byte
 }
 
-func (s mockConfigRPCServer) UpdateConfiguration(_ context.Context, version int32, config []byte) *tunnelpogs.UpdateConfigurationResponse {
+func (s mockConfigRPCServer) UpdateConfiguration(_ context.Context, version int32, config []byte) *pogs.UpdateConfigurationResponse {
 	if s.version != version {
-		return &tunnelpogs.UpdateConfigurationResponse{
+		return &pogs.UpdateConfigurationResponse{
 			Err: fmt.Errorf("expect version %d, got %d", s.version, version),
 		}
 	}
 	if !bytes.Equal(s.config, config) {
-		return &tunnelpogs.UpdateConfigurationResponse{
+		return &pogs.UpdateConfigurationResponse{
 			Err: fmt.Errorf("expect config %v, got %v", s.config, config),
 		}
 	}
-	return &tunnelpogs.UpdateConfigurationResponse{LastAppliedVersion: version}
+	return &pogs.UpdateConfigurationResponse{LastAppliedVersion: version}
 }
 
 type mockRPCStream struct {

tunnelrpc/quic/session_client.go

@@ -0,0 +1,73 @@
+package quic
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net"
+	"time"
+
+	"github.com/google/uuid"
+	"zombiezen.com/go/capnproto2/rpc"
+
+	"github.com/cloudflare/cloudflared/tunnelrpc"
+	"github.com/cloudflare/cloudflared/tunnelrpc/metrics"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+)
+
+// SessionClient calls capnp rpc methods of SessionManager.
+type SessionClient struct {
+	client         pogs.SessionManager_PogsClient
+	transport      rpc.Transport
+	requestTimeout time.Duration
+}
+
+func NewSessionClient(ctx context.Context, stream io.ReadWriteCloser, requestTimeout time.Duration) (*SessionClient, error) {
+	n, err := stream.Write(rpcStreamProtocolSignature[:])
+	if err != nil {
+		return nil, err
+	}
+	if n != len(rpcStreamProtocolSignature) {
+		return nil, fmt.Errorf("expect to write %d bytes for RPC stream protocol signature, wrote %d", len(rpcStreamProtocolSignature), n)
+	}
+	transport := tunnelrpc.SafeTransport(stream)
+	conn := rpc.NewConn(transport)
+	return &SessionClient{
+		client:         pogs.NewSessionManager_PogsClient(conn.Bootstrap(ctx), conn),
+		transport:      transport,
+		requestTimeout: requestTimeout,
+	}, nil
+}
+
+func (c *SessionClient) RegisterUdpSession(ctx context.Context, sessionID uuid.UUID, dstIP net.IP, dstPort uint16, closeIdleAfterHint time.Duration, traceContext string) (*pogs.RegisterUdpSessionResponse, error) {
+	ctx, cancel := context.WithTimeout(ctx, c.requestTimeout)
+	defer cancel()
+	defer metrics.CapnpMetrics.ClientOperations.WithLabelValues(metrics.SessionManager, metrics.OperationRegisterUdpSession).Inc()
+	timer := metrics.NewClientOperationLatencyObserver(metrics.SessionManager, metrics.OperationRegisterUdpSession)
+	defer timer.ObserveDuration()
+
+	resp, err := c.client.RegisterUdpSession(ctx, sessionID, dstIP, dstPort, closeIdleAfterHint, traceContext)
+	if err != nil {
+		metrics.CapnpMetrics.ClientFailures.WithLabelValues(metrics.SessionManager, metrics.OperationRegisterUdpSession).Inc()
+	}
+	return resp, err
+}
+
+func (c *SessionClient) UnregisterUdpSession(ctx context.Context, sessionID uuid.UUID, message string) error {
+	ctx, cancel := context.WithTimeout(ctx, c.requestTimeout)
+	defer cancel()
+	defer metrics.CapnpMetrics.ClientOperations.WithLabelValues(metrics.SessionManager, metrics.OperationUnregisterUdpSession).Inc()
+	timer := metrics.NewClientOperationLatencyObserver(metrics.SessionManager, metrics.OperationUnregisterUdpSession)
+	defer timer.ObserveDuration()
+
+	err := c.client.UnregisterUdpSession(ctx, sessionID, message)
+	if err != nil {
+		metrics.CapnpMetrics.ClientFailures.WithLabelValues(metrics.SessionManager, metrics.OperationUnregisterUdpSession).Inc()
+	}
+	return err
+}
+
+func (c *SessionClient) Close() {
+	_ = c.client.Close()
+	_ = c.transport.Close()
+}

tunnelrpc/quic/session_server.go

@@ -0,0 +1,60 @@
+package quic
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"time"
+
+	"zombiezen.com/go/capnproto2/rpc"
+
+	"github.com/cloudflare/cloudflared/tunnelrpc"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+)
+
+// SessionManagerServer handles streams with the SessionManager RPCs.
+type SessionManagerServer struct {
+	sessionManager  pogs.SessionManager
+	responseTimeout time.Duration
+}
+
+func NewSessionManagerServer(sessionManager pogs.SessionManager, responseTimeout time.Duration) *SessionManagerServer {
+	return &SessionManagerServer{
+		sessionManager:  sessionManager,
+		responseTimeout: responseTimeout,
+	}
+}
+
+func (s *SessionManagerServer) Serve(ctx context.Context, stream io.ReadWriteCloser) error {
+	signature, err := determineProtocol(stream)
+	if err != nil {
+		return err
+	}
+	switch signature {
+	case rpcStreamProtocolSignature:
+		break
+	case dataStreamProtocolSignature:
+		return errDataStreamNotSupported
+	default:
+		return fmt.Errorf("unknown protocol %v", signature)
+	}
+
+	// Every new quic.Stream request aligns to a new RPC request, this is why there is a timeout for the server-side
+	// of the RPC request.
+	ctx, cancel := context.WithTimeout(ctx, s.responseTimeout)
+	defer cancel()
+
+	transport := tunnelrpc.SafeTransport(stream)
+	defer transport.Close()
+
+	main := pogs.SessionManager_ServerToClient(s.sessionManager)
+	rpcConn := rpc.NewConn(transport, rpc.MainInterface(main.Client))
+	defer rpcConn.Close()
+
+	select {
+	case <-rpcConn.Done():
+		return rpcConn.Err()
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+}

tunnelrpc/registration_client.go

@@ -0,0 +1,99 @@
+package tunnelrpc
+
+import (
+	"context"
+	"io"
+	"net"
+	"time"
+
+	"github.com/google/uuid"
+	"zombiezen.com/go/capnproto2/rpc"
+
+	"github.com/cloudflare/cloudflared/tunnelrpc/metrics"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+)
+
+type RegistrationClient interface {
+	RegisterConnection(
+		ctx context.Context,
+		auth pogs.TunnelAuth,
+		tunnelID uuid.UUID,
+		options *pogs.ConnectionOptions,
+		connIndex uint8,
+		edgeAddress net.IP,
+	) (*pogs.ConnectionDetails, error)
+	SendLocalConfiguration(ctx context.Context, config []byte) error
+	GracefulShutdown(ctx context.Context, gracePeriod time.Duration)
+	Close()
+}
+
+type registrationClient struct {
+	client         pogs.RegistrationServer_PogsClient
+	transport      rpc.Transport
+	requestTimeout time.Duration
+}
+
+func NewRegistrationClient(ctx context.Context, stream io.ReadWriteCloser, requestTimeout time.Duration) RegistrationClient {
+	transport := SafeTransport(stream)
+	conn := rpc.NewConn(transport)
+	client := pogs.NewRegistrationServer_PogsClient(conn.Bootstrap(ctx), conn)
+	return &registrationClient{
+		client:         client,
+		transport:      transport,
+		requestTimeout: requestTimeout,
+	}
+}
+
+func (r *registrationClient) RegisterConnection(
+	ctx context.Context,
+	auth pogs.TunnelAuth,
+	tunnelID uuid.UUID,
+	options *pogs.ConnectionOptions,
+	connIndex uint8,
+	edgeAddress net.IP,
+) (*pogs.ConnectionDetails, error) {
+	ctx, cancel := context.WithTimeout(ctx, r.requestTimeout)
+	defer cancel()
+	defer metrics.CapnpMetrics.ClientOperations.WithLabelValues(metrics.Registration, metrics.OperationRegisterConnection).Inc()
+	timer := metrics.NewClientOperationLatencyObserver(metrics.Registration, metrics.OperationRegisterConnection)
+	defer timer.ObserveDuration()
+
+	conn, err := r.client.RegisterConnection(ctx, auth, tunnelID, connIndex, options)
+	if err != nil {
+		metrics.CapnpMetrics.ClientFailures.WithLabelValues(metrics.Registration, metrics.OperationRegisterConnection).Inc()
+	}
+	return conn, err
+}
+
+func (r *registrationClient) SendLocalConfiguration(ctx context.Context, config []byte) error {
+	ctx, cancel := context.WithTimeout(ctx, r.requestTimeout)
+	defer cancel()
+	defer metrics.CapnpMetrics.ClientOperations.WithLabelValues(metrics.Registration, metrics.OperationUpdateLocalConfiguration).Inc()
+	timer := metrics.NewClientOperationLatencyObserver(metrics.Registration, metrics.OperationUpdateLocalConfiguration)
+	defer timer.ObserveDuration()
+
+	err := r.client.SendLocalConfiguration(ctx, config)
+	if err != nil {
+		metrics.CapnpMetrics.ClientFailures.WithLabelValues(metrics.Registration, metrics.OperationUpdateLocalConfiguration).Inc()
+	}
+	return err
+}
+
+func (r *registrationClient) GracefulShutdown(ctx context.Context, gracePeriod time.Duration) {
+	ctx, cancel := context.WithTimeout(ctx, gracePeriod)
+	defer cancel()
+	defer metrics.CapnpMetrics.ClientOperations.WithLabelValues(metrics.Registration, metrics.OperationUnregisterConnection).Inc()
+	timer := metrics.NewClientOperationLatencyObserver(metrics.Registration, metrics.OperationUnregisterConnection)
+	defer timer.ObserveDuration()
+	err := r.client.UnregisterConnection(ctx)
+	if err != nil {
+		metrics.CapnpMetrics.ClientFailures.WithLabelValues(metrics.Registration, metrics.OperationUnregisterConnection).Inc()
+	}
+}
+
+func (r *registrationClient) Close() {
+	// Closing the client will also close the connection
+	_ = r.client.Close()
+	// Closing the transport also closes the stream
+	_ = r.transport.Close()
+}

tunnelrpc/registration_server.go

@@ -0,0 +1,40 @@
+package tunnelrpc
+
+import (
+	"context"
+	"io"
+
+	"zombiezen.com/go/capnproto2/rpc"
+
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+)
+
+// RegistrationServer provides a handler interface for a client to provide methods to handle the different types of
+// requests that can be communicated by the stream.
+type RegistrationServer struct {
+	registrationServer pogs.RegistrationServer
+}
+
+func NewRegistrationServer(registrationServer pogs.RegistrationServer) *RegistrationServer {
+	return &RegistrationServer{
+		registrationServer: registrationServer,
+	}
+}
+
+// Serve listens for all RegistrationServer RPCs, including UnregisterConnection until the underlying connection
+// is terminated.
+func (s *RegistrationServer) Serve(ctx context.Context, stream io.ReadWriteCloser) error {
+	transport := SafeTransport(stream)
+	defer transport.Close()
+
+	main := pogs.RegistrationServer_ServerToClient(s.registrationServer)
+	rpcConn := rpc.NewConn(transport, rpc.MainInterface(main.Client))
+	defer rpcConn.Close()
+
+	select {
+	case <-rpcConn.Done():
+		return rpcConn.Wait()
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+}

tunnelrpc/utils.go

@@ -0,0 +1,69 @@
+package tunnelrpc
+
+import (
+	"io"
+	"time"
+
+	"github.com/pkg/errors"
+	"zombiezen.com/go/capnproto2/rpc"
+)
+
+const (
+	// These default values are here so that we give some time for the underlying connection/stream
+	// to recover in the face of what we believe to be temporarily errors.
+	// We don't want to be too aggressive, as the end result of giving a final error (non-temporary)
+	// will result in the connection to be dropped.
+	// In turn, the other side will probably reconnect, which will put again more pressure in the overall system.
+	// So, the best solution is to give it some conservative time to recover.
+	defaultSleepBetweenTemporaryError = 500 * time.Millisecond
+	defaultMaxRetries                 = 3
+)
+
+type readWriterSafeTemporaryErrorCloser struct {
+	io.ReadWriteCloser
+
+	retries             int
+	sleepBetweenRetries time.Duration
+	maxRetries          int
+}
+
+func (r *readWriterSafeTemporaryErrorCloser) Read(p []byte) (n int, err error) {
+	n, err = r.ReadWriteCloser.Read(p)
+
+	// if there was a failure reading from the read closer, and the error is temporary, try again in some seconds
+	// otherwise, just fail without a temporary error.
+	if n == 0 && err != nil && isTemporaryError(err) {
+		if r.retries >= r.maxRetries {
+			return 0, errors.Wrap(err, "failed read from capnproto ReaderWriter after multiple temporary errors")
+		} else {
+			r.retries += 1
+
+			// sleep for some time to prevent quick read loops that cause exhaustion of CPU resources
+			time.Sleep(r.sleepBetweenRetries)
+		}
+	}
+
+	if err == nil {
+		r.retries = 0
+	}
+
+	return n, err
+}
+
+func SafeTransport(rw io.ReadWriteCloser) rpc.Transport {
+	return rpc.StreamTransport(&readWriterSafeTemporaryErrorCloser{
+		ReadWriteCloser:     rw,
+		maxRetries:          defaultMaxRetries,
+		sleepBetweenRetries: defaultSleepBetweenTemporaryError,
+	})
+}
+
+// isTemporaryError reports whether e has a Temporary() method that
+// returns true.
+func isTemporaryError(e error) bool {
+	type temp interface {
+		Temporary() bool
+	}
+	t, ok := e.(temp)
+	return ok && t.Temporary()
+}