Release 2024.6.0

cloudflared_udp_total_sessions was incorrectly a gauge when it
represents the total since the cloudflared process started and will
only ever increase.

Additionally adds new ICMP metrics for requests and replies.

.github/workflows/check.yaml

@@ -4,15 +4,15 @@ jobs:
   check:
     strategy:
       matrix:
-        go-version: [1.21.x]
+        go-version: [1.22.x]
         os: [ubuntu-latest, macos-latest, windows-latest]
     runs-on: ${{ matrix.os }}
     steps:
     - name: Install Go
-      uses: actions/setup-go@v3
+      uses: actions/setup-go@v5
       with:
         go-version: ${{ matrix.go-version }}
     - name: Checkout code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
     - name: Test
       run: make test

.teamcity/install-cloudflare-go.sh

@@ -3,6 +3,6 @@
 cd /tmp
 git clone -q https://github.com/cloudflare/go
 cd go/src
-# https://github.com/cloudflare/go/tree/34129e47042e214121b6bbff0ded4712debed18e is version go1.21.5-devel-cf
-git checkout -q 34129e47042e214121b6bbff0ded4712debed18e
+# https://github.com/cloudflare/go/tree/ec0a014545f180b0c74dfd687698657a9e86e310 is version go1.22.2-devel-cf
+git checkout -q ec0a014545f180b0c74dfd687698657a9e86e310
 ./make.bash

.teamcity/windows/install-cloudflare-go.ps1

@@ -9,8 +9,8 @@ Set-Location "$Env:Temp"
 git clone -q https://github.com/cloudflare/go
 Write-Output "Building go..."
 cd go/src
-# https://github.com/cloudflare/go/tree/34129e47042e214121b6bbff0ded4712debed18e is version go1.21.5-devel-cf
-git checkout -q 34129e47042e214121b6bbff0ded4712debed18e
+# https://github.com/cloudflare/go/tree/ec0a014545f180b0c74dfd687698657a9e86e310 is version go1.22.2-devel-cf
+git checkout -q ec0a014545f180b0c74dfd687698657a9e86e310
 & ./make.bat
 
 Write-Output "Installed"

.teamcity/windows/install-go-msi.ps1

@@ -1,20 +1,20 @@
 $ErrorActionPreference = "Stop"
 $ProgressPreference = "SilentlyContinue"
-$GoMsiVersion = "go1.21.5.windows-amd64.msi"
- 
+$GoMsiVersion = "go1.22.2.windows-amd64.msi"
+
 Write-Output "Downloading go installer..."
- 
+
 Set-Location "$Env:Temp"
- 
+
 (New-Object System.Net.WebClient).DownloadFile(
     "https://go.dev/dl/$GoMsiVersion",
     "$Env:Temp\$GoMsiVersion"
 )
- 
+
 Write-Output "Installing go..."
 Install-Package "$Env:Temp\$GoMsiVersion" -Force
- 
+
 # Go installer updates global $PATH
 go env
- 
+
 Write-Output "Installed"

CHANGES.md

@@ -1,3 +1,7 @@
+## 2024.2.1
+### Notices
+- Starting from this version, tunnel diagnostics will be enabled by default. This will allow the engineering team to remotely get diagnostics from cloudflared during debug activities. Users still have the capability to opt-out of this feature by defining `--management-diagnostics=false` (or env `TUNNEL_MANAGEMENT_DIAGNOSTICS`).
+
 ## 2023.9.0
 ### Notices
 - The `warp-routing` `enabled: boolean` flag is no longer supported in the configuration file. Warp Routing traffic (eg TCP, UDP, ICMP) traffic is proxied to cloudflared if routes to the target tunnel are configured. This change does not affect remotely managed tunnels, but for locally managed tunnels, users that might be relying on this feature flag to block traffic should instead guarantee that tunnel has no Private Routes configured for the tunnel.

Dockerfile

@@ -1,7 +1,7 @@
 # use a builder image for building cloudflare
 ARG TARGET_GOOS
 ARG TARGET_GOARCH
-FROM golang:1.21.5 as builder
+FROM golang:1.22.2 as builder
 ENV GO111MODULE=on \
     CGO_ENABLED=0 \
     TARGET_GOOS=${TARGET_GOOS} \

Dockerfile.amd64

@@ -1,8 +1,8 @@
 # use a builder image for building cloudflare
-FROM golang:1.21.5 as builder
+FROM golang:1.22.2 as builder
 ENV GO111MODULE=on \
-    CGO_ENABLED=0 
-    
+    CGO_ENABLED=0
+
 WORKDIR /go/src/github.com/cloudflare/cloudflared/
 
 # copy our sources into the builder image

Dockerfile.arm64

@@ -1,8 +1,8 @@
 # use a builder image for building cloudflare
-FROM golang:1.21.5 as builder
+FROM golang:1.22.2 as builder
 ENV GO111MODULE=on \
-    CGO_ENABLED=0 
-    
+    CGO_ENABLED=0
+
 WORKDIR /go/src/github.com/cloudflare/cloudflared/
 
 # copy our sources into the builder image

Makefile

@@ -251,17 +251,11 @@ github-windows-upload:
 	python3 github_release.py --path built_artifacts/cloudflared-windows-386.exe --release-version $(VERSION) --name cloudflared-windows-386.exe
 	python3 github_release.py --path built_artifacts/cloudflared-windows-386.msi --release-version $(VERSION) --name cloudflared-windows-386.msi
 
-.PHONY: tunnelrpc-deps
-tunnelrpc-deps:
+.PHONY: capnp
+capnp:
 	which capnp  # https://capnproto.org/install.html
 	which capnpc-go  # go install zombiezen.com/go/capnproto2/capnpc-go@latest
-	capnp compile -ogo tunnelrpc/tunnelrpc.capnp
-
-.PHONY: quic-deps
-quic-deps:
-	which capnp
-	which capnpc-go
-	capnp compile -ogo quic/schema/quic_metadata_protocol.capnp
+	capnp compile -ogo tunnelrpc/proto/tunnelrpc.capnp tunnelrpc/proto/quic_metadata_protocol.capnp
 
 .PHONY: vet
 vet:
@@ -269,4 +263,4 @@ vet:
 
 .PHONY: fmt
 fmt:
-	goimports -l -w -local github.com/cloudflare/cloudflared $$(go list -mod=vendor -f '{{.Dir}}' -a ./... | fgrep -v tunnelrpc)
+	goimports -l -w -local github.com/cloudflare/cloudflared $$(go list -mod=vendor -f '{{.Dir}}' -a ./... | fgrep -v tunnelrpc/proto)

RELEASE_NOTES

@@ -1,3 +1,61 @@
+2024.6.0
+- 2024-05-30 TUN-8441: Correct UDP total sessions metric to a counter and add new ICMP metrics
+- 2024-05-28 TUN-8422: Add metrics for capnp method calls
+- 2024-05-24 TUN-8424: Refactor capnp registration server
+- 2024-05-23 TUN-8427: Fix BackoffHandler's internally shared clock structure
+- 2024-05-21 TUN-8425: Remove ICMP binding for quick tunnels
+- 2024-05-20 TUN-8423: Deprecate older legacy tunnel capnp interfaces
+- 2024-05-15 TUN-8419: Add capnp safe transport
+- 2024-05-13 TUN-8415: Refactor capnp rpc into a single module
+
+2024.5.0
+- 2024-05-07 TUN-8407: Upgrade go to version 1.22.2
+
+2024.4.1
+- 2024-04-22 TUN-8380: Add sleep before requesting quick tunnel as temporary fix for component tests
+- 2024-04-19 TUN-8374: Close UDP socket if registration fails
+- 2024-04-18 TUN-8371: Bump quic-go to v0.42.0
+- 2024-04-03 TUN-8333: Bump go-jose dependency to v4
+- 2024-04-02 TUN-8331: Add unit testing for AccessJWTValidator middleware
+
+2024.4.0
+- 2024-04-02 feat: provide short version (#1206)
+- 2024-04-02 Format code
+- 2024-01-18 feat: auto tls sni
+- 2023-12-24 fix checkInPingGroup bugs
+- 2023-12-15 Add environment variables for TCP tunnel hostname / destination / URL.
+
+2024.3.0
+- 2024-03-14 TUN-8281: Run cloudflared query list tunnels/routes endpoint in a paginated way
+- 2024-03-13 TUN-8297: Improve write timeout logging on safe_stream.go
+- 2024-03-07 TUN-8290: Remove `|| true` from postrm.sh
+- 2024-03-05 TUN-8275: Skip write timeout log on "no network activity"
+- 2024-01-23 Update postrm.sh to fix incomplete uninstall
+- 2024-01-05 fix typo in errcheck for response parsing logic in CreateTunnel routine
+- 2023-12-23 Update linux_service.go
+- 2023-12-07 ci: bump actions/checkout to v4
+- 2023-12-07 ci/check: bump actions/setup-go to v5
+- 2023-04-28 check.yaml: bump actions/setup-go to v4
+
+2024.2.1
+- 2024-02-20 TUN-8242: Update Changes.md file with new remote diagnostics behaviour
+- 2024-02-19 TUN-8238: Fix type mismatch introduced by fast-forward
+- 2024-02-16 TUN-8243: Collect metrics on the number of QUIC frames sent/received
+- 2024-02-15 TUN-8238: Refactor proxy logging
+- 2024-02-14 TUN-8242: Enable remote diagnostics by default
+- 2024-02-12 TUN-8236: Add write timeout to quic and tcp connections
+- 2024-02-09 TUN-8224: Fix safety of TCP stream logging, separate connect and ack log messages
+
+2024.2.0
+- 2024-02-07 TUN-8224: Count and collect metrics on stream connect successes/errors
+
+2024.1.5
+- 2024-01-22 TUN-8176: Support ARM platforms that don't have an FPU or have it enabled in kernel
+- 2024-01-15 TUN-8158: Bring back commit e6537418859afcac29e56a39daa08bcabc09e048 and fixes infinite loop on linux when the socket is closed
+
+2024.1.4
+- 2024-01-19 Revert "TUN-8158: Add logging to confirm when ICMP reply is returned to the edge"
+
 2024.1.3
 - 2024-01-15 TUN-8161: Fix broken ARM build for armv6
 - 2024-01-15 TUN-8158: Add logging to confirm when ICMP reply is returned to the edge

build-packages.sh

@@ -16,10 +16,10 @@ for arch in ${linuxArchs[@]}; do
     unset TARGET_ARM
     export TARGET_ARCH=$arch
 
-    ## Support for armv6 builds
+    ## Support for arm platforms without hardware FPU enabled
     if [[ $arch == arm ]] ; then
         export TARGET_ARCH=arm
-        export TARGET_ARM=6
+        export TARGET_ARM=5
     fi
     
     ## Support for armhf builds 

cfapi/base_client.go

@@ -109,20 +109,34 @@ func (r *RESTClient) sendRequest(method string, url url.URL, body interface{}) (
 	return r.client.Do(req)
 }
 
-func parseResponse(reader io.Reader, data interface{}) error {
+func parseResponseEnvelope(reader io.Reader) (*response, error) {
 	// Schema for Tunnelstore responses in the v1 API.
 	// Roughly, it's a wrapper around a particular result that adds failures/errors/etc
 	var result response
 	// First, parse the wrapper and check the API call succeeded
 	if err := json.NewDecoder(reader).Decode(&result); err != nil {
-		return errors.Wrap(err, "failed to decode response")
+		return nil, errors.Wrap(err, "failed to decode response")
 	}
 	if err := result.checkErrors(); err != nil {
-		return err
+		return nil, err
 	}
 	if !result.Success {
-		return ErrAPINoSuccess
+		return nil, ErrAPINoSuccess
 	}
+
+	return &result, nil
+}
+
+func parseResponse(reader io.Reader, data interface{}) error {
+	result, err := parseResponseEnvelope(reader)
+	if err != nil {
+		return err
+	}
+
+	return parseResponseBody(result, data)
+}
+
+func parseResponseBody(result *response, data interface{}) error {
 	// At this point we know the API call succeeded, so, parse out the inner
 	// result into the datatype provided as a parameter.
 	if err := json.Unmarshal(result.Result, &data); err != nil {
@@ -131,11 +145,58 @@ func parseResponse(reader io.Reader, data interface{}) error {
 	return nil
 }
 
+func fetchExhaustively[T any](requestFn func(int) (*http.Response, error)) ([]*T, error) {
+	page := 0
+	var fullResponse []*T
+
+	for {
+		page += 1
+		envelope, parsedBody, err := fetchPage[T](requestFn, page)
+
+		if err != nil {
+			return nil, errors.Wrap(err, fmt.Sprintf("Error Parsing page %d", page))
+		}
+
+		fullResponse = append(fullResponse, parsedBody...)
+		if envelope.Pagination.Count < envelope.Pagination.PerPage || len(fullResponse) >= envelope.Pagination.TotalCount {
+			break
+		}
+
+	}
+	return fullResponse, nil
+}
+
+func fetchPage[T any](requestFn func(int) (*http.Response, error), page int) (*response, []*T, error) {
+	pageResp, err := requestFn(page)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "REST request failed")
+	}
+	defer pageResp.Body.Close()
+	if pageResp.StatusCode == http.StatusOK {
+		envelope, err := parseResponseEnvelope(pageResp.Body)
+		if err != nil {
+			return nil, nil, err
+		}
+		var parsedRspBody []*T
+		return envelope, parsedRspBody, parseResponseBody(envelope, &parsedRspBody)
+
+	}
+	return nil, nil, errors.New(fmt.Sprintf("Failed to fetch page. Server returned: %d", pageResp.StatusCode))
+}
+
 type response struct {
-	Success  bool            `json:"success,omitempty"`
-	Errors   []apiErr        `json:"errors,omitempty"`
-	Messages []string        `json:"messages,omitempty"`
-	Result   json.RawMessage `json:"result,omitempty"`
+	Success    bool            `json:"success,omitempty"`
+	Errors     []apiErr        `json:"errors,omitempty"`
+	Messages   []string        `json:"messages,omitempty"`
+	Result     json.RawMessage `json:"result,omitempty"`
+	Pagination Pagination      `json:"result_info,omitempty"`
+}
+
+type Pagination struct {
+	Count      int `json:"count,omitempty"`
+	Page       int `json:"page,omitempty"`
+	PerPage    int `json:"per_page,omitempty"`
+	TotalCount int `json:"total_count,omitempty"`
 }
 
 func (r *response) checkErrors() error {

cfapi/ip_route.go

@@ -137,20 +137,24 @@ type GetRouteByIpParams struct {
 }
 
 // ListRoutes calls the Tunnelstore GET endpoint for all routes under an account.
+// Due to pagination on the server side it will call the endpoint multiple times if needed.
 func (r *RESTClient) ListRoutes(filter *IpRouteFilter) ([]*DetailedRoute, error) {
-	endpoint := r.baseEndpoints.accountRoutes
-	endpoint.RawQuery = filter.Encode()
-	resp, err := r.sendRequest("GET", endpoint, nil)
-	if err != nil {
-		return nil, errors.Wrap(err, "REST request failed")
-	}
-	defer resp.Body.Close()
+	fetchFn := func(page int) (*http.Response, error) {
+		endpoint := r.baseEndpoints.accountRoutes
+		filter.Page(page)
+		endpoint.RawQuery = filter.Encode()
+		rsp, err := r.sendRequest("GET", endpoint, nil)
 
-	if resp.StatusCode == http.StatusOK {
-		return parseListDetailedRoutes(resp.Body)
+		if err != nil {
+			return nil, errors.Wrap(err, "REST request failed")
+		}
+		if rsp.StatusCode != http.StatusOK {
+			rsp.Body.Close()
+			return nil, r.statusCodeToError("list routes", rsp)
+		}
+		return rsp, nil
 	}
-
-	return nil, r.statusCodeToError("list routes", resp)
+	return fetchExhaustively[DetailedRoute](fetchFn)
 }
 
 // AddRoute calls the Tunnelstore POST endpoint for a given route.
@@ -208,12 +212,6 @@ func (r *RESTClient) GetByIP(params GetRouteByIpParams) (DetailedRoute, error) {
 	return DetailedRoute{}, r.statusCodeToError("get route by IP", resp)
 }
 
-func parseListDetailedRoutes(body io.ReadCloser) ([]*DetailedRoute, error) {
-	var routes []*DetailedRoute
-	err := parseResponse(body, &routes)
-	return routes, err
-}
-
 func parseRoute(body io.ReadCloser) (Route, error) {
 	var route Route
 	err := parseResponse(body, &route)

cfapi/ip_route_filter.go

@@ -167,6 +167,10 @@ func (f *IpRouteFilter) MaxFetchSize(max uint) {
 	f.queryParams.Set("per_page", strconv.Itoa(int(max)))
 }
 
+func (f *IpRouteFilter) Page(page int) {
+	f.queryParams.Set("page", strconv.Itoa(page))
+}
+
 func (f IpRouteFilter) Encode() string {
 	return f.queryParams.Encode()
 }

cfapi/tunnel.go

@@ -93,7 +93,7 @@ func (r *RESTClient) CreateTunnel(name string, tunnelSecret []byte) (*TunnelWith
 	switch resp.StatusCode {
 	case http.StatusOK:
 		var tunnel TunnelWithToken
-		if serdeErr := parseResponse(resp.Body, &tunnel); err != nil {
+		if serdeErr := parseResponse(resp.Body, &tunnel); serdeErr != nil {
 			return nil, serdeErr
 		}
 		return &tunnel, nil
@@ -177,25 +177,22 @@ func (r *RESTClient) DeleteTunnel(tunnelID uuid.UUID, cascade bool) error {
 }
 
 func (r *RESTClient) ListTunnels(filter *TunnelFilter) ([]*Tunnel, error) {
-	endpoint := r.baseEndpoints.accountLevel
-	endpoint.RawQuery = filter.encode()
-	resp, err := r.sendRequest("GET", endpoint, nil)
-	if err != nil {
-		return nil, errors.Wrap(err, "REST request failed")
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode == http.StatusOK {
-		return parseListTunnels(resp.Body)
+	fetchFn := func(page int) (*http.Response, error) {
+		endpoint := r.baseEndpoints.accountLevel
+		filter.Page(page)
+		endpoint.RawQuery = filter.encode()
+		rsp, err := r.sendRequest("GET", endpoint, nil)
+		if err != nil {
+			return nil, errors.Wrap(err, "REST request failed")
+		}
+		if rsp.StatusCode != http.StatusOK {
+			rsp.Body.Close()
+			return nil, r.statusCodeToError("list tunnels", rsp)
+		}
+		return rsp, nil
 	}
 
-	return nil, r.statusCodeToError("list tunnels", resp)
-}
-
-func parseListTunnels(body io.ReadCloser) ([]*Tunnel, error) {
-	var tunnels []*Tunnel
-	err := parseResponse(body, &tunnels)
-	return tunnels, err
+	return fetchExhaustively[Tunnel](fetchFn)
 }
 
 func (r *RESTClient) ListActiveClients(tunnelID uuid.UUID) ([]*ActiveClient, error) {

cfapi/tunnel_filter.go

@@ -50,6 +50,10 @@ func (f *TunnelFilter) MaxFetchSize(max uint) {
 	f.queryParams.Set("per_page", strconv.Itoa(int(max)))
 }
 
+func (f *TunnelFilter) Page(page int) {
+	f.queryParams.Set("page", strconv.Itoa(page))
+}
+
 func (f TunnelFilter) encode() string {
 	return f.queryParams.Encode()
 }

cfapi/tunnel_test.go

@@ -3,7 +3,6 @@ package cfapi
 import (
 	"bytes"
 	"fmt"
-	"io"
 	"net"
 	"reflect"
 	"strings"
@@ -16,52 +15,6 @@ import (
 
 var loc, _ = time.LoadLocation("UTC")
 
-func Test_parseListTunnels(t *testing.T) {
-	type args struct {
-		body string
-	}
-	tests := []struct {
-		name    string
-		args    args
-		want    []*Tunnel
-		wantErr bool
-	}{
-		{
-			name: "empty list",
-			args: args{body: `{"success": true, "result": []}`},
-			want: []*Tunnel{},
-		},
-		{
-			name:    "success is false",
-			args:    args{body: `{"success": false, "result": []}`},
-			wantErr: true,
-		},
-		{
-			name:    "errors are present",
-			args:    args{body: `{"errors": [{"code": 1003, "message":"An A, AAAA or CNAME record already exists with that host"}], "result": []}`},
-			wantErr: true,
-		},
-		{
-			name:    "invalid response",
-			args:    args{body: `abc`},
-			wantErr: true,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			body := io.NopCloser(bytes.NewReader([]byte(tt.args.body)))
-			got, err := parseListTunnels(body)
-			if (err != nil) != tt.wantErr {
-				t.Errorf("parseListTunnels() error = %v, wantErr %v", err, tt.wantErr)
-				return
-			}
-			if !reflect.DeepEqual(got, tt.want) {
-				t.Errorf("parseListTunnels() = %v, want %v", got, tt.want)
-			}
-		})
-	}
-}
-
 func Test_unmarshalTunnel(t *testing.T) {
 	type args struct {
 		body string

cfsetup.yaml

@@ -1,4 +1,4 @@
-pinned_go: &pinned_go go-boring=1.21.5-1
+pinned_go: &pinned_go go-boring=1.22.2-1
 
 build_dir: &build_dir /cfsetup_build
 default-flavor: bullseye
@@ -33,11 +33,11 @@ buster: &buster
       - ./build-packages-fips.sh
       # Build binary for component test
       - GOOS=linux GOARCH=amd64 make cloudflared
-  cover: 
+  cover:
     build_dir: *build_dir
     builddeps: *build_deps
     pre-cache: *build_pre_cache
-    post-cache: 
+    post-cache:
      - make cover
   # except FIPS (handled in github-fips-release-pkgs) and macos (handled in github-release-macos-amd64)
   github-release-pkgs:
@@ -96,7 +96,7 @@ buster: &buster
       - make github-release-built-pkgs
   generate-versions-file:
     build_dir: *build_dir
-    builddeps: 
+    builddeps:
       - *pinned_go
       - build-essential
     post-cache:

cmd/cloudflared/access/cmd.go

@@ -132,15 +132,18 @@ func Commands() []*cli.Command {
 							Name:    sshHostnameFlag,
 							Aliases: []string{"tunnel-host", "T"},
 							Usage:   "specify the hostname of your application.",
+							EnvVars: []string{"TUNNEL_SERVICE_HOSTNAME"},
 						},
 						&cli.StringFlag{
-							Name:  sshDestinationFlag,
-							Usage: "specify the destination address of your SSH server.",
+							Name:    sshDestinationFlag,
+							Usage:   "specify the destination address of your SSH server.",
+							EnvVars: []string{"TUNNEL_SERVICE_DESTINATION"},
 						},
 						&cli.StringFlag{
 							Name:    sshURLFlag,
 							Aliases: []string{"listener", "L"},
 							Usage:   "specify the host:port to forward data to Cloudflare edge.",
+							EnvVars: []string{"TUNNEL_SERVICE_URL"},
 						},
 						&cli.StringSliceFlag{
 							Name:    sshHeaderFlag,

cmd/cloudflared/linux_service.go

@@ -55,7 +55,8 @@ var systemdAllTemplates = map[string]ServiceTemplate{
 		Path: fmt.Sprintf("/etc/systemd/system/%s", cloudflaredService),
 		Content: `[Unit]
 Description=cloudflared
-After=network.target
+After=network-online.target
+Wants=network-online.target
 
 [Service]
 TimeoutStartSec=0
@@ -72,7 +73,8 @@ WantedBy=multi-user.target
 		Path: fmt.Sprintf("/etc/systemd/system/%s", cloudflaredUpdateService),
 		Content: `[Unit]
 Description=Update cloudflared
-After=network.target
+After=network-online.target
+Wants=network-online.target
 
 [Service]
 ExecStart=/bin/bash -c '{{ .Path }} update; code=$?; if [ $code -eq 11 ]; then systemctl restart cloudflared; exit 0; fi; exit $code'

cmd/cloudflared/main.go

@@ -134,11 +134,22 @@ To determine if an update happened in a script, check for error code 11.`,
 		{
 			Name: "version",
 			Action: func(c *cli.Context) (err error) {
+				if c.Bool("short") {
+					fmt.Println(strings.Split(c.App.Version, " ")[0])
+					return nil
+				}
 				version(c)
 				return nil
 			},
 			Usage:       versionText,
 			Description: versionText,
+			Flags: []cli.Flag{
+				&cli.BoolFlag{
+					Name:    "short",
+					Aliases: []string{"s"},
+					Usage:   "print just the version number",
+				},
+			},
 		},
 	}
 	cmds = append(cmds, tunnel.Commands()...)

cmd/cloudflared/tunnel/cmd.go

@@ -78,8 +78,11 @@ const (
 	// hostKeyPath is the path of the dir to save SSH host keys too
 	hostKeyPath = "host-key-path"
 
-	// udpUnregisterSessionTimeout is how long we wait before we stop trying to unregister a UDP session from the edge
-	udpUnregisterSessionTimeoutFlag = "udp-unregister-session-timeout"
+	// rpcTimeout is how long to wait for a Capnp RPC request to the edge
+	rpcTimeout = "rpc-timeout"
+
+	// writeStreamTimeout sets if we should have a timeout when writing data to a stream towards the destination (edge/origin).
+	writeStreamTimeout = "write-stream-timeout"
 
 	// quicDisablePathMTUDiscovery sets if QUIC should not perform PTMU discovery and use a smaller (safe) packet size.
 	// Packets will then be at most 1252 (IPv4) / 1232 (IPv6) bytes in size.
@@ -284,7 +287,7 @@ func routeFromFlag(c *cli.Context) (route cfapi.HostnameRoute, ok bool) {
 func StartServer(
 	c *cli.Context,
 	info *cliutil.BuildInfo,
-	namedTunnel *connection.NamedTunnelProperties,
+	namedTunnel *connection.TunnelProperties,
 	log *zerolog.Logger,
 ) error {
 	err := sentry.Init(sentry.ClientOptions{
@@ -406,6 +409,11 @@ func StartServer(
 		}
 	}
 
+	// Disable ICMP packet routing for quick tunnels
+	if quickTunnelURL != "" {
+		tunnelConfig.PacketConfig = nil
+	}
+
 	internalRules := []ingress.Rule{}
 	if features.Contains(features.FeatureManagementLogs) {
 		serviceIP := c.String("service-op-ip")
@@ -655,9 +663,9 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 		}),
 		altsrc.NewStringSliceFlag(&cli.StringSliceFlag{
 			Name:    "tag",
-			Usage:   "Custom tags used to identify this tunnel, in format `KEY=VALUE`. Multiple tags may be specified",
+			Usage:   "Custom tags used to identify this tunnel via added HTTP request headers to the origin, in format `KEY=VALUE`. Multiple tags may be specified.",
 			EnvVars: []string{"TUNNEL_TAG"},
-			Hidden:  shouldHide,
+			Hidden:  true,
 		}),
 		altsrc.NewDurationFlag(&cli.DurationFlag{
 			Name:   "heartbeat-interval",
@@ -692,10 +700,17 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			Hidden: true,
 		}),
 		altsrc.NewDurationFlag(&cli.DurationFlag{
-			Name:   udpUnregisterSessionTimeoutFlag,
+			Name:   rpcTimeout,
 			Value:  5 * time.Second,
 			Hidden: true,
 		}),
+		altsrc.NewDurationFlag(&cli.DurationFlag{
+			Name:    writeStreamTimeout,
+			EnvVars: []string{"TUNNEL_STREAM_WRITE_TIMEOUT"},
+			Usage:   "Use this option to add a stream write timeout for connections when writing towards the origin or edge. Default is 0 which disables the write timeout.",
+			Value:   0 * time.Second,
+			Hidden:  true,
+		}),
 		altsrc.NewBoolFlag(&cli.BoolFlag{
 			Name:    quicDisablePathMTUDiscovery,
 			EnvVars: []string{"TUNNEL_DISABLE_QUIC_PMTU"},
@@ -780,7 +795,7 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			Name:    "management-diagnostics",
 			Usage:   "Enables the in-depth diagnostic routes to be made available over the management service (/debug/pprof, /metrics, etc.)",
 			EnvVars: []string{"TUNNEL_MANAGEMENT_DIAGNOSTICS"},
-			Value:   false,
+			Value:   true,
 		}),
 		selectProtocolFlag,
 		overwriteDNSFlag,

cmd/cloudflared/tunnel/configuration.go

@@ -27,10 +27,13 @@ import (
 	"github.com/cloudflare/cloudflared/orchestration"
 	"github.com/cloudflare/cloudflared/supervisor"
 	"github.com/cloudflare/cloudflared/tlsconfig"
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
-const secretValue = "*****"
+const (
+	secretValue       = "*****"
+	icmpFunnelTimeout = time.Second * 10
+)
 
 var (
 	developerPortal = "https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/install-and-setup"
@@ -105,7 +108,7 @@ func isSecretEnvVar(key string) bool {
 	return false
 }
 
-func dnsProxyStandAlone(c *cli.Context, namedTunnel *connection.NamedTunnelProperties) bool {
+func dnsProxyStandAlone(c *cli.Context, namedTunnel *connection.TunnelProperties) bool {
 	return c.IsSet("proxy-dns") &&
 		!(c.IsSet("name") || // adhoc-named tunnel
 			c.IsSet(ingress.HelloWorldFlag) || // quick or named tunnel
@@ -118,7 +121,7 @@ func prepareTunnelConfig(
 	info *cliutil.BuildInfo,
 	log, logTransport *zerolog.Logger,
 	observer *connection.Observer,
-	namedTunnel *connection.NamedTunnelProperties,
+	namedTunnel *connection.TunnelProperties,
 ) (*supervisor.TunnelConfig, *orchestration.Config, error) {
 	clientID, err := uuid.NewRandom()
 	if err != nil {
@@ -130,7 +133,7 @@ func prepareTunnelConfig(
 		log.Err(err).Msg("Tag parse failure")
 		return nil, nil, errors.Wrap(err, "Tag parse failure")
 	}
-	tags = append(tags, tunnelpogs.Tag{Name: "ID", Value: clientID.String()})
+	tags = append(tags, pogs.Tag{Name: "ID", Value: clientID.String()})
 
 	transportProtocol := c.String("protocol")
 
@@ -163,7 +166,7 @@ func prepareTunnelConfig(
 		)
 	}
 
-	namedTunnel.Client = tunnelpogs.ClientInfo{
+	namedTunnel.Client = pogs.ClientInfo{
 		ClientID: clientID[:],
 		Features: clientFeatures,
 		Version:  info.Version(),
@@ -243,7 +246,8 @@ func prepareTunnelConfig(
 		EdgeTLSConfigs:              edgeTLSConfigs,
 		FeatureSelector:             featureSelector,
 		MaxEdgeAddrRetries:          uint8(c.Int("max-edge-addr-retries")),
-		UDPUnregisterSessionTimeout: c.Duration(udpUnregisterSessionTimeoutFlag),
+		RPCTimeout:                  c.Duration(rpcTimeout),
+		WriteStreamTimeout:          c.Duration(writeStreamTimeout),
 		DisableQUICPathMTUDiscovery: c.Bool(quicDisablePathMTUDiscovery),
 	}
 	packetConfig, err := newPacketConfig(c, log)
@@ -256,6 +260,7 @@ func prepareTunnelConfig(
 		Ingress:            &ingressRules,
 		WarpRouting:        ingress.NewWarpRoutingConfig(&cfg.WarpRouting),
 		ConfigurationFlags: parseConfigFlags(c),
+		WriteTimeout:       c.Duration(writeStreamTimeout),
 	}
 	return tunnelConfig, orchestratorConfig, nil
 }
@@ -361,7 +366,7 @@ func newPacketConfig(c *cli.Context, logger *zerolog.Logger) (*ingress.GlobalRou
 		logger.Info().Msgf("ICMP proxy will use %s as source for IPv6", ipv6Src)
 	}
 
-	icmpRouter, err := ingress.NewICMPRouter(ipv4Src, ipv6Src, zone, logger)
+	icmpRouter, err := ingress.NewICMPRouter(ipv4Src, ipv6Src, zone, logger, icmpFunnelTimeout)
 	if err != nil {
 		return nil, err
 	}

cmd/cloudflared/tunnel/quick_tunnel.go

@@ -79,7 +79,7 @@ func RunQuickTunnel(sc *subcommandContext) error {
 	return StartServer(
 		sc.c,
 		buildInfo,
-		&connection.NamedTunnelProperties{Credentials: credentials, QuickTunnelUrl: data.Result.Hostname},
+		&connection.TunnelProperties{Credentials: credentials, QuickTunnelUrl: data.Result.Hostname},
 		sc.log,
 	)
 }

cmd/cloudflared/tunnel/subcommand_context.go

@@ -261,7 +261,7 @@ func (sc *subcommandContext) runWithCredentials(credentials connection.Credentia
 	return StartServer(
 		sc.c,
 		buildInfo,
-		&connection.NamedTunnelProperties{Credentials: credentials},
+		&connection.TunnelProperties{Credentials: credentials},
 		sc.log,
 	)
 }

cmd/cloudflared/tunnel/tag.go

@@ -4,23 +4,23 @@ import (
 	"fmt"
 	"regexp"
 
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
 // Restrict key names to characters allowed in an HTTP header name.
 // Restrict key values to printable characters (what is recognised as data in an HTTP header value).
 var tagRegexp = regexp.MustCompile("^([a-zA-Z0-9!#$%&'*+\\-.^_`|~]+)=([[:print:]]+)$")
 
-func NewTagFromCLI(compoundTag string) (tunnelpogs.Tag, bool) {
+func NewTagFromCLI(compoundTag string) (pogs.Tag, bool) {
 	matches := tagRegexp.FindStringSubmatch(compoundTag)
 	if len(matches) == 0 {
-		return tunnelpogs.Tag{}, false
+		return pogs.Tag{}, false
 	}
-	return tunnelpogs.Tag{Name: matches[1], Value: matches[2]}, true
+	return pogs.Tag{Name: matches[1], Value: matches[2]}, true
 }
 
-func NewTagSliceFromCLI(tags []string) ([]tunnelpogs.Tag, error) {
-	var tagSlice []tunnelpogs.Tag
+func NewTagSliceFromCLI(tags []string) ([]pogs.Tag, error) {
+	var tagSlice []pogs.Tag
 	for _, compoundTag := range tags {
 		if tag, ok := NewTagFromCLI(compoundTag); ok {
 			tagSlice = append(tagSlice, tag)

cmd/cloudflared/tunnel/tag_test.go

@@ -3,7 +3,7 @@ package tunnel
 import (
 	"testing"
 
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 
 	"github.com/stretchr/testify/assert"
 )
@@ -11,12 +11,12 @@ import (
 func TestSingleTag(t *testing.T) {
 	testCases := []struct {
 		Input  string
-		Output tunnelpogs.Tag
+		Output pogs.Tag
 		Fail   bool
 	}{
-		{Input: "x=y", Output: tunnelpogs.Tag{Name: "x", Value: "y"}},
-		{Input: "More-Complex=Tag Values", Output: tunnelpogs.Tag{Name: "More-Complex", Value: "Tag Values"}},
-		{Input: "First=Equals=Wins", Output: tunnelpogs.Tag{Name: "First", Value: "Equals=Wins"}},
+		{Input: "x=y", Output: pogs.Tag{Name: "x", Value: "y"}},
+		{Input: "More-Complex=Tag Values", Output: pogs.Tag{Name: "More-Complex", Value: "Tag Values"}},
+		{Input: "First=Equals=Wins", Output: pogs.Tag{Name: "First", Value: "Equals=Wins"}},
 		{Input: "x=", Fail: true},
 		{Input: "=y", Fail: true},
 		{Input: "=", Fail: true},

component-tests/test_management.py

@@ -55,7 +55,7 @@ class TestManagement:
         config = component_tests_config(cfd_mode=CfdModes.NAMED, run_proxy_dns=False, provide_ingress=False)
         LOGGER.debug(config)
         config_path = write_config(tmp_path, config.full_config)
-        with start_cloudflared(tmp_path, config, cfd_pre_args=["tunnel", "--ha-connections", "1", "--management-diagnostics"], new_process=True):
+        with start_cloudflared(tmp_path, config, cfd_pre_args=["tunnel", "--ha-connections", "1"], new_process=True):
             wait_tunnel_ready(require_min_connections=1)
             cfd_cli = CloudflaredCli(config, config_path, LOGGER)
             url = cfd_cli.get_management_url("metrics", config, config_path)
@@ -76,7 +76,7 @@ class TestManagement:
         config = component_tests_config(cfd_mode=CfdModes.NAMED, run_proxy_dns=False, provide_ingress=False)
         LOGGER.debug(config)
         config_path = write_config(tmp_path, config.full_config)
-        with start_cloudflared(tmp_path, config, cfd_pre_args=["tunnel", "--ha-connections", "1", "--management-diagnostics"], new_process=True):
+        with start_cloudflared(tmp_path, config, cfd_pre_args=["tunnel", "--ha-connections", "1"], new_process=True):
             wait_tunnel_ready(require_min_connections=1)
             cfd_cli = CloudflaredCli(config, config_path, LOGGER)
             url = cfd_cli.get_management_url("debug/pprof/heap", config, config_path)
@@ -97,7 +97,7 @@ class TestManagement:
         config = component_tests_config(cfd_mode=CfdModes.NAMED, run_proxy_dns=False, provide_ingress=False)
         LOGGER.debug(config)
         config_path = write_config(tmp_path, config.full_config)
-        with start_cloudflared(tmp_path, config, cfd_pre_args=["tunnel", "--ha-connections", "1"], new_process=True):
+        with start_cloudflared(tmp_path, config, cfd_pre_args=["tunnel", "--ha-connections", "1", "--management-diagnostics=false"], new_process=True):
             wait_tunnel_ready(require_min_connections=1)
             cfd_cli = CloudflaredCli(config, config_path, LOGGER)
             url = cfd_cli.get_management_url("metrics", config, config_path)

component-tests/test_quicktunnels.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 from conftest import CfdModes
 from constants import METRICS_PORT
+import time
 from util import LOGGER, start_cloudflared, wait_tunnel_ready, get_quicktunnel_url, send_requests
 
 class TestQuickTunnels:
@@ -9,6 +10,7 @@ class TestQuickTunnels:
         LOGGER.debug(config)
         with start_cloudflared(tmp_path, config, cfd_pre_args=["tunnel", "--ha-connections", "1"], cfd_args=["--hello-world"], new_process=True):
             wait_tunnel_ready(require_min_connections=1)
+            time.sleep(10)
             url = get_quicktunnel_url()
             send_requests(url, 3, True)
     
@@ -17,6 +19,7 @@ class TestQuickTunnels:
         LOGGER.debug(config)
         with start_cloudflared(tmp_path, config, cfd_pre_args=["tunnel", "--ha-connections", "1"], cfd_args=["--url", f"http://localhost:{METRICS_PORT}/"], new_process=True):
             wait_tunnel_ready(require_min_connections=1)
+            time.sleep(10)
             url = get_quicktunnel_url()
             send_requests(url+"/ready", 3, True)
 

config/configuration.go

@@ -205,6 +205,8 @@ type OriginRequestConfig struct {
 	HTTPHostHeader *string `yaml:"httpHostHeader" json:"httpHostHeader,omitempty"`
 	// Hostname on the origin server certificate.
 	OriginServerName *string `yaml:"originServerName" json:"originServerName,omitempty"`
+	// Auto configure the Hostname on the origin server certificate.
+	MatchSNIToHost *bool `yaml:"matchSNItoHost" json:"matchSNItoHost,omitempty"`
 	// Path to the CA for the certificate of your origin.
 	// This option should be used only if your certificate is not signed by Cloudflare.
 	CAPool *string `yaml:"caPool" json:"caPool,omitempty"`

connection/connection.go

@@ -42,7 +42,7 @@ type Orchestrator interface {
 	GetOriginProxy() (OriginProxy, error)
 }
 
-type NamedTunnelProperties struct {
+type TunnelProperties struct {
 	Credentials    Credentials
 	Client         pogs.ClientInfo
 	QuickTunnelUrl string

connection/control.go

@@ -6,25 +6,25 @@ import (
 	"net"
 	"time"
 
-	"github.com/rs/zerolog"
-
 	"github.com/cloudflare/cloudflared/management"
+	"github.com/cloudflare/cloudflared/tunnelrpc"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
-// RPCClientFunc derives a named tunnel rpc client that can then be used to register and unregister connections.
-type RPCClientFunc func(context.Context, io.ReadWriteCloser, *zerolog.Logger) NamedTunnelRPCClient
+// registerClient derives a named tunnel rpc client that can then be used to register and unregister connections.
+type registerClientFunc func(context.Context, io.ReadWriteCloser, time.Duration) tunnelrpc.RegistrationClient
 
 type controlStream struct {
 	observer *Observer
 
-	connectedFuse         ConnectedFuse
-	namedTunnelProperties *NamedTunnelProperties
-	connIndex             uint8
-	edgeAddress           net.IP
-	protocol              Protocol
+	connectedFuse    ConnectedFuse
+	tunnelProperties *TunnelProperties
+	connIndex        uint8
+	edgeAddress      net.IP
+	protocol         Protocol
 
-	newRPCClientFunc RPCClientFunc
+	registerClientFunc registerClientFunc
+	registerTimeout    time.Duration
 
 	gracefulShutdownC <-chan struct{}
 	gracePeriod       time.Duration
@@ -47,27 +47,29 @@ type TunnelConfigJSONGetter interface {
 func NewControlStream(
 	observer *Observer,
 	connectedFuse ConnectedFuse,
-	namedTunnelConfig *NamedTunnelProperties,
+	tunnelProperties *TunnelProperties,
 	connIndex uint8,
 	edgeAddress net.IP,
-	newRPCClientFunc RPCClientFunc,
+	registerClientFunc registerClientFunc,
+	registerTimeout time.Duration,
 	gracefulShutdownC <-chan struct{},
 	gracePeriod time.Duration,
 	protocol Protocol,
 ) ControlStreamHandler {
-	if newRPCClientFunc == nil {
-		newRPCClientFunc = newRegistrationRPCClient
+	if registerClientFunc == nil {
+		registerClientFunc = tunnelrpc.NewRegistrationClient
 	}
 	return &controlStream{
-		observer:              observer,
-		connectedFuse:         connectedFuse,
-		namedTunnelProperties: namedTunnelConfig,
-		newRPCClientFunc:      newRPCClientFunc,
-		connIndex:             connIndex,
-		edgeAddress:           edgeAddress,
-		gracefulShutdownC:     gracefulShutdownC,
-		gracePeriod:           gracePeriod,
-		protocol:              protocol,
+		observer:           observer,
+		connectedFuse:      connectedFuse,
+		tunnelProperties:   tunnelProperties,
+		registerClientFunc: registerClientFunc,
+		registerTimeout:    registerTimeout,
+		connIndex:          connIndex,
+		edgeAddress:        edgeAddress,
+		gracefulShutdownC:  gracefulShutdownC,
+		gracePeriod:        gracePeriod,
+		protocol:           protocol,
 	}
 }
 
@@ -77,13 +79,25 @@ func (c *controlStream) ServeControlStream(
 	connOptions *tunnelpogs.ConnectionOptions,
 	tunnelConfigGetter TunnelConfigJSONGetter,
 ) error {
-	rpcClient := c.newRPCClientFunc(ctx, rw, c.observer.log)
+	registrationClient := c.registerClientFunc(ctx, rw, c.registerTimeout)
 
-	registrationDetails, err := rpcClient.RegisterConnection(ctx, c.namedTunnelProperties, connOptions, c.connIndex, c.edgeAddress, c.observer)
+	registrationDetails, err := registrationClient.RegisterConnection(
+		ctx,
+		c.tunnelProperties.Credentials.Auth(),
+		c.tunnelProperties.Credentials.TunnelID,
+		connOptions,
+		c.connIndex,
+		c.edgeAddress)
 	if err != nil {
-		rpcClient.Close()
-		return err
+		defer registrationClient.Close()
+		if err.Error() == DuplicateConnectionError {
+			c.observer.metrics.regFail.WithLabelValues("dup_edge_conn", "registerConnection").Inc()
+			return errDuplicationConnection
+		}
+		c.observer.metrics.regFail.WithLabelValues("server_error", "registerConnection").Inc()
+		return serverRegistrationErrorFromRPC(err)
 	}
+	c.observer.metrics.regSuccess.WithLabelValues("registerConnection").Inc()
 
 	c.observer.logConnected(registrationDetails.UUID, c.connIndex, registrationDetails.Location, c.edgeAddress, c.protocol)
 	c.observer.sendConnectedEvent(c.connIndex, c.protocol, registrationDetails.Location)
@@ -92,21 +106,23 @@ func (c *controlStream) ServeControlStream(
 	// if conn index is 0 and tunnel is not remotely managed, then send local ingress rules configuration
 	if c.connIndex == 0 && !registrationDetails.TunnelIsRemotelyManaged {
 		if tunnelConfig, err := tunnelConfigGetter.GetConfigJSON(); err == nil {
-			if err := rpcClient.SendLocalConfiguration(ctx, tunnelConfig, c.observer); err != nil {
+			if err := registrationClient.SendLocalConfiguration(ctx, tunnelConfig); err != nil {
+				c.observer.metrics.localConfigMetrics.pushesErrors.Inc()
 				c.observer.log.Err(err).Msg("unable to send local configuration")
 			}
+			c.observer.metrics.localConfigMetrics.pushes.Inc()
 		} else {
 			c.observer.log.Err(err).Msg("failed to obtain current configuration")
 		}
 	}
 
-	c.waitForUnregister(ctx, rpcClient)
+	c.waitForUnregister(ctx, registrationClient)
 	return nil
 }
 
-func (c *controlStream) waitForUnregister(ctx context.Context, rpcClient NamedTunnelRPCClient) {
+func (c *controlStream) waitForUnregister(ctx context.Context, registrationClient tunnelrpc.RegistrationClient) {
 	// wait for connection termination or start of graceful shutdown
-	defer rpcClient.Close()
+	defer registrationClient.Close()
 	select {
 	case <-ctx.Done():
 		break
@@ -115,7 +131,7 @@ func (c *controlStream) waitForUnregister(ctx context.Context, rpcClient NamedTu
 	}
 
 	c.observer.sendUnregisteringEvent(c.connIndex)
-	rpcClient.GracefulShutdown(ctx, c.gracePeriod)
+	registrationClient.GracefulShutdown(ctx, c.gracePeriod)
 	c.observer.log.Info().
 		Int(management.EventTypeKey, int(management.Cloudflared)).
 		Uint8(LogFieldConnIndex, c.connIndex).

connection/http2.go

@@ -40,8 +40,6 @@ type HTTP2Connection struct {
 	connOptions  *tunnelpogs.ConnectionOptions
 	observer     *Observer
 	connIndex    uint8
-	// newRPCClientFunc allows us to mock RPCs during testing
-	newRPCClientFunc func(context.Context, io.ReadWriteCloser, *zerolog.Logger) NamedTunnelRPCClient
 
 	log                  *zerolog.Logger
 	activeRequestsWG     sync.WaitGroup
@@ -69,7 +67,6 @@ func NewHTTP2Connection(
 		connOptions:          connOptions,
 		observer:             observer,
 		connIndex:            connIndex,
-		newRPCClientFunc:     newRegistrationRPCClient,
 		controlStreamHandler: controlStreamHandler,
 		log:                  log,
 	}

connection/http2_test.go

@@ -20,8 +20,8 @@ import (
 	"github.com/stretchr/testify/require"
 	"golang.org/x/net/http2"
 
+	"github.com/cloudflare/cloudflared/tunnelrpc"
 	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
 var (
@@ -36,10 +36,11 @@ func newTestHTTP2Connection() (*HTTP2Connection, net.Conn) {
 	controlStream := NewControlStream(
 		obs,
 		mockConnectedFuse{},
-		&NamedTunnelProperties{},
+		&TunnelProperties{},
 		connIndex,
 		nil,
 		nil,
+		1*time.Second,
 		nil,
 		1*time.Second,
 		HTTP2,
@@ -168,23 +169,23 @@ type mockNamedTunnelRPCClient struct {
 	unregistered chan struct{}
 }
 
-func (mc mockNamedTunnelRPCClient) SendLocalConfiguration(c context.Context, config []byte, observer *Observer) error {
+func (mc mockNamedTunnelRPCClient) SendLocalConfiguration(c context.Context, config []byte) error {
 	return nil
 }
 
 func (mc mockNamedTunnelRPCClient) RegisterConnection(
-	c context.Context,
-	properties *NamedTunnelProperties,
-	options *tunnelpogs.ConnectionOptions,
+	ctx context.Context,
+	auth pogs.TunnelAuth,
+	tunnelID uuid.UUID,
+	options *pogs.ConnectionOptions,
 	connIndex uint8,
 	edgeAddress net.IP,
-	observer *Observer,
-) (*tunnelpogs.ConnectionDetails, error) {
+) (*pogs.ConnectionDetails, error) {
 	if mc.shouldFail != nil {
 		return nil, mc.shouldFail
 	}
 	close(mc.registered)
-	return &tunnelpogs.ConnectionDetails{
+	return &pogs.ConnectionDetails{
 		Location:                "LIS",
 		UUID:                    uuid.New(),
 		TunnelIsRemotelyManaged: false,
@@ -203,8 +204,8 @@ type mockRPCClientFactory struct {
 	unregistered chan struct{}
 }
 
-func (mf *mockRPCClientFactory) newMockRPCClient(context.Context, io.ReadWriteCloser, *zerolog.Logger) NamedTunnelRPCClient {
-	return mockNamedTunnelRPCClient{
+func (mf *mockRPCClientFactory) newMockRPCClient(context.Context, io.ReadWriteCloser, time.Duration) tunnelrpc.RegistrationClient {
+	return &mockNamedTunnelRPCClient{
 		shouldFail:   mf.shouldFail,
 		registered:   mf.registered,
 		unregistered: mf.unregistered,
@@ -360,10 +361,11 @@ func TestServeControlStream(t *testing.T) {
 	controlStream := NewControlStream(
 		obs,
 		mockConnectedFuse{},
-		&NamedTunnelProperties{},
+		&TunnelProperties{},
 		1,
 		nil,
 		rpcClientFactory.newMockRPCClient,
+		1*time.Second,
 		nil,
 		1*time.Second,
 		HTTP2,
@@ -412,10 +414,11 @@ func TestFailRegistration(t *testing.T) {
 	controlStream := NewControlStream(
 		obs,
 		mockConnectedFuse{},
-		&NamedTunnelProperties{},
+		&TunnelProperties{},
 		http2Conn.connIndex,
 		nil,
 		rpcClientFactory.newMockRPCClient,
+		1*time.Second,
 		nil,
 		1*time.Second,
 		HTTP2,
@@ -460,10 +463,11 @@ func TestGracefulShutdownHTTP2(t *testing.T) {
 	controlStream := NewControlStream(
 		obs,
 		mockConnectedFuse{},
-		&NamedTunnelProperties{},
+		&TunnelProperties{},
 		http2Conn.connIndex,
 		nil,
 		rpcClientFactory.newMockRPCClient,
+		1*time.Second,
 		shutdownC,
 		1*time.Second,
 		HTTP2,

connection/metrics.go

@@ -43,7 +43,6 @@ type localConfigMetrics struct {
 }
 
 type tunnelMetrics struct {
-	timerRetries    prometheus.Gauge
 	serverLocations *prometheus.GaugeVec
 	// locationLock is a mutex for oldServerLocations
 	locationLock sync.Mutex
@@ -351,15 +350,6 @@ func initTunnelMetrics() *tunnelMetrics {
 	)
 	prometheus.MustRegister(maxConcurrentRequestsPerTunnel)
 
-	timerRetries := prometheus.NewGauge(
-		prometheus.GaugeOpts{
-			Namespace: MetricsNamespace,
-			Subsystem: TunnelSubsystem,
-			Name:      "timer_retries",
-			Help:      "Unacknowledged heart beats count",
-		})
-	prometheus.MustRegister(timerRetries)
-
 	serverLocations := prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Namespace: MetricsNamespace,
@@ -416,7 +406,6 @@ func initTunnelMetrics() *tunnelMetrics {
 	prometheus.MustRegister(registerSuccess)
 
 	return &tunnelMetrics{
-		timerRetries:        timerRetries,
 		serverLocations:     serverLocations,
 		oldServerLocations:  make(map[string]string),
 		muxerMetrics:        newMuxerMetrics(),

connection/quic.go

@@ -28,9 +28,11 @@ import (
 	"github.com/cloudflare/cloudflared/ingress"
 	"github.com/cloudflare/cloudflared/management"
 	"github.com/cloudflare/cloudflared/packet"
-	quicpogs "github.com/cloudflare/cloudflared/quic"
+	cfdquic "github.com/cloudflare/cloudflared/quic"
 	"github.com/cloudflare/cloudflared/tracing"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	rpcquic "github.com/cloudflare/cloudflared/tunnelrpc/quic"
 )
 
 const (
@@ -59,13 +61,14 @@ type QUICConnection struct {
 	// sessionManager tracks active sessions. It receives datagrams from quic connection via datagramMuxer
 	sessionManager datagramsession.Manager
 	// datagramMuxer mux/demux datagrams from quic connection
-	datagramMuxer        *quicpogs.DatagramMuxerV2
+	datagramMuxer        *cfdquic.DatagramMuxerV2
 	packetRouter         *ingress.PacketRouter
 	controlStreamHandler ControlStreamHandler
 	connOptions          *tunnelpogs.ConnectionOptions
 	connIndex            uint8
 
-	udpUnregisterTimeout time.Duration
+	rpcTimeout         time.Duration
+	streamWriteTimeout time.Duration
 }
 
 // NewQUICConnection returns a new instance of QUICConnection.
@@ -81,7 +84,8 @@ func NewQUICConnection(
 	controlStreamHandler ControlStreamHandler,
 	logger *zerolog.Logger,
 	packetRouterConfig *ingress.GlobalRouterConfig,
-	udpUnregisterTimeout time.Duration,
+	rpcTimeout time.Duration,
+	streamWriteTimeout time.Duration,
 ) (*QUICConnection, error) {
 	udpConn, err := createUDPConnForConnIndex(connIndex, localAddr, logger)
 	if err != nil {
@@ -102,7 +106,7 @@ func NewQUICConnection(
 	}
 
 	sessionDemuxChan := make(chan *packet.Session, demuxChanCapacity)
-	datagramMuxer := quicpogs.NewDatagramMuxerV2(session, logger, sessionDemuxChan)
+	datagramMuxer := cfdquic.NewDatagramMuxerV2(session, logger, sessionDemuxChan)
 	sessionManager := datagramsession.NewManager(logger, datagramMuxer.SendToSession, sessionDemuxChan)
 	packetRouter := ingress.NewPacketRouter(packetRouterConfig, datagramMuxer, logger)
 
@@ -116,7 +120,8 @@ func NewQUICConnection(
 		controlStreamHandler: controlStreamHandler,
 		connOptions:          connOptions,
 		connIndex:            connIndex,
-		udpUnregisterTimeout: udpUnregisterTimeout,
+		rpcTimeout:           rpcTimeout,
+		streamWriteTimeout:   streamWriteTimeout,
 	}, nil
 }
 
@@ -195,7 +200,7 @@ func (q *QUICConnection) acceptStream(ctx context.Context) error {
 
 func (q *QUICConnection) runStream(quicStream quic.Stream) {
 	ctx := quicStream.Context()
-	stream := quicpogs.NewSafeStreamCloser(quicStream)
+	stream := cfdquic.NewSafeStreamCloser(quicStream, q.streamWriteTimeout, q.logger)
 	defer stream.Close()
 
 	// we are going to fuse readers/writers from stream <- cloudflared -> origin, and we want to guarantee that
@@ -203,7 +208,8 @@ func (q *QUICConnection) runStream(quicStream quic.Stream) {
 	// So, we wrap the stream with a no-op write closer and only this method can actually close write side of the stream.
 	// A call to close will simulate a close to the read-side, which will fail subsequent reads.
 	noCloseStream := &nopCloserReadWriter{ReadWriteCloser: stream}
-	if err := q.handleStream(ctx, noCloseStream); err != nil {
+	ss := rpcquic.NewCloudflaredServer(q.handleDataStream, q, q, q.rpcTimeout)
+	if err := ss.Serve(ctx, noCloseStream); err != nil {
 		q.logger.Debug().Err(err).Msg("Failed to handle QUIC stream")
 
 		// if we received an error at this level, then close write side of stream with an error, which will result in
@@ -212,30 +218,7 @@ func (q *QUICConnection) runStream(quicStream quic.Stream) {
 	}
 }
 
-func (q *QUICConnection) handleStream(ctx context.Context, stream io.ReadWriteCloser) error {
-	signature, err := quicpogs.DetermineProtocol(stream)
-	if err != nil {
-		return err
-	}
-	switch signature {
-	case quicpogs.DataStreamProtocolSignature:
-		reqServerStream, err := quicpogs.NewRequestServerStream(stream, signature)
-		if err != nil {
-			return err
-		}
-		return q.handleDataStream(ctx, reqServerStream)
-	case quicpogs.RPCStreamProtocolSignature:
-		rpcStream, err := quicpogs.NewRPCServerStream(stream, signature)
-		if err != nil {
-			return err
-		}
-		return q.handleRPCStream(rpcStream)
-	default:
-		return fmt.Errorf("unknown protocol %v", signature)
-	}
-}
-
-func (q *QUICConnection) handleDataStream(ctx context.Context, stream *quicpogs.RequestServerStream) error {
+func (q *QUICConnection) handleDataStream(ctx context.Context, stream *rpcquic.RequestServerStream) error {
 	request, err := stream.ReadConnectRequestData()
 	if err != nil {
 		return err
@@ -261,22 +244,22 @@ func (q *QUICConnection) handleDataStream(ctx context.Context, stream *quicpogs.
 // dispatchRequest will dispatch the request depending on the type and returns an error if it occurs.
 // More importantly, it also tells if the during processing of the request the ConnectResponse metadata was sent downstream.
 // This is important since it informs
-func (q *QUICConnection) dispatchRequest(ctx context.Context, stream *quicpogs.RequestServerStream, err error, request *quicpogs.ConnectRequest) (error, bool) {
+func (q *QUICConnection) dispatchRequest(ctx context.Context, stream *rpcquic.RequestServerStream, err error, request *pogs.ConnectRequest) (error, bool) {
 	originProxy, err := q.orchestrator.GetOriginProxy()
 	if err != nil {
 		return err, false
 	}
 
 	switch request.Type {
-	case quicpogs.ConnectionTypeHTTP, quicpogs.ConnectionTypeWebsocket:
+	case pogs.ConnectionTypeHTTP, pogs.ConnectionTypeWebsocket:
 		tracedReq, err := buildHTTPRequest(ctx, request, stream, q.connIndex, q.logger)
 		if err != nil {
 			return err, false
 		}
 		w := newHTTPResponseAdapter(stream)
-		return originProxy.ProxyHTTP(&w, tracedReq, request.Type == quicpogs.ConnectionTypeWebsocket), w.connectResponseSent
+		return originProxy.ProxyHTTP(&w, tracedReq, request.Type == pogs.ConnectionTypeWebsocket), w.connectResponseSent
 
-	case quicpogs.ConnectionTypeTCP:
+	case pogs.ConnectionTypeTCP:
 		rwa := &streamReadWriteAcker{RequestServerStream: stream}
 		metadata := request.MetadataMap()
 		return originProxy.ProxyTCP(ctx, rwa, &TCPRequest{
@@ -290,14 +273,6 @@ func (q *QUICConnection) dispatchRequest(ctx context.Context, stream *quicpogs.R
 	}
 }
 
-func (q *QUICConnection) handleRPCStream(rpcStream *quicpogs.RPCServerStream) error {
-	if err := rpcStream.Serve(q, q, q.logger); err != nil {
-		q.logger.Err(err).Msg("failed handling RPC stream")
-	}
-
-	return nil
-}
-
 // RegisterUdpSession is the RPC method invoked by edge to register and run a session
 func (q *QUICConnection) RegisterUdpSession(ctx context.Context, sessionID uuid.UUID, dstIP net.IP, dstPort uint16, closeAfterIdleHint time.Duration, traceContext string) (*tunnelpogs.RegisterUdpSessionResponse, error) {
 	traceCtx := tracing.NewTracedContext(ctx, traceContext, q.logger)
@@ -321,6 +296,7 @@ func (q *QUICConnection) RegisterUdpSession(ctx context.Context, sessionID uuid.
 
 	session, err := q.sessionManager.RegisterSession(ctx, sessionID, originProxy)
 	if err != nil {
+		originProxy.Close()
 		log.Err(err).Str("sessionID", sessionID.String()).Msgf("Failed to register udp session")
 		tracing.EndWithErrorStatus(registerSpan, err)
 		return nil, err
@@ -373,9 +349,9 @@ func (q *QUICConnection) closeUDPSession(ctx context.Context, sessionID uuid.UUI
 		return
 	}
 
-	stream := quicpogs.NewSafeStreamCloser(quicStream)
+	stream := cfdquic.NewSafeStreamCloser(quicStream, q.streamWriteTimeout, q.logger)
 	defer stream.Close()
-	rpcClientStream, err := quicpogs.NewRPCClientStream(ctx, stream, q.udpUnregisterTimeout, q.logger)
+	rpcClientStream, err := rpcquic.NewSessionClient(ctx, stream, q.rpcTimeout)
 	if err != nil {
 		// Log this at debug because this is not an error if session was closed due to lost connection
 		// with edge
@@ -404,16 +380,16 @@ func (q *QUICConnection) UpdateConfiguration(ctx context.Context, version int32,
 // streamReadWriteAcker is a light wrapper over QUIC streams with a callback to send response back to
 // the client.
 type streamReadWriteAcker struct {
-	*quicpogs.RequestServerStream
+	*rpcquic.RequestServerStream
 	connectResponseSent bool
 }
 
 // AckConnection acks response back to the proxy.
 func (s *streamReadWriteAcker) AckConnection(tracePropagation string) error {
-	metadata := []quicpogs.Metadata{}
+	metadata := []pogs.Metadata{}
 	// Only add tracing if provided by origintunneld
 	if tracePropagation != "" {
-		metadata = append(metadata, quicpogs.Metadata{
+		metadata = append(metadata, pogs.Metadata{
 			Key: tracing.CanonicalCloudflaredTracingHeader,
 			Val: tracePropagation,
 		})
@@ -424,12 +400,12 @@ func (s *streamReadWriteAcker) AckConnection(tracePropagation string) error {
 
 // httpResponseAdapter translates responses written by the HTTP Proxy into ones that can be used in QUIC.
 type httpResponseAdapter struct {
-	*quicpogs.RequestServerStream
+	*rpcquic.RequestServerStream
 	headers             http.Header
 	connectResponseSent bool
 }
 
-func newHTTPResponseAdapter(s *quicpogs.RequestServerStream) httpResponseAdapter {
+func newHTTPResponseAdapter(s *rpcquic.RequestServerStream) httpResponseAdapter {
 	return httpResponseAdapter{RequestServerStream: s, headers: make(http.Header)}
 }
 
@@ -438,12 +414,12 @@ func (hrw *httpResponseAdapter) AddTrailer(trailerName, trailerValue string) {
 }
 
 func (hrw *httpResponseAdapter) WriteRespHeaders(status int, header http.Header) error {
-	metadata := make([]quicpogs.Metadata, 0)
-	metadata = append(metadata, quicpogs.Metadata{Key: "HttpStatus", Val: strconv.Itoa(status)})
+	metadata := make([]pogs.Metadata, 0)
+	metadata = append(metadata, pogs.Metadata{Key: "HttpStatus", Val: strconv.Itoa(status)})
 	for k, vv := range header {
 		for _, v := range vv {
 			httpHeaderKey := fmt.Sprintf("%s:%s", HTTPHeaderKey, k)
-			metadata = append(metadata, quicpogs.Metadata{Key: httpHeaderKey, Val: v})
+			metadata = append(metadata, pogs.Metadata{Key: httpHeaderKey, Val: v})
 		}
 	}
 
@@ -479,17 +455,17 @@ func (hrw *httpResponseAdapter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
 }
 
 func (hrw *httpResponseAdapter) WriteErrorResponse(err error) {
-	hrw.WriteConnectResponseData(err, quicpogs.Metadata{Key: "HttpStatus", Val: strconv.Itoa(http.StatusBadGateway)})
+	hrw.WriteConnectResponseData(err, pogs.Metadata{Key: "HttpStatus", Val: strconv.Itoa(http.StatusBadGateway)})
 }
 
-func (hrw *httpResponseAdapter) WriteConnectResponseData(respErr error, metadata ...quicpogs.Metadata) error {
+func (hrw *httpResponseAdapter) WriteConnectResponseData(respErr error, metadata ...pogs.Metadata) error {
 	hrw.connectResponseSent = true
 	return hrw.RequestServerStream.WriteConnectResponseData(respErr, metadata...)
 }
 
 func buildHTTPRequest(
 	ctx context.Context,
-	connectRequest *quicpogs.ConnectRequest,
+	connectRequest *pogs.ConnectRequest,
 	body io.ReadCloser,
 	connIndex uint8,
 	log *zerolog.Logger,
@@ -498,7 +474,7 @@ func buildHTTPRequest(
 	dest := connectRequest.Dest
 	method := metadata[HTTPMethodKey]
 	host := metadata[HTTPHostKey]
-	isWebsocket := connectRequest.Type == quicpogs.ConnectionTypeWebsocket
+	isWebsocket := connectRequest.Type == pogs.ConnectionTypeWebsocket
 
 	req, err := http.NewRequestWithContext(ctx, method, dest, body)
 	if err != nil {
@@ -593,11 +569,11 @@ func (np *nopCloserReadWriter) Close() error {
 
 // muxerWrapper wraps DatagramMuxerV2 to satisfy the packet.FunnelUniPipe interface
 type muxerWrapper struct {
-	muxer *quicpogs.DatagramMuxerV2
+	muxer *cfdquic.DatagramMuxerV2
 }
 
 func (rp *muxerWrapper) SendPacket(dst netip.Addr, pk packet.RawPacket) error {
-	return rp.muxer.SendPacket(quicpogs.RawPacket(pk))
+	return rp.muxer.SendPacket(cfdquic.RawPacket(pk))
 }
 
 func (rp *muxerWrapper) ReceivePacket(ctx context.Context) (packet.RawPacket, error) {
@@ -605,7 +581,7 @@ func (rp *muxerWrapper) ReceivePacket(ctx context.Context) (packet.RawPacket, er
 	if err != nil {
 		return packet.RawPacket{}, err
 	}
-	rawPacket, ok := pk.(quicpogs.RawPacket)
+	rawPacket, ok := pk.(cfdquic.RawPacket)
 	if ok {
 		return packet.RawPacket(rawPacket), nil
 	}

connection/quic_test.go

@@ -3,9 +3,14 @@ package connection
 import (
 	"bytes"
 	"context"
+	"crypto/rand"
+	"crypto/rsa"
 	"crypto/tls"
+	"crypto/x509"
+	"encoding/pem"
 	"fmt"
 	"io"
+	"math/big"
 	"net"
 	"net/http"
 	"net/url"
@@ -23,18 +28,20 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/cloudflare/cloudflared/datagramsession"
-	quicpogs "github.com/cloudflare/cloudflared/quic"
+	cfdquic "github.com/cloudflare/cloudflared/quic"
 	"github.com/cloudflare/cloudflared/tracing"
 	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	rpcquic "github.com/cloudflare/cloudflared/tunnelrpc/quic"
 )
 
 var (
-	testTLSServerConfig = quicpogs.GenerateTLSConfig()
+	testTLSServerConfig = GenerateTLSConfig()
 	testQUICConfig      = &quic.Config{
 		KeepAlivePeriod: 5 * time.Second,
 		EnableDatagrams: true,
 	}
+	defaultQUICTimeout = 30 * time.Second
 )
 
 var _ ReadWriteAcker = (*streamReadWriteAcker)(nil)
@@ -49,16 +56,16 @@ func TestQUICServer(t *testing.T) {
 	var tests = []struct {
 		desc             string
 		dest             string
-		connectionType   quicpogs.ConnectionType
-		metadata         []quicpogs.Metadata
+		connectionType   pogs.ConnectionType
+		metadata         []pogs.Metadata
 		message          []byte
 		expectedResponse []byte
 	}{
 		{
 			desc:           "test http proxy",
 			dest:           "/ok",
-			connectionType: quicpogs.ConnectionTypeHTTP,
-			metadata: []quicpogs.Metadata{
+			connectionType: pogs.ConnectionTypeHTTP,
+			metadata: []pogs.Metadata{
 				{
 					Key: "HttpHeader:Cf-Ray",
 					Val: "123123123",
@@ -77,8 +84,8 @@ func TestQUICServer(t *testing.T) {
 		{
 			desc:           "test http body request streaming",
 			dest:           "/slow_echo_body",
-			connectionType: quicpogs.ConnectionTypeHTTP,
-			metadata: []quicpogs.Metadata{
+			connectionType: pogs.ConnectionTypeHTTP,
+			metadata: []pogs.Metadata{
 				{
 					Key: "HttpHeader:Cf-Ray",
 					Val: "123123123",
@@ -102,8 +109,8 @@ func TestQUICServer(t *testing.T) {
 		{
 			desc:           "test ws proxy",
 			dest:           "/ws/echo",
-			connectionType: quicpogs.ConnectionTypeWebsocket,
-			metadata: []quicpogs.Metadata{
+			connectionType: pogs.ConnectionTypeWebsocket,
+			metadata: []pogs.Metadata{
 				{
 					Key: "HttpHeader:Cf-Cloudflared-Proxy-Connection-Upgrade",
 					Val: "Websocket",
@@ -126,8 +133,8 @@ func TestQUICServer(t *testing.T) {
 		},
 		{
 			desc:             "test tcp proxy",
-			connectionType:   quicpogs.ConnectionTypeTCP,
-			metadata:         []quicpogs.Metadata{},
+			connectionType:   pogs.ConnectionTypeTCP,
+			metadata:         []pogs.Metadata{},
 			message:          []byte("Here is some tcp data"),
 			expectedResponse: []byte("Here is some tcp data"),
 		},
@@ -174,7 +181,7 @@ type fakeControlStream struct {
 	ControlStreamHandler
 }
 
-func (fakeControlStream) ServeControlStream(ctx context.Context, rw io.ReadWriteCloser, connOptions *tunnelpogs.ConnectionOptions, tunnelConfigGetter TunnelConfigJSONGetter) error {
+func (fakeControlStream) ServeControlStream(ctx context.Context, rw io.ReadWriteCloser, connOptions *pogs.ConnectionOptions, tunnelConfigGetter TunnelConfigJSONGetter) error {
 	<-ctx.Done()
 	return nil
 }
@@ -187,8 +194,8 @@ func quicServer(
 	t *testing.T,
 	listener *quic.Listener,
 	dest string,
-	connectionType quicpogs.ConnectionType,
-	metadata []quicpogs.Metadata,
+	connectionType pogs.ConnectionType,
+	metadata []pogs.Metadata,
 	message []byte,
 	expectedResponse []byte,
 ) {
@@ -197,9 +204,9 @@ func quicServer(
 
 	quicStream, err := session.OpenStreamSync(context.Background())
 	require.NoError(t, err)
-	stream := quicpogs.NewSafeStreamCloser(quicStream)
+	stream := cfdquic.NewSafeStreamCloser(quicStream, defaultQUICTimeout, &log)
 
-	reqClientStream := quicpogs.RequestClientStream{ReadWriteCloser: stream}
+	reqClientStream := rpcquic.RequestClientStream{ReadWriteCloser: stream}
 	err = reqClientStream.WriteConnectRequestData(dest, connectionType, metadata...)
 	require.NoError(t, err)
 
@@ -264,15 +271,15 @@ func (moc *mockOriginProxyWithRequest) ProxyHTTP(w ResponseWriter, tr *tracing.T
 func TestBuildHTTPRequest(t *testing.T) {
 	var tests = []struct {
 		name           string
-		connectRequest *quicpogs.ConnectRequest
+		connectRequest *pogs.ConnectRequest
 		body           io.ReadCloser
 		req            *http.Request
 	}{
 		{
 			name: "check if http.Request is built correctly with content length",
-			connectRequest: &quicpogs.ConnectRequest{
+			connectRequest: &pogs.ConnectRequest{
 				Dest: "http://test.com",
-				Metadata: []quicpogs.Metadata{
+				Metadata: []pogs.Metadata{
 					{
 						Key: "HttpHeader:Cf-Cloudflared-Proxy-Connection-Upgrade",
 						Val: "Websocket",
@@ -316,9 +323,9 @@ func TestBuildHTTPRequest(t *testing.T) {
 		},
 		{
 			name: "if content length isn't part of request headers, then it's not set",
-			connectRequest: &quicpogs.ConnectRequest{
+			connectRequest: &pogs.ConnectRequest{
 				Dest: "http://test.com",
-				Metadata: []quicpogs.Metadata{
+				Metadata: []pogs.Metadata{
 					{
 						Key: "HttpHeader:Cf-Cloudflared-Proxy-Connection-Upgrade",
 						Val: "Websocket",
@@ -357,9 +364,9 @@ func TestBuildHTTPRequest(t *testing.T) {
 		},
 		{
 			name: "if content length is 0, but transfer-encoding is chunked, body is not nil",
-			connectRequest: &quicpogs.ConnectRequest{
+			connectRequest: &pogs.ConnectRequest{
 				Dest: "http://test.com",
-				Metadata: []quicpogs.Metadata{
+				Metadata: []pogs.Metadata{
 					{
 						Key: "HttpHeader:Another-Header",
 						Val: "Misc",
@@ -399,9 +406,9 @@ func TestBuildHTTPRequest(t *testing.T) {
 		},
 		{
 			name: "if content length is 0, but transfer-encoding is gzip,chunked, body is not nil",
-			connectRequest: &quicpogs.ConnectRequest{
+			connectRequest: &pogs.ConnectRequest{
 				Dest: "http://test.com",
-				Metadata: []quicpogs.Metadata{
+				Metadata: []pogs.Metadata{
 					{
 						Key: "HttpHeader:Another-Header",
 						Val: "Misc",
@@ -441,10 +448,10 @@ func TestBuildHTTPRequest(t *testing.T) {
 		},
 		{
 			name: "if content length is 0, and connect request is a websocket, body is not nil",
-			connectRequest: &quicpogs.ConnectRequest{
-				Type: quicpogs.ConnectionTypeWebsocket,
+			connectRequest: &pogs.ConnectRequest{
+				Type: pogs.ConnectionTypeWebsocket,
 				Dest: "http://test.com",
-				Metadata: []quicpogs.Metadata{
+				Metadata: []pogs.Metadata{
 					{
 						Key: "HttpHeader:Another-Header",
 						Val: "Misc",
@@ -616,9 +623,9 @@ func serveSession(ctx context.Context, qc *QUICConnection, edgeQUICSession quic.
 	}()
 
 	// Send a message to the quic session on edge side, it should be deumx to this datagram v2 session
-	muxedPayload, err := quicpogs.SuffixSessionID(sessionID, payload)
+	muxedPayload, err := cfdquic.SuffixSessionID(sessionID, payload)
 	require.NoError(t, err)
-	muxedPayload, err = quicpogs.SuffixType(muxedPayload, quicpogs.DatagramTypeUDP)
+	muxedPayload, err = cfdquic.SuffixType(muxedPayload, cfdquic.DatagramTypeUDP)
 	require.NoError(t, err)
 
 	err = edgeQUICSession.SendDatagram(muxedPayload)
@@ -664,7 +671,7 @@ const (
 	closedByTimeout
 )
 
-func runRPCServer(ctx context.Context, session quic.Connection, sessionRPCServer tunnelpogs.SessionManager, configRPCServer tunnelpogs.ConfigurationManager, t *testing.T) {
+func runRPCServer(ctx context.Context, session quic.Connection, sessionRPCServer pogs.SessionManager, configRPCServer pogs.ConfigurationManager, t *testing.T) {
 	stream, err := session.AcceptStream(ctx)
 	require.NoError(t, err)
 
@@ -673,13 +680,15 @@ func runRPCServer(ctx context.Context, session quic.Connection, sessionRPCServer
 		stream, err = session.AcceptStream(ctx)
 		require.NoError(t, err)
 	}
-	protocol, err := quicpogs.DetermineProtocol(stream)
-	assert.NoError(t, err)
-	rpcServerStream, err := quicpogs.NewRPCServerStream(stream, protocol)
-	assert.NoError(t, err)
-
-	log := zerolog.New(os.Stdout)
-	err = rpcServerStream.Serve(sessionRPCServer, configRPCServer, &log)
+	ss := rpcquic.NewCloudflaredServer(
+		func(_ context.Context, _ *rpcquic.RequestServerStream) error {
+			return nil
+		},
+		sessionRPCServer,
+		configRPCServer,
+		10*time.Second,
+	)
+	err = ss.Serve(ctx, stream)
 	assert.NoError(t, err)
 }
 
@@ -725,7 +734,8 @@ func testQUICConnection(udpListenerAddr net.Addr, t *testing.T, index uint8) *QU
 		fakeControlStream{},
 		&log,
 		nil,
-		5*time.Second,
+		15*time.Second,
+		0*time.Second,
 	)
 	require.NoError(t, err)
 	return qc
@@ -742,3 +752,27 @@ func (m *mockReaderNoopWriter) Write(p []byte) (n int, err error) {
 func (m *mockReaderNoopWriter) Close() error {
 	return nil
 }
+
+// GenerateTLSConfig sets up a bare-bones TLS config for a QUIC server
+func GenerateTLSConfig() *tls.Config {
+	key, err := rsa.GenerateKey(rand.Reader, 1024)
+	if err != nil {
+		panic(err)
+	}
+	template := x509.Certificate{SerialNumber: big.NewInt(1)}
+	certDER, err := x509.CreateCertificate(rand.Reader, &template, &template, &key.PublicKey, key)
+	if err != nil {
+		panic(err)
+	}
+	keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)})
+	certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER})
+
+	tlsCert, err := tls.X509KeyPair(certPEM, keyPEM)
+	if err != nil {
+		panic(err)
+	}
+	return &tls.Config{
+		Certificates: []tls.Certificate{tlsCert},
+		NextProtos:   []string{"argotunnel"},
+	}
+}

connection/rpc.go

@@ -1,153 +0,0 @@
-package connection
-
-import (
-	"context"
-	"io"
-	"net"
-	"time"
-
-	"github.com/rs/zerolog"
-	"zombiezen.com/go/capnproto2/rpc"
-
-	"github.com/cloudflare/cloudflared/tunnelrpc"
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
-)
-
-type tunnelServerClient struct {
-	client    tunnelpogs.TunnelServer_PogsClient
-	transport rpc.Transport
-}
-
-// NewTunnelRPCClient creates and returns a new RPC client, which will communicate using a stream on the given muxer.
-// This method is exported for supervisor to call Authenticate RPC
-func NewTunnelServerClient(
-	ctx context.Context,
-	stream io.ReadWriteCloser,
-	log *zerolog.Logger,
-) *tunnelServerClient {
-	transport := tunnelrpc.NewTransportLogger(log, rpc.StreamTransport(stream))
-	conn := rpc.NewConn(
-		transport,
-		tunnelrpc.ConnLog(log),
-	)
-	registrationClient := tunnelpogs.RegistrationServer_PogsClient{Client: conn.Bootstrap(ctx), Conn: conn}
-	return &tunnelServerClient{
-		client:    tunnelpogs.TunnelServer_PogsClient{RegistrationServer_PogsClient: registrationClient, Client: conn.Bootstrap(ctx), Conn: conn},
-		transport: transport,
-	}
-}
-
-func (tsc *tunnelServerClient) Authenticate(ctx context.Context, classicTunnel *ClassicTunnelProperties, registrationOptions *tunnelpogs.RegistrationOptions) (tunnelpogs.AuthOutcome, error) {
-	authResp, err := tsc.client.Authenticate(ctx, classicTunnel.OriginCert, classicTunnel.Hostname, registrationOptions)
-	if err != nil {
-		return nil, err
-	}
-	return authResp.Outcome(), nil
-}
-
-func (tsc *tunnelServerClient) Close() {
-	// Closing the client will also close the connection
-	_ = tsc.client.Close()
-	_ = tsc.transport.Close()
-}
-
-type NamedTunnelRPCClient interface {
-	RegisterConnection(
-		c context.Context,
-		config *NamedTunnelProperties,
-		options *tunnelpogs.ConnectionOptions,
-		connIndex uint8,
-		edgeAddress net.IP,
-		observer *Observer,
-	) (*tunnelpogs.ConnectionDetails, error)
-	SendLocalConfiguration(
-		c context.Context,
-		config []byte,
-		observer *Observer,
-	) error
-	GracefulShutdown(ctx context.Context, gracePeriod time.Duration)
-	Close()
-}
-
-type registrationServerClient struct {
-	client    tunnelpogs.RegistrationServer_PogsClient
-	transport rpc.Transport
-}
-
-func newRegistrationRPCClient(
-	ctx context.Context,
-	stream io.ReadWriteCloser,
-	log *zerolog.Logger,
-) NamedTunnelRPCClient {
-	transport := tunnelrpc.NewTransportLogger(log, rpc.StreamTransport(stream))
-	conn := rpc.NewConn(
-		transport,
-		tunnelrpc.ConnLog(log),
-	)
-	return &registrationServerClient{
-		client:    tunnelpogs.RegistrationServer_PogsClient{Client: conn.Bootstrap(ctx), Conn: conn},
-		transport: transport,
-	}
-}
-
-func (rsc *registrationServerClient) RegisterConnection(
-	ctx context.Context,
-	properties *NamedTunnelProperties,
-	options *tunnelpogs.ConnectionOptions,
-	connIndex uint8,
-	edgeAddress net.IP,
-	observer *Observer,
-) (*tunnelpogs.ConnectionDetails, error) {
-	conn, err := rsc.client.RegisterConnection(
-		ctx,
-		properties.Credentials.Auth(),
-		properties.Credentials.TunnelID,
-		connIndex,
-		options,
-	)
-	if err != nil {
-		if err.Error() == DuplicateConnectionError {
-			observer.metrics.regFail.WithLabelValues("dup_edge_conn", "registerConnection").Inc()
-			return nil, errDuplicationConnection
-		}
-		observer.metrics.regFail.WithLabelValues("server_error", "registerConnection").Inc()
-		return nil, serverRegistrationErrorFromRPC(err)
-	}
-
-	observer.metrics.regSuccess.WithLabelValues("registerConnection").Inc()
-
-	return conn, nil
-}
-
-func (rsc *registrationServerClient) SendLocalConfiguration(ctx context.Context, config []byte, observer *Observer) (err error) {
-	observer.metrics.localConfigMetrics.pushes.Inc()
-	defer func() {
-		if err != nil {
-			observer.metrics.localConfigMetrics.pushesErrors.Inc()
-		}
-	}()
-
-	return rsc.client.SendLocalConfiguration(ctx, config)
-}
-
-func (rsc *registrationServerClient) GracefulShutdown(ctx context.Context, gracePeriod time.Duration) {
-	ctx, cancel := context.WithTimeout(ctx, gracePeriod)
-	defer cancel()
-	_ = rsc.client.UnregisterConnection(ctx)
-}
-
-func (rsc *registrationServerClient) Close() {
-	// Closing the client will also close the connection
-	_ = rsc.client.Close()
-	// Closing the transport also closes the stream
-	_ = rsc.transport.Close()
-}
-
-type rpcName string
-
-const (
-	register     rpcName = "register"
-	reconnect    rpcName = "reconnect"
-	unregister   rpcName = "unregister"
-	authenticate rpcName = " authenticate"
-)

datagramsession/metrics.go

@@ -15,7 +15,7 @@ var (
 		Name:      "active_sessions",
 		Help:      "Concurrent count of UDP sessions that are being proxied to any origin",
 	})
-	totalUDPSessions = prometheus.NewGauge(prometheus.GaugeOpts{
+	totalUDPSessions = prometheus.NewCounter(prometheus.CounterOpts{
 		Namespace: namespace,
 		Subsystem: "udp",
 		Name:      "total_sessions",

dev.Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.21.5 as builder
+FROM golang:1.22.2 as builder
 ENV GO111MODULE=on \
     CGO_ENABLED=0
 WORKDIR /go/src/github.com/cloudflare/cloudflared/

go.mod

@@ -1,21 +1,22 @@
 module github.com/cloudflare/cloudflared
 
-go 1.21
+go 1.22
 
 require (
 	github.com/coredns/coredns v1.10.0
-	github.com/coreos/go-oidc/v3 v3.6.0
+	github.com/coreos/go-oidc/v3 v3.10.0
 	github.com/coreos/go-systemd/v22 v22.5.0
 	github.com/facebookgo/grace v0.0.0-20180706040059-75cf19382434
+	github.com/fortytw2/leaktest v1.3.0
 	github.com/fsnotify/fsnotify v1.4.9
 	github.com/getsentry/sentry-go v0.16.0
 	github.com/go-chi/chi/v5 v5.0.8
 	github.com/go-chi/cors v1.2.1
-	github.com/go-jose/go-jose/v3 v3.0.0
+	github.com/go-jose/go-jose/v4 v4.0.1
 	github.com/gobwas/ws v1.0.4
 	github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3
 	github.com/google/gopacket v1.1.19
-	github.com/google/uuid v1.3.1
+	github.com/google/uuid v1.6.0
 	github.com/gorilla/websocket v1.4.2
 	github.com/json-iterator/go v1.1.12
 	github.com/mattn/go-colorable v0.1.13
@@ -24,23 +25,23 @@ require (
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.13.0
 	github.com/prometheus/client_model v0.2.0
-	github.com/quic-go/quic-go v0.40.1-0.20240101045026-22b7f7744eb6
+	github.com/quic-go/quic-go v0.42.0
 	github.com/rs/zerolog v1.20.0
-	github.com/stretchr/testify v1.8.4
+	github.com/stretchr/testify v1.9.0
 	github.com/urfave/cli/v2 v2.3.0
 	go.opentelemetry.io/contrib/propagators v0.22.0
-	go.opentelemetry.io/otel v1.21.0
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0
-	go.opentelemetry.io/otel/sdk v1.21.0
-	go.opentelemetry.io/otel/trace v1.21.0
-	go.opentelemetry.io/proto/otlp v1.0.0
+	go.opentelemetry.io/otel v1.26.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.26.0
+	go.opentelemetry.io/otel/sdk v1.26.0
+	go.opentelemetry.io/otel/trace v1.26.0
+	go.opentelemetry.io/proto/otlp v1.2.0
 	go.uber.org/automaxprocs v1.4.0
-	golang.org/x/crypto v0.16.0
-	golang.org/x/net v0.19.0
-	golang.org/x/sync v0.4.0
-	golang.org/x/sys v0.15.0
-	golang.org/x/term v0.15.0
-	google.golang.org/protobuf v1.31.0
+	golang.org/x/crypto v0.23.0
+	golang.org/x/net v0.25.0
+	golang.org/x/sync v0.6.0
+	golang.org/x/sys v0.20.0
+	golang.org/x/term v0.20.0
+	google.golang.org/protobuf v1.34.1
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
 	gopkg.in/yaml.v3 v3.0.1
 	nhooyr.io/websocket v1.8.7
@@ -60,14 +61,14 @@ require (
 	github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
 	github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870 // indirect
 	github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 // indirect
-	github.com/go-logr/logr v1.3.0 // indirect
+	github.com/go-logr/logr v1.4.1 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
 	github.com/gobwas/httphead v0.0.0-20200921212729-da3d93bc3c58 // indirect
 	github.com/gobwas/pool v0.2.1 // indirect
-	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect
-	github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 // indirect
 	github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 // indirect
 	github.com/klauspost/compress v1.15.11 // indirect
 	github.com/kr/text v0.2.0 // indirect
@@ -81,19 +82,18 @@ require (
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/common v0.37.0 // indirect
 	github.com/prometheus/procfs v0.8.0 // indirect
-	github.com/quic-go/qtls-go1-20 v0.4.1 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
-	go.opentelemetry.io/otel/metric v1.21.0 // indirect
-	go.uber.org/mock v0.3.0 // indirect
+	go.opentelemetry.io/otel/metric v1.26.0 // indirect
+	go.uber.org/mock v0.4.0 // indirect
 	golang.org/x/exp v0.0.0-20221205204356-47842c84f3db // indirect
 	golang.org/x/mod v0.11.0 // indirect
-	golang.org/x/oauth2 v0.13.0 // indirect
-	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/oauth2 v0.17.0 // indirect
+	golang.org/x/text v0.15.0 // indirect
 	golang.org/x/tools v0.9.1 // indirect
 	google.golang.org/appengine v1.6.8 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20231002182017-d307bd883b97 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97 // indirect
-	google.golang.org/grpc v1.60.0 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de // indirect
+	google.golang.org/grpc v1.63.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 )
 

go.sum

@@ -60,8 +60,8 @@ github.com/coredns/caddy v1.1.1 h1:2eYKZT7i6yxIfGP3qLJoJ7HAsDJqYB+X68g4NYjSrE0=
 github.com/coredns/caddy v1.1.1/go.mod h1:A6ntJQlAWuQfFlsd9hvigKbo2WS0VUs2l1e2F+BawD4=
 github.com/coredns/coredns v1.10.0 h1:jCfuWsBjTs0dapkkhISfPCzn5LqvSRtrFtaf/Tjj4DI=
 github.com/coredns/coredns v1.10.0/go.mod h1:CIfRU5TgpuoIiJBJ4XrofQzfFQpPFh32ERpUevrSlaw=
-github.com/coreos/go-oidc/v3 v3.6.0 h1:AKVxfYw1Gmkn/w96z0DbT/B/xFnzTd3MkZvWLjF4n/o=
-github.com/coreos/go-oidc/v3 v3.6.0/go.mod h1:ZpHUsHBucTUj6WOkrP4E20UPynbLZzhTQ1XKCXkxyPc=
+github.com/coreos/go-oidc/v3 v3.10.0 h1:tDnXHnLyiTVyT/2zLDGj09pFPkhND8Gl8lnTRhoEaJU=
+github.com/coreos/go-oidc/v3 v3.10.0/go.mod h1:5j11xcw0D3+SGxn6Z/WFADsgcWVMyNAlSQupk0KK3ac=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs=
 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
@@ -88,6 +88,8 @@ github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870 h1:E2s37DuLxFhQD
 github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
+github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
+github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/getsentry/sentry-go v0.16.0 h1:owk+S+5XcgJLlGR/3+3s6N4d+uKwqYvh/eS0AIMjPWo=
@@ -106,8 +108,8 @@ github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3Bop
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
-github.com/go-jose/go-jose/v3 v3.0.0 h1:s6rrhirfEP/CGIoc6p+PZAeogN2SxKav6Wp7+dyMWVo=
-github.com/go-jose/go-jose/v3 v3.0.0/go.mod h1:RNkWWRld676jZEYoV3+XK8L2ZnNSvIsxFMht0mSX+u8=
+github.com/go-jose/go-jose/v4 v4.0.1 h1:QVEPDE3OluqXBQZDcnNvQrInro2h0e4eqNbnZSWqS6U=
+github.com/go-jose/go-jose/v4 v4.0.1/go.mod h1:WVf9LFMHh/QVrmqrOfqun0C45tMe3RoiKJMPvgWwLfY=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
@@ -117,8 +119,8 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V
 github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
 github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
-github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY=
-github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ=
+github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
@@ -150,8 +152,6 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a
 github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3 h1:zN2lZNZRflqFyxVaTIU61KNKQ9C0055u9CAfpmqUvo4=
 github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3/go.mod h1:nPpo7qLxd6XL3hWJG/O60sR8ZKfMCiIoNap5GvD12KU=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/glog v1.1.2 h1:DVjP2PbBOzHyzA+dn3WhHIq4NdVu3Q+pvivFICf/7fo=
-github.com/golang/glog v1.1.2/go.mod h1:zR+okUeTbrL6EL3xHUDxZuEtGv04p5shwip1+mL/rLQ=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -178,8 +178,8 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw
 github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
-github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
-github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
+github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
@@ -210,15 +210,15 @@ github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJY
 github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4=
-github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
-github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms=
-github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 h1:/c3QmbOGMGTOumP2iT/rCwB7b0QDGLKzqOmktBjT+Is=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1/go.mod h1:5SN9VR2LTsRFsrEC6FHgRbTWrTHu6tqPeKxEQv15giM=
 github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 h1:MJG/KsmcqMwFAkh8mTnAwhyKoB+sTAnY4CACC110tbU=
 github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645/go.mod h1:6iZfnjpejD4L/4DwD7NryNaJyCQdzwWwH2MWhCA90Kw=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
@@ -320,12 +320,8 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1
 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
 github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo=
 github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4=
-github.com/quic-go/qtls-go1-20 v0.4.1 h1:D33340mCNDAIKBqXuAvexTNMUByrYmFYVfKfDN5nfFs=
-github.com/quic-go/qtls-go1-20 v0.4.1/go.mod h1:X9Nh97ZL80Z+bX/gUXMbipO6OxdiDi58b/fMC9mAL+k=
-github.com/quic-go/quic-go v0.40.1-0.20231203135336-87ef8ec48d55 h1:I4N3ZRnkZPbDN935Tg8QDf8fRpHp3bZ0U0/L42jBgNE=
-github.com/quic-go/quic-go v0.40.1-0.20231203135336-87ef8ec48d55/go.mod h1:PeN7kuVJ4xZbxSv/4OX6S1USOX8MJvydwpTx31vx60c=
-github.com/quic-go/quic-go v0.40.1-0.20240101045026-22b7f7744eb6 h1:OI4WiysowCcxLtcZMGBZildo12di3ljcMN4vWdUQpoU=
-github.com/quic-go/quic-go v0.40.1-0.20240101045026-22b7f7744eb6/go.mod h1:qCkNjqczPEvgsOnxZ0eCD14lv+B2LHlFAB++CNOh9hA=
+github.com/quic-go/quic-go v0.42.0 h1:uSfdap0eveIl8KXnipv9K7nlwZ5IqLlYOpJ58u5utpM=
+github.com/quic-go/quic-go v0.42.0/go.mod h1:132kz4kL3F9vxhW3CtQJLDVwcFe5wdWeJXXijhsO57M=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
 github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
@@ -346,8 +342,8 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
-github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/tinylib/msgp v1.1.2 h1:gWmO7n0Ys2RBEb7GPYB9Ujq8Mk5p2U08lRnmMcGy6BQ=
 github.com/tinylib/msgp v1.1.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
 github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
@@ -368,33 +364,32 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opentelemetry.io/contrib/propagators v0.22.0 h1:KGdv58M2//veiYLIhb31mofaI2LgkIPXXAZVeYVyfd8=
 go.opentelemetry.io/contrib/propagators v0.22.0/go.mod h1:xGOuXr6lLIF9BXipA4pm6UuOSI0M98U6tsI3khbOiwU=
 go.opentelemetry.io/otel v1.0.0-RC2/go.mod h1:w1thVQ7qbAy8MHb0IFj8a5Q2QU0l2ksf8u/CN8m3NOM=
-go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc=
-go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0 h1:cl5P5/GIfFh4t6xyruOgJP5QiA1pw4fYYdv6nc6CBWw=
-go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0/go.mod h1:zgBdWWAu7oEEMC06MMKc5NLbA/1YDXV1sMpSqEeLQLg=
-go.opentelemetry.io/otel/metric v1.21.0 h1:tlYWfeo+Bocx5kLEloTjbcDwBuELRrIFxwdQ36PlJu4=
-go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM=
-go.opentelemetry.io/otel/sdk v1.21.0 h1:FTt8qirL1EysG6sTQRZ5TokkU8d0ugCj8htOgThZXQ8=
-go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E=
+go.opentelemetry.io/otel v1.26.0 h1:LQwgL5s/1W7YiiRwxf03QGnWLb2HW4pLiAhaA5cZXBs=
+go.opentelemetry.io/otel v1.26.0/go.mod h1:UmLkJHUAidDval2EICqBMbnAd0/m2vmpf/dAM+fvFs4=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.26.0 h1:1u/AyyOqAWzy+SkPxDpahCNZParHV8Vid1RnI2clyDE=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.26.0/go.mod h1:z46paqbJ9l7c9fIPCXTqTGwhQZ5XoTIsfeFYWboizjs=
+go.opentelemetry.io/otel/metric v1.26.0 h1:7S39CLuY5Jgg9CrnA9HHiEjGMF/X2VHvoXGgSllRz30=
+go.opentelemetry.io/otel/metric v1.26.0/go.mod h1:SY+rHOI4cEawI9a7N1A4nIg/nTQXe1ccCNWYOJUrpX4=
+go.opentelemetry.io/otel/sdk v1.26.0 h1:Y7bumHf5tAiDlRYFmGqetNcLaVUZmh4iYfmGxtmz7F8=
+go.opentelemetry.io/otel/sdk v1.26.0/go.mod h1:0p8MXpqLeJ0pzcszQQN4F0S5FVjBLgypeGSngLsmirs=
 go.opentelemetry.io/otel/trace v1.0.0-RC2/go.mod h1:JPQ+z6nNw9mqEGT8o3eoPTdnNI+Aj5JcxEsVGREIAy4=
-go.opentelemetry.io/otel/trace v1.21.0 h1:WD9i5gzvoUPuXIXH24ZNBudiarZDKuekPqi/E8fpfLc=
-go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ=
-go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I=
-go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM=
+go.opentelemetry.io/otel/trace v1.26.0 h1:1ieeAUb4y0TE26jUFrCIXKpTuVK7uJGN9/Z/2LP5sQA=
+go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZuWCBllGV2U2y0=
+go.opentelemetry.io/proto/otlp v1.2.0 h1:pVeZGk7nXDC9O2hncA6nHldxEjm6LByfA2aN8IOkz94=
+go.opentelemetry.io/proto/otlp v1.2.0/go.mod h1:gGpR8txAl5M03pDhMC79G6SdqNV26naRm/KDsgaHD8A=
 go.uber.org/automaxprocs v1.4.0 h1:CpDZl6aOlLhReez+8S3eEotD7Jx0Os++lemPlMULQP0=
 go.uber.org/automaxprocs v1.4.0/go.mod h1:/mTEdr7LvHhs0v7mjdxDreTz1OG5zdZGqgOnhWiR/+Q=
-go.uber.org/mock v0.3.0 h1:3mUxI1No2/60yUYax92Pt8eNOEecx2D3lcXZh2NEZJo=
-go.uber.org/mock v0.3.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc=
+go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU=
+go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY=
-golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
+golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=
+golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -466,8 +461,8 @@ golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qx
 golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c=
-golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U=
+golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac=
+golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -475,8 +470,8 @@ golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4Iltr
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
-golang.org/x/oauth2 v0.13.0 h1:jDDenyj+WgFtmV3zYVoi8aE2BwtXFLWOA67ZfNWftiY=
-golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0=
+golang.org/x/oauth2 v0.17.0 h1:6m3ZPmLEFdVxKKWnKq4VqZ60gutO35zm+zrAHVmHyDQ=
+golang.org/x/oauth2 v0.17.0/go.mod h1:OzPDGQiuQMguemayvdylqddI7qcD9lnSDb+1FiwQ5HA=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -488,8 +483,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ=
-golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
+golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
+golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -536,12 +531,12 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
-golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
+golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4=
-golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
+golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw=
+golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -550,11 +545,13 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
-golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
-golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk=
+golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
+golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
@@ -657,12 +654,12 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc
 google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97 h1:SeZZZx0cP0fqUyA+oRzP9k7cSwJlvDFiROO72uwD6i0=
-google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97/go.mod h1:t1VqOqqvce95G3hIDCT5FeO3YUc6Q4Oe24L/+rNMxRk=
-google.golang.org/genproto/googleapis/api v0.0.0-20231002182017-d307bd883b97 h1:W18sezcAYs+3tDZX4F80yctqa12jcP1PUS2gQu1zTPU=
-google.golang.org/genproto/googleapis/api v0.0.0-20231002182017-d307bd883b97/go.mod h1:iargEX0SFPm3xcfMI0d1domjg0ZF4Aa0p2awqyxhvF0=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97 h1:6GQBEOdGkX6MMTLT9V+TjtIRZCw9VPD5Z+yHY9wMgS0=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97/go.mod h1:v7nGkzlmW8P3n/bKmWBn2WpBjpOEx8Q6gMueudAmKfY=
+google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de h1:F6qOa9AZTYJXOUEr4jDysRDLrm4PHePlge4v4TGAlxY=
+google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:VUhTRKeHn9wwcdrk73nvdC9gF178Tzhmt/qyaFcPLSo=
+google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de h1:jFNzHPIeuzhdRwVhbZdiym9q0ory/xY3sA+v2wPg8I0=
+google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:5iCWqnniDlqZHrd3neWVTOwvh/v6s3232omMecelax8=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de h1:cZGRis4/ot9uVm639a+rHCUaG0JJHEsdyzSQTMX+suY=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:H4O17MA/PE9BsGx3w+a+W2VOLLD1Qf7oJneAoU6WktY=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
@@ -675,8 +672,8 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa
 google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
 google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
 google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
-google.golang.org/grpc v1.60.0 h1:6FQAR0kM31P6MRdeluor2w2gPaS4SVNrD/DNTxrQ15k=
-google.golang.org/grpc v1.60.0/go.mod h1:OlCHIeLYqSSsLi6i49B5QGdzaMZK9+M7LXN2FKz4eGM=
+google.golang.org/grpc v1.63.0 h1:WjKe+dnvABXyPJMD7KDNLxtoGk5tgk+YFWN6cBWjZE8=
+google.golang.org/grpc v1.63.0/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
@@ -689,8 +686,8 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
-google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg=
+google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

ingress/config.go

@@ -32,6 +32,7 @@ const (
 	ProxyKeepAliveTimeoutFlag     = "proxy-keepalive-timeout"
 	HTTPHostHeaderFlag            = "http-host-header"
 	OriginServerNameFlag          = "origin-server-name"
+	MatchSNIToHostFlag            = "match-sni-to-host"
 	NoTLSVerifyFlag               = "no-tls-verify"
 	NoChunkedEncodingFlag         = "no-chunked-encoding"
 	ProxyAddressFlag              = "proxy-address"
@@ -118,6 +119,7 @@ func originRequestFromSingleRule(c *cli.Context) OriginRequestConfig {
 	var keepAliveTimeout = defaultKeepAliveTimeout
 	var httpHostHeader string
 	var originServerName string
+	var matchSNItoHost bool
 	var caPool string
 	var noTLSVerify bool
 	var disableChunkedEncoding bool
@@ -150,6 +152,9 @@ func originRequestFromSingleRule(c *cli.Context) OriginRequestConfig {
 	if flag := OriginServerNameFlag; c.IsSet(flag) {
 		originServerName = c.String(flag)
 	}
+	if flag := MatchSNIToHostFlag; c.IsSet(flag) {
+		matchSNItoHost = c.Bool(flag)
+	}
 	if flag := tlsconfig.OriginCAPoolFlag; c.IsSet(flag) {
 		caPool = c.String(flag)
 	}
@@ -185,6 +190,7 @@ func originRequestFromSingleRule(c *cli.Context) OriginRequestConfig {
 		KeepAliveTimeout:       keepAliveTimeout,
 		HTTPHostHeader:         httpHostHeader,
 		OriginServerName:       originServerName,
+		MatchSNIToHost:         matchSNItoHost,
 		CAPool:                 caPool,
 		NoTLSVerify:            noTLSVerify,
 		DisableChunkedEncoding: disableChunkedEncoding,
@@ -229,6 +235,9 @@ func originRequestFromConfig(c config.OriginRequestConfig) OriginRequestConfig {
 	if c.OriginServerName != nil {
 		out.OriginServerName = *c.OriginServerName
 	}
+	if c.MatchSNIToHost != nil {
+		out.MatchSNIToHost = *c.MatchSNIToHost
+	}
 	if c.CAPool != nil {
 		out.CAPool = *c.CAPool
 	}
@@ -287,6 +296,8 @@ type OriginRequestConfig struct {
 	HTTPHostHeader string `yaml:"httpHostHeader" json:"httpHostHeader"`
 	// Hostname on the origin server certificate.
 	OriginServerName string `yaml:"originServerName" json:"originServerName"`
+	// Auto configure the Hostname on the origin server certificate.
+	MatchSNIToHost bool `yaml:"matchSNItoHost" json:"matchSNItoHost"`
 	// Path to the CA for the certificate of your origin.
 	// This option should be used only if your certificate is not signed by Cloudflare.
 	CAPool string `yaml:"caPool" json:"caPool"`
@@ -362,6 +373,12 @@ func (defaults *OriginRequestConfig) setOriginServerName(overrides config.Origin
 	}
 }
 
+func (defaults *OriginRequestConfig) setMatchSNIToHost(overrides config.OriginRequestConfig) {
+	if val := overrides.MatchSNIToHost; val != nil {
+		defaults.MatchSNIToHost = *val
+	}
+}
+
 func (defaults *OriginRequestConfig) setCAPool(overrides config.OriginRequestConfig) {
 	if val := overrides.CAPool; val != nil {
 		defaults.CAPool = *val
@@ -447,6 +464,7 @@ func setConfig(defaults OriginRequestConfig, overrides config.OriginRequestConfi
 	cfg.setTCPKeepAlive(overrides)
 	cfg.setHTTPHostHeader(overrides)
 	cfg.setOriginServerName(overrides)
+	cfg.setMatchSNIToHost(overrides)
 	cfg.setCAPool(overrides)
 	cfg.setNoTLSVerify(overrides)
 	cfg.setDisableChunkedEncoding(overrides)
@@ -501,6 +519,7 @@ func ConvertToRawOriginConfig(c OriginRequestConfig) config.OriginRequestConfig
 		KeepAliveTimeout:       keepAliveTimeout,
 		HTTPHostHeader:         emptyStringToNil(c.HTTPHostHeader),
 		OriginServerName:       emptyStringToNil(c.OriginServerName),
+		MatchSNIToHost:         defaultBoolToNil(c.MatchSNIToHost),
 		CAPool:                 emptyStringToNil(c.CAPool),
 		NoTLSVerify:            defaultBoolToNil(c.NoTLSVerify),
 		DisableChunkedEncoding: defaultBoolToNil(c.DisableChunkedEncoding),

ingress/constants_test.go

@@ -0,0 +1,7 @@
+package ingress
+
+import "github.com/cloudflare/cloudflared/logger"
+
+var (
+	TestLogger = logger.Create(nil)
+)

ingress/icmp_linux.go

@@ -78,19 +78,19 @@ func checkInPingGroup() error {
 	if err != nil {
 		return err
 	}
-	groupID := os.Getgid()
+	groupID := uint64(os.Getegid())
 	// Example content: 999	   59999
 	found := findGroupIDRegex.FindAll(file, 2)
 	if len(found) == 2 {
-		groupMin, err := strconv.ParseInt(string(found[0]), 10, 32)
+		groupMin, err := strconv.ParseUint(string(found[0]), 10, 32)
 		if err != nil {
 			return errors.Wrapf(err, "failed to determine minimum ping group ID")
 		}
-		groupMax, err := strconv.ParseInt(string(found[1]), 10, 32)
+		groupMax, err := strconv.ParseUint(string(found[1]), 10, 32)
 		if err != nil {
-			return errors.Wrapf(err, "failed to determine minimum ping group ID")
+			return errors.Wrapf(err, "failed to determine maximum ping group ID")
 		}
-		if groupID < int(groupMin) || groupID > int(groupMax) {
+		if groupID < groupMin || groupID > groupMax {
 			return fmt.Errorf("Group ID %d is not between ping group %d to %d", groupID, groupMin, groupMax)
 		}
 		return nil
@@ -153,14 +153,8 @@ func (ip *icmpProxy) Request(ctx context.Context, pk *packet.ICMP, responder *pa
 			Int("originalEchoID", originalEcho.ID).
 			Msg("New flow")
 		go func() {
-			defer ip.srcFunnelTracker.Unregister(funnelID, icmpFlow)
-			if err := ip.listenResponse(ctx, icmpFlow); err != nil {
-				ip.logger.Debug().Err(err).
-					Str("src", pk.Src.String()).
-					Str("dst", pk.Dst.String()).
-					Int("originalEchoID", originalEcho.ID).
-					Msg("Failed to listen for ICMP echo response")
-			}
+			ip.listenResponse(ctx, icmpFlow)
+			ip.srcFunnelTracker.Unregister(funnelID, icmpFlow)
 		}()
 	}
 	if err := icmpFlow.sendToDst(pk.Dst, pk.Message); err != nil {
@@ -176,17 +170,17 @@ func (ip *icmpProxy) Serve(ctx context.Context) error {
 	return ctx.Err()
 }
 
-func (ip *icmpProxy) listenResponse(ctx context.Context, flow *icmpEchoFlow) error {
+func (ip *icmpProxy) listenResponse(ctx context.Context, flow *icmpEchoFlow) {
 	buf := make([]byte, mtu)
 	for {
-		retryable, err := ip.handleResponse(ctx, flow, buf)
-		if err != nil && !retryable {
-			return err
+		if done := ip.handleResponse(ctx, flow, buf); done {
+			return
 		}
 	}
 }
 
-func (ip *icmpProxy) handleResponse(ctx context.Context, flow *icmpEchoFlow, buf []byte) (retryableErr bool, err error) {
+// Listens for ICMP response and handles error logging
+func (ip *icmpProxy) handleResponse(ctx context.Context, flow *icmpEchoFlow, buf []byte) (done bool) {
 	_, span := flow.responder.replySpan(ctx, ip.logger)
 	defer flow.responder.exportSpan()
 
@@ -198,33 +192,34 @@ func (ip *icmpProxy) handleResponse(ctx context.Context, flow *icmpEchoFlow, buf
 	if err != nil {
 		if flow.IsClosed() {
 			tracing.EndWithErrorStatus(span, fmt.Errorf("flow was closed"))
-			return false, nil
+			return true
 		}
+		ip.logger.Error().Err(err).Str("socket", flow.originConn.LocalAddr().String()).Msg("Failed to read from ICMP socket")
 		tracing.EndWithErrorStatus(span, err)
-		return false, err
+		return true
 	}
 	reply, err := parseReply(from, buf[:n])
 	if err != nil {
 		ip.logger.Error().Err(err).Str("dst", from.String()).Msg("Failed to parse ICMP reply")
 		tracing.EndWithErrorStatus(span, err)
-		return true, err
+		return false
 	}
 	if !isEchoReply(reply.msg) {
 		err := fmt.Errorf("Expect ICMP echo reply, got %s", reply.msg.Type)
 		ip.logger.Debug().Str("dst", from.String()).Msgf("Drop ICMP %s from reply", reply.msg.Type)
 		tracing.EndWithErrorStatus(span, err)
-		return true, err
+		return false
 	}
 
 	if err := flow.returnToSrc(reply); err != nil {
 		ip.logger.Error().Err(err).Str("dst", from.String()).Msg("Failed to send ICMP reply")
 		tracing.EndWithErrorStatus(span, err)
-		return true, err
+		return false
 	}
 
 	observeICMPReply(ip.logger, span, from.String(), reply.echo.ID, reply.echo.Seq)
 	tracing.End(span)
-	return true, nil
+	return false
 }
 
 // Only linux uses flow3Tuple as FunnelID

ingress/icmp_metrics.go

@@ -0,0 +1,39 @@
+package ingress
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const (
+	namespace = "cloudflared"
+)
+
+var (
+	icmpRequests = prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: namespace,
+		Subsystem: "icmp",
+		Name:      "total_requests",
+		Help:      "Total count of ICMP requests that have been proxied to any origin",
+	})
+	icmpReplies = prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: namespace,
+		Subsystem: "icmp",
+		Name:      "total_replies",
+		Help:      "Total count of ICMP replies that have been proxied from any origin",
+	})
+)
+
+func init() {
+	prometheus.MustRegister(
+		icmpRequests,
+		icmpReplies,
+	)
+}
+
+func incrementICMPRequest() {
+	icmpRequests.Inc()
+}
+
+func incrementICMPReply() {
+	icmpReplies.Inc()
+}

ingress/icmp_posix_test.go

@@ -8,6 +8,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/fortytw2/leaktest"
 	"github.com/google/gopacket/layers"
 	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/require"
@@ -18,6 +19,8 @@ import (
 )
 
 func TestFunnelIdleTimeout(t *testing.T) {
+	defer leaktest.Check(t)()
+
 	const (
 		idleTimeout = time.Second
 		echoID      = 42573
@@ -73,13 +76,16 @@ func TestFunnelIdleTimeout(t *testing.T) {
 	require.NoError(t, proxy.Request(ctx, &pk, &newResponder))
 	validateEchoFlow(t, <-newMuxer.cfdToEdge, &pk)
 
+	time.Sleep(idleTimeout * 2)
 	cancel()
 	<-proxyDone
 }
 
 func TestReuseFunnel(t *testing.T) {
+	defer leaktest.Check(t)()
+
 	const (
-		idleTimeout = time.Second
+		idleTimeout = time.Millisecond * 100
 		echoID      = 42573
 		startSeq    = 8129
 	)
@@ -135,6 +141,8 @@ func TestReuseFunnel(t *testing.T) {
 	require.True(t, found)
 	require.Equal(t, funnel1, funnel2)
 
+	time.Sleep(idleTimeout * 2)
+
 	cancel()
 	<-proxyDone
 }

ingress/middleware/jwtvalidator_test.go

@@ -0,0 +1,126 @@
+package middleware
+
+import (
+	"context"
+	"crypto"
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"encoding/json"
+	"fmt"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/coreos/go-oidc/v3/oidc"
+	"github.com/go-jose/go-jose/v4"
+	"github.com/go-jose/go-jose/v4/jwt"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+var (
+	issuer = fmt.Sprintf(cloudflareAccessCertsURL, "testteam")
+)
+
+type accessTokenClaims struct {
+	Email string `json:"email"`
+	Type  string `json:"type"`
+	jwt.Claims
+}
+
+func TestJWTValidator(t *testing.T) {
+	req := httptest.NewRequest("GET", "http://example.com", nil)
+
+	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	require.NoError(t, err)
+	issued := time.Now()
+	claims := accessTokenClaims{
+		Email: "test@example.com",
+		Type:  "app",
+		Claims: jwt.Claims{
+			Issuer:   issuer,
+			Subject:  "ee239b7a-e3e6-4173-972a-8fbe9d99c04f",
+			Audience: []string{""},
+			Expiry:   jwt.NewNumericDate(issued.Add(time.Hour)),
+			IssuedAt: jwt.NewNumericDate(issued),
+		},
+	}
+	token := signToken(t, claims, key)
+	req.Header.Add(headerKeyAccessJWTAssertion, token)
+
+	keySet := oidc.StaticKeySet{PublicKeys: []crypto.PublicKey{key.Public()}}
+	config := &oidc.Config{
+		SkipClientIDCheck:    true,
+		SupportedSigningAlgs: []string{string(jose.ES256)},
+	}
+	verifier := oidc.NewVerifier(issuer, &keySet, config)
+
+	tests := []struct {
+		name    string
+		audTags []string
+		aud     jwt.Audience
+		error   bool
+	}{
+		{
+			name: "valid",
+			audTags: []string{
+				"0bc545634b1732494b3f9472794a549c883fabd48de9dfe0e0413e59c3f96c38",
+				"d7ec5b7fda23ffa8f8c8559fb37c66a2278208a78dbe376a3394b5ffec6911ba",
+			},
+			aud:   jwt.Audience{"d7ec5b7fda23ffa8f8c8559fb37c66a2278208a78dbe376a3394b5ffec6911ba"},
+			error: false,
+		},
+		{
+			name: "invalid no match",
+			audTags: []string{
+				"0bc545634b1732494b3f9472794a549c883fabd48de9dfe0e0413e59c3f96c38",
+				"d7ec5b7fda23ffa8f8c8559fb37c66a2278208a78dbe376a3394b5ffec6911ba",
+			},
+			aud:   jwt.Audience{"09dc377143841843ecca28b196bdb1ec1675af38c8b7b60c7def5876c8877157"},
+			error: true,
+		},
+		{
+			name:    "invalid empty check",
+			audTags: []string{},
+			aud:     jwt.Audience{"09dc377143841843ecca28b196bdb1ec1675af38c8b7b60c7def5876c8877157"},
+			error:   true,
+		},
+		{
+			name: "invalid absent aud",
+			audTags: []string{
+				"0bc545634b1732494b3f9472794a549c883fabd48de9dfe0e0413e59c3f96c38",
+				"d7ec5b7fda23ffa8f8c8559fb37c66a2278208a78dbe376a3394b5ffec6911ba",
+			},
+			aud:   jwt.Audience{""},
+			error: true,
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			validator := JWTValidator{
+				IDTokenVerifier: verifier,
+				audTags:         test.audTags,
+			}
+			claims.Audience = test.aud
+			token := signToken(t, claims, key)
+			req.Header.Set(headerKeyAccessJWTAssertion, token)
+
+			result, err := validator.Handle(context.Background(), req)
+			assert.NoError(t, err)
+			assert.Equal(t, test.error, result.ShouldFilterRequest)
+		})
+	}
+}
+
+func signToken(t *testing.T, token accessTokenClaims, key *ecdsa.PrivateKey) string {
+	signer, err := jose.NewSigner(jose.SigningKey{Algorithm: jose.ES256, Key: key}, &jose.SignerOptions{})
+	require.NoError(t, err)
+	payload, err := json.Marshal(token)
+	require.NoError(t, err)
+	jws, err := signer.Sign(payload)
+	require.NoError(t, err)
+	jwt, err := jws.CompactSerialize()
+	require.NoError(t, err)
+	return jwt
+}

ingress/origin_connection.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"io"
 	"net"
+	"time"
 
 	"github.com/rs/zerolog"
 
@@ -31,15 +32,32 @@ func DefaultStreamHandler(originConn io.ReadWriter, remoteConn net.Conn, log *ze
 
 // tcpConnection is an OriginConnection that directly streams to raw TCP.
 type tcpConnection struct {
-	conn net.Conn
+	net.Conn
+	writeTimeout time.Duration
+	logger       *zerolog.Logger
 }
 
-func (tc *tcpConnection) Stream(ctx context.Context, tunnelConn io.ReadWriter, log *zerolog.Logger) {
-	stream.Pipe(tunnelConn, tc.conn, log)
+func (tc *tcpConnection) Stream(_ context.Context, tunnelConn io.ReadWriter, _ *zerolog.Logger) {
+	stream.Pipe(tunnelConn, tc, tc.logger)
+}
+
+func (tc *tcpConnection) Write(b []byte) (int, error) {
+	if tc.writeTimeout > 0 {
+		if err := tc.Conn.SetWriteDeadline(time.Now().Add(tc.writeTimeout)); err != nil {
+			tc.logger.Err(err).Msg("Error setting write deadline for TCP connection")
+		}
+	}
+
+	nBytes, err := tc.Conn.Write(b)
+	if err != nil {
+		tc.logger.Err(err).Msg("Error writing to the TCP connection")
+	}
+
+	return nBytes, err
 }
 
 func (tc *tcpConnection) Close() {
-	tc.conn.Close()
+	tc.Conn.Close()
 }
 
 // tcpOverWSConnection is an OriginConnection that streams to TCP over WS.

ingress/origin_connection_test.go

@@ -19,7 +19,6 @@ import (
 	"golang.org/x/net/proxy"
 	"golang.org/x/sync/errgroup"
 
-	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/socks"
 	"github.com/cloudflare/cloudflared/stream"
 	"github.com/cloudflare/cloudflared/websocket"
@@ -31,7 +30,6 @@ const (
 )
 
 var (
-	testLogger   = logger.Create(nil)
 	testMessage  = []byte("TestStreamOriginConnection")
 	testResponse = []byte(fmt.Sprintf("echo-%s", testMessage))
 )
@@ -39,7 +37,8 @@ var (
 func TestStreamTCPConnection(t *testing.T) {
 	cfdConn, originConn := net.Pipe()
 	tcpConn := tcpConnection{
-		conn: cfdConn,
+		Conn:         cfdConn,
+		writeTimeout: 30 * time.Second,
 	}
 
 	eyeballConn, edgeConn := net.Pipe()
@@ -66,7 +65,7 @@ func TestStreamTCPConnection(t *testing.T) {
 		return nil
 	})
 
-	tcpConn.Stream(ctx, edgeConn, testLogger)
+	tcpConn.Stream(ctx, edgeConn, TestLogger)
 	require.NoError(t, errGroup.Wait())
 }
 
@@ -93,7 +92,7 @@ func TestDefaultStreamWSOverTCPConnection(t *testing.T) {
 		return nil
 	})
 
-	tcpOverWSConn.Stream(ctx, edgeConn, testLogger)
+	tcpOverWSConn.Stream(ctx, edgeConn, TestLogger)
 	require.NoError(t, errGroup.Wait())
 }
 
@@ -147,7 +146,7 @@ func TestSocksStreamWSOverTCPConnection(t *testing.T) {
 
 		errGroup, ctx := errgroup.WithContext(ctx)
 		errGroup.Go(func() error {
-			tcpOverWSConn.Stream(ctx, edgeConn, testLogger)
+			tcpOverWSConn.Stream(ctx, edgeConn, TestLogger)
 			return nil
 		})
 
@@ -159,7 +158,7 @@ func TestSocksStreamWSOverTCPConnection(t *testing.T) {
 			require.NoError(t, err)
 			defer wsForwarderInConn.Close()
 
-			stream.Pipe(wsForwarderInConn, &wsEyeball{wsForwarderOutConn}, testLogger)
+			stream.Pipe(wsForwarderInConn, &wsEyeball{wsForwarderOutConn}, TestLogger)
 			return nil
 		})
 
@@ -209,7 +208,7 @@ func TestWsConnReturnsBeforeStreamReturns(t *testing.T) {
 			originConn.Close()
 		}()
 		ctx := context.WithValue(r.Context(), websocket.PingPeriodContextKey, time.Microsecond)
-		tcpOverWSConn.Stream(ctx, eyeballConn, testLogger)
+		tcpOverWSConn.Stream(ctx, eyeballConn, TestLogger)
 	})
 	server := httptest.NewServer(handler)
 	defer server.Close()

ingress/origin_icmp_proxy.go

@@ -17,9 +17,7 @@ import (
 )
 
 const (
-	// funnelIdleTimeout controls how long to wait to close a funnel without send/return
-	funnelIdleTimeout = time.Second * 10
-	mtu               = 1500
+	mtu = 1500
 	// icmpRequestTimeoutMs controls how long to wait for a reply
 	icmpRequestTimeoutMs = 1000
 )
@@ -34,8 +32,9 @@ type icmpRouter struct {
 }
 
 // NewICMPRouter doesn't return an error if either ipv4 proxy or ipv6 proxy can be created. The machine might only
-// support one of them
-func NewICMPRouter(ipv4Addr, ipv6Addr netip.Addr, ipv6Zone string, logger *zerolog.Logger) (*icmpRouter, error) {
+// support one of them.
+// funnelIdleTimeout controls how long to wait to close a funnel without send/return
+func NewICMPRouter(ipv4Addr, ipv6Addr netip.Addr, ipv6Zone string, logger *zerolog.Logger, funnelIdleTimeout time.Duration) (*icmpRouter, error) {
 	ipv4Proxy, ipv4Err := newICMPProxy(ipv4Addr, "", logger, funnelIdleTimeout)
 	ipv6Proxy, ipv6Err := newICMPProxy(ipv6Addr, ipv6Zone, logger, funnelIdleTimeout)
 	if ipv4Err != nil && ipv6Err != nil {
@@ -106,6 +105,7 @@ func isEchoReply(msg *icmp.Message) bool {
 }
 
 func observeICMPRequest(logger *zerolog.Logger, span trace.Span, src string, dst string, echoID int, seq int) {
+	incrementICMPRequest()
 	logger.Debug().
 		Str("src", src).
 		Str("dst", dst).
@@ -119,6 +119,7 @@ func observeICMPRequest(logger *zerolog.Logger, span trace.Span, src string, dst
 }
 
 func observeICMPReply(logger *zerolog.Logger, span trace.Span, dst string, echoID int, seq int) {
+	incrementICMPReply()
 	logger.Debug().Str("dst", dst).Int("echoID", echoID).Int("seq", seq).Msg("Sent ICMP reply to edge")
 	span.SetAttributes(
 		attribute.String("dst", dst),

ingress/origin_icmp_proxy_test.go

@@ -9,7 +9,9 @@ import (
 	"strings"
 	"sync"
 	"testing"
+	"time"
 
+	"github.com/fortytw2/leaktest"
 	"github.com/google/gopacket/layers"
 	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/require"
@@ -23,9 +25,10 @@ import (
 )
 
 var (
-	noopLogger    = zerolog.Nop()
-	localhostIP   = netip.MustParseAddr("127.0.0.1")
-	localhostIPv6 = netip.MustParseAddr("::1")
+	noopLogger            = zerolog.Nop()
+	localhostIP           = netip.MustParseAddr("127.0.0.1")
+	localhostIPv6         = netip.MustParseAddr("::1")
+	testFunnelIdleTimeout = time.Millisecond * 10
 )
 
 // TestICMPProxyEcho makes sure we can send ICMP echo via the Request method and receives response via the
@@ -40,12 +43,14 @@ func TestICMPRouterEcho(t *testing.T) {
 }
 
 func testICMPRouterEcho(t *testing.T, sendIPv4 bool) {
+	defer leaktest.Check(t)()
+
 	const (
 		echoID = 36571
 		endSeq = 20
 	)
 
-	router, err := NewICMPRouter(localhostIP, localhostIPv6, "", &noopLogger)
+	router, err := NewICMPRouter(localhostIP, localhostIPv6, "", &noopLogger, testFunnelIdleTimeout)
 	require.NoError(t, err)
 
 	proxyDone := make(chan struct{})
@@ -97,14 +102,19 @@ func testICMPRouterEcho(t *testing.T, sendIPv4 bool) {
 			validateEchoFlow(t, <-muxer.cfdToEdge, &pk)
 		}
 	}
+
+	// Make sure funnel cleanup kicks in
+	time.Sleep(testFunnelIdleTimeout * 2)
 	cancel()
 	<-proxyDone
 }
 
 func TestTraceICMPRouterEcho(t *testing.T) {
+	defer leaktest.Check(t)()
+
 	tracingCtx := "ec31ad8a01fde11fdcabe2efdce36873:52726f6cabc144f5:0:1"
 
-	router, err := NewICMPRouter(localhostIP, localhostIPv6, "", &noopLogger)
+	router, err := NewICMPRouter(localhostIP, localhostIPv6, "", &noopLogger, testFunnelIdleTimeout)
 	require.NoError(t, err)
 
 	proxyDone := make(chan struct{})
@@ -196,6 +206,7 @@ func TestTraceICMPRouterEcho(t *testing.T) {
 	default:
 	}
 
+	time.Sleep(testFunnelIdleTimeout * 2)
 	cancel()
 	<-proxyDone
 }
@@ -203,12 +214,14 @@ func TestTraceICMPRouterEcho(t *testing.T) {
 // TestConcurrentRequests makes sure icmpRouter can send concurrent requests to the same destination with different
 // echo ID. This simulates concurrent ping to the same destination.
 func TestConcurrentRequestsToSameDst(t *testing.T) {
+	defer leaktest.Check(t)()
+
 	const (
 		concurrentPings = 5
 		endSeq          = 5
 	)
 
-	router, err := NewICMPRouter(localhostIP, localhostIPv6, "", &noopLogger)
+	router, err := NewICMPRouter(localhostIP, localhostIPv6, "", &noopLogger, testFunnelIdleTimeout)
 	require.NoError(t, err)
 
 	proxyDone := make(chan struct{})
@@ -282,12 +295,16 @@ func TestConcurrentRequestsToSameDst(t *testing.T) {
 		}()
 	}
 	wg.Wait()
+
+	time.Sleep(testFunnelIdleTimeout * 2)
 	cancel()
 	<-proxyDone
 }
 
 // TestICMPProxyRejectNotEcho makes sure it rejects messages other than echo
 func TestICMPRouterRejectNotEcho(t *testing.T) {
+	defer leaktest.Check(t)()
+
 	msgs := []icmp.Message{
 		{
 			Type: ipv4.ICMPTypeDestinationUnreachable,
@@ -341,7 +358,7 @@ func TestICMPRouterRejectNotEcho(t *testing.T) {
 }
 
 func testICMPRouterRejectNotEcho(t *testing.T, srcDstIP netip.Addr, msgs []icmp.Message) {
-	router, err := NewICMPRouter(localhostIP, localhostIPv6, "", &noopLogger)
+	router, err := NewICMPRouter(localhostIP, localhostIPv6, "", &noopLogger, testFunnelIdleTimeout)
 	require.NoError(t, err)
 
 	muxer := newMockMuxer(1)

ingress/origin_proxy.go

@@ -2,8 +2,12 @@ package ingress
 
 import (
 	"context"
+	"crypto/tls"
 	"fmt"
+	"net"
 	"net/http"
+
+	"github.com/rs/zerolog"
 )
 
 // HTTPOriginProxy can be implemented by origin services that want to proxy http requests.
@@ -14,7 +18,7 @@ type HTTPOriginProxy interface {
 
 // StreamBasedOriginProxy can be implemented by origin services that want to proxy ws/TCP.
 type StreamBasedOriginProxy interface {
-	EstablishConnection(ctx context.Context, dest string) (OriginConnection, error)
+	EstablishConnection(ctx context.Context, dest string, log *zerolog.Logger) (OriginConnection, error)
 }
 
 // HTTPLocalProxy can be implemented by cloudflared services that want to handle incoming http requests.
@@ -46,9 +50,28 @@ func (o *httpService) RoundTrip(req *http.Request) (*http.Response, error) {
 		req.Header.Set("X-Forwarded-Host", req.Host)
 		req.Host = o.hostHeader
 	}
+
+	if o.matchSNIToHost {
+		o.SetOriginServerName(req)
+	}
+
 	return o.transport.RoundTrip(req)
 }
 
+func (o *httpService) SetOriginServerName(req *http.Request) {
+	o.transport.DialTLSContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
+		conn, err := o.transport.DialContext(ctx, network, addr)
+		if err != nil {
+			return nil, err
+		}
+		return tls.Client(conn, &tls.Config{
+			RootCAs:            o.transport.TLSClientConfig.RootCAs,
+			InsecureSkipVerify: o.transport.TLSClientConfig.InsecureSkipVerify,
+			ServerName:         req.Host,
+		}), nil
+	}
+}
+
 func (o *statusCode) RoundTrip(_ *http.Request) (*http.Response, error) {
 	if o.defaultResp {
 		o.log.Warn().Msgf(ErrNoIngressRulesCLI.Error())
@@ -62,19 +85,21 @@ func (o *statusCode) RoundTrip(_ *http.Request) (*http.Response, error) {
 	return resp, nil
 }
 
-func (o *rawTCPService) EstablishConnection(ctx context.Context, dest string) (OriginConnection, error) {
+func (o *rawTCPService) EstablishConnection(ctx context.Context, dest string, logger *zerolog.Logger) (OriginConnection, error) {
 	conn, err := o.dialer.DialContext(ctx, "tcp", dest)
 	if err != nil {
 		return nil, err
 	}
 
 	originConn := &tcpConnection{
-		conn: conn,
+		Conn:         conn,
+		writeTimeout: o.writeTimeout,
+		logger:       logger,
 	}
 	return originConn, nil
 }
 
-func (o *tcpOverWSService) EstablishConnection(ctx context.Context, dest string) (OriginConnection, error) {
+func (o *tcpOverWSService) EstablishConnection(ctx context.Context, dest string, _ *zerolog.Logger) (OriginConnection, error) {
 	var err error
 	if !o.isBastion {
 		dest = o.dest
@@ -92,6 +117,6 @@ func (o *tcpOverWSService) EstablishConnection(ctx context.Context, dest string)
 
 }
 
-func (o *socksProxyOverWSService) EstablishConnection(_ctx context.Context, _dest string) (OriginConnection, error) {
+func (o *socksProxyOverWSService) EstablishConnection(_ context.Context, _ string, _ *zerolog.Logger) (OriginConnection, error) {
 	return o.conn, nil
 }

ingress/origin_proxy_test.go

@@ -36,7 +36,7 @@ func TestRawTCPServiceEstablishConnection(t *testing.T) {
 	require.NoError(t, err)
 
 	// Origin not listening for new connection, should return an error
-	_, err = rawTCPService.EstablishConnection(context.Background(), req.URL.String())
+	_, err = rawTCPService.EstablishConnection(context.Background(), req.URL.String(), TestLogger)
 	require.Error(t, err)
 }
 
@@ -87,7 +87,7 @@ func TestTCPOverWSServiceEstablishConnection(t *testing.T) {
 		t.Run(test.testCase, func(t *testing.T) {
 			if test.expectErr {
 				bastionHost, _ := carrier.ResolveBastionDest(test.req)
-				_, err := test.service.EstablishConnection(context.Background(), bastionHost)
+				_, err := test.service.EstablishConnection(context.Background(), bastionHost, TestLogger)
 				assert.Error(t, err)
 			}
 		})
@@ -99,7 +99,7 @@ func TestTCPOverWSServiceEstablishConnection(t *testing.T) {
 	for _, service := range []*tcpOverWSService{newTCPOverWSService(originURL), newBastionService()} {
 		// Origin not listening for new connection, should return an error
 		bastionHost, _ := carrier.ResolveBastionDest(bastionReq)
-		_, err := service.EstablishConnection(context.Background(), bastionHost)
+		_, err := service.EstablishConnection(context.Background(), bastionHost, TestLogger)
 		assert.Error(t, err)
 	}
 }
@@ -132,7 +132,7 @@ func TestHTTPServiceHostHeaderOverride(t *testing.T) {
 		url: originURL,
 	}
 	shutdownC := make(chan struct{})
-	require.NoError(t, httpService.start(testLogger, shutdownC, cfg))
+	require.NoError(t, httpService.start(TestLogger, shutdownC, cfg))
 
 	req, err := http.NewRequest(http.MethodGet, originURL.String(), nil)
 	require.NoError(t, err)
@@ -167,7 +167,7 @@ func TestHTTPServiceUsesIngressRuleScheme(t *testing.T) {
 		url: originURL,
 	}
 	shutdownC := make(chan struct{})
-	require.NoError(t, httpService.start(testLogger, shutdownC, cfg))
+	require.NoError(t, httpService.start(TestLogger, shutdownC, cfg))
 
 	// Tunnel uses scheme defined in the service field of the ingress rule, independent of the X-Forwarded-Proto header
 	protos := []string{"https", "http", "dne"}

ingress/origin_service.go

@@ -68,9 +68,10 @@ func (o unixSocketPath) MarshalJSON() ([]byte, error) {
 }
 
 type httpService struct {
-	url        *url.URL
-	hostHeader string
-	transport  *http.Transport
+	url            *url.URL
+	hostHeader     string
+	transport      *http.Transport
+	matchSNIToHost bool
 }
 
 func (o *httpService) start(log *zerolog.Logger, _ <-chan struct{}, cfg OriginRequestConfig) error {
@@ -80,6 +81,7 @@ func (o *httpService) start(log *zerolog.Logger, _ <-chan struct{}, cfg OriginRe
 	}
 	o.hostHeader = cfg.HTTPHostHeader
 	o.transport = transport
+	o.matchSNIToHost = cfg.MatchSNIToHost
 	return nil
 }
 
@@ -94,15 +96,17 @@ func (o httpService) MarshalJSON() ([]byte, error) {
 // rawTCPService dials TCP to the destination specified by the client
 // It's used by warp routing
 type rawTCPService struct {
-	name   string
-	dialer net.Dialer
+	name         string
+	dialer       net.Dialer
+	writeTimeout time.Duration
+	logger       *zerolog.Logger
 }
 
 func (o *rawTCPService) String() string {
 	return o.name
 }
 
-func (o *rawTCPService) start(log *zerolog.Logger, _ <-chan struct{}, cfg OriginRequestConfig) error {
+func (o *rawTCPService) start(_ *zerolog.Logger, _ <-chan struct{}, _ OriginRequestConfig) error {
 	return nil
 }
 
@@ -285,13 +289,14 @@ type WarpRoutingService struct {
 	Proxy StreamBasedOriginProxy
 }
 
-func NewWarpRoutingService(config WarpRoutingConfig) *WarpRoutingService {
+func NewWarpRoutingService(config WarpRoutingConfig, writeTimeout time.Duration) *WarpRoutingService {
 	svc := &rawTCPService{
 		name: ServiceWarpRouting,
 		dialer: net.Dialer{
 			Timeout:   config.ConnectTimeout.Duration,
 			KeepAlive: config.TCPKeepAlive.Duration,
 		},
+		writeTimeout: writeTimeout,
 	}
 
 	return &WarpRoutingService{Proxy: svc}

ingress/rule_test.go

@@ -204,25 +204,25 @@ func TestMarshalJSON(t *testing.T) {
 		{
 			name:     "Nil",
 			path:     nil,
-			expected: `{"hostname":"example.com","path":null,"service":"https://localhost:8000","Handlers":null,"originRequest":{"connectTimeout":30,"tlsTimeout":10,"tcpKeepAlive":30,"noHappyEyeballs":false,"keepAliveTimeout":90,"keepAliveConnections":100,"httpHostHeader":"","originServerName":"","caPool":"","noTLSVerify":false,"disableChunkedEncoding":false,"bastionMode":false,"proxyAddress":"127.0.0.1","proxyPort":0,"proxyType":"","ipRules":null,"http2Origin":false,"access":{"teamName":"","audTag":null}}}`,
+			expected: `{"hostname":"example.com","path":null,"service":"https://localhost:8000","Handlers":null,"originRequest":{"connectTimeout":30,"tlsTimeout":10,"tcpKeepAlive":30,"noHappyEyeballs":false,"keepAliveTimeout":90,"keepAliveConnections":100,"httpHostHeader":"","originServerName":"","matchSNItoHost":false,"caPool":"","noTLSVerify":false,"disableChunkedEncoding":false,"bastionMode":false,"proxyAddress":"127.0.0.1","proxyPort":0,"proxyType":"","ipRules":null,"http2Origin":false,"access":{"teamName":"","audTag":null}}}`,
 			want:     true,
 		},
 		{
 			name:     "Nil regex",
 			path:     &Regexp{Regexp: nil},
-			expected: `{"hostname":"example.com","path":null,"service":"https://localhost:8000","Handlers":null,"originRequest":{"connectTimeout":30,"tlsTimeout":10,"tcpKeepAlive":30,"noHappyEyeballs":false,"keepAliveTimeout":90,"keepAliveConnections":100,"httpHostHeader":"","originServerName":"","caPool":"","noTLSVerify":false,"disableChunkedEncoding":false,"bastionMode":false,"proxyAddress":"127.0.0.1","proxyPort":0,"proxyType":"","ipRules":null,"http2Origin":false,"access":{"teamName":"","audTag":null}}}`,
+			expected: `{"hostname":"example.com","path":null,"service":"https://localhost:8000","Handlers":null,"originRequest":{"connectTimeout":30,"tlsTimeout":10,"tcpKeepAlive":30,"noHappyEyeballs":false,"keepAliveTimeout":90,"keepAliveConnections":100,"httpHostHeader":"","originServerName":"","matchSNItoHost":false,"caPool":"","noTLSVerify":false,"disableChunkedEncoding":false,"bastionMode":false,"proxyAddress":"127.0.0.1","proxyPort":0,"proxyType":"","ipRules":null,"http2Origin":false,"access":{"teamName":"","audTag":null}}}`,
 			want:     true,
 		},
 		{
 			name:     "Empty",
 			path:     &Regexp{Regexp: regexp.MustCompile("")},
-			expected: `{"hostname":"example.com","path":"","service":"https://localhost:8000","Handlers":null,"originRequest":{"connectTimeout":30,"tlsTimeout":10,"tcpKeepAlive":30,"noHappyEyeballs":false,"keepAliveTimeout":90,"keepAliveConnections":100,"httpHostHeader":"","originServerName":"","caPool":"","noTLSVerify":false,"disableChunkedEncoding":false,"bastionMode":false,"proxyAddress":"127.0.0.1","proxyPort":0,"proxyType":"","ipRules":null,"http2Origin":false,"access":{"teamName":"","audTag":null}}}`,
+			expected: `{"hostname":"example.com","path":"","service":"https://localhost:8000","Handlers":null,"originRequest":{"connectTimeout":30,"tlsTimeout":10,"tcpKeepAlive":30,"noHappyEyeballs":false,"keepAliveTimeout":90,"keepAliveConnections":100,"httpHostHeader":"","originServerName":"","matchSNItoHost":false,"caPool":"","noTLSVerify":false,"disableChunkedEncoding":false,"bastionMode":false,"proxyAddress":"127.0.0.1","proxyPort":0,"proxyType":"","ipRules":null,"http2Origin":false,"access":{"teamName":"","audTag":null}}}`,
 			want:     true,
 		},
 		{
 			name:     "Basic",
 			path:     &Regexp{Regexp: regexp.MustCompile("/echo")},
-			expected: `{"hostname":"example.com","path":"/echo","service":"https://localhost:8000","Handlers":null,"originRequest":{"connectTimeout":30,"tlsTimeout":10,"tcpKeepAlive":30,"noHappyEyeballs":false,"keepAliveTimeout":90,"keepAliveConnections":100,"httpHostHeader":"","originServerName":"","caPool":"","noTLSVerify":false,"disableChunkedEncoding":false,"bastionMode":false,"proxyAddress":"127.0.0.1","proxyPort":0,"proxyType":"","ipRules":null,"http2Origin":false,"access":{"teamName":"","audTag":null}}}`,
+			expected: `{"hostname":"example.com","path":"/echo","service":"https://localhost:8000","Handlers":null,"originRequest":{"connectTimeout":30,"tlsTimeout":10,"tcpKeepAlive":30,"noHappyEyeballs":false,"keepAliveTimeout":90,"keepAliveConnections":100,"httpHostHeader":"","originServerName":"","matchSNItoHost":false,"caPool":"","noTLSVerify":false,"disableChunkedEncoding":false,"bastionMode":false,"proxyAddress":"127.0.0.1","proxyPort":0,"proxyType":"","ipRules":null,"http2Origin":false,"access":{"teamName":"","audTag":null}}}`,
 			want:     true,
 		},
 	}

management/token.go

@@ -3,7 +3,8 @@ package management
 import (
 	"fmt"
 
-	"github.com/go-jose/go-jose/v3/jwt"
+	"github.com/go-jose/go-jose/v4"
+	"github.com/go-jose/go-jose/v4/jwt"
 )
 
 type managementTokenClaims struct {
@@ -37,7 +38,7 @@ func (t *actor) verify() bool {
 }
 
 func parseToken(token string) (*managementTokenClaims, error) {
-	jwt, err := jwt.ParseSigned(token)
+	jwt, err := jwt.ParseSigned(token, []jose.SignatureAlgorithm{jose.ES256})
 	if err != nil {
 		return nil, fmt.Errorf("malformed jwt: %v", err)
 	}

management/token_test.go

@@ -7,7 +7,7 @@ import (
 	"errors"
 	"testing"
 
-	"github.com/go-jose/go-jose/v3"
+	"github.com/go-jose/go-jose/v4"
 	"github.com/stretchr/testify/require"
 )
 

orchestration/config.go

@@ -2,6 +2,7 @@ package orchestration
 
 import (
 	"encoding/json"
+	"time"
 
 	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/ingress"
@@ -19,8 +20,9 @@ type newLocalConfig struct {
 
 // Config is the original config as read and parsed by cloudflared.
 type Config struct {
-	Ingress     *ingress.Ingress
-	WarpRouting ingress.WarpRoutingConfig
+	Ingress      *ingress.Ingress
+	WarpRouting  ingress.WarpRoutingConfig
+	WriteTimeout time.Duration
 
 	// Extra settings used to configure this instance but that are not eligible for remotely management
 	// ie. (--protocol, --loglevel, ...)

orchestration/orchestrator.go

@@ -14,13 +14,13 @@ import (
 	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/ingress"
 	"github.com/cloudflare/cloudflared/proxy"
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
-// Orchestrator manages configurations so they can be updatable during runtime
+// Orchestrator manages configurations, so they can be updatable during runtime
 // properties are static, so it can be read without lock
 // currentVersion and config are read/write infrequently, so their access are synchronized with RWMutex
-// access to proxy is synchronized with atmoic.Value, because it uses copy-on-write to provide scalable frequently
+// access to proxy is synchronized with atomic.Value, because it uses copy-on-write to provide scalable frequently
 // read when update is infrequent
 type Orchestrator struct {
 	currentVersion int32
@@ -30,9 +30,10 @@ type Orchestrator struct {
 	proxy atomic.Value
 	// Set of internal ingress rules defined at cloudflared startup (separate from user-defined ingress rules)
 	internalRules []ingress.Rule
-	config        *Config
-	tags          []tunnelpogs.Tag
-	log           *zerolog.Logger
+	// cloudflared Configuration
+	config *Config
+	tags   []pogs.Tag
+	log    *zerolog.Logger
 
 	// orchestrator must not handle any more updates after shutdownC is closed
 	shutdownC <-chan struct{}
@@ -40,7 +41,11 @@ type Orchestrator struct {
 	proxyShutdownC chan<- struct{}
 }
 
-func NewOrchestrator(ctx context.Context, config *Config, tags []tunnelpogs.Tag, internalRules []ingress.Rule, log *zerolog.Logger) (*Orchestrator, error) {
+func NewOrchestrator(ctx context.Context,
+	config *Config,
+	tags []pogs.Tag,
+	internalRules []ingress.Rule,
+	log *zerolog.Logger) (*Orchestrator, error) {
 	o := &Orchestrator{
 		// Lowest possible version, any remote configuration will have version higher than this
 		// Starting at -1 allows a configuration migration (local to remote) to override the current configuration as it
@@ -60,7 +65,7 @@ func NewOrchestrator(ctx context.Context, config *Config, tags []tunnelpogs.Tag,
 }
 
 // UpdateConfig creates a new proxy with the new ingress rules
-func (o *Orchestrator) UpdateConfig(version int32, config []byte) *tunnelpogs.UpdateConfigurationResponse {
+func (o *Orchestrator) UpdateConfig(version int32, config []byte) *pogs.UpdateConfigurationResponse {
 	o.lock.Lock()
 	defer o.lock.Unlock()
 
@@ -69,7 +74,7 @@ func (o *Orchestrator) UpdateConfig(version int32, config []byte) *tunnelpogs.Up
 			Int32("current_version", o.currentVersion).
 			Int32("received_version", version).
 			Msg("Current version is equal or newer than received version")
-		return &tunnelpogs.UpdateConfigurationResponse{
+		return &pogs.UpdateConfigurationResponse{
 			LastAppliedVersion: o.currentVersion,
 		}
 	}
@@ -79,7 +84,7 @@ func (o *Orchestrator) UpdateConfig(version int32, config []byte) *tunnelpogs.Up
 			Int32("version", version).
 			Str("config", string(config)).
 			Msgf("Failed to deserialize new configuration")
-		return &tunnelpogs.UpdateConfigurationResponse{
+		return &pogs.UpdateConfigurationResponse{
 			LastAppliedVersion: o.currentVersion,
 			Err:                err,
 		}
@@ -90,7 +95,7 @@ func (o *Orchestrator) UpdateConfig(version int32, config []byte) *tunnelpogs.Up
 			Int32("version", version).
 			Str("config", string(config)).
 			Msgf("Failed to update ingress")
-		return &tunnelpogs.UpdateConfigurationResponse{
+		return &pogs.UpdateConfigurationResponse{
 			LastAppliedVersion: o.currentVersion,
 			Err:                err,
 		}
@@ -102,7 +107,7 @@ func (o *Orchestrator) UpdateConfig(version int32, config []byte) *tunnelpogs.Up
 		Str("config", string(config)).
 		Msg("Updated to new configuration")
 	configVersion.Set(float64(version))
-	return &tunnelpogs.UpdateConfigurationResponse{
+	return &pogs.UpdateConfigurationResponse{
 		LastAppliedVersion: o.currentVersion,
 	}
 }
@@ -131,7 +136,7 @@ func (o *Orchestrator) updateIngress(ingressRules ingress.Ingress, warpRouting i
 	if err := ingressRules.StartOrigins(o.log, proxyShutdownC); err != nil {
 		return errors.Wrap(err, "failed to start origin")
 	}
-	proxy := proxy.NewOriginProxy(ingressRules, warpRouting, o.tags, o.log)
+	proxy := proxy.NewOriginProxy(ingressRules, warpRouting, o.tags, o.config.WriteTimeout, o.log)
 	o.proxy.Store(proxy)
 	o.config.Ingress = &ingressRules
 	o.config.WarpRouting = warpRouting

orchestration/orchestrator_test.go

@@ -23,12 +23,12 @@ import (
 	"github.com/cloudflare/cloudflared/ingress"
 	"github.com/cloudflare/cloudflared/management"
 	"github.com/cloudflare/cloudflared/tracing"
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
 var (
-	testLogger = zerolog.Logger{}
-	testTags   = []tunnelpogs.Tag{
+	testLogger = zerolog.Nop()
+	testTags   = []pogs.Tag{
 		{
 			Name:  "package",
 			Value: "orchestration",

postrm.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
 set -eu
-rm /usr/local/bin/cloudflared
-rm /usr/local/etc/cloudflared/.installedFromPackageManager || true
+rm -f /usr/local/bin/cloudflared
+rm -f /usr/local/etc/cloudflared/.installedFromPackageManager

proxy/logger.go

@@ -0,0 +1,78 @@
+package proxy
+
+import (
+	"net/http"
+	"strconv"
+
+	"github.com/rs/zerolog"
+
+	"github.com/cloudflare/cloudflared/connection"
+	"github.com/cloudflare/cloudflared/ingress"
+	"github.com/cloudflare/cloudflared/management"
+)
+
+const (
+	logFieldCFRay         = "cfRay"
+	logFieldLBProbe       = "lbProbe"
+	logFieldRule          = "ingressRule"
+	logFieldOriginService = "originService"
+	logFieldFlowID        = "flowID"
+	logFieldConnIndex     = "connIndex"
+	logFieldDestAddr      = "destAddr"
+)
+
+// newHTTPLogger creates a child zerolog.Logger from the provided with added context from the HTTP request, ingress
+// services, and connection index.
+func newHTTPLogger(logger *zerolog.Logger, connIndex uint8, req *http.Request, rule int, serviceName string) zerolog.Logger {
+	ctx := logger.With().
+		Int(management.EventTypeKey, int(management.HTTP)).
+		Uint8(logFieldConnIndex, connIndex)
+	cfRay := connection.FindCfRayHeader(req)
+	lbProbe := connection.IsLBProbeRequest(req)
+	if cfRay != "" {
+		ctx.Str(logFieldCFRay, cfRay)
+	}
+	if lbProbe {
+		ctx.Bool(logFieldLBProbe, lbProbe)
+	}
+	return ctx.
+		Str(logFieldOriginService, serviceName).
+		Interface(logFieldRule, rule).
+		Logger()
+}
+
+// newTCPLogger creates a child zerolog.Logger from the provided with added context from the TCPRequest.
+func newTCPLogger(logger *zerolog.Logger, req *connection.TCPRequest) zerolog.Logger {
+	return logger.With().
+		Int(management.EventTypeKey, int(management.TCP)).
+		Uint8(logFieldConnIndex, req.ConnIndex).
+		Str(logFieldOriginService, ingress.ServiceWarpRouting).
+		Str(logFieldFlowID, req.FlowID).
+		Str(logFieldDestAddr, req.Dest).
+		Uint8(logFieldConnIndex, req.ConnIndex).
+		Logger()
+}
+
+// logHTTPRequest logs a Debug message with the corresponding HTTP request details from the eyeball.
+func logHTTPRequest(logger *zerolog.Logger, r *http.Request) {
+	logger.Debug().
+		Str("host", r.Host).
+		Str("path", r.URL.Path).
+		Interface("headers", r.Header).
+		Int64("content-length", r.ContentLength).
+		Msgf("%s %s %s", r.Method, r.URL, r.Proto)
+}
+
+// logOriginHTTPResponse logs a Debug message of the origin response.
+func logOriginHTTPResponse(logger *zerolog.Logger, resp *http.Response) {
+	responseByCode.WithLabelValues(strconv.Itoa(resp.StatusCode)).Inc()
+	logger.Debug().
+		Int64("content-length", resp.ContentLength).
+		Msgf("%s", resp.Status)
+}
+
+// logRequestError logs an error for the proxied request.
+func logRequestError(logger *zerolog.Logger, err error) {
+	requestErrors.Inc()
+	logger.Error().Err(err).Send()
+}

proxy/metrics.go

@@ -59,6 +59,23 @@ var (
 			Help:      "Total count of TCP sessions that have been proxied to any origin",
 		},
 	)
+	connectLatency = prometheus.NewHistogram(
+		prometheus.HistogramOpts{
+			Namespace: connection.MetricsNamespace,
+			Subsystem: "proxy",
+			Name:      "connect_latency",
+			Help:      "Time it takes to establish and acknowledge connections in milliseconds",
+			Buckets:   []float64{1, 10, 25, 50, 100, 500, 1000, 5000},
+		},
+	)
+	connectStreamErrors = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Namespace: connection.MetricsNamespace,
+			Subsystem: "proxy",
+			Name:      "connect_streams_errors",
+			Help:      "Total count of failure to establish and acknowledge connections",
+		},
+	)
 )
 
 func init() {
@@ -69,6 +86,8 @@ func init() {
 		requestErrors,
 		activeTCPSessions,
 		totalTCPSessions,
+		connectLatency,
+		connectStreamErrors,
 	)
 }
 

proxy/proxy.go

@@ -6,6 +6,7 @@ import (
 	"io"
 	"net/http"
 	"strconv"
+	"time"
 
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
@@ -16,24 +17,15 @@ import (
 	"github.com/cloudflare/cloudflared/cfio"
 	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/ingress"
-	"github.com/cloudflare/cloudflared/management"
 	"github.com/cloudflare/cloudflared/stream"
 	"github.com/cloudflare/cloudflared/tracing"
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
 const (
 	// TagHeaderNamePrefix indicates a Cloudflared Warp Tag prefix that gets appended for warp traffic stream headers.
-	TagHeaderNamePrefix   = "Cf-Warp-Tag-"
-	LogFieldCFRay         = "cfRay"
-	LogFieldLBProbe       = "lbProbe"
-	LogFieldRule          = "ingressRule"
-	LogFieldOriginService = "originService"
-	LogFieldFlowID        = "flowID"
-	LogFieldConnIndex     = "connIndex"
-	LogFieldDestAddr      = "destAddr"
-
-	trailerHeaderName = "Trailer"
+	TagHeaderNamePrefix = "Cf-Warp-Tag-"
+	trailerHeaderName   = "Trailer"
 )
 
 // Proxy represents a means to Proxy between cloudflared and the origin services.
@@ -41,7 +33,7 @@ type Proxy struct {
 	ingressRules ingress.Ingress
 	warpRouting  *ingress.WarpRoutingService
 	management   *ingress.ManagementService
-	tags         []tunnelpogs.Tag
+	tags         []pogs.Tag
 	log          *zerolog.Logger
 }
 
@@ -49,7 +41,8 @@ type Proxy struct {
 func NewOriginProxy(
 	ingressRules ingress.Ingress,
 	warpRouting ingress.WarpRoutingConfig,
-	tags []tunnelpogs.Tag,
+	tags []pogs.Tag,
+	writeTimeout time.Duration,
 	log *zerolog.Logger,
 ) *Proxy {
 	proxy := &Proxy{
@@ -58,7 +51,7 @@ func NewOriginProxy(
 		log:          log,
 	}
 
-	proxy.warpRouting = ingress.NewWarpRoutingService(warpRouting)
+	proxy.warpRouting = ingress.NewWarpRoutingService(warpRouting, writeTimeout)
 
 	return proxy
 }
@@ -89,26 +82,18 @@ func (p *Proxy) ProxyHTTP(
 	defer decrementConcurrentRequests()
 
 	req := tr.Request
-	cfRay := connection.FindCfRayHeader(req)
-	lbProbe := connection.IsLBProbeRequest(req)
 	p.appendTagHeaders(req)
 
 	_, ruleSpan := tr.Tracer().Start(req.Context(), "ingress_match",
 		trace.WithAttributes(attribute.String("req-host", req.Host)))
 	rule, ruleNum := p.ingressRules.FindMatchingRule(req.Host, req.URL.Path)
-	logFields := logFields{
-		cfRay:     cfRay,
-		lbProbe:   lbProbe,
-		rule:      ruleNum,
-		connIndex: tr.ConnIndex,
-	}
-	p.logRequest(req, logFields)
 	ruleSpan.SetAttributes(attribute.Int("rule-num", ruleNum))
 	ruleSpan.End()
+	logger := newHTTPLogger(p.log, tr.ConnIndex, req, ruleNum, rule.Service.String())
+	logHTTPRequest(&logger, req)
 	if err, applied := p.applyIngressMiddleware(rule, req, w); err != nil {
 		if applied {
-			rule, srv := ruleField(p.ingressRules, ruleNum)
-			p.logRequestError(err, cfRay, "", rule, srv)
+			logRequestError(&logger, err)
 			return nil
 		}
 		return err
@@ -122,10 +107,9 @@ func (p *Proxy) ProxyHTTP(
 			originProxy,
 			isWebsocket,
 			rule.Config.DisableChunkedEncoding,
-			logFields,
+			&logger,
 		); err != nil {
-			rule, srv := ruleField(p.ingressRules, ruleNum)
-			p.logRequestError(err, cfRay, "", rule, srv)
+			logRequestError(&logger, err)
 			return err
 		}
 		return nil
@@ -139,9 +123,9 @@ func (p *Proxy) ProxyHTTP(
 			return fmt.Errorf("response writer is not a flusher")
 		}
 		rws := connection.NewHTTPResponseReadWriterAcker(w, flusher, req)
-		if err := p.proxyStream(tr.ToTracedContext(), rws, dest, originProxy); err != nil {
-			rule, srv := ruleField(p.ingressRules, ruleNum)
-			p.logRequestError(err, cfRay, "", rule, srv)
+		logger := logger.With().Str(logFieldDestAddr, dest).Logger()
+		if err := p.proxyStream(tr.ToTracedContext(), rws, dest, originProxy, &logger); err != nil {
+			logRequestError(&logger, err)
 			return err
 		}
 		return nil
@@ -171,38 +155,20 @@ func (p *Proxy) ProxyTCP(
 	serveCtx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
-	tracedCtx := tracing.NewTracedContext(serveCtx, req.CfTraceID, p.log)
+	logger := newTCPLogger(p.log, req)
+	tracedCtx := tracing.NewTracedContext(serveCtx, req.CfTraceID, &logger)
+	logger.Debug().Msg("tcp proxy stream started")
 
-	p.log.Debug().
-		Int(management.EventTypeKey, int(management.TCP)).
-		Str(LogFieldFlowID, req.FlowID).
-		Str(LogFieldDestAddr, req.Dest).
-		Uint8(LogFieldConnIndex, req.ConnIndex).
-		Msg("tcp proxy stream started")
-
-	if err := p.proxyStream(tracedCtx, rwa, req.Dest, p.warpRouting.Proxy); err != nil {
-		p.logRequestError(err, req.CFRay, req.FlowID, "", ingress.ServiceWarpRouting)
+	if err := p.proxyStream(tracedCtx, rwa, req.Dest, p.warpRouting.Proxy, &logger); err != nil {
+		logRequestError(&logger, err)
 		return err
 	}
 
-	p.log.Debug().
-		Int(management.EventTypeKey, int(management.TCP)).
-		Str(LogFieldFlowID, req.FlowID).
-		Str(LogFieldDestAddr, req.Dest).
-		Uint8(LogFieldConnIndex, req.ConnIndex).
-		Msg("tcp proxy stream finished successfully")
+	logger.Debug().Msg("tcp proxy stream finished successfully")
 
 	return nil
 }
 
-func ruleField(ing ingress.Ingress, ruleNum int) (ruleID string, srv string) {
-	srv = ing.Rules[ruleNum].Service.String()
-	if ing.IsSingleRule() {
-		return "", srv
-	}
-	return fmt.Sprintf("%d", ruleNum), srv
-}
-
 // ProxyHTTPRequest proxies requests of underlying type http and websocket to the origin service.
 func (p *Proxy) proxyHTTPRequest(
 	w connection.ResponseWriter,
@@ -210,7 +176,7 @@ func (p *Proxy) proxyHTTPRequest(
 	httpService ingress.HTTPOriginProxy,
 	isWebsocket bool,
 	disableChunkedEncoding bool,
-	fields logFields,
+	logger *zerolog.Logger,
 ) error {
 	roundTripReq := tr.Request
 	if isWebsocket {
@@ -277,7 +243,7 @@ func (p *Proxy) proxyHTTPRequest(
 			reader: tr.Request.Body,
 		}
 
-		stream.Pipe(eyeballStream, rwc, p.log)
+		stream.Pipe(eyeballStream, rwc, logger)
 		return nil
 	}
 
@@ -288,35 +254,45 @@ func (p *Proxy) proxyHTTPRequest(
 	// copy trailers
 	copyTrailers(w, resp)
 
-	p.logOriginResponse(resp, fields)
+	logOriginHTTPResponse(logger, resp)
 	return nil
 }
 
 // proxyStream proxies type TCP and other underlying types if the connection is defined as a stream oriented
 // ingress rule.
+// connectedLogger is used to log when the connection is acknowledged
 func (p *Proxy) proxyStream(
 	tr *tracing.TracedContext,
 	rwa connection.ReadWriteAcker,
 	dest string,
 	connectionProxy ingress.StreamBasedOriginProxy,
+	logger *zerolog.Logger,
 ) error {
 	ctx := tr.Context
 	_, connectSpan := tr.Tracer().Start(ctx, "stream-connect")
-	originConn, err := connectionProxy.EstablishConnection(ctx, dest)
+
+	start := time.Now()
+	originConn, err := connectionProxy.EstablishConnection(ctx, dest, logger)
 	if err != nil {
+		connectStreamErrors.Inc()
 		tracing.EndWithErrorStatus(connectSpan, err)
 		return err
 	}
 	connectSpan.End()
 	defer originConn.Close()
+	logger.Debug().Msg("origin connection established")
 
 	encodedSpans := tr.GetSpans()
 
 	if err := rwa.AckConnection(encodedSpans); err != nil {
+		connectStreamErrors.Inc()
 		return err
 	}
 
-	originConn.Stream(ctx, rwa, p.log)
+	connectLatency.Observe(float64(time.Since(start).Milliseconds()))
+	logger.Debug().Msg("proxy stream acknowledged")
+
+	originConn.Stream(ctx, rwa, logger)
 	return nil
 }
 
@@ -350,14 +326,6 @@ func (p *Proxy) appendTagHeaders(r *http.Request) {
 	}
 }
 
-type logFields struct {
-	cfRay     string
-	lbProbe   bool
-	rule      int
-	flowID    string
-	connIndex uint8
-}
-
 func copyTrailers(w connection.ResponseWriter, response *http.Response) {
 	for trailerHeader, trailerValues := range response.Trailer {
 		for _, trailerValue := range trailerValues {
@@ -366,64 +334,6 @@ func copyTrailers(w connection.ResponseWriter, response *http.Response) {
 	}
 }
 
-func (p *Proxy) logRequest(r *http.Request, fields logFields) {
-	log := p.log.With().Int(management.EventTypeKey, int(management.HTTP)).Logger()
-	event := log.Debug()
-	if fields.cfRay != "" {
-		event = event.Str(LogFieldCFRay, fields.cfRay)
-	}
-	if fields.lbProbe {
-		event = event.Bool(LogFieldLBProbe, fields.lbProbe)
-	}
-	if fields.cfRay == "" && !fields.lbProbe {
-		log.Debug().Msgf("All requests should have a CF-RAY header. Please open a support ticket with Cloudflare. %s %s %s ", r.Method, r.URL, r.Proto)
-	}
-	event.
-		Uint8(LogFieldConnIndex, fields.connIndex).
-		Str("host", r.Host).
-		Str("path", r.URL.Path).
-		Interface(LogFieldRule, fields.rule).
-		Interface("headers", r.Header).
-		Int64("content-length", r.ContentLength).
-		Msgf("%s %s %s", r.Method, r.URL, r.Proto)
-}
-
-func (p *Proxy) logOriginResponse(resp *http.Response, fields logFields) {
-	responseByCode.WithLabelValues(strconv.Itoa(resp.StatusCode)).Inc()
-	event := p.log.Debug()
-	if fields.cfRay != "" {
-		event = event.Str(LogFieldCFRay, fields.cfRay)
-	}
-	if fields.lbProbe {
-		event = event.Bool(LogFieldLBProbe, fields.lbProbe)
-	}
-	event.
-		Int(management.EventTypeKey, int(management.HTTP)).
-		Uint8(LogFieldConnIndex, fields.connIndex).
-		Int64("content-length", resp.ContentLength).
-		Msgf("%s", resp.Status)
-}
-
-func (p *Proxy) logRequestError(err error, cfRay string, flowID string, rule, service string) {
-	requestErrors.Inc()
-	log := p.log.Error().Err(err)
-	if cfRay != "" {
-		log = log.Str(LogFieldCFRay, cfRay)
-	}
-	if flowID != "" {
-		log = log.Str(LogFieldFlowID, flowID).Int(management.EventTypeKey, int(management.TCP))
-	} else {
-		log = log.Int(management.EventTypeKey, int(management.HTTP))
-	}
-	if rule != "" {
-		log = log.Str(LogFieldRule, rule)
-	}
-	if service != "" {
-		log = log.Str(LogFieldOriginService, service)
-	}
-	log.Send()
-}
-
 func getDestFromRule(rule *ingress.Rule, req *http.Request) (string, error) {
 	switch rule.Service.String() {
 	case ingress.ServiceBastion:

proxy/proxy_test.go

@@ -30,11 +30,11 @@ import (
 	"github.com/cloudflare/cloudflared/ingress"
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/tracing"
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
 var (
-	testTags        = []tunnelpogs.Tag{{Name: "Name", Value: "value"}}
+	testTags        = []pogs.Tag{{Name: "Name", Value: "value"}}
 	noWarpRouting   = ingress.WarpRoutingConfig{}
 	testWarpRouting = ingress.WarpRoutingConfig{
 		ConnectTimeout: config.CustomDuration{Duration: time.Second},
@@ -162,7 +162,7 @@ func TestProxySingleOrigin(t *testing.T) {
 
 	require.NoError(t, ingressRule.StartOrigins(&log, ctx.Done()))
 
-	proxy := NewOriginProxy(ingressRule, noWarpRouting, testTags, &log)
+	proxy := NewOriginProxy(ingressRule, noWarpRouting, testTags, time.Duration(0), &log)
 	t.Run("testProxyHTTP", testProxyHTTP(proxy))
 	t.Run("testProxyWebsocket", testProxyWebsocket(proxy))
 	t.Run("testProxySSE", testProxySSE(proxy))
@@ -366,7 +366,7 @@ func runIngressTestScenarios(t *testing.T, unvalidatedIngress []config.Unvalidat
 	ctx, cancel := context.WithCancel(context.Background())
 	require.NoError(t, ingress.StartOrigins(&log, ctx.Done()))
 
-	proxy := NewOriginProxy(ingress, noWarpRouting, testTags, &log)
+	proxy := NewOriginProxy(ingress, noWarpRouting, testTags, time.Duration(0), &log)
 
 	for _, test := range tests {
 		responseWriter := newMockHTTPRespWriter()
@@ -414,7 +414,7 @@ func TestProxyError(t *testing.T) {
 
 	log := zerolog.Nop()
 
-	proxy := NewOriginProxy(ing, noWarpRouting, testTags, &log)
+	proxy := NewOriginProxy(ing, noWarpRouting, testTags, time.Duration(0), &log)
 
 	responseWriter := newMockHTTPRespWriter()
 	req, err := http.NewRequest(http.MethodGet, "http://127.0.0.1", nil)
@@ -530,7 +530,7 @@ func TestConnections(t *testing.T) {
 				originService:         runEchoTCPService,
 				eyeballResponseWriter: newTCPRespWriter(replayer),
 				eyeballRequestBody:    newTCPRequestBody([]byte("test2")),
-				warpRoutingService:    ingress.NewWarpRoutingService(testWarpRouting),
+				warpRoutingService:    ingress.NewWarpRoutingService(testWarpRouting, time.Duration(0)),
 				connectionType:        connection.TypeTCP,
 				requestHeaders: map[string][]string{
 					"Cf-Cloudflared-Proxy-Src": {"non-blank-value"},
@@ -548,7 +548,7 @@ func TestConnections(t *testing.T) {
 				originService:        runEchoWSService,
 				// eyeballResponseWriter gets set after roundtrip dial.
 				eyeballRequestBody: newPipedWSRequestBody([]byte("test3")),
-				warpRoutingService: ingress.NewWarpRoutingService(testWarpRouting),
+				warpRoutingService: ingress.NewWarpRoutingService(testWarpRouting, time.Duration(0)),
 				requestHeaders: map[string][]string{
 					"Cf-Cloudflared-Proxy-Src": {"non-blank-value"},
 				},
@@ -675,7 +675,7 @@ func TestConnections(t *testing.T) {
 
 			ingressRule := createSingleIngressConfig(t, test.args.ingressServiceScheme+ln.Addr().String())
 			ingressRule.StartOrigins(logger, ctx.Done())
-			proxy := NewOriginProxy(ingressRule, testWarpRouting, testTags, logger)
+			proxy := NewOriginProxy(ingressRule, testWarpRouting, testTags, time.Duration(0), logger)
 			proxy.warpRouting = test.args.warpRoutingService
 
 			dest := ln.Addr().String()

quic/constants.go

@@ -0,0 +1,12 @@
+package quic
+
+import "time"
+
+const (
+	HandshakeIdleTimeout = 5 * time.Second
+	MaxIdleTimeout       = 5 * time.Second
+	MaxIdlePingPeriod    = 1 * time.Second
+
+	// MaxIncomingStreams is 2^60, which is the maximum supported value by Quic-Go
+	MaxIncomingStreams = 1 << 60
+)

quic/conversion.go

@@ -7,17 +7,6 @@ import (
 	"github.com/quic-go/quic-go/logging"
 )
 
-func perspectiveString(p logging.Perspective) string {
-	switch p {
-	case logging.PerspectiveClient:
-		return "client"
-	case logging.PerspectiveServer:
-		return "server"
-	default:
-		return ""
-	}
-}
-
 // Helper to convert logging.ByteCount(alias for int64) to float64 used in prometheus
 func byteCountToPromCount(count logging.ByteCount) float64 {
 	return float64(count)

quic/metrics.go

@@ -1,6 +1,8 @@
 package quic
 
 import (
+	"reflect"
+	"strings"
 	"sync"
 
 	"github.com/prometheus/client_golang/prometheus"
@@ -16,10 +18,10 @@ var (
 	clientMetrics    = struct {
 		totalConnections  prometheus.Counter
 		closedConnections prometheus.Counter
-		sentPackets       *prometheus.CounterVec
+		sentFrames        *prometheus.CounterVec
 		sentBytes         *prometheus.CounterVec
-		receivePackets    *prometheus.CounterVec
-		receiveBytes      *prometheus.CounterVec
+		receivedFrames    *prometheus.CounterVec
+		receivedBytes     *prometheus.CounterVec
 		bufferedPackets   *prometheus.CounterVec
 		droppedPackets    *prometheus.CounterVec
 		lostPackets       *prometheus.CounterVec
@@ -28,43 +30,88 @@ var (
 		smoothedRTT       *prometheus.GaugeVec
 	}{
 		totalConnections: prometheus.NewCounter(
-			totalConnectionsOpts(logging.PerspectiveClient),
+			prometheus.CounterOpts{
+				Namespace: namespace,
+				Subsystem: "client",
+				Name:      "total_connections",
+				Help:      "Number of connections initiated",
+			},
 		),
 		closedConnections: prometheus.NewCounter(
-			closedConnectionsOpts(logging.PerspectiveClient),
+			prometheus.CounterOpts{
+				Namespace: namespace,
+				Subsystem: "client",
+				Name:      "closed_connections",
+				Help:      "Number of connections that has been closed",
+			},
 		),
-		sentPackets: prometheus.NewCounterVec(
-			sentPacketsOpts(logging.PerspectiveClient),
-			clientConnLabels,
+		sentFrames: prometheus.NewCounterVec(
+			prometheus.CounterOpts{
+				Namespace: namespace,
+				Subsystem: "client",
+				Name:      "sent_frames",
+				Help:      "Number of frames that have been sent through a connection",
+			},
+			append(clientConnLabels, "frame_type"),
 		),
 		sentBytes: prometheus.NewCounterVec(
-			sentBytesOpts(logging.PerspectiveClient),
+			prometheus.CounterOpts{
+				Namespace: namespace,
+				Subsystem: "client",
+				Name:      "sent_bytes",
+				Help:      "Number of bytes that have been sent through a connection",
+			},
 			clientConnLabels,
 		),
-		receivePackets: prometheus.NewCounterVec(
-			receivePacketsOpts(logging.PerspectiveClient),
-			clientConnLabels,
+		receivedFrames: prometheus.NewCounterVec(
+			prometheus.CounterOpts{
+				Namespace: namespace,
+				Subsystem: "client",
+				Name:      "received_frames",
+				Help:      "Number of frames that have been received through a connection",
+			},
+			append(clientConnLabels, "frame_type"),
 		),
-		receiveBytes: prometheus.NewCounterVec(
-			receiveBytesOpts(logging.PerspectiveClient),
+		receivedBytes: prometheus.NewCounterVec(
+			prometheus.CounterOpts{
+				Namespace: namespace,
+				Subsystem: "client",
+				Name:      "receive_bytes",
+				Help:      "Number of bytes that have been received through a connection",
+			},
 			clientConnLabels,
 		),
 		bufferedPackets: prometheus.NewCounterVec(
-			bufferedPacketsOpts(logging.PerspectiveClient),
+			prometheus.CounterOpts{
+				Namespace: namespace,
+				Subsystem: "client",
+				Name:      "buffered_packets",
+				Help:      "Number of bytes that have been buffered on a connection",
+			},
 			append(clientConnLabels, "packet_type"),
 		),
 		droppedPackets: prometheus.NewCounterVec(
-			droppedPacketsOpts(logging.PerspectiveClient),
+			prometheus.CounterOpts{
+				Namespace: namespace,
+				Subsystem: "client",
+				Name:      "dropped_packets",
+				Help:      "Number of bytes that have been dropped on a connection",
+			},
 			append(clientConnLabels, "packet_type", "reason"),
 		),
 		lostPackets: prometheus.NewCounterVec(
-			lostPacketsOpts(logging.PerspectiveClient),
+			prometheus.CounterOpts{
+				Namespace: namespace,
+				Subsystem: "client",
+				Name:      "lost_packets",
+				Help:      "Number of packets that have been lost from a connection",
+			},
 			append(clientConnLabels, "reason"),
 		),
 		minRTT: prometheus.NewGaugeVec(
 			prometheus.GaugeOpts{
 				Namespace: namespace,
-				Subsystem: perspectiveString(logging.PerspectiveClient),
+				Subsystem: "client",
 				Name:      "min_rtt",
 				Help:      "Lowest RTT measured on a connection in millisec",
 			},
@@ -73,7 +120,7 @@ var (
 		latestRTT: prometheus.NewGaugeVec(
 			prometheus.GaugeOpts{
 				Namespace: namespace,
-				Subsystem: perspectiveString(logging.PerspectiveClient),
+				Subsystem: "client",
 				Name:      "latest_rtt",
 				Help:      "Latest RTT measured on a connection",
 			},
@@ -82,188 +129,37 @@ var (
 		smoothedRTT: prometheus.NewGaugeVec(
 			prometheus.GaugeOpts{
 				Namespace: namespace,
-				Subsystem: perspectiveString(logging.PerspectiveClient),
+				Subsystem: "client",
 				Name:      "smoothed_rtt",
 				Help:      "Calculated smoothed RTT measured on a connection in millisec",
 			},
 			clientConnLabels,
 		),
 	}
-	// The server has many QUIC connections. Adding per connection label incurs high memory cost
-	serverMetrics = struct {
-		totalConnections  prometheus.Counter
-		closedConnections prometheus.Counter
-		sentPackets       prometheus.Counter
-		sentBytes         prometheus.Counter
-		receivePackets    prometheus.Counter
-		receiveBytes      prometheus.Counter
-		bufferedPackets   *prometheus.CounterVec
-		droppedPackets    *prometheus.CounterVec
-		lostPackets       *prometheus.CounterVec
-		rtt               prometheus.Histogram
-	}{
-		totalConnections: prometheus.NewCounter(
-			totalConnectionsOpts(logging.PerspectiveServer),
-		),
-		closedConnections: prometheus.NewCounter(
-			closedConnectionsOpts(logging.PerspectiveServer),
-		),
-		sentPackets: prometheus.NewCounter(
-			sentPacketsOpts(logging.PerspectiveServer),
-		),
-		sentBytes: prometheus.NewCounter(
-			sentBytesOpts(logging.PerspectiveServer),
-		),
-		receivePackets: prometheus.NewCounter(
-			receivePacketsOpts(logging.PerspectiveServer),
-		),
-		receiveBytes: prometheus.NewCounter(
-			receiveBytesOpts(logging.PerspectiveServer),
-		),
-		bufferedPackets: prometheus.NewCounterVec(
-			bufferedPacketsOpts(logging.PerspectiveServer),
-			[]string{"packet_type"},
-		),
-		droppedPackets: prometheus.NewCounterVec(
-			droppedPacketsOpts(logging.PerspectiveServer),
-			[]string{"packet_type", "reason"},
-		),
-		lostPackets: prometheus.NewCounterVec(
-			lostPacketsOpts(logging.PerspectiveServer),
-			[]string{"reason"},
-		),
-		rtt: prometheus.NewHistogram(
-			prometheus.HistogramOpts{
-				Namespace: namespace,
-				Subsystem: perspectiveString(logging.PerspectiveServer),
-				Name:      "rtt",
-				Buckets:   []float64{5, 10, 20, 30, 40, 50, 75, 100},
-			},
-		),
-	}
+
 	registerClient = sync.Once{}
-	registerServer = sync.Once{}
 
 	packetTooBigDropped = prometheus.NewCounter(prometheus.CounterOpts{
 		Namespace: namespace,
-		Subsystem: perspectiveString(logging.PerspectiveClient),
+		Subsystem: "client",
 		Name:      "packet_too_big_dropped",
 		Help:      "Count of packets received from origin that are too big to send to the edge and are dropped as a result",
 	})
 )
 
-// MetricsCollector abstracts the difference between client and server metrics from connTracer
-type MetricsCollector interface {
-	startedConnection()
-	closedConnection(err error)
-	sentPackets(logging.ByteCount)
-	receivedPackets(logging.ByteCount)
-	bufferedPackets(logging.PacketType)
-	droppedPackets(logging.PacketType, logging.ByteCount, logging.PacketDropReason)
-	lostPackets(logging.PacketLossReason)
-	updatedRTT(*logging.RTTStats)
-}
-
-func totalConnectionsOpts(p logging.Perspective) prometheus.CounterOpts {
-	var help string
-	if p == logging.PerspectiveClient {
-		help = "Number of connections initiated. For all quic metrics, client means the side initiating the connection"
-	} else {
-		help = "Number of connections accepted. For all quic metrics, server means the side accepting connections"
-	}
-	return prometheus.CounterOpts{
-		Namespace: namespace,
-		Subsystem: perspectiveString(p),
-		Name:      "total_connections",
-		Help:      help,
-	}
-}
-
-func closedConnectionsOpts(p logging.Perspective) prometheus.CounterOpts {
-	return prometheus.CounterOpts{
-		Namespace: namespace,
-		Subsystem: perspectiveString(p),
-		Name:      "closed_connections",
-		Help:      "Number of connections that has been closed",
-	}
-}
-
-func sentPacketsOpts(p logging.Perspective) prometheus.CounterOpts {
-	return prometheus.CounterOpts{
-		Namespace: namespace,
-		Subsystem: perspectiveString(p),
-		Name:      "sent_packets",
-		Help:      "Number of packets that have been sent through a connection",
-	}
-}
-
-func sentBytesOpts(p logging.Perspective) prometheus.CounterOpts {
-	return prometheus.CounterOpts{
-		Namespace: namespace,
-		Subsystem: perspectiveString(p),
-		Name:      "sent_bytes",
-		Help:      "Number of bytes that have been sent through a connection",
-	}
-}
-
-func receivePacketsOpts(p logging.Perspective) prometheus.CounterOpts {
-	return prometheus.CounterOpts{
-		Namespace: namespace,
-		Subsystem: perspectiveString(p),
-		Name:      "receive_packets",
-		Help:      "Number of packets that have been received through a connection",
-	}
-}
-
-func receiveBytesOpts(p logging.Perspective) prometheus.CounterOpts {
-	return prometheus.CounterOpts{
-		Namespace: namespace,
-		Subsystem: perspectiveString(p),
-		Name:      "receive_bytes",
-		Help:      "Number of bytes that have been received through a connection",
-	}
-}
-
-func bufferedPacketsOpts(p logging.Perspective) prometheus.CounterOpts {
-	return prometheus.CounterOpts{
-		Namespace: namespace,
-		Subsystem: perspectiveString(p),
-		Name:      "buffered_packets",
-		Help:      "Number of bytes that have been buffered on a connection",
-	}
-}
-
-func droppedPacketsOpts(p logging.Perspective) prometheus.CounterOpts {
-	return prometheus.CounterOpts{
-		Namespace: namespace,
-		Subsystem: perspectiveString(p),
-		Name:      "dropped_packets",
-		Help:      "Number of bytes that have been dropped on a connection",
-	}
-}
-
-func lostPacketsOpts(p logging.Perspective) prometheus.CounterOpts {
-	return prometheus.CounterOpts{
-		Namespace: namespace,
-		Subsystem: perspectiveString(p),
-		Name:      "lost_packets",
-		Help:      "Number of packets that have been lost from a connection",
-	}
-}
-
 type clientCollector struct {
 	index string
 }
 
-func newClientCollector(index uint8) MetricsCollector {
+func newClientCollector(index uint8) *clientCollector {
 	registerClient.Do(func() {
 		prometheus.MustRegister(
 			clientMetrics.totalConnections,
 			clientMetrics.closedConnections,
-			clientMetrics.sentPackets,
+			clientMetrics.sentFrames,
 			clientMetrics.sentBytes,
-			clientMetrics.receivePackets,
-			clientMetrics.receiveBytes,
+			clientMetrics.receivedFrames,
+			clientMetrics.receivedBytes,
 			clientMetrics.bufferedPackets,
 			clientMetrics.droppedPackets,
 			clientMetrics.lostPackets,
@@ -286,14 +182,12 @@ func (cc *clientCollector) closedConnection(err error) {
 	clientMetrics.closedConnections.Inc()
 }
 
-func (cc *clientCollector) sentPackets(size logging.ByteCount) {
-	clientMetrics.sentPackets.WithLabelValues(cc.index).Inc()
-	clientMetrics.sentBytes.WithLabelValues(cc.index).Add(byteCountToPromCount(size))
+func (cc *clientCollector) sentPackets(size logging.ByteCount, frames []logging.Frame) {
+	cc.collectPackets(size, frames, clientMetrics.sentFrames, clientMetrics.sentBytes)
 }
 
-func (cc *clientCollector) receivedPackets(size logging.ByteCount) {
-	clientMetrics.receivePackets.WithLabelValues(cc.index).Inc()
-	clientMetrics.receiveBytes.WithLabelValues(cc.index).Add(byteCountToPromCount(size))
+func (cc *clientCollector) receivedPackets(size logging.ByteCount, frames []logging.Frame) {
+	cc.collectPackets(size, frames, clientMetrics.receivedFrames, clientMetrics.receivedBytes)
 }
 
 func (cc *clientCollector) bufferedPackets(packetType logging.PacketType) {
@@ -318,63 +212,18 @@ func (cc *clientCollector) updatedRTT(rtt *logging.RTTStats) {
 	clientMetrics.smoothedRTT.WithLabelValues(cc.index).Set(durationToPromGauge(rtt.SmoothedRTT()))
 }
 
-type serverCollector struct{}
-
-func newServiceCollector() MetricsCollector {
-	registerServer.Do(func() {
-		prometheus.MustRegister(
-			serverMetrics.totalConnections,
-			serverMetrics.closedConnections,
-			serverMetrics.sentPackets,
-			serverMetrics.sentBytes,
-			serverMetrics.receivePackets,
-			serverMetrics.receiveBytes,
-			serverMetrics.bufferedPackets,
-			serverMetrics.droppedPackets,
-			serverMetrics.lostPackets,
-			serverMetrics.rtt,
-		)
-	})
-	return &serverCollector{}
+func (cc *clientCollector) collectPackets(size logging.ByteCount, frames []logging.Frame, counter, bandwidth *prometheus.CounterVec) {
+	for _, frame := range frames {
+		counter.WithLabelValues(cc.index, frameName(frame)).Inc()
+	}
+	bandwidth.WithLabelValues(cc.index).Add(byteCountToPromCount(size))
 }
 
-func (sc *serverCollector) startedConnection() {
-	serverMetrics.totalConnections.Inc()
-}
-
-func (sc *serverCollector) closedConnection(err error) {
-	serverMetrics.closedConnections.Inc()
-}
-
-func (sc *serverCollector) sentPackets(size logging.ByteCount) {
-	serverMetrics.sentPackets.Inc()
-	serverMetrics.sentBytes.Add(byteCountToPromCount(size))
-}
-
-func (sc *serverCollector) receivedPackets(size logging.ByteCount) {
-	serverMetrics.receivePackets.Inc()
-	serverMetrics.receiveBytes.Add(byteCountToPromCount(size))
-}
-
-func (sc *serverCollector) bufferedPackets(packetType logging.PacketType) {
-	serverMetrics.bufferedPackets.WithLabelValues(packetTypeString(packetType)).Inc()
-}
-
-func (sc *serverCollector) droppedPackets(packetType logging.PacketType, size logging.ByteCount, reason logging.PacketDropReason) {
-	serverMetrics.droppedPackets.WithLabelValues(
-		packetTypeString(packetType),
-		packetDropReasonString(reason),
-	).Add(byteCountToPromCount(size))
-}
-
-func (sc *serverCollector) lostPackets(reason logging.PacketLossReason) {
-	serverMetrics.lostPackets.WithLabelValues(packetLossReasonString(reason)).Inc()
-}
-
-func (sc *serverCollector) updatedRTT(rtt *logging.RTTStats) {
-	latestRTT := rtt.LatestRTT()
-	// May return 0 if no valid updates have occurred
-	if latestRTT > 0 {
-		serverMetrics.rtt.Observe(durationToPromGauge(latestRTT))
+func frameName(frame logging.Frame) string {
+	if frame == nil {
+		return "nil"
+	} else {
+		name := reflect.TypeOf(frame).Elem().Name()
+		return strings.TrimSuffix(name, "Frame")
 	}
 }

quic/quic_protocol.go

@@ -1,274 +0,0 @@
-package quic
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"net"
-	"time"
-
-	capnp "zombiezen.com/go/capnproto2"
-	"zombiezen.com/go/capnproto2/rpc"
-
-	"github.com/google/uuid"
-	"github.com/rs/zerolog"
-
-	"github.com/cloudflare/cloudflared/tunnelrpc"
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
-)
-
-// ProtocolSignature defines the first 6 bytes of the stream, which is used to distinguish the type of stream. It
-// ensures whoever performs a handshake does not write data before writing the metadata.
-type ProtocolSignature [6]byte
-
-var (
-	// DataStreamProtocolSignature is a custom protocol signature for data stream
-	DataStreamProtocolSignature = ProtocolSignature{0x0A, 0x36, 0xCD, 0x12, 0xA1, 0x3E}
-
-	// RPCStreamProtocolSignature is a custom protocol signature for RPC stream
-	RPCStreamProtocolSignature = ProtocolSignature{0x52, 0xBB, 0x82, 0x5C, 0xDB, 0x65}
-)
-
-type protocolVersion string
-
-const (
-	protocolV1 protocolVersion = "01"
-
-	protocolVersionLength = 2
-
-	HandshakeIdleTimeout = 5 * time.Second
-	MaxIdleTimeout       = 5 * time.Second
-	MaxIdlePingPeriod    = 1 * time.Second
-
-	// MaxIncomingStreams is 2^60, which is the maximum supported value by Quic-Go
-	MaxIncomingStreams = 1 << 60
-)
-
-// RequestServerStream is a stream to serve requests
-type RequestServerStream struct {
-	io.ReadWriteCloser
-}
-
-func NewRequestServerStream(stream io.ReadWriteCloser, signature ProtocolSignature) (*RequestServerStream, error) {
-	if signature != DataStreamProtocolSignature {
-		return nil, fmt.Errorf("RequestClientStream can only be created from data stream")
-	}
-	return &RequestServerStream{stream}, nil
-}
-
-// ReadConnectRequestData reads the handshake data from a QUIC stream.
-func (rss *RequestServerStream) ReadConnectRequestData() (*ConnectRequest, error) {
-	// This is a NO-OP for now. We could cause a branching if we wanted to use multiple versions.
-	if _, err := readVersion(rss); err != nil {
-		return nil, err
-	}
-
-	msg, err := capnp.NewDecoder(rss).Decode()
-	if err != nil {
-		return nil, err
-	}
-
-	r := &ConnectRequest{}
-	if err := r.fromPogs(msg); err != nil {
-		return nil, err
-	}
-	return r, nil
-}
-
-// WriteConnectResponseData writes response to a QUIC stream.
-func (rss *RequestServerStream) WriteConnectResponseData(respErr error, metadata ...Metadata) error {
-	var connectResponse *ConnectResponse
-	if respErr != nil {
-		connectResponse = &ConnectResponse{
-			Error: respErr.Error(),
-		}
-	} else {
-		connectResponse = &ConnectResponse{
-			Metadata: metadata,
-		}
-	}
-
-	msg, err := connectResponse.toPogs()
-	if err != nil {
-		return err
-	}
-
-	if err := writeDataStreamPreamble(rss); err != nil {
-		return err
-	}
-	return capnp.NewEncoder(rss).Encode(msg)
-}
-
-type RequestClientStream struct {
-	io.ReadWriteCloser
-}
-
-// WriteConnectRequestData writes requestMeta to a stream.
-func (rcs *RequestClientStream) WriteConnectRequestData(dest string, connectionType ConnectionType, metadata ...Metadata) error {
-	connectRequest := &ConnectRequest{
-		Dest:     dest,
-		Type:     connectionType,
-		Metadata: metadata,
-	}
-
-	msg, err := connectRequest.toPogs()
-	if err != nil {
-		return err
-	}
-
-	if err := writeDataStreamPreamble(rcs); err != nil {
-		return err
-	}
-	return capnp.NewEncoder(rcs).Encode(msg)
-}
-
-// ReadConnectResponseData reads the response to a RequestMeta in a stream.
-func (rcs *RequestClientStream) ReadConnectResponseData() (*ConnectResponse, error) {
-	signature, err := DetermineProtocol(rcs)
-	if err != nil {
-		return nil, err
-	}
-	if signature != DataStreamProtocolSignature {
-		return nil, fmt.Errorf("wrong protocol signature %v", signature)
-	}
-
-	// This is a NO-OP for now. We could cause a branching if we wanted to use multiple versions.
-	if _, err := readVersion(rcs); err != nil {
-		return nil, err
-	}
-
-	msg, err := capnp.NewDecoder(rcs).Decode()
-	if err != nil {
-		return nil, err
-	}
-
-	r := &ConnectResponse{}
-	if err := r.fromPogs(msg); err != nil {
-		return nil, err
-	}
-	return r, nil
-}
-
-// RPCServerStream is a stream to serve RPCs. It is closed when the RPC client is done
-type RPCServerStream struct {
-	io.ReadWriteCloser
-}
-
-func NewRPCServerStream(stream io.ReadWriteCloser, protocol ProtocolSignature) (*RPCServerStream, error) {
-	if protocol != RPCStreamProtocolSignature {
-		return nil, fmt.Errorf("RPCStream can only be created from rpc stream")
-	}
-	return &RPCServerStream{stream}, nil
-}
-
-func (s *RPCServerStream) Serve(sessionManager tunnelpogs.SessionManager, configManager tunnelpogs.ConfigurationManager, logger *zerolog.Logger) error {
-	// RPC logs are very robust, create a new logger that only logs error to reduce noise
-	rpcLogger := logger.Level(zerolog.ErrorLevel)
-	rpcTransport := tunnelrpc.NewTransportLogger(&rpcLogger, rpc.StreamTransport(s))
-	defer rpcTransport.Close()
-
-	main := tunnelpogs.CloudflaredServer_ServerToClient(sessionManager, configManager)
-	rpcConn := rpc.NewConn(
-		rpcTransport,
-		rpc.MainInterface(main.Client),
-		tunnelrpc.ConnLog(&rpcLogger),
-	)
-	defer rpcConn.Close()
-
-	return rpcConn.Wait()
-}
-
-func DetermineProtocol(stream io.Reader) (ProtocolSignature, error) {
-	signature, err := readSignature(stream)
-	if err != nil {
-		return ProtocolSignature{}, err
-	}
-	switch signature {
-	case DataStreamProtocolSignature:
-		return DataStreamProtocolSignature, nil
-	case RPCStreamProtocolSignature:
-		return RPCStreamProtocolSignature, nil
-	default:
-		return ProtocolSignature{}, fmt.Errorf("unknown signature %v", signature)
-	}
-}
-
-func writeDataStreamPreamble(stream io.Writer) error {
-	if err := writeSignature(stream, DataStreamProtocolSignature); err != nil {
-		return err
-	}
-
-	return writeVersion(stream)
-}
-
-func writeVersion(stream io.Writer) error {
-	_, err := stream.Write([]byte(protocolV1)[:protocolVersionLength])
-	return err
-}
-
-func readVersion(stream io.Reader) (string, error) {
-	version := make([]byte, protocolVersionLength)
-	_, err := stream.Read(version)
-	return string(version), err
-}
-
-func readSignature(stream io.Reader) (ProtocolSignature, error) {
-	var signature ProtocolSignature
-	if _, err := io.ReadFull(stream, signature[:]); err != nil {
-		return ProtocolSignature{}, err
-	}
-	return signature, nil
-}
-
-func writeSignature(stream io.Writer, signature ProtocolSignature) error {
-	_, err := stream.Write(signature[:])
-	return err
-}
-
-// RPCClientStream is a stream to call methods of SessionManager
-type RPCClientStream struct {
-	client    tunnelpogs.CloudflaredServer_PogsClient
-	transport rpc.Transport
-
-	// Time we wait for the server to respond to a request before we close the connection.
-	rpcUnregisterUDPSessionDeadline time.Duration
-}
-
-func NewRPCClientStream(ctx context.Context, stream io.ReadWriteCloser, rpcUnregisterUDPSessionDeadline time.Duration, logger *zerolog.Logger) (*RPCClientStream, error) {
-	n, err := stream.Write(RPCStreamProtocolSignature[:])
-	if err != nil {
-		return nil, err
-	}
-	if n != len(RPCStreamProtocolSignature) {
-		return nil, fmt.Errorf("expect to write %d bytes for RPC stream protocol signature, wrote %d", len(RPCStreamProtocolSignature), n)
-	}
-	transport := tunnelrpc.NewTransportLogger(logger, rpc.StreamTransport(stream))
-	conn := rpc.NewConn(
-		transport,
-		tunnelrpc.ConnLog(logger),
-	)
-	return &RPCClientStream{
-		client:                          tunnelpogs.NewCloudflaredServer_PogsClient(conn.Bootstrap(ctx), conn),
-		transport:                       transport,
-		rpcUnregisterUDPSessionDeadline: rpcUnregisterUDPSessionDeadline,
-	}, nil
-}
-
-func (rcs *RPCClientStream) RegisterUdpSession(ctx context.Context, sessionID uuid.UUID, dstIP net.IP, dstPort uint16, closeIdleAfterHint time.Duration, traceContext string) (*tunnelpogs.RegisterUdpSessionResponse, error) {
-	return rcs.client.RegisterUdpSession(ctx, sessionID, dstIP, dstPort, closeIdleAfterHint, traceContext)
-}
-
-func (rcs *RPCClientStream) UnregisterUdpSession(ctx context.Context, sessionID uuid.UUID, message string) error {
-	ctx, cancel := context.WithTimeout(ctx, rcs.rpcUnregisterUDPSessionDeadline)
-	defer cancel()
-	return rcs.client.UnregisterUdpSession(ctx, sessionID, message)
-}
-
-func (rcs *RPCClientStream) UpdateConfiguration(ctx context.Context, version int32, config []byte) (*tunnelpogs.UpdateConfigurationResponse, error) {
-	return rcs.client.UpdateConfiguration(ctx, version, config)
-}
-
-func (rcs *RPCClientStream) Close() {
-	_ = rcs.client.Close()
-	_ = rcs.transport.Close()
-}

quic/safe_stream.go

@@ -1,20 +1,33 @@
 package quic
 
 import (
+	"errors"
+	"net"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/quic-go/quic-go"
+	"github.com/rs/zerolog"
+	"github.com/rs/zerolog/log"
 )
 
+// The error that is throw by the writer when there is `no network activity`.
+var idleTimeoutError = quic.IdleTimeoutError{}
+
 type SafeStreamCloser struct {
-	lock   sync.Mutex
-	stream quic.Stream
+	lock         sync.Mutex
+	stream       quic.Stream
+	writeTimeout time.Duration
+	log          *zerolog.Logger
+	closing      atomic.Bool
 }
 
-func NewSafeStreamCloser(stream quic.Stream) *SafeStreamCloser {
+func NewSafeStreamCloser(stream quic.Stream, writeTimeout time.Duration, log *zerolog.Logger) *SafeStreamCloser {
 	return &SafeStreamCloser{
-		stream: stream,
+		stream:       stream,
+		writeTimeout: writeTimeout,
+		log:          log,
 	}
 }
 
@@ -25,10 +38,43 @@ func (s *SafeStreamCloser) Read(p []byte) (n int, err error) {
 func (s *SafeStreamCloser) Write(p []byte) (n int, err error) {
 	s.lock.Lock()
 	defer s.lock.Unlock()
-	return s.stream.Write(p)
+	if s.writeTimeout > 0 {
+		err = s.stream.SetWriteDeadline(time.Now().Add(s.writeTimeout))
+		if err != nil {
+			log.Err(err).Msg("Error setting write deadline for QUIC stream")
+		}
+	}
+	nBytes, err := s.stream.Write(p)
+	if err != nil {
+		s.handleWriteError(err)
+	}
+
+	return nBytes, err
+}
+
+// Handles the timeout error in case it happened, by canceling the stream write.
+func (s *SafeStreamCloser) handleWriteError(err error) {
+	// If we are closing the stream we just ignore any write error.
+	if s.closing.Load() {
+		return
+	}
+	var netErr net.Error
+	if errors.As(err, &netErr) {
+		if netErr.Timeout() {
+			// We don't need to log if what cause the timeout was no network activity.
+			if !errors.Is(netErr, &idleTimeoutError) {
+				s.log.Error().Err(netErr).Msg("Closing quic stream due to timeout while writing")
+			}
+			// We need to explicitly cancel the write so that it frees all buffers.
+			s.stream.CancelWrite(0)
+		}
+	}
 }
 
 func (s *SafeStreamCloser) Close() error {
+	// Set this stream to a closing state.
+	s.closing.Store(true)
+
 	// Make sure a possible writer does not block the lock forever. We need it, so we can close the writer
 	// side of the stream safely.
 	_ = s.stream.SetWriteDeadline(time.Now())

quic/safe_stream_test.go

@@ -2,13 +2,20 @@ package quic
 
 import (
 	"context"
+	"crypto/rand"
+	"crypto/rsa"
 	"crypto/tls"
+	"crypto/x509"
+	"encoding/pem"
 	"io"
+	"math/big"
 	"net"
 	"sync"
 	"testing"
 	"time"
 
+	"github.com/rs/zerolog"
+
 	"github.com/quic-go/quic-go"
 	"github.com/stretchr/testify/require"
 )
@@ -70,7 +77,8 @@ func quicClient(t *testing.T, addr net.Addr) {
 		go func(iter int) {
 			defer wg.Done()
 
-			stream := NewSafeStreamCloser(quicStream)
+			log := zerolog.Nop()
+			stream := NewSafeStreamCloser(quicStream, 30*time.Second, &log)
 			defer stream.Close()
 
 			// Do a bunch of round trips over this stream that should work.
@@ -107,7 +115,8 @@ func quicServer(t *testing.T, serverReady *sync.WaitGroup, conn net.PacketConn)
 		go func(iter int) {
 			defer wg.Done()
 
-			stream := NewSafeStreamCloser(quicStream)
+			log := zerolog.Nop()
+			stream := NewSafeStreamCloser(quicStream, 30*time.Second, &log)
 			defer stream.Close()
 
 			// Do a bunch of round trips over this stream that should work.
@@ -143,3 +152,27 @@ func serverRoundTrip(t *testing.T, stream io.ReadWriteCloser, mustWork bool) {
 	}
 	require.NoError(t, err)
 }
+
+// GenerateTLSConfig sets up a bare-bones TLS config for a QUIC server
+func GenerateTLSConfig() *tls.Config {
+	key, err := rsa.GenerateKey(rand.Reader, 1024)
+	if err != nil {
+		panic(err)
+	}
+	template := x509.Certificate{SerialNumber: big.NewInt(1)}
+	certDER, err := x509.CreateCertificate(rand.Reader, &template, &template, &key.PublicKey, key)
+	if err != nil {
+		panic(err)
+	}
+	keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)})
+	certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER})
+
+	tlsCert, err := tls.X509KeyPair(certPEM, keyPEM)
+	if err != nil {
+		panic(err)
+	}
+	return &tls.Config{
+		Certificates: []tls.Certificate{tlsCert},
+		NextProtos:   []string{"argotunnel"},
+	}
+}

quic/schema/quic_metadata_protocol.capnp

@@ -1,28 +0,0 @@
-using Go = import "/go.capnp";
-@0xb29021ef7421cc32;
-
-$Go.package("schema");
-$Go.import("schema");
-
-
-struct ConnectRequest{
-	dest @0 :Text;
-	type @1 :ConnectionType;
-	metadata @2 :List(Metadata);
-}
-
-enum ConnectionType{
-	http @0;
-	websocket @1;
-	tcp @2;
-}
-
-struct Metadata {
-    key @0 :Text;
-	val @1 :Text;
-}
-
-struct ConnectResponse{
-	error @0 :Text;
-	metadata @1 :List(Metadata);
-}

quic/test_utils.go

@@ -1,34 +0,0 @@
-package quic
-
-import (
-	"crypto/rand"
-	"crypto/rsa"
-	"crypto/tls"
-	"crypto/x509"
-	"encoding/pem"
-	"math/big"
-)
-
-// GenerateTLSConfig sets up a bare-bones TLS config for a QUIC server
-func GenerateTLSConfig() *tls.Config {
-	key, err := rsa.GenerateKey(rand.Reader, 1024)
-	if err != nil {
-		panic(err)
-	}
-	template := x509.Certificate{SerialNumber: big.NewInt(1)}
-	certDER, err := x509.CreateCertificate(rand.Reader, &template, &template, &key.PublicKey, key)
-	if err != nil {
-		panic(err)
-	}
-	keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)})
-	certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER})
-
-	tlsCert, err := tls.X509KeyPair(certPEM, keyPEM)
-	if err != nil {
-		panic(err)
-	}
-	return &tls.Config{
-		Certificates: []tls.Certificate{tlsCert},
-		NextProtos:   []string{"argotunnel"},
-	}
-}

quic/tracing.go

@@ -3,7 +3,6 @@ package quic
 import (
 	"context"
 	"net"
-	"time"
 
 	"github.com/quic-go/quic-go/logging"
 	"github.com/rs/zerolog"
@@ -16,8 +15,6 @@ type tracer struct {
 }
 
 type tracerConfig struct {
-	isClient bool
-	// Only client has an index
 	index uint8
 }
 
@@ -25,67 +22,36 @@ func NewClientTracer(logger *zerolog.Logger, index uint8) func(context.Context,
 	t := &tracer{
 		logger: logger,
 		config: &tracerConfig{
-			isClient: true,
-			index:    index,
+			index: index,
 		},
 	}
 	return t.TracerForConnection
 }
 
-func NewServerTracer(logger *zerolog.Logger) *logging.Tracer {
-	return &logging.Tracer{
-		SentPacket:                   func(net.Addr, *logging.Header, logging.ByteCount, []logging.Frame) {},
-		SentVersionNegotiationPacket: func(_ net.Addr, dest, src logging.ArbitraryLenConnectionID, _ []logging.VersionNumber) {},
-		DroppedPacket:                func(net.Addr, logging.PacketType, logging.ByteCount, logging.PacketDropReason) {},
-	}
-}
-
 func (t *tracer) TracerForConnection(_ctx context.Context, _p logging.Perspective, _odcid logging.ConnectionID) *logging.ConnectionTracer {
-	if t.config.isClient {
-		return newConnTracer(newClientCollector(t.config.index))
-	}
-	return newConnTracer(newServiceCollector())
+	return newConnTracer(newClientCollector(t.config.index))
 }
 
 // connTracer collects connection level metrics
 type connTracer struct {
-	metricsCollector MetricsCollector
+	metricsCollector *clientCollector
 }
 
-func newConnTracer(metricsCollector MetricsCollector) *logging.ConnectionTracer {
+func newConnTracer(metricsCollector *clientCollector) *logging.ConnectionTracer {
 	tracer := connTracer{
 		metricsCollector: metricsCollector,
 	}
 	return &logging.ConnectionTracer{
-		StartedConnection:                tracer.StartedConnection,
-		NegotiatedVersion:                tracer.NegotiatedVersion,
-		ClosedConnection:                 tracer.ClosedConnection,
-		SentTransportParameters:          tracer.SentTransportParameters,
-		ReceivedTransportParameters:      tracer.ReceivedTransportParameters,
-		RestoredTransportParameters:      tracer.RestoredTransportParameters,
-		SentLongHeaderPacket:             tracer.SentLongHeaderPacket,
-		SentShortHeaderPacket:            tracer.SentShortHeaderPacket,
-		ReceivedVersionNegotiationPacket: tracer.ReceivedVersionNegotiationPacket,
-		ReceivedRetry:                    tracer.ReceivedRetry,
-		ReceivedLongHeaderPacket:         tracer.ReceivedLongHeaderPacket,
-		ReceivedShortHeaderPacket:        tracer.ReceivedShortHeaderPacket,
-		BufferedPacket:                   tracer.BufferedPacket,
-		DroppedPacket:                    tracer.DroppedPacket,
-		UpdatedMetrics:                   tracer.UpdatedMetrics,
-		AcknowledgedPacket:               tracer.AcknowledgedPacket,
-		LostPacket:                       tracer.LostPacket,
-		UpdatedCongestionState:           tracer.UpdatedCongestionState,
-		UpdatedPTOCount:                  tracer.UpdatedPTOCount,
-		UpdatedKeyFromTLS:                tracer.UpdatedKeyFromTLS,
-		UpdatedKey:                       tracer.UpdatedKey,
-		DroppedEncryptionLevel:           tracer.DroppedEncryptionLevel,
-		DroppedKey:                       tracer.DroppedKey,
-		SetLossTimer:                     tracer.SetLossTimer,
-		LossTimerExpired:                 tracer.LossTimerExpired,
-		LossTimerCanceled:                tracer.LossTimerCanceled,
-		ECNStateUpdated:                  tracer.ECNStateUpdated,
-		Close:                            tracer.Close,
-		Debug:                            tracer.Debug,
+		StartedConnection:         tracer.StartedConnection,
+		ClosedConnection:          tracer.ClosedConnection,
+		SentLongHeaderPacket:      tracer.SentLongHeaderPacket,
+		SentShortHeaderPacket:     tracer.SentShortHeaderPacket,
+		ReceivedLongHeaderPacket:  tracer.ReceivedLongHeaderPacket,
+		ReceivedShortHeaderPacket: tracer.ReceivedShortHeaderPacket,
+		BufferedPacket:            tracer.BufferedPacket,
+		DroppedPacket:             tracer.DroppedPacket,
+		UpdatedMetrics:            tracer.UpdatedMetrics,
+		LostPacket:                tracer.LostPacket,
 	}
 }
 
@@ -97,14 +63,6 @@ func (ct *connTracer) ClosedConnection(err error) {
 	ct.metricsCollector.closedConnection(err)
 }
 
-func (ct *connTracer) SentPacket(hdr *logging.ExtendedHeader, packetSize logging.ByteCount, ack *logging.AckFrame, frames []logging.Frame) {
-	ct.metricsCollector.sentPackets(packetSize)
-}
-
-func (ct *connTracer) ReceivedPacket(hdr *logging.ExtendedHeader, size logging.ByteCount, frames []logging.Frame) {
-	ct.metricsCollector.receivedPackets(size)
-}
-
 func (ct *connTracer) BufferedPacket(pt logging.PacketType, size logging.ByteCount) {
 	ct.metricsCollector.bufferedPackets(pt)
 }
@@ -121,74 +79,20 @@ func (ct *connTracer) UpdatedMetrics(rttStats *logging.RTTStats, cwnd, bytesInFl
 	ct.metricsCollector.updatedRTT(rttStats)
 }
 
-func (ct *connTracer) NegotiatedVersion(chosen logging.VersionNumber, clientVersions, serverVersions []logging.VersionNumber) {
-}
-
-func (ct *connTracer) SentTransportParameters(parameters *logging.TransportParameters) {
-}
-
-func (ct *connTracer) ReceivedTransportParameters(parameters *logging.TransportParameters) {
-}
-
-func (ct *connTracer) RestoredTransportParameters(parameters *logging.TransportParameters) {
-}
-
 func (ct *connTracer) SentLongHeaderPacket(hdr *logging.ExtendedHeader, size logging.ByteCount, ecn logging.ECN, ack *logging.AckFrame, frames []logging.Frame) {
+	ct.metricsCollector.sentPackets(size, frames)
 }
 
 func (ct *connTracer) SentShortHeaderPacket(hdr *logging.ShortHeader, size logging.ByteCount, ecn logging.ECN, ack *logging.AckFrame, frames []logging.Frame) {
-}
-
-func (ct *connTracer) ReceivedVersionNegotiationPacket(dest, src logging.ArbitraryLenConnectionID, _ []logging.VersionNumber) {
-}
-
-func (ct *connTracer) ReceivedRetry(header *logging.Header) {
+	ct.metricsCollector.sentPackets(size, frames)
 }
 
 func (ct *connTracer) ReceivedLongHeaderPacket(hdr *logging.ExtendedHeader, size logging.ByteCount, ecn logging.ECN, frames []logging.Frame) {
+	ct.metricsCollector.receivedPackets(size, frames)
 }
 
 func (ct *connTracer) ReceivedShortHeaderPacket(hdr *logging.ShortHeader, size logging.ByteCount, ecn logging.ECN, frames []logging.Frame) {
-}
-
-func (ct *connTracer) AcknowledgedPacket(level logging.EncryptionLevel, number logging.PacketNumber) {
-}
-
-func (ct *connTracer) UpdatedCongestionState(state logging.CongestionState) {
-}
-
-func (ct *connTracer) UpdatedPTOCount(value uint32) {
-}
-
-func (ct *connTracer) UpdatedKeyFromTLS(level logging.EncryptionLevel, perspective logging.Perspective) {
-}
-
-func (ct *connTracer) UpdatedKey(generation logging.KeyPhase, remote bool) {
-}
-
-func (ct *connTracer) DroppedEncryptionLevel(level logging.EncryptionLevel) {
-}
-
-func (ct *connTracer) DroppedKey(generation logging.KeyPhase) {
-}
-
-func (ct *connTracer) SetLossTimer(timerType logging.TimerType, level logging.EncryptionLevel, time time.Time) {
-}
-
-func (ct *connTracer) LossTimerExpired(timerType logging.TimerType, level logging.EncryptionLevel) {
-}
-
-func (ct *connTracer) LossTimerCanceled() {
-}
-
-func (ct *connTracer) ECNStateUpdated(state logging.ECNState, trigger logging.ECNStateTrigger) {
-
-}
-
-func (ct *connTracer) Close() {
-}
-
-func (ct *connTracer) Debug(name, msg string) {
+	ct.metricsCollector.receivedPackets(size, frames)
 }
 
 type quicLogger struct {

retry/backoffhandler.go

@@ -6,17 +6,16 @@ import (
 	"time"
 )
 
+const (
+	DefaultBaseTime time.Duration = time.Second
+)
+
 // Redeclare time functions so they can be overridden in tests.
-type clock struct {
+type Clock struct {
 	Now   func() time.Time
 	After func(d time.Duration) <-chan time.Time
 }
 
-var Clock = clock{
-	Now:   time.Now,
-	After: time.After,
-}
-
 // BackoffHandler manages exponential backoff and limits the maximum number of retries.
 // The base time period is 1 second, doubling with each retry.
 // After initial success, a grace period can be set to reset the backoff timer if
@@ -25,15 +24,26 @@ var Clock = clock{
 type BackoffHandler struct {
 	// MaxRetries sets the maximum number of retries to perform. The default value
 	// of 0 disables retry completely.
-	MaxRetries uint
+	maxRetries uint
 	// RetryForever caps the exponential backoff period according to MaxRetries
 	// but allows you to retry indefinitely.
-	RetryForever bool
+	retryForever bool
 	// BaseTime sets the initial backoff period.
-	BaseTime time.Duration
+	baseTime time.Duration
 
 	retries       uint
 	resetDeadline time.Time
+
+	Clock Clock
+}
+
+func NewBackoff(maxRetries uint, baseTime time.Duration, retryForever bool) BackoffHandler {
+	return BackoffHandler{
+		maxRetries:   maxRetries,
+		baseTime:     baseTime,
+		retryForever: retryForever,
+		Clock:        Clock{Now: time.Now, After: time.After},
+	}
 }
 
 func (b BackoffHandler) GetMaxBackoffDuration(ctx context.Context) (time.Duration, bool) {
@@ -44,11 +54,11 @@ func (b BackoffHandler) GetMaxBackoffDuration(ctx context.Context) (time.Duratio
 		return time.Duration(0), false
 	default:
 	}
-	if !b.resetDeadline.IsZero() && Clock.Now().After(b.resetDeadline) {
+	if !b.resetDeadline.IsZero() && b.Clock.Now().After(b.resetDeadline) {
 		// b.retries would be set to 0 at this point
 		return time.Second, true
 	}
-	if b.retries >= b.MaxRetries && !b.RetryForever {
+	if b.retries >= b.maxRetries && !b.retryForever {
 		return time.Duration(0), false
 	}
 	maxTimeToWait := b.GetBaseTime() * 1 << (b.retries + 1)
@@ -58,12 +68,12 @@ func (b BackoffHandler) GetMaxBackoffDuration(ctx context.Context) (time.Duratio
 // BackoffTimer returns a channel that sends the current time when the exponential backoff timeout expires.
 // Returns nil if the maximum number of retries have been used.
 func (b *BackoffHandler) BackoffTimer() <-chan time.Time {
-	if !b.resetDeadline.IsZero() && Clock.Now().After(b.resetDeadline) {
+	if !b.resetDeadline.IsZero() && b.Clock.Now().After(b.resetDeadline) {
 		b.retries = 0
 		b.resetDeadline = time.Time{}
 	}
-	if b.retries >= b.MaxRetries {
-		if !b.RetryForever {
+	if b.retries >= b.maxRetries {
+		if !b.retryForever {
 			return nil
 		}
 	} else {
@@ -71,7 +81,7 @@ func (b *BackoffHandler) BackoffTimer() <-chan time.Time {
 	}
 	maxTimeToWait := time.Duration(b.GetBaseTime() * 1 << (b.retries))
 	timeToWait := time.Duration(rand.Int63n(maxTimeToWait.Nanoseconds()))
-	return Clock.After(timeToWait)
+	return b.Clock.After(timeToWait)
 }
 
 // Backoff is used to wait according to exponential backoff. Returns false if the
@@ -94,16 +104,16 @@ func (b *BackoffHandler) Backoff(ctx context.Context) bool {
 func (b *BackoffHandler) SetGracePeriod() time.Duration {
 	maxTimeToWait := b.GetBaseTime() * 2 << (b.retries + 1)
 	timeToWait := time.Duration(rand.Int63n(maxTimeToWait.Nanoseconds()))
-	b.resetDeadline = Clock.Now().Add(timeToWait)
+	b.resetDeadline = b.Clock.Now().Add(timeToWait)
 
 	return timeToWait
 }
 
 func (b BackoffHandler) GetBaseTime() time.Duration {
-	if b.BaseTime == 0 {
-		return time.Second
+	if b.baseTime == 0 {
+		return DefaultBaseTime
 	}
-	return b.BaseTime
+	return b.baseTime
 }
 
 // Retries returns the number of retries consumed so far.
@@ -112,9 +122,10 @@ func (b *BackoffHandler) Retries() int {
 }
 
 func (b *BackoffHandler) ReachedMaxRetries() bool {
-	return b.retries == b.MaxRetries
+	return b.retries == b.maxRetries
 }
 
 func (b *BackoffHandler) ResetNow() {
-	b.resetDeadline = time.Now()
+	b.resetDeadline = b.Clock.Now()
+	b.retries = 0
 }

retry/backoffhandler_test.go

@@ -13,10 +13,9 @@ func immediateTimeAfter(time.Duration) <-chan time.Time {
 }
 
 func TestBackoffRetries(t *testing.T) {
-	// make backoff return immediately
-	Clock.After = immediateTimeAfter
 	ctx := context.Background()
-	backoff := BackoffHandler{MaxRetries: 3}
+	// make backoff return immediately
+	backoff := BackoffHandler{maxRetries: 3, Clock: Clock{time.Now, immediateTimeAfter}}
 	if !backoff.Backoff(ctx) {
 		t.Fatalf("backoff failed immediately")
 	}
@@ -32,10 +31,10 @@ func TestBackoffRetries(t *testing.T) {
 }
 
 func TestBackoffCancel(t *testing.T) {
-	// prevent backoff from returning normally
-	Clock.After = func(time.Duration) <-chan time.Time { return make(chan time.Time) }
 	ctx, cancelFunc := context.WithCancel(context.Background())
-	backoff := BackoffHandler{MaxRetries: 3}
+	// prevent backoff from returning normally
+	after := func(time.Duration) <-chan time.Time { return make(chan time.Time) }
+	backoff := BackoffHandler{maxRetries: 3, Clock: Clock{time.Now, after}}
 	cancelFunc()
 	if backoff.Backoff(ctx) {
 		t.Fatalf("backoff allowed after cancel")
@@ -46,13 +45,12 @@ func TestBackoffCancel(t *testing.T) {
 }
 
 func TestBackoffGracePeriod(t *testing.T) {
+	ctx := context.Background()
 	currentTime := time.Now()
 	// make Clock.Now return whatever we like
-	Clock.Now = func() time.Time { return currentTime }
+	now := func() time.Time { return currentTime }
 	// make backoff return immediately
-	Clock.After = immediateTimeAfter
-	ctx := context.Background()
-	backoff := BackoffHandler{MaxRetries: 1}
+	backoff := BackoffHandler{maxRetries: 1, Clock: Clock{now, immediateTimeAfter}}
 	if !backoff.Backoff(ctx) {
 		t.Fatalf("backoff failed immediately")
 	}
@@ -70,10 +68,9 @@ func TestBackoffGracePeriod(t *testing.T) {
 }
 
 func TestGetMaxBackoffDurationRetries(t *testing.T) {
-	// make backoff return immediately
-	Clock.After = immediateTimeAfter
 	ctx := context.Background()
-	backoff := BackoffHandler{MaxRetries: 3}
+	// make backoff return immediately
+	backoff := BackoffHandler{maxRetries: 3, Clock: Clock{time.Now, immediateTimeAfter}}
 	if _, ok := backoff.GetMaxBackoffDuration(ctx); !ok {
 		t.Fatalf("backoff failed immediately")
 	}
@@ -95,10 +92,9 @@ func TestGetMaxBackoffDurationRetries(t *testing.T) {
 }
 
 func TestGetMaxBackoffDuration(t *testing.T) {
-	// make backoff return immediately
-	Clock.After = immediateTimeAfter
 	ctx := context.Background()
-	backoff := BackoffHandler{MaxRetries: 3}
+	// make backoff return immediately
+	backoff := BackoffHandler{maxRetries: 3, Clock: Clock{time.Now, immediateTimeAfter}}
 	if duration, ok := backoff.GetMaxBackoffDuration(ctx); !ok || duration > time.Second*2 {
 		t.Fatalf("backoff (%s) didn't return < 2 seconds on first retry", duration)
 	}
@@ -117,10 +113,9 @@ func TestGetMaxBackoffDuration(t *testing.T) {
 }
 
 func TestBackoffRetryForever(t *testing.T) {
-	// make backoff return immediately
-	Clock.After = immediateTimeAfter
 	ctx := context.Background()
-	backoff := BackoffHandler{MaxRetries: 3, RetryForever: true}
+	// make backoff return immediately
+	backoff := BackoffHandler{maxRetries: 3, retryForever: true, Clock: Clock{time.Now, immediateTimeAfter}}
 	if duration, ok := backoff.GetMaxBackoffDuration(ctx); !ok || duration > time.Second*2 {
 		t.Fatalf("backoff (%s) didn't return < 2 seconds on first retry", duration)
 	}

sshgen/sshgen.go

@@ -15,7 +15,8 @@ import (
 	"os"
 	"time"
 
-	"github.com/go-jose/go-jose/v3/jwt"
+	"github.com/go-jose/go-jose/v4"
+	"github.com/go-jose/go-jose/v4/jwt"
 	homedir "github.com/mitchellh/go-homedir"
 	"github.com/pkg/errors"
 	gossh "golang.org/x/crypto/ssh"
@@ -51,6 +52,8 @@ type errorResponse struct {
 
 var mockRequest func(url, contentType string, body io.Reader) (*http.Response, error) = nil
 
+var signatureAlgs = []jose.SignatureAlgorithm{jose.RS256}
+
 // GenerateShortLivedCertificate generates and stores a keypair for short lived certs
 func GenerateShortLivedCertificate(appURL *url.URL, token string) error {
 	fullName, err := cfpath.GenerateSSHCertFilePathFromURL(appURL, keyName)
@@ -87,7 +90,7 @@ func SignCert(token, pubKey string) (string, error) {
 		return "", errors.New("invalid token")
 	}
 
-	parsedToken, err := jwt.ParseSigned(token)
+	parsedToken, err := jwt.ParseSigned(token, signatureAlgs)
 	if err != nil {
 		return "", errors.Wrap(err, "failed to parse JWT")
 	}

sshgen/sshgen_test.go

@@ -3,6 +3,8 @@
 package sshgen
 
 import (
+	"crypto/rand"
+	"crypto/rsa"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -14,8 +16,8 @@ import (
 	"testing"
 	"time"
 
-	"github.com/go-jose/go-jose/v3"
-	"github.com/go-jose/go-jose/v3/jwt"
+	"github.com/go-jose/go-jose/v4"
+	"github.com/go-jose/go-jose/v4/jwt"
 	"github.com/stretchr/testify/assert"
 
 	"github.com/cloudflare/cloudflared/config"
@@ -103,13 +105,16 @@ func tokenGenerator() string {
 		Expiry:   jwt.NewNumericDate(exp),
 	}
 
-	key := []byte("secret")
-	signer, err := jose.NewSigner(jose.SigningKey{Algorithm: jose.HS256, Key: key}, (&jose.SignerOptions{}).WithType("JWT"))
+	key, err := rsa.GenerateKey(rand.Reader, 4096)
+	if err != nil {
+		panic(err)
+	}
+	signer, err := jose.NewSigner(jose.SigningKey{Algorithm: jose.RS256, Key: key}, (&jose.SignerOptions{}).WithType("JWT"))
 	if err != nil {
 		panic(err)
 	}
 
-	signedToken, err := jwt.Signed(signer).Claims(claims).CompactSerialize()
+	signedToken, err := jwt.Signed(signer).Claims(claims).Serialize()
 	if err != nil {
 		panic(err)
 	}

supervisor/reconnect.go

@@ -1,138 +0,0 @@
-package supervisor
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"sync"
-	"time"
-
-	"github.com/prometheus/client_golang/prometheus"
-
-	"github.com/cloudflare/cloudflared/retry"
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
-)
-
-var (
-	errJWTUnset = errors.New("JWT unset")
-)
-
-// reconnectTunnelCredentialManager is invoked by functions in tunnel.go to
-// get/set parameters for ReconnectTunnel RPC calls.
-type reconnectCredentialManager struct {
-	mu          sync.RWMutex
-	jwt         []byte
-	eventDigest map[uint8][]byte
-	connDigest  map[uint8][]byte
-	authSuccess prometheus.Counter
-	authFail    *prometheus.CounterVec
-}
-
-func newReconnectCredentialManager(namespace, subsystem string, haConnections int) *reconnectCredentialManager {
-	authSuccess := prometheus.NewCounter(
-		prometheus.CounterOpts{
-			Namespace: namespace,
-			Subsystem: subsystem,
-			Name:      "tunnel_authenticate_success",
-			Help:      "Count of successful tunnel authenticate",
-		},
-	)
-	authFail := prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Namespace: namespace,
-			Subsystem: subsystem,
-			Name:      "tunnel_authenticate_fail",
-			Help:      "Count of tunnel authenticate errors by type",
-		},
-		[]string{"error"},
-	)
-	prometheus.MustRegister(authSuccess, authFail)
-	return &reconnectCredentialManager{
-		eventDigest: make(map[uint8][]byte, haConnections),
-		connDigest:  make(map[uint8][]byte, haConnections),
-		authSuccess: authSuccess,
-		authFail:    authFail,
-	}
-}
-
-func (cm *reconnectCredentialManager) ReconnectToken() ([]byte, error) {
-	cm.mu.RLock()
-	defer cm.mu.RUnlock()
-	if cm.jwt == nil {
-		return nil, errJWTUnset
-	}
-	return cm.jwt, nil
-}
-
-func (cm *reconnectCredentialManager) SetReconnectToken(jwt []byte) {
-	cm.mu.Lock()
-	defer cm.mu.Unlock()
-	cm.jwt = jwt
-}
-
-func (cm *reconnectCredentialManager) EventDigest(connID uint8) ([]byte, error) {
-	cm.mu.RLock()
-	defer cm.mu.RUnlock()
-	digest, ok := cm.eventDigest[connID]
-	if !ok {
-		return nil, fmt.Errorf("no event digest for connection %v", connID)
-	}
-	return digest, nil
-}
-
-func (cm *reconnectCredentialManager) SetEventDigest(connID uint8, digest []byte) {
-	cm.mu.Lock()
-	defer cm.mu.Unlock()
-	cm.eventDigest[connID] = digest
-}
-
-func (cm *reconnectCredentialManager) ConnDigest(connID uint8) ([]byte, error) {
-	cm.mu.RLock()
-	defer cm.mu.RUnlock()
-	digest, ok := cm.connDigest[connID]
-	if !ok {
-		return nil, fmt.Errorf("no connection digest for connection %v", connID)
-	}
-	return digest, nil
-}
-
-func (cm *reconnectCredentialManager) SetConnDigest(connID uint8, digest []byte) {
-	cm.mu.Lock()
-	defer cm.mu.Unlock()
-	cm.connDigest[connID] = digest
-}
-
-func (cm *reconnectCredentialManager) RefreshAuth(
-	ctx context.Context,
-	backoff *retry.BackoffHandler,
-	authenticate func(ctx context.Context, numPreviousAttempts int) (tunnelpogs.AuthOutcome, error),
-) (retryTimer <-chan time.Time, err error) {
-	authOutcome, err := authenticate(ctx, backoff.Retries())
-	if err != nil {
-		cm.authFail.WithLabelValues(err.Error()).Inc()
-		if _, ok := backoff.GetMaxBackoffDuration(ctx); ok {
-			return backoff.BackoffTimer(), nil
-		}
-		return nil, err
-	}
-	// clear backoff timer
-	backoff.SetGracePeriod()
-
-	switch outcome := authOutcome.(type) {
-	case tunnelpogs.AuthSuccess:
-		cm.SetReconnectToken(outcome.JWT())
-		cm.authSuccess.Inc()
-		return retry.Clock.After(outcome.RefreshAfter()), nil
-	case tunnelpogs.AuthUnknown:
-		duration := outcome.RefreshAfter()
-		cm.authFail.WithLabelValues(outcome.Error()).Inc()
-		return retry.Clock.After(duration), nil
-	case tunnelpogs.AuthFail:
-		cm.authFail.WithLabelValues(outcome.Error()).Inc()
-		return nil, outcome
-	default:
-		err := fmt.Errorf("refresh_auth: Unexpected outcome type %T", authOutcome)
-		cm.authFail.WithLabelValues(err.Error()).Inc()
-		return nil, err
-	}
-}

supervisor/reconnect_test.go

@@ -1,120 +0,0 @@
-package supervisor
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-
-	"github.com/cloudflare/cloudflared/retry"
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
-)
-
-func TestRefreshAuthBackoff(t *testing.T) {
-	rcm := newReconnectCredentialManager(t.Name(), t.Name(), 4)
-
-	var wait time.Duration
-	retry.Clock.After = func(d time.Duration) <-chan time.Time {
-		wait = d
-		return time.After(d)
-	}
-	backoff := &retry.BackoffHandler{MaxRetries: 3}
-	auth := func(ctx context.Context, n int) (tunnelpogs.AuthOutcome, error) {
-		return nil, fmt.Errorf("authentication failure")
-	}
-
-	// authentication failures should consume the backoff
-	for i := uint(0); i < backoff.MaxRetries; i++ {
-		retryChan, err := rcm.RefreshAuth(context.Background(), backoff, auth)
-		require.NoError(t, err)
-		require.NotNil(t, retryChan)
-		require.Greater(t, wait.Seconds(), 0.0)
-		require.Less(t, wait.Seconds(), float64((1<<(i+1))*time.Second))
-	}
-	retryChan, err := rcm.RefreshAuth(context.Background(), backoff, auth)
-	require.Error(t, err)
-	require.Nil(t, retryChan)
-
-	// now we actually make contact with the remote server
-	_, _ = rcm.RefreshAuth(context.Background(), backoff, func(ctx context.Context, n int) (tunnelpogs.AuthOutcome, error) {
-		return tunnelpogs.NewAuthUnknown(errors.New("auth unknown"), 19), nil
-	})
-
-	// The backoff timer should have been reset. To confirm this, make timeNow
-	// return a value after the backoff timer's grace period
-	retry.Clock.Now = func() time.Time {
-		expectedGracePeriod := time.Duration(time.Second * 2 << backoff.MaxRetries)
-		return time.Now().Add(expectedGracePeriod * 2)
-	}
-	_, ok := backoff.GetMaxBackoffDuration(context.Background())
-	require.True(t, ok)
-}
-
-func TestRefreshAuthSuccess(t *testing.T) {
-	rcm := newReconnectCredentialManager(t.Name(), t.Name(), 4)
-
-	var wait time.Duration
-	retry.Clock.After = func(d time.Duration) <-chan time.Time {
-		wait = d
-		return time.After(d)
-	}
-
-	backoff := &retry.BackoffHandler{MaxRetries: 3}
-	auth := func(ctx context.Context, n int) (tunnelpogs.AuthOutcome, error) {
-		return tunnelpogs.NewAuthSuccess([]byte("jwt"), 19), nil
-	}
-
-	retryChan, err := rcm.RefreshAuth(context.Background(), backoff, auth)
-	assert.NoError(t, err)
-	assert.NotNil(t, retryChan)
-	assert.Equal(t, 19*time.Hour, wait)
-
-	token, err := rcm.ReconnectToken()
-	assert.NoError(t, err)
-	assert.Equal(t, []byte("jwt"), token)
-}
-
-func TestRefreshAuthUnknown(t *testing.T) {
-	rcm := newReconnectCredentialManager(t.Name(), t.Name(), 4)
-
-	var wait time.Duration
-	retry.Clock.After = func(d time.Duration) <-chan time.Time {
-		wait = d
-		return time.After(d)
-	}
-
-	backoff := &retry.BackoffHandler{MaxRetries: 3}
-	auth := func(ctx context.Context, n int) (tunnelpogs.AuthOutcome, error) {
-		return tunnelpogs.NewAuthUnknown(errors.New("auth unknown"), 19), nil
-	}
-
-	retryChan, err := rcm.RefreshAuth(context.Background(), backoff, auth)
-	assert.NoError(t, err)
-	assert.NotNil(t, retryChan)
-	assert.Equal(t, 19*time.Hour, wait)
-
-	token, err := rcm.ReconnectToken()
-	assert.Equal(t, errJWTUnset, err)
-	assert.Nil(t, token)
-}
-
-func TestRefreshAuthFail(t *testing.T) {
-	rcm := newReconnectCredentialManager(t.Name(), t.Name(), 4)
-
-	backoff := &retry.BackoffHandler{MaxRetries: 3}
-	auth := func(ctx context.Context, n int) (tunnelpogs.AuthOutcome, error) {
-		return tunnelpogs.NewAuthFail(errors.New("auth fail")), nil
-	}
-
-	retryChan, err := rcm.RefreshAuth(context.Background(), backoff, auth)
-	assert.Error(t, err)
-	assert.Nil(t, retryChan)
-
-	token, err := rcm.ReconnectToken()
-	assert.Equal(t, errJWTUnset, err)
-	assert.Nil(t, token)
-}

supervisor/supervisor.go

@@ -49,8 +49,6 @@ type Supervisor struct {
 	log          *ConnAwareLogger
 	logTransport *zerolog.Logger
 
-	reconnectCredentialManager *reconnectCredentialManager
-
 	reconnectCh       chan ReconnectSignal
 	gracefulShutdownC <-chan struct{}
 }
@@ -76,8 +74,6 @@ func NewSupervisor(config *TunnelConfig, orchestrator *orchestration.Orchestrato
 		return nil, err
 	}
 
-	reconnectCredentialManager := newReconnectCredentialManager(connection.MetricsNamespace, connection.TunnelSubsystem, config.HAConnections)
-
 	tracker := tunnelstate.NewConnTracker(config.Log)
 	log := NewConnAwareLogger(config.Log, tracker, config.Observer)
 
@@ -87,7 +83,6 @@ func NewSupervisor(config *TunnelConfig, orchestrator *orchestration.Orchestrato
 	edgeTunnelServer := EdgeTunnelServer{
 		config:            config,
 		orchestrator:      orchestrator,
-		credentialManager: reconnectCredentialManager,
 		edgeAddrs:         edgeIPs,
 		edgeAddrHandler:   edgeAddrHandler,
 		edgeBindAddr:      edgeBindAddr,
@@ -98,18 +93,17 @@ func NewSupervisor(config *TunnelConfig, orchestrator *orchestration.Orchestrato
 	}
 
 	return &Supervisor{
-		config:                     config,
-		orchestrator:               orchestrator,
-		edgeIPs:                    edgeIPs,
-		edgeTunnelServer:           &edgeTunnelServer,
-		tunnelErrors:               make(chan tunnelError),
-		tunnelsConnecting:          map[int]chan struct{}{},
-		tunnelsProtocolFallback:    map[int]*protocolFallback{},
-		log:                        log,
-		logTransport:               config.LogTransport,
-		reconnectCredentialManager: reconnectCredentialManager,
-		reconnectCh:                reconnectCh,
-		gracefulShutdownC:          gracefulShutdownC,
+		config:                  config,
+		orchestrator:            orchestrator,
+		edgeIPs:                 edgeIPs,
+		edgeTunnelServer:        &edgeTunnelServer,
+		tunnelErrors:            make(chan tunnelError),
+		tunnelsConnecting:       map[int]chan struct{}{},
+		tunnelsProtocolFallback: map[int]*protocolFallback{},
+		log:                     log,
+		logTransport:            config.LogTransport,
+		reconnectCh:             reconnectCh,
+		gracefulShutdownC:       gracefulShutdownC,
 	}, nil
 }
 
@@ -138,7 +132,7 @@ func (s *Supervisor) Run(
 	var tunnelsWaiting []int
 	tunnelsActive := s.config.HAConnections
 
-	backoff := retry.BackoffHandler{MaxRetries: s.config.Retries, BaseTime: tunnelRetryDuration, RetryForever: true}
+	backoff := retry.NewBackoff(s.config.Retries, tunnelRetryDuration, true)
 	var backoffTimer <-chan time.Time
 
 	shuttingDown := false
@@ -212,7 +206,7 @@ func (s *Supervisor) initialize(
 		s.config.HAConnections = availableAddrs
 	}
 	s.tunnelsProtocolFallback[0] = &protocolFallback{
-		retry.BackoffHandler{MaxRetries: s.config.Retries, RetryForever: true},
+		retry.NewBackoff(s.config.Retries, retry.DefaultBaseTime, true),
 		s.config.ProtocolSelector.Current(),
 		false,
 	}
@@ -234,7 +228,7 @@ func (s *Supervisor) initialize(
 	// At least one successful connection, so start the rest
 	for i := 1; i < s.config.HAConnections; i++ {
 		s.tunnelsProtocolFallback[i] = &protocolFallback{
-			retry.BackoffHandler{MaxRetries: s.config.Retries, RetryForever: true},
+			retry.NewBackoff(s.config.Retries, retry.DefaultBaseTime, true),
 			// Set the protocol we know the first tunnel connected with.
 			s.tunnelsProtocolFallback[0].protocol,
 			false,

supervisor/tunnel.go

@@ -10,7 +10,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/google/uuid"
 	"github.com/pkg/errors"
 	"github.com/quic-go/quic-go"
 	"github.com/rs/zerolog"
@@ -27,8 +26,7 @@ import (
 	quicpogs "github.com/cloudflare/cloudflared/quic"
 	"github.com/cloudflare/cloudflared/retry"
 	"github.com/cloudflare/cloudflared/signal"
-	"github.com/cloudflare/cloudflared/tunnelrpc"
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 	"github.com/cloudflare/cloudflared/tunnelstate"
 )
 
@@ -49,7 +47,7 @@ type TunnelConfig struct {
 	HAConnections      int
 	IsAutoupdated      bool
 	LBPool             string
-	Tags               []tunnelpogs.Tag
+	Tags               []pogs.Tag
 	Log                *zerolog.Logger
 	LogTransport       *zerolog.Logger
 	Observer           *connection.Observer
@@ -60,46 +58,25 @@ type TunnelConfig struct {
 
 	NeedPQ bool
 
-	NamedTunnel      *connection.NamedTunnelProperties
+	NamedTunnel      *connection.TunnelProperties
 	ProtocolSelector connection.ProtocolSelector
 	EdgeTLSConfigs   map[connection.Protocol]*tls.Config
 	PacketConfig     *ingress.GlobalRouterConfig
 
-	UDPUnregisterSessionTimeout time.Duration
+	RPCTimeout         time.Duration
+	WriteStreamTimeout time.Duration
 
 	DisableQUICPathMTUDiscovery bool
 
 	FeatureSelector *features.FeatureSelector
 }
 
-func (c *TunnelConfig) registrationOptions(connectionID uint8, OriginLocalIP string, uuid uuid.UUID) *tunnelpogs.RegistrationOptions {
-	policy := tunnelrpc.ExistingTunnelPolicy_balance
-	if c.HAConnections <= 1 && c.LBPool == "" {
-		policy = tunnelrpc.ExistingTunnelPolicy_disconnect
-	}
-	return &tunnelpogs.RegistrationOptions{
-		ClientID:             c.ClientID,
-		Version:              c.ReportedVersion,
-		OS:                   c.OSArch,
-		ExistingTunnelPolicy: policy,
-		PoolName:             c.LBPool,
-		Tags:                 c.Tags,
-		ConnectionID:         connectionID,
-		OriginLocalIP:        OriginLocalIP,
-		IsAutoupdated:        c.IsAutoupdated,
-		RunFromTerminal:      c.RunFromTerminal,
-		CompressionQuality:   0,
-		UUID:                 uuid.String(),
-		Features:             c.SupportedFeatures(),
-	}
-}
-
-func (c *TunnelConfig) connectionOptions(originLocalAddr string, numPreviousAttempts uint8) *tunnelpogs.ConnectionOptions {
+func (c *TunnelConfig) connectionOptions(originLocalAddr string, numPreviousAttempts uint8) *pogs.ConnectionOptions {
 	// attempt to parse out origin IP, but don't fail since it's informational field
 	host, _, _ := net.SplitHostPort(originLocalAddr)
 	originIP := net.ParseIP(host)
 
-	return &tunnelpogs.ConnectionOptions{
+	return &pogs.ConnectionOptions{
 		Client:              c.NamedTunnel.Client,
 		OriginLocalIP:       originIP,
 		ReplaceExisting:     c.ReplaceExisting,
@@ -202,7 +179,6 @@ func (f *ipAddrFallback) ShouldGetNewAddress(connIndex uint8, err error) (needsN
 type EdgeTunnelServer struct {
 	config            *TunnelConfig
 	orchestrator      *orchestration.Orchestrator
-	credentialManager *reconnectCredentialManager
 	edgeAddrHandler   EdgeAddrHandler
 	edgeAddrs         *edgediscovery.Edge
 	edgeBindAddr      net.IP
@@ -478,6 +454,7 @@ func (e *EdgeTunnelServer) serveConnection(
 		connIndex,
 		addr.UDP.IP,
 		nil,
+		e.config.RPCTimeout,
 		e.gracefulShutdownC,
 		e.config.GracePeriod,
 		protocol,
@@ -530,7 +507,7 @@ func (e *EdgeTunnelServer) serveHTTP2(
 	ctx context.Context,
 	connLog *ConnAwareLogger,
 	tlsServerConn net.Conn,
-	connOptions *tunnelpogs.ConnectionOptions,
+	connOptions *pogs.ConnectionOptions,
 	controlStreamHandler connection.ControlStreamHandler,
 	connIndex uint8,
 ) error {
@@ -572,7 +549,7 @@ func (e *EdgeTunnelServer) serveQUIC(
 	ctx context.Context,
 	edgeAddr *net.UDPAddr,
 	connLogger *ConnAwareLogger,
-	connOptions *tunnelpogs.ConnectionOptions,
+	connOptions *pogs.ConnectionOptions,
 	controlStreamHandler connection.ControlStreamHandler,
 	connIndex uint8,
 ) (err error, recoverable bool) {
@@ -613,7 +590,8 @@ func (e *EdgeTunnelServer) serveQUIC(
 		controlStreamHandler,
 		connLogger.Logger(),
 		e.config.PacketConfig,
-		e.config.UDPUnregisterSessionTimeout,
+		e.config.RPCTimeout,
+		e.config.WriteStreamTimeout,
 	)
 	if err != nil {
 		connLogger.ConnAwareLogger().Err(err).Msgf("Failed to create new quic connection")

supervisor/tunnel_test.go

@@ -24,14 +24,18 @@ func (dmf *dynamicMockFetcher) fetch() edgediscovery.PercentageFetcher {
 	}
 }
 
+func immediateTimeAfter(time.Duration) <-chan time.Time {
+	c := make(chan time.Time, 1)
+	c <- time.Now()
+	return c
+}
+
 func TestWaitForBackoffFallback(t *testing.T) {
 	maxRetries := uint(3)
-	backoff := retry.BackoffHandler{
-		MaxRetries: maxRetries,
-		BaseTime:   time.Millisecond * 10,
-	}
+	backoff := retry.NewBackoff(maxRetries, 40*time.Millisecond, false)
+	backoff.Clock.After = immediateTimeAfter
 	log := zerolog.Nop()
-	resolveTTL := time.Duration(0)
+	resolveTTL := 10 * time.Second
 	mockFetcher := dynamicMockFetcher{
 		protocolPercents: edgediscovery.ProtocolPercents{edgediscovery.ProtocolPercent{Protocol: "quic", Percentage: 100}},
 	}
@@ -64,21 +68,23 @@ func TestWaitForBackoffFallback(t *testing.T) {
 	}
 
 	// Retry fallback protocol
-	for i := 0; i < int(maxRetries); i++ {
-		protoFallback.BackoffTimer() // simulate retry
-		ok := selectNextProtocol(&log, protoFallback, protocolSelector, nil)
-		assert.True(t, ok)
-		fallback, ok := protocolSelector.Fallback()
-		assert.True(t, ok)
-		assert.Equal(t, fallback, protoFallback.protocol)
-	}
+	protoFallback.BackoffTimer() // simulate retry
+	ok := selectNextProtocol(&log, protoFallback, protocolSelector, nil)
+	assert.True(t, ok)
+	fallback, ok := protocolSelector.Fallback()
+	assert.True(t, ok)
+	assert.Equal(t, fallback, protoFallback.protocol)
+	assert.Equal(t, connection.HTTP2, protoFallback.protocol)
 
 	currentGlobalProtocol := protocolSelector.Current()
 	assert.Equal(t, initProtocol, currentGlobalProtocol)
 
+	// Simulate max retries again (retries reset after protocol switch)
+	for i := 0; i < int(maxRetries); i++ {
+		protoFallback.BackoffTimer()
+	}
 	// No protocol to fallback, return error
-	protoFallback.BackoffTimer() // simulate retry
-	ok := selectNextProtocol(&log, protoFallback, protocolSelector, nil)
+	ok = selectNextProtocol(&log, protoFallback, protocolSelector, nil)
 	assert.False(t, ok)
 
 	protoFallback.reset()

token/token.go

@@ -12,7 +12,7 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/go-jose/go-jose/v3"
+	"github.com/go-jose/go-jose/v4"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
 
@@ -31,7 +31,8 @@ const (
 )
 
 var (
-	userAgent = "DEV"
+	userAgent     = "DEV"
+	signatureAlgs = []jose.SignatureAlgorithm{jose.RS256}
 )
 
 type AppInfo struct {
@@ -93,9 +94,10 @@ func errDeleteTokenFailed(lockFilePath string) error {
 // newLock will get a new file lock
 func newLock(path string) *lock {
 	lockPath := path + ".lock"
+	backoff := retry.NewBackoff(uint(7), retry.DefaultBaseTime, false)
 	return &lock{
 		lockFilePath: lockPath,
-		backoff:      &retry.BackoffHandler{MaxRetries: 7},
+		backoff:      &backoff,
 		sigHandler: &signalHandler{
 			signals: []os.Signal{syscall.SIGINT, syscall.SIGTERM},
 		},
@@ -415,7 +417,7 @@ func getTokenIfExists(path string) (*jose.JSONWebSignature, error) {
 	if err != nil {
 		return nil, err
 	}
-	token, err := jose.ParseSigned(string(content))
+	token, err := jose.ParseSigned(string(content), signatureAlgs)
 	if err != nil {
 		return nil, err
 	}

tunnelrpc/go.capnp

@@ -1,15 +0,0 @@
-# Generate go.capnp.out with:
-# capnp compile -o- go.capnp > go.capnp.out
-# Must run inside this directory to preserve paths.
-
-@0xd12a1c51fedd6c88;
-
-annotation package(file) :Text;
-annotation import(file) :Text;
-annotation doc(struct, field, enum) :Text;
-annotation tag(enumerant) :Text;
-annotation notag(enumerant) :Void;
-annotation customtype(field) :Text;
-annotation name(struct, field, union, enum, enumerant, interface, method, param, annotation, const, group) :Text;
-
-$package("capnp");

tunnelrpc/log.go

@@ -1,43 +0,0 @@
-package tunnelrpc
-
-import (
-	"context"
-
-	"github.com/rs/zerolog"
-	"golang.org/x/net/trace"
-	"zombiezen.com/go/capnproto2/rpc"
-)
-
-// ConnLogger wraps a Zerolog Logger for a connection.
-type ConnLogger struct {
-	Log *zerolog.Logger
-}
-
-func (c ConnLogger) Infof(ctx context.Context, format string, args ...interface{}) {
-	c.Log.Info().Msgf(format, args...)
-}
-
-func (c ConnLogger) Errorf(ctx context.Context, format string, args ...interface{}) {
-	c.Log.Error().Msgf(format, args...)
-}
-
-func ConnLog(log *zerolog.Logger) rpc.ConnOption {
-	return rpc.ConnLog(ConnLogger{log})
-}
-
-// ConnTracer wraps a trace.EventLog for a connection.
-type ConnTracer struct {
-	Events trace.EventLog
-}
-
-func (c ConnTracer) Infof(ctx context.Context, format string, args ...interface{}) {
-	c.Events.Printf(format, args...)
-}
-
-func (c ConnTracer) Errorf(ctx context.Context, format string, args ...interface{}) {
-	c.Events.Errorf(format, args...)
-}
-
-func ConnTrace(events trace.EventLog) rpc.ConnOption {
-	return rpc.ConnLog(ConnTracer{events})
-}

tunnelrpc/logtransport.go

@@ -1,45 +0,0 @@
-// Package logtransport provides a transport that logs all of its messages.
-package tunnelrpc
-
-import (
-	"bytes"
-	"context"
-
-	"github.com/rs/zerolog"
-	"zombiezen.com/go/capnproto2/encoding/text"
-	"zombiezen.com/go/capnproto2/rpc"
-	rpccapnp "zombiezen.com/go/capnproto2/std/capnp/rpc"
-)
-
-type transport struct {
-	rpc.Transport
-	log *zerolog.Logger
-}
-
-// NewTransportLogger creates a new logger that proxies messages to and from t and
-// logs them to log.  If log is nil, then the log package's default
-// logger is used.
-func NewTransportLogger(log *zerolog.Logger, t rpc.Transport) rpc.Transport {
-	return &transport{Transport: t, log: log}
-}
-
-func (t *transport) SendMessage(ctx context.Context, msg rpccapnp.Message) error {
-	t.log.Trace().Msgf("rpc tx: %s", formatMsg(msg))
-	return t.Transport.SendMessage(ctx, msg)
-}
-
-func (t *transport) RecvMessage(ctx context.Context) (rpccapnp.Message, error) {
-	msg, err := t.Transport.RecvMessage(ctx)
-	if err != nil {
-		t.log.Debug().Msgf("rpc rx error: %s", err)
-		return msg, err
-	}
-	t.log.Trace().Msgf("rpc rx: %s", formatMsg(msg))
-	return msg, nil
-}
-
-func formatMsg(m rpccapnp.Message) string {
-	var buf bytes.Buffer
-	_ = text.NewEncoder(&buf).Encode(0x91b79f1f808db032, m.Struct)
-	return buf.String()
-}

tunnelrpc/metrics/metrics.go

@@ -0,0 +1,143 @@
+package metrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const (
+	metricsNamespace = "cloudflared"
+	rpcSubsystem     = "rpc"
+)
+
+// CloudflaredServer operation labels
+// CloudflaredServer is an extension of SessionManager with additional methods, but it's helpful
+// to visualize it separately in the metrics since they are technically different client/servers.
+const (
+	Cloudflared = "cloudflared"
+)
+
+// ConfigurationManager operation labels
+const (
+	ConfigurationManager = "config"
+
+	OperationUpdateConfiguration = "update_configuration"
+)
+
+// SessionManager operation labels
+const (
+	SessionManager = "session"
+
+	OperationRegisterUdpSession   = "register_udp_session"
+	OperationUnregisterUdpSession = "unregister_udp_session"
+)
+
+// RegistrationServer operation labels
+const (
+	Registration = "registration"
+
+	OperationRegisterConnection       = "register_connection"
+	OperationUnregisterConnection     = "unregister_connection"
+	OperationUpdateLocalConfiguration = "update_local_configuration"
+)
+
+type rpcMetrics struct {
+	serverOperations        *prometheus.CounterVec
+	serverFailures          *prometheus.CounterVec
+	serverOperationsLatency *prometheus.HistogramVec
+
+	ClientOperations        *prometheus.CounterVec
+	ClientFailures          *prometheus.CounterVec
+	ClientOperationsLatency *prometheus.HistogramVec
+}
+
+var CapnpMetrics *rpcMetrics = &rpcMetrics{
+	serverOperations: prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: metricsNamespace,
+			Subsystem: rpcSubsystem,
+			Name:      "server_operations",
+			Help:      "Number of rpc methods by handler served",
+		},
+		[]string{"handler", "method"},
+	),
+	serverFailures: prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: metricsNamespace,
+			Subsystem: rpcSubsystem,
+			Name:      "server_failures",
+			Help:      "Number of rpc methods failures by handler served",
+		},
+		[]string{"handler", "method"},
+	),
+	serverOperationsLatency: prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Namespace: metricsNamespace,
+			Subsystem: rpcSubsystem,
+			Name:      "server_latency_secs",
+			Help:      "Latency of rpc methods by handler served",
+			// Bucket starts at 50ms, each bucket grows by a factor of 3, up to 5 buckets and is expressed as seconds:
+			// 50ms, 150ms, 450ms, 1350ms, 4050ms
+			Buckets: prometheus.ExponentialBuckets(0.05, 3, 5),
+		},
+		[]string{"handler", "method"},
+	),
+	ClientOperations: prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: metricsNamespace,
+			Subsystem: rpcSubsystem,
+			Name:      "client_operations",
+			Help:      "Number of rpc methods by handler requested",
+		},
+		[]string{"handler", "method"},
+	),
+	ClientFailures: prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: metricsNamespace,
+			Subsystem: rpcSubsystem,
+			Name:      "client_failures",
+			Help:      "Number of rpc method failures by handler requested",
+		},
+		[]string{"handler", "method"},
+	),
+	ClientOperationsLatency: prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Namespace: metricsNamespace,
+			Subsystem: rpcSubsystem,
+			Name:      "client_latency_secs",
+			Help:      "Latency of rpc methods by handler requested",
+			// Bucket starts at 50ms, each bucket grows by a factor of 3, up to 5 buckets and is expressed as seconds:
+			// 50ms, 150ms, 450ms, 1350ms, 4050ms
+			Buckets: prometheus.ExponentialBuckets(0.05, 3, 5),
+		},
+		[]string{"handler", "method"},
+	),
+}
+
+func ObserveServerHandler(inner func() error, handler, method string) error {
+	defer CapnpMetrics.serverOperations.WithLabelValues(handler, method).Inc()
+	timer := prometheus.NewTimer(prometheus.ObserverFunc(func(s float64) {
+		CapnpMetrics.serverOperationsLatency.WithLabelValues(handler, method).Observe(s)
+	}))
+	defer timer.ObserveDuration()
+
+	err := inner()
+	if err != nil {
+		CapnpMetrics.serverFailures.WithLabelValues(handler, method).Inc()
+	}
+	return err
+}
+
+func NewClientOperationLatencyObserver(server string, method string) *prometheus.Timer {
+	return prometheus.NewTimer(prometheus.ObserverFunc(func(s float64) {
+		CapnpMetrics.ClientOperationsLatency.WithLabelValues(server, method).Observe(s)
+	}))
+}
+
+func init() {
+	prometheus.MustRegister(CapnpMetrics.serverOperations)
+	prometheus.MustRegister(CapnpMetrics.serverFailures)
+	prometheus.MustRegister(CapnpMetrics.serverOperationsLatency)
+	prometheus.MustRegister(CapnpMetrics.ClientOperations)
+	prometheus.MustRegister(CapnpMetrics.ClientFailures)
+	prometheus.MustRegister(CapnpMetrics.ClientOperationsLatency)
+}

tunnelrpc/pogs/auth_outcome.go

@@ -1,131 +0,0 @@
-package pogs
-
-import (
-	"errors"
-	"time"
-)
-
-// AuthenticateResponse is the serialized response from the Authenticate RPC.
-// It's a 1:1 representation of the capnp message, so it's not very useful for programmers.
-// Instead, you should call the `Outcome()` method to get a programmer-friendly sum type, with one
-// case for each possible outcome.
-type AuthenticateResponse struct {
-	PermanentErr      string
-	RetryableErr      string
-	Jwt               []byte
-	HoursUntilRefresh uint8
-}
-
-// Outcome turns the deserialized response of Authenticate into a programmer-friendly sum type.
-func (ar AuthenticateResponse) Outcome() AuthOutcome {
-	// If the user's authentication was unsuccessful, the server will return an error explaining why.
-	// cloudflared should fatal with this error.
-	if ar.PermanentErr != "" {
-		return NewAuthFail(errors.New(ar.PermanentErr))
-	}
-
-	// If there was a network error, then cloudflared should retry later,
-	// because origintunneld couldn't prove whether auth was correct or not.
-	if ar.RetryableErr != "" {
-		return NewAuthUnknown(errors.New(ar.RetryableErr), ar.HoursUntilRefresh)
-	}
-
-	// If auth succeeded, return the token and refresh it when instructed.
-	if len(ar.Jwt) > 0 {
-		return NewAuthSuccess(ar.Jwt, ar.HoursUntilRefresh)
-	}
-
-	// Otherwise the state got messed up.
-	return nil
-}
-
-// AuthOutcome is a programmer-friendly sum type denoting the possible outcomes of Authenticate.
-type AuthOutcome interface {
-	isAuthOutcome()
-	// Serialize into an AuthenticateResponse which can be sent via Capnp
-	Serialize() AuthenticateResponse
-}
-
-// AuthSuccess means the backend successfully authenticated this cloudflared.
-type AuthSuccess struct {
-	jwt               []byte
-	hoursUntilRefresh uint8
-}
-
-func NewAuthSuccess(jwt []byte, hoursUntilRefresh uint8) AuthSuccess {
-	return AuthSuccess{jwt: jwt, hoursUntilRefresh: hoursUntilRefresh}
-}
-
-func (ao AuthSuccess) JWT() []byte {
-	return ao.jwt
-}
-
-// RefreshAfter is how long cloudflared should wait before rerunning Authenticate.
-func (ao AuthSuccess) RefreshAfter() time.Duration {
-	return hoursToTime(ao.hoursUntilRefresh)
-}
-
-// Serialize into an AuthenticateResponse which can be sent via Capnp
-func (ao AuthSuccess) Serialize() AuthenticateResponse {
-	return AuthenticateResponse{
-		Jwt:               ao.jwt,
-		HoursUntilRefresh: ao.hoursUntilRefresh,
-	}
-}
-
-func (ao AuthSuccess) isAuthOutcome() {}
-
-// AuthFail means this cloudflared has the wrong auth and should exit.
-type AuthFail struct {
-	err error
-}
-
-func NewAuthFail(err error) AuthFail {
-	return AuthFail{err: err}
-}
-
-func (ao AuthFail) Error() string {
-	return ao.err.Error()
-}
-
-// Serialize into an AuthenticateResponse which can be sent via Capnp
-func (ao AuthFail) Serialize() AuthenticateResponse {
-	return AuthenticateResponse{
-		PermanentErr: ao.err.Error(),
-	}
-}
-
-func (ao AuthFail) isAuthOutcome() {}
-
-// AuthUnknown means the backend couldn't finish checking authentication. Try again later.
-type AuthUnknown struct {
-	err               error
-	hoursUntilRefresh uint8
-}
-
-func NewAuthUnknown(err error, hoursUntilRefresh uint8) AuthUnknown {
-	return AuthUnknown{err: err, hoursUntilRefresh: hoursUntilRefresh}
-}
-
-func (ao AuthUnknown) Error() string {
-	return ao.err.Error()
-}
-
-// RefreshAfter is how long cloudflared should wait before rerunning Authenticate.
-func (ao AuthUnknown) RefreshAfter() time.Duration {
-	return hoursToTime(ao.hoursUntilRefresh)
-}
-
-// Serialize into an AuthenticateResponse which can be sent via Capnp
-func (ao AuthUnknown) Serialize() AuthenticateResponse {
-	return AuthenticateResponse{
-		RetryableErr:      ao.err.Error(),
-		HoursUntilRefresh: ao.hoursUntilRefresh,
-	}
-}
-
-func (ao AuthUnknown) isAuthOutcome() {}
-
-func hoursToTime(hours uint8) time.Duration {
-	return time.Duration(hours) * time.Hour
-}

tunnelrpc/pogs/auth_serialize.go

@@ -1,78 +0,0 @@
-package pogs
-
-import (
-	"context"
-
-	"zombiezen.com/go/capnproto2/pogs"
-	"zombiezen.com/go/capnproto2/server"
-
-	"github.com/cloudflare/cloudflared/tunnelrpc"
-)
-
-func (i TunnelServer_PogsImpl) Authenticate(p tunnelrpc.TunnelServer_authenticate) error {
-	originCert, err := p.Params.OriginCert()
-	if err != nil {
-		return err
-	}
-	hostname, err := p.Params.Hostname()
-	if err != nil {
-		return err
-	}
-	options, err := p.Params.Options()
-	if err != nil {
-		return err
-	}
-	pogsOptions, err := UnmarshalRegistrationOptions(options)
-	if err != nil {
-		return err
-	}
-
-	server.Ack(p.Options)
-	resp, err := i.impl.Authenticate(p.Ctx, originCert, hostname, pogsOptions)
-	if err != nil {
-		return err
-	}
-	result, err := p.Results.NewResult()
-	if err != nil {
-		return err
-	}
-	return MarshalAuthenticateResponse(result, resp)
-}
-
-func MarshalAuthenticateResponse(s tunnelrpc.AuthenticateResponse, p *AuthenticateResponse) error {
-	return pogs.Insert(tunnelrpc.AuthenticateResponse_TypeID, s.Struct, p)
-}
-
-func (c TunnelServer_PogsClient) Authenticate(ctx context.Context, originCert []byte, hostname string, options *RegistrationOptions) (*AuthenticateResponse, error) {
-	client := tunnelrpc.TunnelServer{Client: c.Client}
-	promise := client.Authenticate(ctx, func(p tunnelrpc.TunnelServer_authenticate_Params) error {
-		err := p.SetOriginCert(originCert)
-		if err != nil {
-			return err
-		}
-		err = p.SetHostname(hostname)
-		if err != nil {
-			return err
-		}
-		registrationOptions, err := p.NewOptions()
-		if err != nil {
-			return err
-		}
-		err = MarshalRegistrationOptions(registrationOptions, options)
-		if err != nil {
-			return err
-		}
-		return nil
-	})
-	retval, err := promise.Result().Struct()
-	if err != nil {
-		return nil, err
-	}
-	return UnmarshalAuthenticateResponse(retval)
-}
-
-func UnmarshalAuthenticateResponse(s tunnelrpc.AuthenticateResponse) (*AuthenticateResponse, error) {
-	p := new(AuthenticateResponse)
-	err := pogs.Extract(p, tunnelrpc.AuthenticateResponse_TypeID, s.Struct)
-	return p, err
-}

tunnelrpc/pogs/auth_test.go

@@ -1,136 +0,0 @@
-package pogs
-
-import (
-	"fmt"
-	"reflect"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-	capnp "zombiezen.com/go/capnproto2"
-
-	"github.com/cloudflare/cloudflared/tunnelrpc"
-)
-
-// Ensure the AuthOutcome sum is correct
-var _ AuthOutcome = &AuthSuccess{}
-var _ AuthOutcome = &AuthFail{}
-var _ AuthOutcome = &AuthUnknown{}
-
-// Unit tests for AuthenticateResponse.Outcome()
-func TestAuthenticateResponseOutcome(t *testing.T) {
-	type fields struct {
-		PermanentErr      string
-		RetryableErr      string
-		Jwt               []byte
-		HoursUntilRefresh uint8
-	}
-	tests := []struct {
-		name   string
-		fields fields
-		want   AuthOutcome
-	}{
-		{"success",
-			fields{Jwt: []byte("asdf"), HoursUntilRefresh: 6},
-			AuthSuccess{jwt: []byte("asdf"), hoursUntilRefresh: 6},
-		},
-		{"fail",
-			fields{PermanentErr: "bad creds"},
-			AuthFail{err: fmt.Errorf("bad creds")},
-		},
-		{"error",
-			fields{RetryableErr: "bad conn", HoursUntilRefresh: 6},
-			AuthUnknown{err: fmt.Errorf("bad conn"), hoursUntilRefresh: 6},
-		},
-		{"nil (no fields are set)",
-			fields{},
-			nil,
-		},
-		{"nil (too few fields are set)",
-			fields{HoursUntilRefresh: 6},
-			nil,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			ar := AuthenticateResponse{
-				PermanentErr:      tt.fields.PermanentErr,
-				RetryableErr:      tt.fields.RetryableErr,
-				Jwt:               tt.fields.Jwt,
-				HoursUntilRefresh: tt.fields.HoursUntilRefresh,
-			}
-			got := ar.Outcome()
-			if !reflect.DeepEqual(got, tt.want) {
-				t.Errorf("AuthenticateResponse.Outcome() = %T, want %v", got, tt.want)
-			}
-			if got != nil && !reflect.DeepEqual(got.Serialize(), ar) {
-				t.Errorf(".Outcome() and .Serialize() should be inverses but weren't. Expected %v, got %v", ar, got.Serialize())
-			}
-		})
-	}
-}
-
-func TestAuthSuccess(t *testing.T) {
-	input := NewAuthSuccess([]byte("asdf"), 6)
-	output, ok := input.Serialize().Outcome().(AuthSuccess)
-	assert.True(t, ok)
-	assert.Equal(t, input, output)
-}
-
-func TestAuthUnknown(t *testing.T) {
-	input := NewAuthUnknown(fmt.Errorf("pdx unreachable"), 6)
-	output, ok := input.Serialize().Outcome().(AuthUnknown)
-	assert.True(t, ok)
-	assert.Equal(t, input, output)
-}
-
-func TestAuthFail(t *testing.T) {
-	input := NewAuthFail(fmt.Errorf("wrong creds"))
-	output, ok := input.Serialize().Outcome().(AuthFail)
-	assert.True(t, ok)
-	assert.Equal(t, input, output)
-}
-
-func TestWhenToRefresh(t *testing.T) {
-	expected := 4 * time.Hour
-	actual := hoursToTime(4)
-	if expected != actual {
-		t.Fatalf("expected %v hours, got %v", expected, actual)
-	}
-}
-
-// Test that serializing and deserializing AuthenticationResponse undo each other.
-func TestSerializeAuthenticationResponse(t *testing.T) {
-
-	tests := []*AuthenticateResponse{
-		{
-			Jwt:               []byte("\xbd\xb2\x3d\xbc\x20\xe2\x8c\x98"),
-			HoursUntilRefresh: 24,
-		},
-		{
-			PermanentErr: "bad auth",
-		},
-		{
-			RetryableErr:      "bad connection",
-			HoursUntilRefresh: 24,
-		},
-	}
-
-	for i, testCase := range tests {
-		_, seg, err := capnp.NewMessage(capnp.SingleSegment(nil))
-		assert.NoError(t, err)
-		capnpEntity, err := tunnelrpc.NewAuthenticateResponse(seg)
-		if !assert.NoError(t, err) {
-			t.Fatal("Couldn't initialize a new message")
-		}
-		err = MarshalAuthenticateResponse(capnpEntity, testCase)
-		if !assert.NoError(t, err, "testCase index %v failed to marshal", i) {
-			continue
-		}
-		result, err := UnmarshalAuthenticateResponse(capnpEntity)
-		if !assert.NoError(t, err, "testCase index %v failed to unmarshal", i) {
-			continue
-		}
-		assert.Equal(t, testCase, result, "testCase index %v didn't preserve struct through marshalling and unmarshalling", i)
-	}
-}

tunnelrpc/pogs/cloudflared_server.go

@@ -1,9 +1,10 @@
 package pogs
 
 import (
-	"github.com/cloudflare/cloudflared/tunnelrpc"
 	capnp "zombiezen.com/go/capnproto2"
 	"zombiezen.com/go/capnproto2/rpc"
+
+	"github.com/cloudflare/cloudflared/tunnelrpc/proto"
 )
 
 type CloudflaredServer interface {
@@ -16,8 +17,8 @@ type CloudflaredServer_PogsImpl struct {
 	ConfigurationManager_PogsImpl
 }
 
-func CloudflaredServer_ServerToClient(s SessionManager, c ConfigurationManager) tunnelrpc.CloudflaredServer {
-	return tunnelrpc.CloudflaredServer_ServerToClient(CloudflaredServer_PogsImpl{
+func CloudflaredServer_ServerToClient(s SessionManager, c ConfigurationManager) proto.CloudflaredServer {
+	return proto.CloudflaredServer_ServerToClient(CloudflaredServer_PogsImpl{
 		SessionManager_PogsImpl:       SessionManager_PogsImpl{s},
 		ConfigurationManager_PogsImpl: ConfigurationManager_PogsImpl{c},
 	})

tunnelrpc/pogs/configuration_manager.go

@@ -4,13 +4,16 @@ import (
 	"context"
 	"fmt"
 
-	"github.com/cloudflare/cloudflared/tunnelrpc"
 	capnp "zombiezen.com/go/capnproto2"
 	"zombiezen.com/go/capnproto2/rpc"
 	"zombiezen.com/go/capnproto2/server"
+
+	"github.com/cloudflare/cloudflared/tunnelrpc/metrics"
+	"github.com/cloudflare/cloudflared/tunnelrpc/proto"
 )
 
 type ConfigurationManager interface {
+	// UpdateConfiguration is the call provided to cloudflared to load the latest remote configuration.
 	UpdateConfiguration(ctx context.Context, version int32, config []byte) *UpdateConfigurationResponse
 }
 
@@ -18,11 +21,15 @@ type ConfigurationManager_PogsImpl struct {
 	impl ConfigurationManager
 }
 
-func ConfigurationManager_ServerToClient(c ConfigurationManager) tunnelrpc.ConfigurationManager {
-	return tunnelrpc.ConfigurationManager_ServerToClient(ConfigurationManager_PogsImpl{c})
+func ConfigurationManager_ServerToClient(c ConfigurationManager) proto.ConfigurationManager {
+	return proto.ConfigurationManager_ServerToClient(ConfigurationManager_PogsImpl{c})
 }
 
-func (i ConfigurationManager_PogsImpl) UpdateConfiguration(p tunnelrpc.ConfigurationManager_updateConfiguration) error {
+func (i ConfigurationManager_PogsImpl) UpdateConfiguration(p proto.ConfigurationManager_updateConfiguration) error {
+	return metrics.ObserveServerHandler(func() error { return i.updateConfiguration(p) }, metrics.ConfigurationManager, metrics.OperationUpdateConfiguration)
+}
+
+func (i ConfigurationManager_PogsImpl) updateConfiguration(p proto.ConfigurationManager_updateConfiguration) error {
 	server.Ack(p.Options)
 
 	version := p.Params.Version()
@@ -51,8 +58,8 @@ func (c ConfigurationManager_PogsClient) Close() error {
 }
 
 func (c ConfigurationManager_PogsClient) UpdateConfiguration(ctx context.Context, version int32, config []byte) (*UpdateConfigurationResponse, error) {
-	client := tunnelrpc.ConfigurationManager{Client: c.Client}
-	promise := client.UpdateConfiguration(ctx, func(p tunnelrpc.ConfigurationManager_updateConfiguration_Params) error {
+	client := proto.ConfigurationManager{Client: c.Client}
+	promise := client.UpdateConfiguration(ctx, func(p proto.ConfigurationManager_updateConfiguration_Params) error {
 		p.SetVersion(version)
 		return p.SetConfig(config)
 	})
@@ -74,7 +81,7 @@ type UpdateConfigurationResponse struct {
 	Err                error `json:"err"`
 }
 
-func (p *UpdateConfigurationResponse) Marshal(s tunnelrpc.UpdateConfigurationResponse) error {
+func (p *UpdateConfigurationResponse) Marshal(s proto.UpdateConfigurationResponse) error {
 	s.SetLatestAppliedVersion(p.LastAppliedVersion)
 	if p.Err != nil {
 		return s.SetErr(p.Err.Error())
@@ -82,7 +89,7 @@ func (p *UpdateConfigurationResponse) Marshal(s tunnelrpc.UpdateConfigurationRes
 	return nil
 }
 
-func (p *UpdateConfigurationResponse) Unmarshal(s tunnelrpc.UpdateConfigurationResponse) error {
+func (p *UpdateConfigurationResponse) Unmarshal(s proto.UpdateConfigurationResponse) error {
 	p.LastAppliedVersion = s.LatestAppliedVersion()
 	respErr, err := s.Err()
 	if err != nil {

tunnelrpc/pogs/quic_metadata_protocol.go

@@ -1,4 +1,4 @@
-package quic
+package pogs
 
 import (
 	"fmt"
@@ -6,7 +6,7 @@ import (
 	capnp "zombiezen.com/go/capnproto2"
 	"zombiezen.com/go/capnproto2/pogs"
 
-	"github.com/cloudflare/cloudflared/quic/schema"
+	"github.com/cloudflare/cloudflared/tunnelrpc/proto"
 )
 
 // ConnectionType indicates the type of underlying connection proxied within the QUIC stream.
@@ -52,26 +52,26 @@ func (r *ConnectRequest) MetadataMap() map[string]string {
 	return metadataMap
 }
 
-func (r *ConnectRequest) fromPogs(msg *capnp.Message) error {
-	metadata, err := schema.ReadRootConnectRequest(msg)
+func (r *ConnectRequest) FromPogs(msg *capnp.Message) error {
+	metadata, err := proto.ReadRootConnectRequest(msg)
 	if err != nil {
 		return err
 	}
-	return pogs.Extract(r, schema.ConnectRequest_TypeID, metadata.Struct)
+	return pogs.Extract(r, proto.ConnectRequest_TypeID, metadata.Struct)
 }
 
-func (r *ConnectRequest) toPogs() (*capnp.Message, error) {
+func (r *ConnectRequest) ToPogs() (*capnp.Message, error) {
 	msg, seg, err := capnp.NewMessage(capnp.SingleSegment(nil))
 	if err != nil {
 		return nil, err
 	}
 
-	root, err := schema.NewRootConnectRequest(seg)
+	root, err := proto.NewRootConnectRequest(seg)
 	if err != nil {
 		return nil, err
 	}
 
-	if err := pogs.Insert(schema.ConnectRequest_TypeID, root.Struct, r); err != nil {
+	if err := pogs.Insert(proto.ConnectRequest_TypeID, root.Struct, r); err != nil {
 		return nil, err
 	}
 
@@ -84,26 +84,26 @@ type ConnectResponse struct {
 	Metadata []Metadata `capnp:"metadata"`
 }
 
-func (r *ConnectResponse) fromPogs(msg *capnp.Message) error {
-	metadata, err := schema.ReadRootConnectResponse(msg)
+func (r *ConnectResponse) FromPogs(msg *capnp.Message) error {
+	metadata, err := proto.ReadRootConnectResponse(msg)
 	if err != nil {
 		return err
 	}
-	return pogs.Extract(r, schema.ConnectResponse_TypeID, metadata.Struct)
+	return pogs.Extract(r, proto.ConnectResponse_TypeID, metadata.Struct)
 }
 
-func (r *ConnectResponse) toPogs() (*capnp.Message, error) {
+func (r *ConnectResponse) ToPogs() (*capnp.Message, error) {
 	msg, seg, err := capnp.NewMessage(capnp.SingleSegment(nil))
 	if err != nil {
 		return nil, err
 	}
 
-	root, err := schema.NewRootConnectResponse(seg)
+	root, err := proto.NewRootConnectResponse(seg)
 	if err != nil {
 		return nil, err
 	}
 
-	if err := pogs.Insert(schema.ConnectResponse_TypeID, root.Struct, r); err != nil {
+	if err := pogs.Insert(proto.ConnectResponse_TypeID, root.Struct, r); err != nil {
 		return nil, err
 	}
 

tunnelrpc/pogs/reconnect_tunnel.go

@@ -1,93 +0,0 @@
-package pogs
-
-import (
-	"context"
-
-	"zombiezen.com/go/capnproto2/server"
-
-	"github.com/cloudflare/cloudflared/tunnelrpc"
-)
-
-func (i TunnelServer_PogsImpl) ReconnectTunnel(p tunnelrpc.TunnelServer_reconnectTunnel) error {
-	jwt, err := p.Params.Jwt()
-	if err != nil {
-		return err
-	}
-	eventDigest, err := p.Params.EventDigest()
-	if err != nil {
-		return err
-	}
-	connDigest, err := p.Params.ConnDigest()
-	if err != nil {
-		return err
-	}
-	hostname, err := p.Params.Hostname()
-	if err != nil {
-		return err
-	}
-	options, err := p.Params.Options()
-	if err != nil {
-		return err
-	}
-	pogsOptions, err := UnmarshalRegistrationOptions(options)
-	if err != nil {
-		return err
-	}
-	server.Ack(p.Options)
-	registration, err := i.impl.ReconnectTunnel(p.Ctx, jwt, eventDigest, connDigest, hostname, pogsOptions)
-	if err != nil {
-		return err
-	}
-	result, err := p.Results.NewResult()
-	if err != nil {
-		return err
-	}
-	return MarshalTunnelRegistration(result, registration)
-}
-
-func (c TunnelServer_PogsClient) ReconnectTunnel(
-	ctx context.Context,
-	jwt,
-	eventDigest []byte,
-	connDigest []byte,
-	hostname string,
-	options *RegistrationOptions,
-) *TunnelRegistration {
-	client := tunnelrpc.TunnelServer{Client: c.Client}
-	promise := client.ReconnectTunnel(ctx, func(p tunnelrpc.TunnelServer_reconnectTunnel_Params) error {
-		err := p.SetJwt(jwt)
-		if err != nil {
-			return err
-		}
-		err = p.SetEventDigest(eventDigest)
-		if err != nil {
-			return err
-		}
-		err = p.SetConnDigest(connDigest)
-		if err != nil {
-			return err
-		}
-		err = p.SetHostname(hostname)
-		if err != nil {
-			return err
-		}
-		registrationOptions, err := p.NewOptions()
-		if err != nil {
-			return err
-		}
-		err = MarshalRegistrationOptions(registrationOptions, options)
-		if err != nil {
-			return err
-		}
-		return nil
-	})
-	retval, err := promise.Result().Struct()
-	if err != nil {
-		return NewRetryableRegistrationError(err, defaultRetryAfterSeconds).Serialize()
-	}
-	registration, err := UnmarshalTunnelRegistration(retval)
-	if err != nil {
-		return NewRetryableRegistrationError(err, defaultRetryAfterSeconds).Serialize()
-	}
-	return registration
-}

tunnelrpc/pogs/registration_server.go

@@ -12,12 +12,18 @@ import (
 	"zombiezen.com/go/capnproto2/rpc"
 	"zombiezen.com/go/capnproto2/server"
 
-	"github.com/cloudflare/cloudflared/tunnelrpc"
+	"github.com/cloudflare/cloudflared/tunnelrpc/metrics"
+	"github.com/cloudflare/cloudflared/tunnelrpc/proto"
 )
 
 type RegistrationServer interface {
+	// RegisterConnection is the call typically handled by the edge to initiate and authenticate a new connection
+	// for cloudflared.
 	RegisterConnection(ctx context.Context, auth TunnelAuth, tunnelID uuid.UUID, connIndex byte, options *ConnectionOptions) (*ConnectionDetails, error)
+	// UnregisterConnection is the call typically handled by the edge to close an existing connection for cloudflared.
 	UnregisterConnection(ctx context.Context)
+	// UpdateLocalConfiguration is the call typically handled by the edge for cloudflared to provide the current
+	// configuration it is operating with.
 	UpdateLocalConfiguration(ctx context.Context, config []byte) error
 }
 
@@ -25,11 +31,15 @@ type RegistrationServer_PogsImpl struct {
 	impl RegistrationServer
 }
 
-func RegistrationServer_ServerToClient(s RegistrationServer) tunnelrpc.RegistrationServer {
-	return tunnelrpc.RegistrationServer_ServerToClient(RegistrationServer_PogsImpl{s})
+func RegistrationServer_ServerToClient(s RegistrationServer) proto.RegistrationServer {
+	return proto.RegistrationServer_ServerToClient(RegistrationServer_PogsImpl{s})
 }
 
-func (i RegistrationServer_PogsImpl) RegisterConnection(p tunnelrpc.RegistrationServer_registerConnection) error {
+func (i RegistrationServer_PogsImpl) RegisterConnection(p proto.RegistrationServer_registerConnection) error {
+	return metrics.ObserveServerHandler(func() error { return i.registerConnection(p) }, metrics.Registration, metrics.OperationRegisterConnection)
+}
+
+func (i RegistrationServer_PogsImpl) registerConnection(p proto.RegistrationServer_registerConnection) error {
 	server.Ack(p.Options)
 
 	auth, err := p.Params.Auth()
@@ -82,14 +92,19 @@ func (i RegistrationServer_PogsImpl) RegisterConnection(p tunnelrpc.Registration
 	}
 }
 
-func (i RegistrationServer_PogsImpl) UnregisterConnection(p tunnelrpc.RegistrationServer_unregisterConnection) error {
-	server.Ack(p.Options)
-
-	i.impl.UnregisterConnection(p.Ctx)
-	return nil
+func (i RegistrationServer_PogsImpl) UnregisterConnection(p proto.RegistrationServer_unregisterConnection) error {
+	return metrics.ObserveServerHandler(func() error {
+		server.Ack(p.Options)
+		i.impl.UnregisterConnection(p.Ctx)
+		return nil // No metrics will be reported for failure as this method has no return value
+	}, metrics.Registration, metrics.OperationUnregisterConnection)
 }
 
-func (i RegistrationServer_PogsImpl) UpdateLocalConfiguration(c tunnelrpc.RegistrationServer_updateLocalConfiguration) error {
+func (i RegistrationServer_PogsImpl) UpdateLocalConfiguration(p proto.RegistrationServer_updateLocalConfiguration) error {
+	return metrics.ObserveServerHandler(func() error { return i.updateLocalConfiguration(p) }, metrics.Registration, metrics.OperationUpdateLocalConfiguration)
+}
+
+func (i RegistrationServer_PogsImpl) updateLocalConfiguration(c proto.RegistrationServer_updateLocalConfiguration) error {
 	server.Ack(c.Options)
 
 	configBytes, err := c.Params.Config()
@@ -105,14 +120,21 @@ type RegistrationServer_PogsClient struct {
 	Conn   *rpc.Conn
 }
 
+func NewRegistrationServer_PogsClient(client capnp.Client, conn *rpc.Conn) RegistrationServer_PogsClient {
+	return RegistrationServer_PogsClient{
+		Client: client,
+		Conn:   conn,
+	}
+}
+
 func (c RegistrationServer_PogsClient) Close() error {
 	c.Client.Close()
 	return c.Conn.Close()
 }
 
 func (c RegistrationServer_PogsClient) RegisterConnection(ctx context.Context, auth TunnelAuth, tunnelID uuid.UUID, connIndex byte, options *ConnectionOptions) (*ConnectionDetails, error) {
-	client := tunnelrpc.TunnelServer{Client: c.Client}
-	promise := client.RegisterConnection(ctx, func(p tunnelrpc.RegistrationServer_registerConnection_Params) error {
+	client := proto.TunnelServer{Client: c.Client}
+	promise := client.RegisterConnection(ctx, func(p proto.RegistrationServer_registerConnection_Params) error {
 		tunnelAuth, err := p.NewAuth()
 		if err != nil {
 			return err
@@ -145,7 +167,7 @@ func (c RegistrationServer_PogsClient) RegisterConnection(ctx context.Context, a
 	}
 	result := response.Result()
 	switch result.Which() {
-	case tunnelrpc.ConnectionResponse_result_Which_error:
+	case proto.ConnectionResponse_result_Which_error:
 		resultError, err := result.Error()
 		if err != nil {
 			return nil, wrapRPCError(err)
@@ -160,7 +182,7 @@ func (c RegistrationServer_PogsClient) RegisterConnection(ctx context.Context, a
 		}
 		return nil, err
 
-	case tunnelrpc.ConnectionResponse_result_Which_connectionDetails:
+	case proto.ConnectionResponse_result_Which_connectionDetails:
 		connDetails, err := result.ConnectionDetails()
 		if err != nil {
 			return nil, wrapRPCError(err)
@@ -176,8 +198,8 @@ func (c RegistrationServer_PogsClient) RegisterConnection(ctx context.Context, a
 }
 
 func (c RegistrationServer_PogsClient) SendLocalConfiguration(ctx context.Context, config []byte) error {
-	client := tunnelrpc.TunnelServer{Client: c.Client}
-	promise := client.UpdateLocalConfiguration(ctx, func(p tunnelrpc.RegistrationServer_updateLocalConfiguration_Params) error {
+	client := proto.TunnelServer{Client: c.Client}
+	promise := client.UpdateLocalConfiguration(ctx, func(p proto.RegistrationServer_updateLocalConfiguration_Params) error {
 		if err := p.SetConfig(config); err != nil {
 			return err
 		}
@@ -194,8 +216,8 @@ func (c RegistrationServer_PogsClient) SendLocalConfiguration(ctx context.Contex
 }
 
 func (c RegistrationServer_PogsClient) UnregisterConnection(ctx context.Context) error {
-	client := tunnelrpc.TunnelServer{Client: c.Client}
-	promise := client.UnregisterConnection(ctx, func(p tunnelrpc.RegistrationServer_unregisterConnection_Params) error {
+	client := proto.TunnelServer{Client: c.Client}
+	promise := client.UnregisterConnection(ctx, func(p proto.RegistrationServer_unregisterConnection_Params) error {
 		return nil
 	})
 	_, err := promise.Struct()
@@ -225,20 +247,20 @@ type TunnelAuth struct {
 	TunnelSecret []byte
 }
 
-func (p *ConnectionOptions) MarshalCapnproto(s tunnelrpc.ConnectionOptions) error {
-	return pogs.Insert(tunnelrpc.ConnectionOptions_TypeID, s.Struct, p)
+func (p *ConnectionOptions) MarshalCapnproto(s proto.ConnectionOptions) error {
+	return pogs.Insert(proto.ConnectionOptions_TypeID, s.Struct, p)
 }
 
-func (p *ConnectionOptions) UnmarshalCapnproto(s tunnelrpc.ConnectionOptions) error {
-	return pogs.Extract(p, tunnelrpc.ConnectionOptions_TypeID, s.Struct)
+func (p *ConnectionOptions) UnmarshalCapnproto(s proto.ConnectionOptions) error {
+	return pogs.Extract(p, proto.ConnectionOptions_TypeID, s.Struct)
 }
 
-func (a *TunnelAuth) MarshalCapnproto(s tunnelrpc.TunnelAuth) error {
-	return pogs.Insert(tunnelrpc.TunnelAuth_TypeID, s.Struct, a)
+func (a *TunnelAuth) MarshalCapnproto(s proto.TunnelAuth) error {
+	return pogs.Insert(proto.TunnelAuth_TypeID, s.Struct, a)
 }
 
-func (a *TunnelAuth) UnmarshalCapnproto(s tunnelrpc.TunnelAuth) error {
-	return pogs.Extract(a, tunnelrpc.TunnelAuth_TypeID, s.Struct)
+func (a *TunnelAuth) UnmarshalCapnproto(s proto.TunnelAuth) error {
+	return pogs.Extract(a, proto.TunnelAuth_TypeID, s.Struct)
 }
 
 type ConnectionDetails struct {
@@ -247,7 +269,7 @@ type ConnectionDetails struct {
 	TunnelIsRemotelyManaged bool
 }
 
-func (details *ConnectionDetails) MarshalCapnproto(s tunnelrpc.ConnectionDetails) error {
+func (details *ConnectionDetails) MarshalCapnproto(s proto.ConnectionDetails) error {
 	if err := s.SetUuid(details.UUID[:]); err != nil {
 		return err
 	}
@@ -259,7 +281,7 @@ func (details *ConnectionDetails) MarshalCapnproto(s tunnelrpc.ConnectionDetails
 	return nil
 }
 
-func (details *ConnectionDetails) UnmarshalCapnproto(s tunnelrpc.ConnectionDetails) error {
+func (details *ConnectionDetails) UnmarshalCapnproto(s proto.ConnectionDetails) error {
 	uuidBytes, err := s.Uuid()
 	if err != nil {
 		return err
@@ -277,7 +299,7 @@ func (details *ConnectionDetails) UnmarshalCapnproto(s tunnelrpc.ConnectionDetai
 	return err
 }
 
-func MarshalError(s tunnelrpc.ConnectionError, err error) error {
+func MarshalError(s proto.ConnectionError, err error) error {
 	if err := s.SetCause(err.Error()); err != nil {
 		return err
 	}

tunnelrpc/pogs/registration_server_test.go

@@ -13,7 +13,7 @@ import (
 	capnp "zombiezen.com/go/capnproto2"
 	"zombiezen.com/go/capnproto2/rpc"
 
-	"github.com/cloudflare/cloudflared/tunnelrpc"
+	"github.com/cloudflare/cloudflared/tunnelrpc/proto"
 )
 
 const testAccountTag = "abc123"
@@ -34,7 +34,7 @@ func TestMarshalConnectionOptions(t *testing.T) {
 
 	_, seg, err := capnp.NewMessage(capnp.SingleSegment(nil))
 	require.NoError(t, err)
-	capnpOpts, err := tunnelrpc.NewConnectionOptions(seg)
+	capnpOpts, err := proto.NewConnectionOptions(seg)
 	require.NoError(t, err)
 
 	err = orig.MarshalCapnproto(capnpOpts)
@@ -54,18 +54,14 @@ func TestConnectionRegistrationRPC(t *testing.T) {
 
 	// Server-side
 	testImpl := testConnectionRegistrationServer{}
-	srv := TunnelServer_ServerToClient(&testImpl)
+	srv := RegistrationServer_ServerToClient(&testImpl)
 	serverConn := rpc.NewConn(t1, rpc.MainInterface(srv.Client))
 	defer serverConn.Wait()
 
 	ctx := context.Background()
 	clientConn := rpc.NewConn(t2)
 	defer clientConn.Close()
-	client := TunnelServer_PogsClient{
-		RegistrationServer_PogsClient: RegistrationServer_PogsClient{
-			Client: clientConn.Bootstrap(ctx),
-			Conn:   clientConn,
-		},
+	client := RegistrationServer_PogsClient{
 		Client: clientConn.Bootstrap(ctx),
 		Conn:   clientConn,
 	}
@@ -123,8 +119,6 @@ func TestConnectionRegistrationRPC(t *testing.T) {
 }
 
 type testConnectionRegistrationServer struct {
-	mockTunnelServerBase
-
 	details *ConnectionDetails
 	err     error
 }
@@ -147,3 +141,7 @@ func (t *testConnectionRegistrationServer) RegisterConnection(ctx context.Contex
 
 	panic("either details or err mush be set")
 }
+
+func (t *testConnectionRegistrationServer) UnregisterConnection(ctx context.Context) {
+	panic("unimplemented: UnregisterConnection")
+}

tunnelrpc/pogs/session_manager.go

@@ -6,11 +6,13 @@ import (
 	"net"
 	"time"
 
-	"github.com/cloudflare/cloudflared/tunnelrpc"
 	"github.com/google/uuid"
 	capnp "zombiezen.com/go/capnproto2"
 	"zombiezen.com/go/capnproto2/rpc"
 	"zombiezen.com/go/capnproto2/server"
+
+	"github.com/cloudflare/cloudflared/tunnelrpc/metrics"
+	"github.com/cloudflare/cloudflared/tunnelrpc/proto"
 )
 
 type SessionManager interface {
@@ -26,11 +28,15 @@ type SessionManager_PogsImpl struct {
 	impl SessionManager
 }
 
-func SessionManager_ServerToClient(s SessionManager) tunnelrpc.SessionManager {
-	return tunnelrpc.SessionManager_ServerToClient(SessionManager_PogsImpl{s})
+func SessionManager_ServerToClient(s SessionManager) proto.SessionManager {
+	return proto.SessionManager_ServerToClient(SessionManager_PogsImpl{s})
 }
 
-func (i SessionManager_PogsImpl) RegisterUdpSession(p tunnelrpc.SessionManager_registerUdpSession) error {
+func (i SessionManager_PogsImpl) RegisterUdpSession(p proto.SessionManager_registerUdpSession) error {
+	return metrics.ObserveServerHandler(func() error { return i.registerUdpSession(p) }, metrics.SessionManager, metrics.OperationRegisterUdpSession)
+}
+
+func (i SessionManager_PogsImpl) registerUdpSession(p proto.SessionManager_registerUdpSession) error {
 	server.Ack(p.Options)
 
 	sessionIDRaw, err := p.Params.SessionId()
@@ -76,7 +82,11 @@ func (i SessionManager_PogsImpl) RegisterUdpSession(p tunnelrpc.SessionManager_r
 	return resp.Marshal(result)
 }
 
-func (i SessionManager_PogsImpl) UnregisterUdpSession(p tunnelrpc.SessionManager_unregisterUdpSession) error {
+func (i SessionManager_PogsImpl) UnregisterUdpSession(p proto.SessionManager_unregisterUdpSession) error {
+	return metrics.ObserveServerHandler(func() error { return i.unregisterUdpSession(p) }, metrics.SessionManager, metrics.OperationUnregisterUdpSession)
+}
+
+func (i SessionManager_PogsImpl) unregisterUdpSession(p proto.SessionManager_unregisterUdpSession) error {
 	server.Ack(p.Options)
 
 	sessionIDRaw, err := p.Params.SessionId()
@@ -101,7 +111,7 @@ type RegisterUdpSessionResponse struct {
 	Spans []byte // Spans in protobuf format
 }
 
-func (p *RegisterUdpSessionResponse) Marshal(s tunnelrpc.RegisterUdpSessionResponse) error {
+func (p *RegisterUdpSessionResponse) Marshal(s proto.RegisterUdpSessionResponse) error {
 	if p.Err != nil {
 		return s.SetErr(p.Err.Error())
 	}
@@ -111,7 +121,7 @@ func (p *RegisterUdpSessionResponse) Marshal(s tunnelrpc.RegisterUdpSessionRespo
 	return nil
 }
 
-func (p *RegisterUdpSessionResponse) Unmarshal(s tunnelrpc.RegisterUdpSessionResponse) error {
+func (p *RegisterUdpSessionResponse) Unmarshal(s proto.RegisterUdpSessionResponse) error {
 	respErr, err := s.Err()
 	if err != nil {
 		return err
@@ -131,14 +141,21 @@ type SessionManager_PogsClient struct {
 	Conn   *rpc.Conn
 }
 
+func NewSessionManager_PogsClient(client capnp.Client, conn *rpc.Conn) SessionManager_PogsClient {
+	return SessionManager_PogsClient{
+		Client: client,
+		Conn:   conn,
+	}
+}
+
 func (c SessionManager_PogsClient) Close() error {
 	c.Client.Close()
 	return c.Conn.Close()
 }
 
 func (c SessionManager_PogsClient) RegisterUdpSession(ctx context.Context, sessionID uuid.UUID, dstIP net.IP, dstPort uint16, closeAfterIdleHint time.Duration, traceContext string) (*RegisterUdpSessionResponse, error) {
-	client := tunnelrpc.SessionManager{Client: c.Client}
-	promise := client.RegisterUdpSession(ctx, func(p tunnelrpc.SessionManager_registerUdpSession_Params) error {
+	client := proto.SessionManager{Client: c.Client}
+	promise := client.RegisterUdpSession(ctx, func(p proto.SessionManager_registerUdpSession_Params) error {
 		if err := p.SetSessionId(sessionID[:]); err != nil {
 			return err
 		}
@@ -164,8 +181,8 @@ func (c SessionManager_PogsClient) RegisterUdpSession(ctx context.Context, sessi
 }
 
 func (c SessionManager_PogsClient) UnregisterUdpSession(ctx context.Context, sessionID uuid.UUID, message string) error {
-	client := tunnelrpc.SessionManager{Client: c.Client}
-	promise := client.UnregisterUdpSession(ctx, func(p tunnelrpc.SessionManager_unregisterUdpSession_Params) error {
+	client := proto.SessionManager{Client: c.Client}
+	promise := client.UnregisterUdpSession(ctx, func(p proto.SessionManager_unregisterUdpSession_Params) error {
 		if err := p.SetSessionId(sessionID[:]); err != nil {
 			return err
 		}

tunnelrpc/pogs/support_test.go

@@ -1,40 +0,0 @@
-package pogs
-
-import (
-	"context"
-
-	"github.com/google/uuid"
-)
-
-// mockTunnelServerBase provides a placeholder implementation
-// for TunnelServer interface that can be used to build
-// mocks for specific unit tests without having to implement every method
-type mockTunnelServerBase struct{}
-
-func (mockTunnelServerBase) RegisterConnection(ctx context.Context, auth TunnelAuth, tunnelID uuid.UUID, connIndex byte, options *ConnectionOptions) (*ConnectionDetails, error) {
-	panic("unexpected call to RegisterConnection")
-}
-
-func (mockTunnelServerBase) UnregisterConnection(ctx context.Context) {
-	panic("unexpected call to UnregisterConnection")
-}
-
-func (mockTunnelServerBase) RegisterTunnel(ctx context.Context, originCert []byte, hostname string, options *RegistrationOptions) *TunnelRegistration {
-	panic("unexpected call to RegisterTunnel")
-}
-
-func (mockTunnelServerBase) GetServerInfo(ctx context.Context) (*ServerInfo, error) {
-	panic("unexpected call to GetServerInfo")
-}
-
-func (mockTunnelServerBase) UnregisterTunnel(ctx context.Context, gracePeriodNanoSec int64) error {
-	panic("unexpected call to UnregisterTunnel")
-}
-
-func (mockTunnelServerBase) Authenticate(ctx context.Context, originCert []byte, hostname string, options *RegistrationOptions) (*AuthenticateResponse, error) {
-	panic("unexpected call to Authenticate")
-}
-
-func (mockTunnelServerBase) ReconnectTunnel(ctx context.Context, jwt, eventDigest, connDigest []byte, hostname string, options *RegistrationOptions) (*TunnelRegistration, error) {
-	panic("unexpected call to ReconnectTunnel")
-}