Release 2023.10.0

This changes guarantees that the coommand to report rule matches when
testing local config reports the rule number using the 0-based indexing.
This is to be consistent with the 0-based indexing on the log lines when
proxying requests.

.github/workflows/check.yaml

@@ -4,7 +4,7 @@ jobs:
   check:
     strategy:
       matrix:
-        go-version: [1.19.x]
+        go-version: [1.20.x]
         os: [ubuntu-latest, macos-latest, windows-latest]
     runs-on: ${{ matrix.os }}
     steps:

CHANGES.md

@@ -1,3 +1,10 @@
+## 2023.9.0
+### Notices
+- The `warp-routing` `enabled: boolean` flag is no longer supported in the configuration file. Warp Routing traffic (eg TCP, UDP, ICMP) traffic is proxied to cloudflared if routes to the target tunnel are configured. This change does not affect remotely managed tunnels, but for locally managed tunnels, users that might be relying on this feature flag to block traffic should instead guarantee that tunnel has no Private Routes configured for the tunnel.
+## 2023.7.0
+### New Features
+- You can now enable additional diagnostics over the management.argotunnel.com service for your active cloudflared connectors via a new runtime flag `--management-diagnostics` (or env `TUNNEL_MANAGEMENT_DIAGNOSTICS`). This feature is provided as opt-in and requires the flag to enable. Endpoints such as /metrics provides your prometheus metrics endpoint another mechanism to be reached. Additionally /debug/pprof/(goroutine|heap) are also introduced to allow for remotely retrieving active pprof information from a running cloudflared connector.
+
 ## 2023.4.1
 ### New Features
 - You can now stream your logs from your remote cloudflared to your local terminal with `cloudflared tail <TUNNEL-ID>`. This new feature requires the remote cloudflared to be version 2023.4.1 or higher.

Dockerfile

@@ -1,7 +1,7 @@
 # use a builder image for building cloudflare
 ARG TARGET_GOOS
 ARG TARGET_GOARCH
-FROM golang:1.19 as builder
+FROM golang:1.20.6 as builder
 ENV GO111MODULE=on \
     CGO_ENABLED=0 \
     TARGET_GOOS=${TARGET_GOOS} \

Dockerfile.amd64

@@ -1,5 +1,5 @@
 # use a builder image for building cloudflare
-FROM golang:1.19 as builder
+FROM golang:1.20.6 as builder
 ENV GO111MODULE=on \
     CGO_ENABLED=0 
     

Dockerfile.arm64

@@ -1,5 +1,5 @@
 # use a builder image for building cloudflare
-FROM golang:1.19 as builder
+FROM golang:1.20.6 as builder
 ENV GO111MODULE=on \
     CGO_ENABLED=0 
     

Makefile

@@ -209,58 +209,6 @@ cloudflared-darwin-amd64.tgz: cloudflared
 	tar czf cloudflared-darwin-amd64.tgz cloudflared
 	rm cloudflared
 
-.PHONY: cloudflared-junos
-cloudflared-junos: cloudflared jetez-certificate.pem jetez-key.pem
-	jetez --source . \
-		  -j jet.yaml \
-		  --key jetez-key.pem \
-		  --cert jetez-certificate.pem \
-		  --version $(VERSION)
-	rm jetez-*.pem
-
-jetez-certificate.pem:
-ifndef JETEZ_CERT
-	$(error JETEZ_CERT not defined)
-endif
-	@echo "Writing JetEZ certificate"
-	@echo "$$JETEZ_CERT" > jetez-certificate.pem
-
-jetez-key.pem:
-ifndef JETEZ_KEY
-	$(error JETEZ_KEY not defined)
-endif
-	@echo "Writing JetEZ key"
-	@echo "$$JETEZ_KEY" > jetez-key.pem
-
-.PHONY: publish-cloudflared-junos
-publish-cloudflared-junos: cloudflared-junos cloudflared-x86-64.latest.s3
-ifndef S3_ENDPOINT
-	$(error S3_HOST not defined)
-endif
-ifndef S3_URI
-	$(error S3_URI not defined)
-endif
-ifndef S3_ACCESS_KEY
-	$(error S3_ACCESS_KEY not defined)
-endif
-ifndef S3_SECRET_KEY
-	$(error S3_SECRET_KEY not defined)
-endif
-	sha256sum cloudflared-x86-64-$(VERSION).tgz | awk '{printf $$1}' > cloudflared-x86-64-$(VERSION).tgz.shasum
-	s4cmd --endpoint-url $(S3_ENDPOINT) --force --API-GrantRead=uri=http://acs.amazonaws.com/groups/global/AllUsers \
-		put cloudflared-x86-64-$(VERSION).tgz $(S3_URI)/cloudflared-x86-64-$(VERSION).tgz
-	s4cmd --endpoint-url $(S3_ENDPOINT) --force --API-GrantRead=uri=http://acs.amazonaws.com/groups/global/AllUsers \
-		put cloudflared-x86-64-$(VERSION).tgz.shasum $(S3_URI)/cloudflared-x86-64-$(VERSION).tgz.shasum
-	dpkg --compare-versions "$(VERSION)" gt "$(shell cat cloudflared-x86-64.latest.s3)" && \
-		echo -n "$(VERSION)" > cloudflared-x86-64.latest && \
-		s4cmd --endpoint-url $(S3_ENDPOINT) --force --API-GrantRead=uri=http://acs.amazonaws.com/groups/global/AllUsers \
-			put cloudflared-x86-64.latest $(S3_URI)/cloudflared-x86-64.latest || \
-		echo "Latest version not updated"
-
-cloudflared-x86-64.latest.s3:
-	s4cmd --endpoint-url $(S3_ENDPOINT) --force \
-		get $(S3_URI)/cloudflared-x86-64.latest cloudflared-x86-64.latest.s3
-
 .PHONY: homebrew-upload
 homebrew-upload: cloudflared-darwin-amd64.tgz
 	aws s3 --endpoint-url $(S3_ENDPOINT) cp --acl public-read $$^ $(S3_URI)/cloudflared-$$(VERSION)-$1.tgz

README.md

@@ -53,9 +53,6 @@ Want to test Cloudflare Tunnel before adding a website to Cloudflare? You can do
 
 ## Deprecated versions
 
-Cloudflare currently supports versions of `cloudflared` 2020.5.1 and later. Breaking changes unrelated to feature availability may be introduced that will impact versions released prior to 2020.5.1. You can read more about upgrading `cloudflared` in our [developer documentation](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/install-and-setup/installation#updating-cloudflared).
+Cloudflare currently supports versions of cloudflared that are **within one year** of the most recent release. Breaking changes unrelated to feature availability may be introduced that will impact versions released more than one year ago. You can read more about upgrading cloudflared in our [developer documentation](https://developers.cloudflare.com/cloudflare-one/connections/connect-networks/downloads/#updating-cloudflared).
 
-| Version(s) | Deprecation status |
-|---|---|
-| 2020.5.1 and later | Supported |
-| Versions prior to 2020.5.1 | No longer supported |
+For example, as of January 2023 Cloudflare will support cloudflared version 2023.1.1 to cloudflared 2022.1.1.

RELEASE_NOTES

@@ -1,3 +1,60 @@
+2023.10.0
+- 2023-10-06 TUN-7864: Document cloudflared versions support
+- 2023-10-03 CUSTESC-33731: Make rule match test report rule in 0-index base
+- 2023-09-22 TUN-7824: Fix usage of systemctl status to detect which services are installed
+- 2023-09-20 TUN-7813: Improve tunnel delete command to use cascade delete
+- 2023-09-20 TUN-7787: cloudflared only list ip routes targeted for cfd_tunnel
+- 2023-09-15 TUN-7787: Refactor cloudflared to use new route endpoints based on route IDs
+- 2023-09-08 TUN-7776: Remove warp-routing flag from cloudflared
+- 2023-09-05 TUN-7756: Clarify that QUIC is mandatory to support ICMP proxying
+
+2023.8.2
+- 2023-08-25 TUN-7700: Implement feature selector to determine if connections will prefer post quantum cryptography
+- 2023-08-22 TUN-7707: Use X25519Kyber768Draft00 curve when post-quantum feature is enabled
+
+2023.8.1
+- 2023-08-23 TUN-7718: Update R2 Token to no longer encode secret
+
+2023.8.0
+- 2023-07-26 TUN-7584: Bump go 1.20.6
+
+2023.7.3
+- 2023-07-25 TUN-7628: Correct Host parsing for Access
+- 2023-07-24 TUN-7624: Fix flaky TestBackoffGracePeriod test in cloudflared
+
+2023.7.2
+- 2023-07-19 TUN-7599: Onboard cloudflared to Software Dashboard
+- 2023-07-19 TUN-7587: Remove junos builds
+- 2023-07-18 TUN-7597: Add flag to disable auto-update services to be installed
+- 2023-07-17 TUN-7594: Add nightly arm64 cloudflared internal deb publishes
+- 2023-07-14 TUN-7586: Upgrade go-jose/go-jose/v3 and core-os/go-oidc/v3
+- 2023-07-14 TUN-7589: Remove legacy golang.org/x/crypto/ssh/terminal package usage
+- 2023-07-14 TUN-7590: Remove usages of ioutil
+- 2023-07-14 TUN-7585: Remove h2mux compression
+- 2023-07-14 TUN-7588: Update package coreos/go-systemd
+
+2023.7.1
+- 2023-07-13 TUN-7582: Correct changelog wording for --management-diagnostics
+- 2023-07-12 TUN-7575: Add option to disable PTMU discovery over QUIC
+
+2023.7.0
+- 2023-07-06 TUN-7558: Flush on Writes for StreamBasedOriginProxy
+- 2023-07-05 TUN-7553: Add flag to enable management diagnostic services
+- 2023-07-05 TUN-7564: Support cf-trace-id for cloudflared access
+- 2023-07-05 TUN-7477: Decrement UDP sessions on shutdown
+- 2023-07-03 TUN-7545: Add support for full bidirectionally streaming with close signal propagation
+- 2023-06-30 TUN-7549: Add metrics route to management service
+- 2023-06-30 TUN-7551: Complete removal of raven-go to sentry-go
+- 2023-06-30 TUN-7550: Add pprof endpoint to management service
+- 2023-06-29 TUN-7543: Add --debug-stream flag to cloudflared access ssh
+- 2023-06-26 TUN-6011: Remove docker networks from ICMP Proxy test
+- 2023-06-20 AUTH-5328 Pass cloudflared_token_check param when running cloudflared access login
+
+2023.6.1
+- 2023-06-19 TUN-7480: Added a timeout for unregisterUDP.
+- 2023-06-16 TUN-7477: Add UDP/TCP session metrics
+- 2023-06-14 TUN-7468: Increase the limit of incoming streams
+
 2023.6.0
 - 2023-06-15 TUN-7471: Fixes cloudflared not closing the quic stream on unregister UDP session
 - 2023-06-09 TUN-7463: Add default ingress rule if no ingress rules are provided when updating the configuration

catalog-info.yaml

@@ -0,0 +1,16 @@
+apiVersion: backstage.io/v1alpha1
+kind: Component
+metadata:
+  name: cloudflared
+  description: Client for Cloudflare Tunnels
+  annotations:
+    backstage.io/source-location: url:https://bitbucket.cfdata.org/projects/TUN/repos/cloudflared/browse
+    cloudflare.com/software-excellence-opt-in: "true"
+    cloudflare.com/jira-project-key: "TUN"
+    cloudflare.com/jira-project-component: "Cloudflare Tunnel"
+  tags:
+    - internal
+spec:
+  type: "service"
+  lifecycle: "Active"
+  owner: "teams/tunnel-teams-routing"

cfapi/client.go

@@ -9,7 +9,7 @@ type TunnelClient interface {
 	GetTunnel(tunnelID uuid.UUID) (*Tunnel, error)
 	GetTunnelToken(tunnelID uuid.UUID) (string, error)
 	GetManagementToken(tunnelID uuid.UUID) (string, error)
-	DeleteTunnel(tunnelID uuid.UUID) error
+	DeleteTunnel(tunnelID uuid.UUID, cascade bool) error
 	ListTunnels(filter *TunnelFilter) ([]*Tunnel, error)
 	ListActiveClients(tunnelID uuid.UUID) ([]*ActiveClient, error)
 	CleanupConnections(tunnelID uuid.UUID, params *CleanupParams) error
@@ -22,7 +22,7 @@ type HostnameClient interface {
 type IPRouteClient interface {
 	ListRoutes(filter *IpRouteFilter) ([]*DetailedRoute, error)
 	AddRoute(newRoute NewRoute) (Route, error)
-	DeleteRoute(params DeleteRouteParams) error
+	DeleteRoute(id uuid.UUID) error
 	GetByIP(params GetRouteByIpParams) (DetailedRoute, error)
 }
 

cfapi/ip_route.go

@@ -75,10 +75,12 @@ type NewRoute struct {
 // MarshalJSON handles fields with non-JSON types (e.g. net.IPNet).
 func (r NewRoute) MarshalJSON() ([]byte, error) {
 	return json.Marshal(&struct {
+		Network  string     `json:"network"`
 		TunnelID uuid.UUID  `json:"tunnel_id"`
 		Comment  string     `json:"comment"`
 		VNetID   *uuid.UUID `json:"virtual_network_id,omitempty"`
 	}{
+		Network:  r.Network.String(),
 		TunnelID: r.TunnelID,
 		Comment:  r.Comment,
 		VNetID:   r.VNetID,
@@ -87,6 +89,7 @@ func (r NewRoute) MarshalJSON() ([]byte, error) {
 
 // DetailedRoute is just a Route with some extra fields, e.g. TunnelName.
 type DetailedRoute struct {
+	ID       uuid.UUID `json:"id"`
 	Network  CIDR      `json:"network"`
 	TunnelID uuid.UUID `json:"tunnel_id"`
 	// Optional field. When unset, it means the DetailedRoute belongs to the default virtual network.
@@ -115,7 +118,8 @@ func (r DetailedRoute) TableString() string {
 	}
 
 	return fmt.Sprintf(
-		"%s\t%s\t%s\t%s\t%s\t%s\t%s\t",
+		"%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t",
+		r.ID,
 		r.Network.String(),
 		vnetColumn,
 		r.Comment,
@@ -126,12 +130,6 @@ func (r DetailedRoute) TableString() string {
 	)
 }
 
-type DeleteRouteParams struct {
-	Network net.IPNet
-	// Optional field. If unset, backend will assume the default vnet for the account.
-	VNetID *uuid.UUID
-}
-
 type GetRouteByIpParams struct {
 	Ip net.IP
 	// Optional field. If unset, backend will assume the default vnet for the account.
@@ -158,7 +156,7 @@ func (r *RESTClient) ListRoutes(filter *IpRouteFilter) ([]*DetailedRoute, error)
 // AddRoute calls the Tunnelstore POST endpoint for a given route.
 func (r *RESTClient) AddRoute(newRoute NewRoute) (Route, error) {
 	endpoint := r.baseEndpoints.accountRoutes
-	endpoint.Path = path.Join(endpoint.Path, "network", url.PathEscape(newRoute.Network.String()))
+	endpoint.Path = path.Join(endpoint.Path)
 	resp, err := r.sendRequest("POST", endpoint, newRoute)
 	if err != nil {
 		return Route{}, errors.Wrap(err, "REST request failed")
@@ -173,10 +171,9 @@ func (r *RESTClient) AddRoute(newRoute NewRoute) (Route, error) {
 }
 
 // DeleteRoute calls the Tunnelstore DELETE endpoint for a given route.
-func (r *RESTClient) DeleteRoute(params DeleteRouteParams) error {
+func (r *RESTClient) DeleteRoute(id uuid.UUID) error {
 	endpoint := r.baseEndpoints.accountRoutes
-	endpoint.Path = path.Join(endpoint.Path, "network", url.PathEscape(params.Network.String()))
-	setVnetParam(&endpoint, params.VNetID)
+	endpoint.Path = path.Join(endpoint.Path, url.PathEscape(id.String()))
 
 	resp, err := r.sendRequest("DELETE", endpoint, nil)
 	if err != nil {

cfapi/ip_route_filter.go

@@ -58,31 +58,29 @@ type IpRouteFilter struct {
 
 // NewIpRouteFilterFromCLI parses CLI flags to discover which filters should get applied.
 func NewIpRouteFilterFromCLI(c *cli.Context) (*IpRouteFilter, error) {
-	f := &IpRouteFilter{
-		queryParams: url.Values{},
-	}
+	f := NewIPRouteFilter()
 
 	// Set deletion filter
 	if flag := filterIpRouteDeleted.Name; c.IsSet(flag) && c.Bool(flag) {
-		f.deleted()
+		f.Deleted()
 	} else {
-		f.notDeleted()
+		f.NotDeleted()
 	}
 
 	if subset, err := cidrFromFlag(c, filterSubsetIpRoute); err != nil {
 		return nil, err
 	} else if subset != nil {
-		f.networkIsSupersetOf(*subset)
+		f.NetworkIsSupersetOf(*subset)
 	}
 
 	if superset, err := cidrFromFlag(c, filterSupersetIpRoute); err != nil {
 		return nil, err
 	} else if superset != nil {
-		f.networkIsSupersetOf(*superset)
+		f.NetworkIsSupersetOf(*superset)
 	}
 
 	if comment := c.String(filterIpRouteComment.Name); comment != "" {
-		f.commentIs(comment)
+		f.CommentIs(comment)
 	}
 
 	if tunnelID := c.String(filterIpRouteTunnelID.Name); tunnelID != "" {
@@ -90,7 +88,7 @@ func NewIpRouteFilterFromCLI(c *cli.Context) (*IpRouteFilter, error) {
 		if err != nil {
 			return nil, errors.Wrapf(err, "Couldn't parse UUID from %s", filterIpRouteTunnelID.Name)
 		}
-		f.tunnelID(u)
+		f.TunnelID(u)
 	}
 
 	if vnetId := c.String(filterIpRouteByVnet.Name); vnetId != "" {
@@ -98,7 +96,7 @@ func NewIpRouteFilterFromCLI(c *cli.Context) (*IpRouteFilter, error) {
 		if err != nil {
 			return nil, errors.Wrapf(err, "Couldn't parse UUID from %s", filterIpRouteByVnet.Name)
 		}
-		f.vnetID(u)
+		f.VNetID(u)
 	}
 
 	if maxFetch := c.Int("max-fetch-size"); maxFetch > 0 {
@@ -124,35 +122,44 @@ func cidrFromFlag(c *cli.Context, flag cli.StringFlag) (*net.IPNet, error) {
 	return subset, nil
 }
 
-func (f *IpRouteFilter) commentIs(comment string) {
+func NewIPRouteFilter() *IpRouteFilter {
+	values := &IpRouteFilter{queryParams: url.Values{}}
+
+	// always list cfd_tunnel routes only
+	values.queryParams.Set("tun_types", "cfd_tunnel")
+
+	return values
+}
+
+func (f *IpRouteFilter) CommentIs(comment string) {
 	f.queryParams.Set("comment", comment)
 }
 
-func (f *IpRouteFilter) notDeleted() {
+func (f *IpRouteFilter) NotDeleted() {
 	f.queryParams.Set("is_deleted", "false")
 }
 
-func (f *IpRouteFilter) deleted() {
+func (f *IpRouteFilter) Deleted() {
 	f.queryParams.Set("is_deleted", "true")
 }
 
-func (f *IpRouteFilter) networkIsSubsetOf(superset net.IPNet) {
+func (f *IpRouteFilter) NetworkIsSubsetOf(superset net.IPNet) {
 	f.queryParams.Set("network_subset", superset.String())
 }
 
-func (f *IpRouteFilter) networkIsSupersetOf(subset net.IPNet) {
+func (f *IpRouteFilter) NetworkIsSupersetOf(subset net.IPNet) {
 	f.queryParams.Set("network_superset", subset.String())
 }
 
-func (f *IpRouteFilter) existedAt(existedAt time.Time) {
+func (f *IpRouteFilter) ExistedAt(existedAt time.Time) {
 	f.queryParams.Set("existed_at", existedAt.Format(time.RFC3339))
 }
 
-func (f *IpRouteFilter) tunnelID(id uuid.UUID) {
+func (f *IpRouteFilter) TunnelID(id uuid.UUID) {
 	f.queryParams.Set("tunnel_id", id.String())
 }
 
-func (f *IpRouteFilter) vnetID(id uuid.UUID) {
+func (f *IpRouteFilter) VNetID(id uuid.UUID) {
 	f.queryParams.Set("virtual_network_id", id.String())
 }
 

cfapi/ip_route_test.go

@@ -69,6 +69,7 @@ func TestDetailedRouteJsonRoundtrip(t *testing.T) {
 	}{
 		{
 			`{
+				"id":"91ebc578-cc99-4641-9937-0fb630505fa0",
 				"network":"10.1.2.40/29",
 				"tunnel_id":"fba6ffea-807f-4e7a-a740-4184ee1b82c8",
 				"comment":"test",
@@ -80,6 +81,7 @@ func TestDetailedRouteJsonRoundtrip(t *testing.T) {
 		},
 		{
 			`{
+				"id":"91ebc578-cc99-4641-9937-0fb630505fa0",
 				"network":"10.1.2.40/29",
 				"tunnel_id":"fba6ffea-807f-4e7a-a740-4184ee1b82c8",
 				"virtual_network_id":"38c95083-8191-4110-8339-3f438d44fdb9",
@@ -167,9 +169,10 @@ func TestRouteTableString(t *testing.T) {
 	require.NoError(t, err)
 	require.NotNil(t, network)
 	r := DetailedRoute{
+		ID:      uuid.Nil,
 		Network: CIDR(*network),
 	}
 	row := r.TableString()
 	fmt.Println(row)
-	require.True(t, strings.HasPrefix(row, "1.2.3.4/32"))
+	require.True(t, strings.HasPrefix(row, "00000000-0000-0000-0000-000000000000\t1.2.3.4/32"))
 }

cfapi/tunnel.go

@@ -159,9 +159,14 @@ func (r *RESTClient) GetManagementToken(tunnelID uuid.UUID) (token string, err e
 	return "", r.statusCodeToError("get tunnel token", resp)
 }
 
-func (r *RESTClient) DeleteTunnel(tunnelID uuid.UUID) error {
+func (r *RESTClient) DeleteTunnel(tunnelID uuid.UUID, cascade bool) error {
 	endpoint := r.baseEndpoints.accountLevel
 	endpoint.Path = path.Join(endpoint.Path, fmt.Sprintf("%v", tunnelID))
+	// Cascade will delete all tunnel dependencies (connections, routes, etc.) that
+	// are linked to the deleted tunnel.
+	if cascade {
+		endpoint.RawQuery = "cascade=true"
+	}
 	resp, err := r.sendRequest("DELETE", endpoint, nil)
 	if err != nil {
 		return errors.Wrap(err, "REST request failed")

cfapi/tunnel_test.go

@@ -3,7 +3,7 @@ package cfapi
 import (
 	"bytes"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net"
 	"reflect"
 	"strings"
@@ -49,7 +49,7 @@ func Test_parseListTunnels(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			body := ioutil.NopCloser(bytes.NewReader([]byte(tt.args.body)))
+			body := io.NopCloser(bytes.NewReader([]byte(tt.args.body)))
 			got, err := parseListTunnels(body)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("parseListTunnels() error = %v, wantErr %v", err, tt.wantErr)

cfsetup.yaml

@@ -1,5 +1,5 @@
-pinned_go: &pinned_go go=1.19.6-1
-pinned_go_fips: &pinned_go_fips go-boring=1.19.6-1
+pinned_go: &pinned_go go=1.20.6-1
+pinned_go_fips: &pinned_go_fips go-boring=1.20.6-1
 
 build_dir: &build_dir /cfsetup_build
 default-flavor: bullseye
@@ -121,7 +121,7 @@ buster: &buster
       - export FIPS=true
       - export ORIGINAL_NAME=true
       - make cloudflared-deb
-  build-fips-internal-deb-nightly:
+  build-internal-deb-nightly-amd64:
     build_dir: *build_dir
     builddeps: *build_fips_deb_deps
     post-cache:
@@ -131,6 +131,16 @@ buster: &buster
       - export FIPS=true
       - export ORIGINAL_NAME=true
       - make cloudflared-deb
+  build-internal-deb-nightly-arm64:
+    build_dir: *build_dir
+    builddeps: *build_fips_deb_deps
+    post-cache:
+      - export GOOS=linux
+      - export GOARCH=arm64
+      - export NIGHTLY=true
+      #- export FIPS=true # TUN-7595
+      - export ORIGINAL_NAME=true
+      - make cloudflared-deb
   build-deb-arm64:
     build_dir: *build_dir
     builddeps: *build_deb_deps
@@ -244,35 +254,6 @@ buster: &buster
     pre-cache: *install_pygithub
     post-cache:
       - make github-message
-  build-junos:
-    build_dir: *build_dir
-    builddeps:
-      - *pinned_go
-      - build-essential
-      - python3
-      - genisoimage
-    pre-cache:
-      - ln -s /usr/bin/genisoimage /usr/bin/mkisofs
-    post-cache:
-      - export CGO_ENABLED=0
-      - export GOOS=freebsd
-      - export GOARCH=amd64
-      - make cloudflared
-  publish-junos:
-    build_dir: *build_dir
-    builddeps:
-      - *pinned_go
-      - build-essential
-      - python3
-      - genisoimage
-      - jetez
-      - s4cmd
-    pre-cache:
-      - ln -s /usr/bin/genisoimage /usr/bin/mkisofs
-    post-cache:
-      - export GOOS=freebsd
-      - export GOARCH=amd64
-      - make publish-cloudflared-junos
 
 bullseye: *buster
 bookworm: *buster

cmd/cloudflared/access/carrier.go

@@ -3,6 +3,7 @@ package access
 import (
 	"crypto/tls"
 	"fmt"
+	"io"
 	"net/http"
 	"strings"
 
@@ -13,6 +14,7 @@ import (
 	"github.com/cloudflare/cloudflared/carrier"
 	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/logger"
+	"github.com/cloudflare/cloudflared/stream"
 	"github.com/cloudflare/cloudflared/validation"
 )
 
@@ -38,6 +40,7 @@ func StartForwarder(forwarder config.Forwarder, shutdown <-chan struct{}, log *z
 	if forwarder.TokenSecret != "" {
 		headers.Set(cfAccessClientSecretHeader, forwarder.TokenSecret)
 	}
+	headers.Set("User-Agent", userAgent)
 
 	carrier.SetBastionDest(headers, forwarder.Destination)
 
@@ -58,31 +61,37 @@ func StartForwarder(forwarder config.Forwarder, shutdown <-chan struct{}, log *z
 // useful for proxying other protocols (like ssh) over websockets
 // (which you can put Access in front of)
 func ssh(c *cli.Context) error {
-	log := logger.CreateSSHLoggerFromContext(c, logger.EnableTerminalLog)
+	// If not running as a forwarder, disable terminal logs as it collides with the stdin/stdout of the parent process
+	outputTerminal := logger.DisableTerminalLog
+	if c.IsSet(sshURLFlag) {
+		outputTerminal = logger.EnableTerminalLog
+	}
+	log := logger.CreateSSHLoggerFromContext(c, outputTerminal)
 
 	// get the hostname from the cmdline and error out if its not provided
 	rawHostName := c.String(sshHostnameFlag)
-	hostname, err := validation.ValidateHostname(rawHostName)
-	if err != nil || rawHostName == "" {
+	url, err := parseURL(rawHostName)
+	if err != nil {
+		log.Err(err).Send()
 		return cli.ShowCommandHelp(c, "ssh")
 	}
-	originURL := ensureURLScheme(hostname)
 
 	// get the headers from the cmdline and add them
-	headers := buildRequestHeaders(c.StringSlice(sshHeaderFlag))
+	headers := parseRequestHeaders(c.StringSlice(sshHeaderFlag))
 	if c.IsSet(sshTokenIDFlag) {
 		headers.Set(cfAccessClientIDHeader, c.String(sshTokenIDFlag))
 	}
 	if c.IsSet(sshTokenSecretFlag) {
 		headers.Set(cfAccessClientSecretHeader, c.String(sshTokenSecretFlag))
 	}
+	headers.Set("User-Agent", userAgent)
 
 	carrier.SetBastionDest(headers, c.String(sshDestinationFlag))
 
 	options := &carrier.StartOptions{
-		OriginURL: originURL,
+		OriginURL: url.String(),
 		Headers:   headers,
-		Host:      hostname,
+		Host:      url.Host,
 	}
 
 	if connectTo := c.String(sshConnectTo); connectTo != "" {
@@ -121,16 +130,17 @@ func ssh(c *cli.Context) error {
 		return err
 	}
 
-	return carrier.StartClient(wsConn, &carrier.StdinoutStream{}, options)
-}
-
-func buildRequestHeaders(values []string) http.Header {
-	headers := make(http.Header)
-	for _, valuePair := range values {
-		split := strings.Split(valuePair, ":")
-		if len(split) > 1 {
-			headers.Add(strings.TrimSpace(split[0]), strings.TrimSpace(split[1]))
+	var s io.ReadWriter
+	s = &carrier.StdinoutStream{}
+	if c.IsSet(sshDebugStream) {
+		maxMessages := c.Uint64(sshDebugStream)
+		if maxMessages == 0 {
+			// default to 10 if provided but unset
+			maxMessages = 10
 		}
+		logger := log.With().Str("host", url.Host).Logger()
+		s = stream.NewDebugStream(s, &logger, maxMessages)
 	}
-	return headers
+	carrier.StartClient(wsConn, s, options)
+	return nil
 }

cmd/cloudflared/access/carrier_test.go

@@ -1,18 +0,0 @@
-package access
-
-import (
-	"net/http"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestBuildRequestHeaders(t *testing.T) {
-	headers := make(http.Header)
-	headers.Add("client", "value")
-	headers.Add("secret", "safe-value")
-
-	values := buildRequestHeaders([]string{"client: value", "secret: safe-value", "trash"})
-	assert.Equal(t, headers.Get("client"), values.Get("client"))
-	assert.Equal(t, headers.Get("secret"), values.Get("secret"))
-}

cmd/cloudflared/access/cmd.go

@@ -34,6 +34,7 @@ const (
 	sshTokenSecretFlag = "service-token-secret"
 	sshGenCertFlag     = "short-lived-cert"
 	sshConnectTo       = "connect-to"
+	sshDebugStream     = "debug-stream"
 	sshConfigTemplate  = `
 Add to your {{.Home}}/.ssh/config:
 
@@ -151,9 +152,12 @@ func Commands() []*cli.Command {
 							EnvVars: []string{"TUNNEL_SERVICE_TOKEN_SECRET"},
 						},
 						&cli.StringFlag{
-							Name:    logger.LogSSHDirectoryFlag,
-							Aliases: []string{"logfile"}, //added to match the tunnel side
-							Usage:   "Save application log to this directory for reporting issues.",
+							Name:  logger.LogFileFlag,
+							Usage: "Save application log to this file for reporting issues.",
+						},
+						&cli.StringFlag{
+							Name:  logger.LogSSHDirectoryFlag,
+							Usage: "Save application log to this directory for reporting issues.",
 						},
 						&cli.StringFlag{
 							Name:    logger.LogSSHLevelFlag,
@@ -165,6 +169,11 @@ func Commands() []*cli.Command {
 							Hidden: true,
 							Usage:  "Connect to alternate location for testing, value is host, host:port, or sni:port:host",
 						},
+						&cli.Uint64Flag{
+							Name:   sshDebugStream,
+							Hidden: true,
+							Usage:  "Writes up-to the max provided stream payloads to the logger as debug statements.",
+						},
 					},
 				},
 				{
@@ -213,8 +222,7 @@ func login(c *cli.Context) error {
 	log := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
 
 	args := c.Args()
-	rawURL := ensureURLScheme(args.First())
-	appURL, err := url.Parse(rawURL)
+	appURL, err := parseURL(args.First())
 	if args.Len() < 1 || err != nil {
 		log.Error().Msg("Please provide the url of the Access application")
 		return err
@@ -243,16 +251,6 @@ func login(c *cli.Context) error {
 	return nil
 }
 
-// ensureURLScheme prepends a URL with https:// if it doesn't have a scheme. http:// URLs will not be converted.
-func ensureURLScheme(url string) string {
-	url = strings.Replace(strings.ToLower(url), "http://", "https://", 1)
-	if !strings.HasPrefix(url, "https://") {
-		url = fmt.Sprintf("https://%s", url)
-
-	}
-	return url
-}
-
 // curl provides a wrapper around curl, passing Access JWT along in request
 func curl(c *cli.Context) error {
 	err := sentry.Init(sentry.ClientOptions{
@@ -336,7 +334,7 @@ func generateToken(c *cli.Context) error {
 	if err != nil {
 		return err
 	}
-	appURL, err := url.Parse(ensureURLScheme(c.String("app")))
+	appURL, err := parseURL(c.String("app"))
 	if err != nil || c.NumFlags() < 1 {
 		fmt.Fprintln(os.Stderr, "Please provide a url.")
 		return err
@@ -389,7 +387,7 @@ func sshGen(c *cli.Context) error {
 		return cli.ShowCommandHelp(c, "ssh-gen")
 	}
 
-	originURL, err := url.Parse(ensureURLScheme(hostname))
+	originURL, err := parseURL(hostname)
 	if err != nil {
 		return err
 	}
@@ -490,7 +488,7 @@ func isFileThere(candidate string) bool {
 // Then makes a request to to the origin with the token to ensure it is valid.
 // Returns nil if token is valid.
 func verifyTokenAtEdge(appUrl *url.URL, appInfo *token.AppInfo, c *cli.Context, log *zerolog.Logger) error {
-	headers := buildRequestHeaders(c.StringSlice(sshHeaderFlag))
+	headers := parseRequestHeaders(c.StringSlice(sshHeaderFlag))
 	if c.IsSet(sshTokenIDFlag) {
 		headers.Add(cfAccessClientIDHeader, c.String(sshTokenIDFlag))
 	}
@@ -525,6 +523,11 @@ func isTokenValid(options *carrier.StartOptions, log *zerolog.Logger) (bool, err
 		return false, errors.Wrap(err, "Could not create access request")
 	}
 	req.Header.Set("User-Agent", userAgent)
+
+	query := req.URL.Query()
+	query.Set("cloudflared_token_check", "true")
+	req.URL.RawQuery = query.Encode()
+
 	// Do not follow redirects
 	client := &http.Client{
 		CheckRedirect: func(req *http.Request, via []*http.Request) error {

cmd/cloudflared/access/cmd_test.go

@@ -1,25 +0,0 @@
-package access
-
-import "testing"
-
-func Test_ensureURLScheme(t *testing.T) {
-	type args struct {
-		url string
-	}
-	tests := []struct {
-		name string
-		args args
-		want string
-	}{
-		{"no scheme", args{"localhost:123"}, "https://localhost:123"},
-		{"http scheme", args{"http://test"}, "https://test"},
-		{"https scheme", args{"https://test"}, "https://test"},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if got := ensureURLScheme(tt.args.url); got != tt.want {
-				t.Errorf("ensureURLScheme() = %v, want %v", got, tt.want)
-			}
-		})
-	}
-}

cmd/cloudflared/access/validation.go

@@ -0,0 +1,55 @@
+package access
+
+import (
+	"errors"
+	"fmt"
+	"net/http"
+	"net/url"
+	"strings"
+
+	"golang.org/x/net/http/httpguts"
+)
+
+// parseRequestHeaders will take user-provided header values as strings "Content-Type: application/json" and create
+// a http.Header object.
+func parseRequestHeaders(values []string) http.Header {
+	headers := make(http.Header)
+	for _, valuePair := range values {
+		header, value, found := strings.Cut(valuePair, ":")
+		if found {
+			headers.Add(strings.TrimSpace(header), strings.TrimSpace(value))
+		}
+	}
+	return headers
+}
+
+// parseHostname will attempt to convert a user provided URL string into a string with some light error checking on
+// certain expectations from the URL.
+// Will convert all HTTP URLs to HTTPS
+func parseURL(input string) (*url.URL, error) {
+	if input == "" {
+		return nil, errors.New("no input provided")
+	}
+	if !strings.HasPrefix(input, "https://") && !strings.HasPrefix(input, "http://") {
+		input = fmt.Sprintf("https://%s", input)
+	}
+	url, err := url.ParseRequestURI(input)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse as URL: %w", err)
+	}
+	if url.Scheme != "https" {
+		url.Scheme = "https"
+	}
+	if url.Host == "" {
+		return nil, errors.New("failed to parse Host")
+	}
+	host, err := httpguts.PunycodeHostPort(url.Host)
+	if err != nil || host == "" {
+		return nil, err
+	}
+	if !httpguts.ValidHostHeader(host) {
+		return nil, errors.New("invalid Host provided")
+	}
+	url.Host = host
+	return url, nil
+}

cmd/cloudflared/access/validation_test.go

@@ -0,0 +1,80 @@
+package access
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestParseRequestHeaders(t *testing.T) {
+	values := parseRequestHeaders([]string{"client: value", "secret: safe-value", "trash", "cf-trace-id: 000:000:0:1:asd"})
+	assert.Len(t, values, 3)
+	assert.Equal(t, "value", values.Get("client"))
+	assert.Equal(t, "safe-value", values.Get("secret"))
+	assert.Equal(t, "000:000:0:1:asd", values.Get("cf-trace-id"))
+}
+
+func TestParseURL(t *testing.T) {
+	schemes := []string{
+		"http://",
+		"https://",
+		"",
+	}
+	hosts := []struct {
+		input    string
+		expected string
+	}{
+		{"localhost", "localhost"},
+		{"127.0.0.1", "127.0.0.1"},
+		{"127.0.0.1:9090", "127.0.0.1:9090"},
+		{"::1", "::1"},
+		{"::1:8080", "::1:8080"},
+		{"[::1]", "[::1]"},
+		{"[::1]:8080", "[::1]:8080"},
+		{":8080", ":8080"},
+		{"example.com", "example.com"},
+		{"hello.example.com", "hello.example.com"},
+		{"bücher.example.com", "xn--bcher-kva.example.com"},
+	}
+	paths := []string{
+		"",
+		"/test",
+		"/example.com?qwe=123",
+	}
+	for i, scheme := range schemes {
+		for j, host := range hosts {
+			for k, path := range paths {
+				t.Run(fmt.Sprintf("%d_%d_%d", i, j, k), func(t *testing.T) {
+					input := fmt.Sprintf("%s%s%s", scheme, host.input, path)
+					expected := fmt.Sprintf("%s%s%s", "https://", host.expected, path)
+					url, err := parseURL(input)
+					assert.NoError(t, err, "input: %s\texpected: %s", input, expected)
+					assert.Equal(t, expected, url.String())
+					assert.Equal(t, host.expected, url.Host)
+					assert.Equal(t, "https", url.Scheme)
+				})
+			}
+		}
+	}
+
+	t.Run("no input", func(t *testing.T) {
+		_, err := parseURL("")
+		assert.ErrorContains(t, err, "no input provided")
+	})
+
+	t.Run("missing host", func(t *testing.T) {
+		_, err := parseURL("https:///host")
+		assert.ErrorContains(t, err, "failed to parse Host")
+	})
+
+	t.Run("invalid path only", func(t *testing.T) {
+		_, err := parseURL("/host")
+		assert.ErrorContains(t, err, "failed to parse Host")
+	})
+
+	t.Run("invalid parse URL", func(t *testing.T) {
+		_, err := parseURL("https://host\\host")
+		assert.ErrorContains(t, err, "failed to parse as URL")
+	})
+}

cmd/cloudflared/generic_service.go

@@ -1,5 +1,4 @@
 //go:build !windows && !darwin && !linux
-// +build !windows,!darwin,!linux
 
 package main
 

cmd/cloudflared/linux_service.go

@@ -1,5 +1,4 @@
 //go:build linux
-// +build linux
 
 package main
 
@@ -25,6 +24,9 @@ func runApp(app *cli.App, graceShutdownC chan struct{}) {
 				Name:   "install",
 				Usage:  "Install cloudflared as a system service",
 				Action: cliutil.ConfiguredAction(installLinuxService),
+				Flags: []cli.Flag{
+					noUpdateServiceFlag,
+				},
 			},
 			{
 				Name:   "uninstall",
@@ -39,15 +41,17 @@ func runApp(app *cli.App, graceShutdownC chan struct{}) {
 // The directory and files that are used by the service.
 // These are hard-coded in the templates below.
 const (
-	serviceConfigDir      = "/etc/cloudflared"
-	serviceConfigFile     = "config.yml"
-	serviceCredentialFile = "cert.pem"
-	serviceConfigPath     = serviceConfigDir + "/" + serviceConfigFile
-	cloudflaredService    = "cloudflared.service"
+	serviceConfigDir         = "/etc/cloudflared"
+	serviceConfigFile        = "config.yml"
+	serviceCredentialFile    = "cert.pem"
+	serviceConfigPath        = serviceConfigDir + "/" + serviceConfigFile
+	cloudflaredService       = "cloudflared.service"
+	cloudflaredUpdateService = "cloudflared-update.service"
+	cloudflaredUpdateTimer   = "cloudflared-update.timer"
 )
 
-var systemdTemplates = []ServiceTemplate{
-	{
+var systemdAllTemplates = map[string]ServiceTemplate{
+	cloudflaredService: {
 		Path: fmt.Sprintf("/etc/systemd/system/%s", cloudflaredService),
 		Content: `[Unit]
 Description=cloudflared
@@ -64,8 +68,8 @@ RestartSec=5s
 WantedBy=multi-user.target
 `,
 	},
-	{
-		Path: "/etc/systemd/system/cloudflared-update.service",
+	cloudflaredUpdateService: {
+		Path: fmt.Sprintf("/etc/systemd/system/%s", cloudflaredUpdateService),
 		Content: `[Unit]
 Description=Update cloudflared
 After=network.target
@@ -74,8 +78,8 @@ After=network.target
 ExecStart=/bin/bash -c '{{ .Path }} update; code=$?; if [ $code -eq 11 ]; then systemctl restart cloudflared; exit 0; fi; exit $code'
 `,
 	},
-	{
-		Path: "/etc/systemd/system/cloudflared-update.timer",
+	cloudflaredUpdateTimer: {
+		Path: fmt.Sprintf("/etc/systemd/system/%s", cloudflaredUpdateTimer),
 		Content: `[Unit]
 Description=Update cloudflared
 
@@ -106,7 +110,7 @@ var sysvTemplate = ServiceTemplate{
 # Description:       cloudflared agent
 ### END INIT INFO
 name=$(basename $(readlink -f $0))
-cmd="{{.Path}} --pidfile /var/run/$name.pid --autoupdate-freq 24h0m0s{{ range .ExtraArgs }} {{ . }}{{ end }}"
+cmd="{{.Path}} --pidfile /var/run/$name.pid {{ range .ExtraArgs }} {{ . }}{{ end }}"
 pid_file="/var/run/$name.pid"
 stdout_log="/var/log/$name.log"
 stderr_log="/var/log/$name.err"
@@ -178,6 +182,14 @@ exit 0
 `,
 }
 
+var (
+	noUpdateServiceFlag = &cli.BoolFlag{
+		Name:  "no-update-service",
+		Usage: "Disable auto-update of the cloudflared linux service, which restarts the server to upgrade for new versions.",
+		Value: false,
+	}
+)
+
 func isSystemd() bool {
 	if _, err := os.Stat("/run/systemd/system"); err == nil {
 		return true
@@ -196,6 +208,9 @@ func installLinuxService(c *cli.Context) error {
 		Path: etPath,
 	}
 
+	// Check if the "no update flag" is set
+	autoUpdate := !c.IsSet(noUpdateServiceFlag.Name)
+
 	var extraArgsFunc func(c *cli.Context, log *zerolog.Logger) ([]string, error)
 	if c.NArg() == 0 {
 		extraArgsFunc = buildArgsForConfig
@@ -213,10 +228,10 @@ func installLinuxService(c *cli.Context) error {
 	switch {
 	case isSystemd():
 		log.Info().Msgf("Using Systemd")
-		err = installSystemd(&templateArgs, log)
+		err = installSystemd(&templateArgs, autoUpdate, log)
 	default:
 		log.Info().Msgf("Using SysV")
-		err = installSysv(&templateArgs, log)
+		err = installSysv(&templateArgs, autoUpdate, log)
 	}
 
 	if err == nil {
@@ -261,7 +276,20 @@ credentials-file: CREDENTIALS-FILE
 	}, nil
 }
 
-func installSystemd(templateArgs *ServiceTemplateArgs, log *zerolog.Logger) error {
+func installSystemd(templateArgs *ServiceTemplateArgs, autoUpdate bool, log *zerolog.Logger) error {
+	var systemdTemplates []ServiceTemplate
+	if autoUpdate {
+		systemdTemplates = []ServiceTemplate{
+			systemdAllTemplates[cloudflaredService],
+			systemdAllTemplates[cloudflaredUpdateService],
+			systemdAllTemplates[cloudflaredUpdateTimer],
+		}
+	} else {
+		systemdTemplates = []ServiceTemplate{
+			systemdAllTemplates[cloudflaredService],
+		}
+	}
+
 	for _, serviceTemplate := range systemdTemplates {
 		err := serviceTemplate.Generate(templateArgs)
 		if err != nil {
@@ -273,10 +301,14 @@ func installSystemd(templateArgs *ServiceTemplateArgs, log *zerolog.Logger) erro
 		log.Err(err).Msgf("systemctl enable %s error", cloudflaredService)
 		return err
 	}
-	if err := runCommand("systemctl", "start", "cloudflared-update.timer"); err != nil {
-		log.Err(err).Msg("systemctl start cloudflared-update.timer error")
-		return err
+
+	if autoUpdate {
+		if err := runCommand("systemctl", "start", cloudflaredUpdateTimer); err != nil {
+			log.Err(err).Msgf("systemctl start %s error", cloudflaredUpdateTimer)
+			return err
+		}
 	}
+
 	if err := runCommand("systemctl", "daemon-reload"); err != nil {
 		log.Err(err).Msg("systemctl daemon-reload error")
 		return err
@@ -284,12 +316,19 @@ func installSystemd(templateArgs *ServiceTemplateArgs, log *zerolog.Logger) erro
 	return runCommand("systemctl", "start", cloudflaredService)
 }
 
-func installSysv(templateArgs *ServiceTemplateArgs, log *zerolog.Logger) error {
+func installSysv(templateArgs *ServiceTemplateArgs, autoUpdate bool, log *zerolog.Logger) error {
 	confPath, err := sysvTemplate.ResolvePath()
 	if err != nil {
 		log.Err(err).Msg("error resolving system path")
 		return err
 	}
+
+	if autoUpdate {
+		templateArgs.ExtraArgs = append([]string{"--autoupdate-freq 24h0m0s"}, templateArgs.ExtraArgs...)
+	} else {
+		templateArgs.ExtraArgs = append([]string{"--no-autoupdate"}, templateArgs.ExtraArgs...)
+	}
+
 	if err := sysvTemplate.Generate(templateArgs); err != nil {
 		log.Err(err).Msg("error generating system template")
 		return err
@@ -327,19 +366,35 @@ func uninstallLinuxService(c *cli.Context) error {
 }
 
 func uninstallSystemd(log *zerolog.Logger) error {
-	if err := runCommand("systemctl", "disable", cloudflaredService); err != nil {
-		log.Err(err).Msgf("systemctl disable %s error", cloudflaredService)
-		return err
+	// Get only the installed services
+	installedServices := make(map[string]ServiceTemplate)
+	for serviceName, serviceTemplate := range systemdAllTemplates {
+		if err := runCommand("systemctl", "list-units", "--all", "|", "grep", serviceName); err == nil {
+			installedServices[serviceName] = serviceTemplate
+		} else {
+			log.Info().Msgf("Service '%s' not installed, skipping its uninstall", serviceName)
+		}
 	}
-	if err := runCommand("systemctl", "stop", cloudflaredService); err != nil {
-		log.Err(err).Msgf("systemctl stop %s error", cloudflaredService)
-		return err
+
+	if _, exists := installedServices[cloudflaredService]; exists {
+		if err := runCommand("systemctl", "disable", cloudflaredService); err != nil {
+			log.Err(err).Msgf("systemctl disable %s error", cloudflaredService)
+			return err
+		}
+		if err := runCommand("systemctl", "stop", cloudflaredService); err != nil {
+			log.Err(err).Msgf("systemctl stop %s error", cloudflaredService)
+			return err
+		}
 	}
-	if err := runCommand("systemctl", "stop", "cloudflared-update.timer"); err != nil {
-		log.Err(err).Msg("systemctl stop cloudflared-update.timer error")
-		return err
+
+	if _, exists := installedServices[cloudflaredUpdateTimer]; exists {
+		if err := runCommand("systemctl", "stop", cloudflaredUpdateTimer); err != nil {
+			log.Err(err).Msgf("systemctl stop %s error", cloudflaredUpdateTimer)
+			return err
+		}
 	}
-	for _, serviceTemplate := range systemdTemplates {
+
+	for _, serviceTemplate := range installedServices {
 		if err := serviceTemplate.Remove(); err != nil {
 			log.Err(err).Msg("error removing service template")
 			return err

cmd/cloudflared/macos_service.go

@@ -1,5 +1,4 @@
 //go:build darwin
-// +build darwin
 
 package main
 

cmd/cloudflared/service_template.go

@@ -5,7 +5,6 @@ import (
 	"bytes"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"os/exec"
 	"path"
@@ -64,7 +63,7 @@ func (st *ServiceTemplate) Generate(args *ServiceTemplateArgs) error {
 		return fmt.Errorf("error creating %s: %v", plistFolder, err)
 	}
 
-	err = ioutil.WriteFile(resolvedPath, buffer.Bytes(), fileMode)
+	err = os.WriteFile(resolvedPath, buffer.Bytes(), fileMode)
 	if err != nil {
 		return fmt.Errorf("error writing %s: %v", resolvedPath, err)
 	}
@@ -103,7 +102,7 @@ func runCommand(command string, args ...string) error {
 		return fmt.Errorf("error starting %s: %v", command, err)
 	}
 
-	output, _ := ioutil.ReadAll(stderr)
+	output, _ := io.ReadAll(stderr)
 	err = cmd.Wait()
 	if err != nil {
 		return fmt.Errorf("%s %v returned with error code %v due to: %v", command, args, err, string(output))

cmd/cloudflared/tunnel/cmd.go

@@ -4,7 +4,6 @@ import (
 	"bufio"
 	"context"
 	"fmt"
-	"io/ioutil"
 	"net/url"
 	"os"
 	"runtime/trace"
@@ -12,7 +11,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/coreos/go-systemd/daemon"
+	"github.com/coreos/go-systemd/v22/daemon"
 	"github.com/facebookgo/grace/gracenet"
 	"github.com/getsentry/sentry-go"
 	"github.com/google/uuid"
@@ -79,6 +78,14 @@ const (
 	// hostKeyPath is the path of the dir to save SSH host keys too
 	hostKeyPath = "host-key-path"
 
+	// udpUnregisterSessionTimeout is how long we wait before we stop trying to unregister a UDP session from the edge
+	udpUnregisterSessionTimeoutFlag = "udp-unregister-session-timeout"
+
+	// quicDisablePathMTUDiscovery sets if QUIC should not perform PTMU discovery and use a smaller (safe) packet size.
+	// Packets will then be at most 1252 (IPv4) / 1232 (IPv6) bytes in size.
+	// Note that this may result in packet drops for UDP proxying, since we expect being able to send at least 1280 bytes of inner packets.
+	quicDisablePathMTUDiscovery = "quic-disable-pmtu-discovery"
+
 	// uiFlag is to enable launching cloudflared in interactive UI mode
 	uiFlag = "ui"
 
@@ -297,7 +304,7 @@ func StartServer(
 	}
 
 	if c.IsSet("trace-output") {
-		tmpTraceFile, err := ioutil.TempFile("", "trace")
+		tmpTraceFile, err := os.CreateTemp("", "trace")
 		if err != nil {
 			log.Err(err).Msg("Failed to create new temporary file to save trace output")
 		}
@@ -385,7 +392,7 @@ func StartServer(
 		observer.SendURL(quickTunnelURL)
 	}
 
-	tunnelConfig, orchestratorConfig, err := prepareTunnelConfig(c, info, log, logTransport, observer, namedTunnel)
+	tunnelConfig, orchestratorConfig, err := prepareTunnelConfig(ctx, c, info, log, logTransport, observer, namedTunnel)
 	if err != nil {
 		log.Err(err).Msg("Couldn't start tunnel")
 		return err
@@ -410,6 +417,7 @@ func StartServer(
 
 		mgmt := management.New(
 			c.String("management-hostname"),
+			c.Bool("management-diagnostics"),
 			serviceIP,
 			clientID,
 			c.String(connectorLabelFlag),
@@ -683,6 +691,18 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			Value:  4,
 			Hidden: true,
 		}),
+		altsrc.NewDurationFlag(&cli.DurationFlag{
+			Name:   udpUnregisterSessionTimeoutFlag,
+			Value:  5 * time.Second,
+			Hidden: true,
+		}),
+		altsrc.NewBoolFlag(&cli.BoolFlag{
+			Name:    quicDisablePathMTUDiscovery,
+			EnvVars: []string{"TUNNEL_DISABLE_QUIC_PMTU"},
+			Usage:   "Use this option to disable PTMU discovery for QUIC connections. This will result in lower packet sizes. Not however, that this may cause instability for UDP proxying.",
+			Value:   false,
+			Hidden:  true,
+		}),
 		altsrc.NewStringFlag(&cli.StringFlag{
 			Name:  connectorLabelFlag,
 			Usage: "Use this option to give a meaningful label to a specific connector. When a tunnel starts up, a connector id unique to the tunnel is generated. This is a uuid. To make it easier to identify a connector, we will use the hostname of the machine the tunnel is running on along with the connector ID. This option exists if one wants to have more control over what their individual connectors are called.",
@@ -756,6 +776,12 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			EnvVars: []string{"TUNNEL_POST_QUANTUM"},
 			Hidden:  FipsEnabled,
 		}),
+		altsrc.NewBoolFlag(&cli.BoolFlag{
+			Name:    "management-diagnostics",
+			Usage:   "Enables the in-depth diagnostic routes to be made available over the management service (/debug/pprof, /metrics, etc.)",
+			EnvVars: []string{"TUNNEL_MANAGEMENT_DIAGNOSTICS"},
+			Value:   false,
+		}),
 		selectProtocolFlag,
 		overwriteDNSFlag,
 	}...)

cmd/cloudflared/tunnel/config_test.go

@@ -4,10 +4,12 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/require"
+
+	"github.com/cloudflare/cloudflared/features"
 )
 
 func TestDedup(t *testing.T) {
 	expected := []string{"a", "b"}
-	actual := dedup([]string{"a", "b", "a"})
+	actual := features.Dedup([]string{"a", "b", "a"})
 	require.ElementsMatch(t, expected, actual)
 }

cmd/cloudflared/tunnel/configuration.go

@@ -1,9 +1,9 @@
 package tunnel
 
 import (
+	"context"
 	"crypto/tls"
 	"fmt"
-	mathRand "math/rand"
 	"net"
 	"net/netip"
 	"os"
@@ -15,7 +15,7 @@ import (
 	"github.com/rs/zerolog"
 	"github.com/urfave/cli/v2"
 	"github.com/urfave/cli/v2/altsrc"
-	"golang.org/x/crypto/ssh/terminal"
+	"golang.org/x/term"
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
 	"github.com/cloudflare/cloudflared/config"
@@ -113,6 +113,7 @@ func dnsProxyStandAlone(c *cli.Context, namedTunnel *connection.NamedTunnelPrope
 }
 
 func prepareTunnelConfig(
+	ctx context.Context,
 	c *cli.Context,
 	info *cliutil.BuildInfo,
 	log, logTransport *zerolog.Logger,
@@ -132,22 +133,36 @@ func prepareTunnelConfig(
 	tags = append(tags, tunnelpogs.Tag{Name: "ID", Value: clientID.String()})
 
 	transportProtocol := c.String("protocol")
-	needPQ := c.Bool("post-quantum")
-	if needPQ {
+
+	clientFeatures := features.Dedup(append(c.StringSlice("features"), features.DefaultFeatures...))
+
+	staticFeatures := features.StaticFeatures{}
+	if c.Bool("post-quantum") {
 		if FipsEnabled {
 			return nil, nil, fmt.Errorf("post-quantum not supported in FIPS mode")
 		}
+		pqMode := features.PostQuantumStrict
+		staticFeatures.PostQuantumMode = &pqMode
+	}
+	featureSelector, err := features.NewFeatureSelector(ctx, namedTunnel.Credentials.AccountTag, staticFeatures, log)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "Failed to create feature selector")
+	}
+	pqMode := featureSelector.PostQuantumMode()
+	if pqMode == features.PostQuantumStrict {
 		// Error if the user tries to force a non-quic transport protocol
 		if transportProtocol != connection.AutoSelectFlag && transportProtocol != connection.QUIC.String() {
 			return nil, nil, fmt.Errorf("post-quantum is only supported with the quic transport")
 		}
 		transportProtocol = connection.QUIC.String()
+		clientFeatures = append(clientFeatures, features.FeaturePostQuantum)
+
+		log.Info().Msgf(
+			"Using hybrid post-quantum key agreement %s",
+			supervisor.PQKexName,
+		)
 	}
 
-	clientFeatures := dedup(append(c.StringSlice("features"), features.DefaultFeatures...))
-	if needPQ {
-		clientFeatures = append(clientFeatures, features.FeaturePostQuantum)
-	}
 	namedTunnel.Client = tunnelpogs.ClientInfo{
 		ClientID: clientID[:],
 		Features: clientFeatures,
@@ -203,15 +218,6 @@ func prepareTunnelConfig(
 		log.Warn().Str("edgeIPVersion", edgeIPVersion.String()).Err(err).Msg("Overriding edge-ip-version")
 	}
 
-	var pqKexIdx int
-	if needPQ {
-		pqKexIdx = mathRand.Intn(len(supervisor.PQKexes))
-		log.Info().Msgf(
-			"Using experimental hybrid post-quantum key agreement %s",
-			supervisor.PQKexNames[supervisor.PQKexes[pqKexIdx]],
-		)
-	}
-
 	tunnelConfig := &supervisor.TunnelConfig{
 		GracePeriod:     gracePeriod,
 		ReplaceExisting: c.Bool("force"),
@@ -231,14 +237,15 @@ func prepareTunnelConfig(
 		Observer:        observer,
 		ReportedVersion: info.Version(),
 		// Note TUN-3758 , we use Int because UInt is not supported with altsrc
-		Retries:            uint(c.Int("retries")),
-		RunFromTerminal:    isRunningFromTerminal(),
-		NamedTunnel:        namedTunnel,
-		ProtocolSelector:   protocolSelector,
-		EdgeTLSConfigs:     edgeTLSConfigs,
-		NeedPQ:             needPQ,
-		PQKexIdx:           pqKexIdx,
-		MaxEdgeAddrRetries: uint8(c.Int("max-edge-addr-retries")),
+		Retries:                     uint(c.Int("retries")),
+		RunFromTerminal:             isRunningFromTerminal(),
+		NamedTunnel:                 namedTunnel,
+		ProtocolSelector:            protocolSelector,
+		EdgeTLSConfigs:              edgeTLSConfigs,
+		FeatureSelector:             featureSelector,
+		MaxEdgeAddrRetries:          uint8(c.Int("max-edge-addr-retries")),
+		UDPUnregisterSessionTimeout: c.Duration(udpUnregisterSessionTimeoutFlag),
+		DisableQUICPathMTUDiscovery: c.Bool(quicDisablePathMTUDiscovery),
 	}
 	packetConfig, err := newPacketConfig(c, log)
 	if err != nil {
@@ -275,26 +282,7 @@ func gracePeriod(c *cli.Context) (time.Duration, error) {
 }
 
 func isRunningFromTerminal() bool {
-	return terminal.IsTerminal(int(os.Stdout.Fd()))
-}
-
-// Remove any duplicates from the slice
-func dedup(slice []string) []string {
-
-	// Convert the slice into a set
-	set := make(map[string]bool, 0)
-	for _, str := range slice {
-		set[str] = true
-	}
-
-	// Convert the set back into a slice
-	keys := make([]string, len(set))
-	i := 0
-	for str := range set {
-		keys[i] = str
-		i++
-	}
-	return keys
+	return term.IsTerminal(int(os.Stdout.Fd()))
 }
 
 // ParseConfigIPVersion returns the IP version from possible expected values from config

cmd/cloudflared/tunnel/configuration_test.go

@@ -1,5 +1,4 @@
 //go:build ignore
-// +build ignore
 
 // TODO: Remove the above build tag and include this test when we start compiling with Golang 1.10.0+
 

cmd/cloudflared/tunnel/filesystem.go

@@ -1,7 +1,6 @@
 package tunnel
 
 import (
-	"io/ioutil"
 	"os"
 )
 
@@ -23,5 +22,5 @@ func (fs realFileSystem) validFilePath(path string) bool {
 }
 
 func (fs realFileSystem) readFile(filePath string) ([]byte, error) {
-	return ioutil.ReadFile(filePath)
+	return os.ReadFile(filePath)
 }

cmd/cloudflared/tunnel/ingress_subcommands.go

@@ -139,7 +139,7 @@ func testURLCommand(c *cli.Context) error {
 	}
 
 	_, i := ing.FindMatchingRule(requestURL.Hostname(), requestURL.Path)
-	fmt.Printf("Matched rule #%d\n", i+1)
+	fmt.Printf("Matched rule #%d\n", i)
 	fmt.Println(ing.Rules[i].MultiLineString())
 	return nil
 }

cmd/cloudflared/tunnel/login.go

@@ -2,7 +2,6 @@ package tunnel
 
 import (
 	"fmt"
-	"io/ioutil"
 	"net/url"
 	"os"
 	"path/filepath"
@@ -66,7 +65,7 @@ func login(c *cli.Context) error {
 		return err
 	}
 
-	if err := ioutil.WriteFile(path, resourceData, 0600); err != nil {
+	if err := os.WriteFile(path, resourceData, 0600); err != nil {
 		return errors.Wrap(err, fmt.Sprintf("error writing cert to %s", path))
 	}
 

cmd/cloudflared/tunnel/signal_test.go

@@ -1,5 +1,4 @@
 //go:build !windows
-// +build !windows
 
 package tunnel
 

cmd/cloudflared/tunnel/subcommand_context.go

@@ -156,7 +156,7 @@ func (sc *subcommandContext) create(name string, credentialsFilePath string, sec
 		var errorLines []string
 		errorLines = append(errorLines, fmt.Sprintf("Your tunnel '%v' was created with ID %v. However, cloudflared couldn't write tunnel credentials to %s.", tunnel.Name, tunnel.ID, credentialsFilePath))
 		errorLines = append(errorLines, fmt.Sprintf("The file-writing error is: %v", writeFileErr))
-		if deleteErr := client.DeleteTunnel(tunnel.ID); deleteErr != nil {
+		if deleteErr := client.DeleteTunnel(tunnel.ID, true); deleteErr != nil {
 			errorLines = append(errorLines, fmt.Sprintf("Cloudflared tried to delete the tunnel for you, but encountered an error. You should use `cloudflared tunnel delete %v` to delete the tunnel yourself, because the tunnel can't be run without the tunnelfile.", tunnel.ID))
 			errorLines = append(errorLines, fmt.Sprintf("The delete tunnel error is: %v", deleteErr))
 		} else {
@@ -206,13 +206,8 @@ func (sc *subcommandContext) delete(tunnelIDs []uuid.UUID) error {
 		if !tunnel.DeletedAt.IsZero() {
 			return fmt.Errorf("Tunnel %s has already been deleted", tunnel.ID)
 		}
-		if forceFlagSet {
-			if err := client.CleanupConnections(tunnel.ID, cfapi.NewCleanupParams()); err != nil {
-				return errors.Wrapf(err, "Error cleaning up connections for tunnel %s", tunnel.ID)
-			}
-		}
 
-		if err := client.DeleteTunnel(tunnel.ID); err != nil {
+		if err := client.DeleteTunnel(tunnel.ID, forceFlagSet); err != nil {
 			return errors.Wrapf(err, "Error deleting tunnel %s", tunnel.ID)
 		}
 

cmd/cloudflared/tunnel/subcommand_context_teamnet.go

@@ -1,6 +1,9 @@
 package tunnel
 
 import (
+	"net"
+
+	"github.com/google/uuid"
 	"github.com/pkg/errors"
 
 	"github.com/cloudflare/cloudflared/cfapi"
@@ -24,12 +27,12 @@ func (sc *subcommandContext) addRoute(newRoute cfapi.NewRoute) (cfapi.Route, err
 	return client.AddRoute(newRoute)
 }
 
-func (sc *subcommandContext) deleteRoute(params cfapi.DeleteRouteParams) error {
+func (sc *subcommandContext) deleteRoute(id uuid.UUID) error {
 	client, err := sc.client()
 	if err != nil {
 		return errors.Wrap(err, noClientMsg)
 	}
-	return client.DeleteRoute(params)
+	return client.DeleteRoute(id)
 }
 
 func (sc *subcommandContext) getRouteByIP(params cfapi.GetRouteByIpParams) (cfapi.DetailedRoute, error) {
@@ -39,3 +42,25 @@ func (sc *subcommandContext) getRouteByIP(params cfapi.GetRouteByIpParams) (cfap
 	}
 	return client.GetByIP(params)
 }
+
+func (sc *subcommandContext) getRouteId(network net.IPNet, vnetId *uuid.UUID) (uuid.UUID, error) {
+	filters := cfapi.NewIPRouteFilter()
+	filters.NotDeleted()
+	filters.NetworkIsSubsetOf(network)
+	filters.NetworkIsSupersetOf(network)
+
+	if vnetId != nil {
+		filters.VNetID(*vnetId)
+	}
+
+	result, err := sc.listRoutes(filters)
+	if err != nil {
+		return uuid.Nil, err
+	}
+
+	if len(result) != 1 {
+		return uuid.Nil, errors.New("unable to find route for provided network and vnet")
+	}
+
+	return result[0].ID, nil
+}

cmd/cloudflared/tunnel/subcommand_context_test.go

@@ -219,7 +219,7 @@ func (d *deleteMockTunnelStore) GetTunnelToken(tunnelID uuid.UUID) (string, erro
 	return "token", nil
 }
 
-func (d *deleteMockTunnelStore) DeleteTunnel(tunnelID uuid.UUID) error {
+func (d *deleteMockTunnelStore) DeleteTunnel(tunnelID uuid.UUID, cascade bool) error {
 	tunnel, ok := d.mockTunnels[tunnelID]
 	if !ok {
 		return fmt.Errorf("Couldn't find tunnel: %v", tunnelID)

cmd/cloudflared/tunnel/subcommands.go

@@ -5,7 +5,6 @@ import (
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"regexp"
@@ -120,8 +119,8 @@ var (
 	forceDeleteFlag = &cli.BoolFlag{
 		Name:    "force",
 		Aliases: []string{"f"},
-		Usage: "Cleans up any stale connections before the tunnel is deleted. cloudflared will not " +
-			"delete a tunnel with connections without this flag.",
+		Usage: "Deletes a tunnel even if tunnel is connected and it has dependencies associated to it. (eg. IP routes)." +
+			" It is not possible to delete tunnels that have connections or non-deleted dependencies, without this flag.",
 		EnvVars: []string{"TUNNEL_RUN_FORCE_OVERWRITE"},
 	}
 	selectProtocolFlag = altsrc.NewStringFlag(&cli.StringFlag{
@@ -241,7 +240,7 @@ func writeTunnelCredentials(filePath string, credentials *connection.Credentials
 	if err != nil {
 		return errors.Wrap(err, "Unable to marshal tunnel credentials to JSON")
 	}
-	return ioutil.WriteFile(filePath, body, 400)
+	return os.WriteFile(filePath, body, 0400)
 }
 
 func buildListCommand() *cli.Command {

cmd/cloudflared/tunnel/teamnet_subcommands.go

@@ -21,6 +21,8 @@ var (
 		Aliases: []string{"vn"},
 		Usage:   "The ID or name of the virtual network to which the route is associated to.",
 	}
+
+	routeAddError = errors.New("You must supply exactly one argument, the ID or CIDR of the route you want to delete")
 )
 
 func buildRouteIPSubcommand() *cli.Command {
@@ -68,11 +70,9 @@ which virtual network's routing table you want to add the route to with:
 				Name:      "delete",
 				Action:    cliutil.ConfiguredAction(deleteRouteCommand),
 				Usage:     "Delete a row from your organization's private routing table",
-				UsageText: "cloudflared tunnel [--config FILEPATH] route ip delete [flags] [CIDR]",
-				Description: `Deletes the row for a given CIDR from your routing table. That portion of your network
-will no longer be reachable by the WARP clients. Note that if you use virtual
-networks, then you have to tell which virtual network whose routing table you
-have a row deleted from.`,
+				UsageText: "cloudflared tunnel [--config FILEPATH] route ip delete [flags] [Route ID or CIDR]",
+				Description: `Deletes the row for the given route ID from your routing table. That portion of your network
+will no longer be reachable.`,
 				Flags: []cli.Flag{vnetFlag},
 			},
 			{
@@ -187,33 +187,36 @@ func deleteRouteCommand(c *cli.Context) error {
 	}
 
 	if c.NArg() != 1 {
-		return errors.New("You must supply exactly one argument, the network whose route you want to delete (in CIDR form e.g. 1.2.3.4/32)")
+		return routeAddError
 	}
 
-	_, network, err := net.ParseCIDR(c.Args().First())
+	var routeId uuid.UUID
+	routeId, err = uuid.Parse(c.Args().First())
 	if err != nil {
-		return errors.Wrap(err, "Invalid network CIDR")
-	}
-	if network == nil {
-		return errors.New("Invalid network CIDR")
-	}
+		_, network, err := net.ParseCIDR(c.Args().First())
+		if err != nil || network == nil {
+			return routeAddError
+		}
 
-	params := cfapi.DeleteRouteParams{
-		Network: *network,
-	}
+		var vnetId *uuid.UUID
+		if c.IsSet(vnetFlag.Name) {
+			id, err := getVnetId(sc, c.String(vnetFlag.Name))
+			if err != nil {
+				return err
+			}
+			vnetId = &id
+		}
 
-	if c.IsSet(vnetFlag.Name) {
-		vnetId, err := getVnetId(sc, c.String(vnetFlag.Name))
+		routeId, err = sc.getRouteId(*network, vnetId)
 		if err != nil {
 			return err
 		}
-		params.VNetID = &vnetId
 	}
 
-	if err := sc.deleteRoute(params); err != nil {
+	if err := sc.deleteRoute(routeId); err != nil {
 		return errors.Wrap(err, "API error")
 	}
-	fmt.Printf("Successfully deleted route for %s\n", network)
+	fmt.Printf("Successfully deleted route with ID %s\n", routeId)
 	return nil
 }
 
@@ -269,7 +272,7 @@ func formatAndPrintRouteList(routes []*cfapi.DetailedRoute) {
 	defer writer.Flush()
 
 	// Print column headers with tabbed columns
-	_, _ = fmt.Fprintln(writer, "NETWORK\tVIRTUAL NET ID\tCOMMENT\tTUNNEL ID\tTUNNEL NAME\tCREATED\tDELETED\t")
+	_, _ = fmt.Fprintln(writer, "ID\tNETWORK\tVIRTUAL NET ID\tCOMMENT\tTUNNEL ID\tTUNNEL NAME\tCREATED\tDELETED\t")
 
 	// Loop through routes, create formatted string for each, and print using tabwriter
 	for _, route := range routes {

cmd/cloudflared/updater/update.go

@@ -12,7 +12,7 @@ import (
 	"github.com/facebookgo/grace/gracenet"
 	"github.com/rs/zerolog"
 	"github.com/urfave/cli/v2"
-	"golang.org/x/crypto/ssh/terminal"
+	"golang.org/x/term"
 
 	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/logger"
@@ -304,7 +304,7 @@ func wasInstalledFromPackageManager() bool {
 }
 
 func isRunningFromTerminal() bool {
-	return terminal.IsTerminal(int(os.Stdout.Fd()))
+	return term.IsTerminal(int(os.Stdout.Fd()))
 }
 
 func IsSysV() bool {

cmd/cloudflared/updater/workers_service_test.go

@@ -1,5 +1,4 @@
 //go:build !windows
-// +build !windows
 
 package updater
 
@@ -11,7 +10,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"io/ioutil"
 	"log"
 	"net/http"
 	"net/http/httptest"
@@ -224,7 +222,7 @@ func TestUpdateService(t *testing.T) {
 	require.Equal(t, v.Version(), mostRecentVersion)
 
 	require.NoError(t, v.Apply())
-	dat, err := ioutil.ReadFile(testFilePath)
+	dat, err := os.ReadFile(testFilePath)
 	require.NoError(t, err)
 
 	require.Equal(t, string(dat), mostRecentVersion)
@@ -243,7 +241,7 @@ func TestBetaUpdateService(t *testing.T) {
 	require.Equal(t, v.Version(), mostRecentBetaVersion)
 
 	require.NoError(t, v.Apply())
-	dat, err := ioutil.ReadFile(testFilePath)
+	dat, err := os.ReadFile(testFilePath)
 	require.NoError(t, err)
 
 	require.Equal(t, string(dat), mostRecentBetaVersion)
@@ -289,7 +287,7 @@ func TestForcedUpdateService(t *testing.T) {
 	require.Equal(t, v.Version(), mostRecentVersion)
 
 	require.NoError(t, v.Apply())
-	dat, err := ioutil.ReadFile(testFilePath)
+	dat, err := os.ReadFile(testFilePath)
 	require.NoError(t, err)
 
 	require.Equal(t, string(dat), mostRecentVersion)
@@ -309,7 +307,7 @@ func TestUpdateSpecificVersionService(t *testing.T) {
 	require.Equal(t, reqVersion, v.Version())
 
 	require.NoError(t, v.Apply())
-	dat, err := ioutil.ReadFile(testFilePath)
+	dat, err := os.ReadFile(testFilePath)
 	require.NoError(t, err)
 
 	require.Equal(t, reqVersion, string(dat))
@@ -328,7 +326,7 @@ func TestCompressedUpdateService(t *testing.T) {
 	require.Equal(t, "2020.09.02", v.Version())
 
 	require.NoError(t, v.Apply())
-	dat, err := ioutil.ReadFile(testFilePath)
+	dat, err := os.ReadFile(testFilePath)
 	require.NoError(t, err)
 
 	require.Equal(t, "2020.09.02", string(dat))

cmd/cloudflared/windows_service.go

@@ -1,5 +1,4 @@
 //go:build windows
-// +build windows
 
 package main
 

component-tests/test_config.py

@@ -36,17 +36,17 @@ class TestConfig:
         _ = start_cloudflared(tmp_path, config, validate_args)
 
         self.match_rule(tmp_path, config,
-                        "http://example.com/index.html", 1)
+                        "http://example.com/index.html", 0)
         self.match_rule(tmp_path, config,
-                        "https://example.com/index.html", 1)
+                        "https://example.com/index.html", 0)
         self.match_rule(tmp_path, config,
-                        "https://api.example.com/login", 2)
+                        "https://api.example.com/login", 1)
         self.match_rule(tmp_path, config,
-                        "https://wss.example.com", 3)
+                        "https://wss.example.com", 2)
         self.match_rule(tmp_path, config,
-                        "https://ssh.example.com", 4)
+                        "https://ssh.example.com", 3)
         self.match_rule(tmp_path, config,
-                        "https://api.example.com", 5)
+                        "https://api.example.com", 4)
 
     # This is used to check that the command tunnel ingress url <url> matches rule number <rule_num>. Note that rule number uses 1-based indexing
 

component-tests/test_management.py

@@ -0,0 +1,113 @@
+#!/usr/bin/env python
+import requests
+from conftest import CfdModes
+from constants import METRICS_PORT, MAX_RETRIES, BACKOFF_SECS
+from retrying import retry
+from cli import CloudflaredCli
+from util import LOGGER, write_config, start_cloudflared, wait_tunnel_ready, send_requests
+import platform
+
+"""
+Each test in TestManagement will:
+1. Acquire a management token from Cloudflare public API
+2. Make a request against the management service for the running tunnel
+"""
+class TestManagement:
+    """
+        test_get_host_details does the following:
+        1. It gets a management token from Tunnelstore using cloudflared tail token <tunnel_id>
+        2. It gets the connector_id after starting a cloudflare tunnel
+        3. It sends a request to the management host with the connector_id and management token
+        4. Asserts that the response has a hostname and ip.
+    """
+    def test_get_host_details(self, tmp_path, component_tests_config):
+        # TUN-7377 : wait_tunnel_ready does not work properly in windows.
+        # Skipping this test for windows for now and will address it as part of tun-7377
+        if platform.system() == "Windows":
+            return
+        config = component_tests_config(cfd_mode=CfdModes.NAMED, run_proxy_dns=False, provide_ingress=False)
+        LOGGER.debug(config)
+        headers = {}
+        headers["Content-Type"] = "application/json"
+        config_path = write_config(tmp_path, config.full_config)
+        with start_cloudflared(tmp_path, config, cfd_pre_args=["tunnel", "--ha-connections", "1", "--label" , "test"], cfd_args=["run", "--hello-world"], new_process=True):
+            wait_tunnel_ready(tunnel_url=config.get_url(),
+                              require_min_connections=1)
+            cfd_cli = CloudflaredCli(config, config_path, LOGGER)
+            connector_id = cfd_cli.get_connector_id(config)[0]
+            url = cfd_cli.get_management_url("host_details", config, config_path)
+            resp = send_request(url, headers=headers)
+            
+            # Assert response json.
+            assert resp.status_code == 200, "Expected cloudflared to return 200 for host details"
+            assert resp.json()["hostname"] == "custom:test", "Expected cloudflared to return hostname"
+            assert resp.json()["ip"] != "", "Expected cloudflared to return ip"
+            assert resp.json()["connector_id"] == connector_id, "Expected cloudflared to return connector_id"
+    
+    """
+        test_get_metrics will verify that the /metrics endpoint returns the prometheus metrics dump
+    """
+    def test_get_metrics(self, tmp_path, component_tests_config):
+        # TUN-7377 : wait_tunnel_ready does not work properly in windows.
+        # Skipping this test for windows for now and will address it as part of tun-7377
+        if platform.system() == "Windows":
+            return
+        config = component_tests_config(cfd_mode=CfdModes.NAMED, run_proxy_dns=False, provide_ingress=False)
+        LOGGER.debug(config)
+        config_path = write_config(tmp_path, config.full_config)
+        with start_cloudflared(tmp_path, config, cfd_pre_args=["tunnel", "--ha-connections", "1", "--management-diagnostics"], new_process=True):
+            wait_tunnel_ready(require_min_connections=1)
+            cfd_cli = CloudflaredCli(config, config_path, LOGGER)
+            url = cfd_cli.get_management_url("metrics", config, config_path)
+            resp = send_request(url)
+            
+            # Assert response.
+            assert resp.status_code == 200, "Expected cloudflared to return 200 for /metrics"
+            assert "# HELP build_info Build and version information" in resp.text, "Expected /metrics to have with the build_info details"
+
+    """
+        test_get_pprof_heap will verify that the /debug/pprof/heap endpoint returns a pprof/heap dump response
+    """
+    def test_get_pprof_heap(self, tmp_path, component_tests_config):
+        # TUN-7377 : wait_tunnel_ready does not work properly in windows.
+        # Skipping this test for windows for now and will address it as part of tun-7377
+        if platform.system() == "Windows":
+            return
+        config = component_tests_config(cfd_mode=CfdModes.NAMED, run_proxy_dns=False, provide_ingress=False)
+        LOGGER.debug(config)
+        config_path = write_config(tmp_path, config.full_config)
+        with start_cloudflared(tmp_path, config, cfd_pre_args=["tunnel", "--ha-connections", "1", "--management-diagnostics"], new_process=True):
+            wait_tunnel_ready(require_min_connections=1)
+            cfd_cli = CloudflaredCli(config, config_path, LOGGER)
+            url = cfd_cli.get_management_url("debug/pprof/heap", config, config_path)
+            resp = send_request(url)
+            
+            # Assert response.
+            assert resp.status_code == 200, "Expected cloudflared to return 200 for /debug/pprof/heap"
+            assert resp.headers["Content-Type"] == "application/octet-stream", "Expected /debug/pprof/heap to have return a binary response"
+    
+    """
+        test_get_metrics_when_disabled will verify that diagnostic endpoints (such as /metrics) return 404 and are unmounted. 
+    """
+    def test_get_metrics_when_disabled(self, tmp_path, component_tests_config):
+        # TUN-7377 : wait_tunnel_ready does not work properly in windows.
+        # Skipping this test for windows for now and will address it as part of tun-7377
+        if platform.system() == "Windows":
+            return
+        config = component_tests_config(cfd_mode=CfdModes.NAMED, run_proxy_dns=False, provide_ingress=False)
+        LOGGER.debug(config)
+        config_path = write_config(tmp_path, config.full_config)
+        with start_cloudflared(tmp_path, config, cfd_pre_args=["tunnel", "--ha-connections", "1"], new_process=True):
+            wait_tunnel_ready(require_min_connections=1)
+            cfd_cli = CloudflaredCli(config, config_path, LOGGER)
+            url = cfd_cli.get_management_url("metrics", config, config_path)
+            resp = send_request(url)
+            
+            # Assert response.
+            assert resp.status_code == 404, "Expected cloudflared to return 404 for /metrics"
+
+
+@retry(stop_max_attempt_number=MAX_RETRIES, wait_fixed=BACKOFF_SECS * 1000)
+def send_request(url, headers={}):
+    with requests.Session() as s:
+        return s.get(url, timeout=BACKOFF_SECS, headers=headers)

component-tests/test_tunnel.py

@@ -16,38 +16,6 @@ class TestTunnel:
         with start_cloudflared(tmp_path, config, cfd_pre_args=["tunnel", "--ha-connections", "1"],  cfd_args=["run", "--hello-world"], new_process=True):
             wait_tunnel_ready(tunnel_url=config.get_url(),
                               require_min_connections=1)
-
-    """
-        test_get_host_details does the following:
-        1. It gets a management token from Tunnelstore using cloudflared tail token <tunnel_id>
-        2. It gets the connector_id after starting a cloudflare tunnel
-        3. It sends a request to the management host with the connector_id and management token
-        4. Asserts that the response has a hostname and ip.
-    """
-    def test_get_host_details(self, tmp_path, component_tests_config):
-        # TUN-7377 : wait_tunnel_ready does not work properly in windows.
-        # Skipping this test for windows for now and will address it as part of tun-7377
-        if platform.system() == "Windows":
-            return
-        config = component_tests_config(cfd_mode=CfdModes.NAMED, run_proxy_dns=False, provide_ingress=False)
-        LOGGER.debug(config)
-        headers = {}
-        headers["Content-Type"] = "application/json"
-        config_path = write_config(tmp_path, config.full_config)
-        with start_cloudflared(tmp_path, config, cfd_pre_args=["tunnel", "--ha-connections", "1", "--label" , "test"], cfd_args=["run", "--hello-world"], new_process=True):
-            wait_tunnel_ready(tunnel_url=config.get_url(),
-                              require_min_connections=1)
-            cfd_cli = CloudflaredCli(config, config_path, LOGGER)
-            connector_id = cfd_cli.get_connector_id(config)[0]
-            url = cfd_cli.get_management_url("host_details", config, config_path)
-            resp = send_request(url, headers=headers)
-            
-            # Assert response json.
-            assert resp.status_code == 200, "Expected cloudflared to return 200 for host details"
-            assert resp.json()["hostname"] == "custom:test", "Expected cloudflared to return hostname"
-            assert resp.json()["ip"] != "", "Expected cloudflared to return ip"
-            assert resp.json()["connector_id"] == connector_id, "Expected cloudflared to return connector_id"
-
     
     def test_tunnel_url(self, tmp_path, component_tests_config):
         config = component_tests_config(cfd_mode=CfdModes.NAMED, run_proxy_dns=False, provide_ingress=False)

config/configuration.go

@@ -257,7 +257,6 @@ type Configuration struct {
 }
 
 type WarpRoutingConfig struct {
-	Enabled        bool            `yaml:"enabled" json:"enabled"`
 	ConnectTimeout *CustomDuration `yaml:"connectTimeout" json:"connectTimeout,omitempty"`
 	TCPKeepAlive   *CustomDuration `yaml:"tcpKeepAlive" json:"tcpKeepAlive,omitempty"`
 }

config/configuration_test.go

@@ -23,7 +23,6 @@ func TestConfigFileSettings(t *testing.T) {
 			Service:  "https://localhost:8001",
 		}
 		warpRouting = WarpRoutingConfig{
-			Enabled:        true,
 			ConnectTimeout: &CustomDuration{Duration: 2 * time.Second},
 			TCPKeepAlive:   &CustomDuration{Duration: 10 * time.Second},
 		}

connection/connection.go

@@ -40,7 +40,6 @@ type Orchestrator interface {
 	UpdateConfig(version int32, config []byte) *pogs.UpdateConfigurationResponse
 	GetConfigJSON() ([]byte, error)
 	GetOriginProxy() (OriginProxy, error)
-	WarpRoutingEnabled() (enabled bool)
 }
 
 type NamedTunnelProperties struct {
@@ -157,14 +156,16 @@ type ReadWriteAcker interface {
 type HTTPResponseReadWriteAcker struct {
 	r   io.Reader
 	w   ResponseWriter
+	f   http.Flusher
 	req *http.Request
 }
 
 // NewHTTPResponseReadWriterAcker returns a new instance of HTTPResponseReadWriteAcker.
-func NewHTTPResponseReadWriterAcker(w ResponseWriter, req *http.Request) *HTTPResponseReadWriteAcker {
+func NewHTTPResponseReadWriterAcker(w ResponseWriter, flusher http.Flusher, req *http.Request) *HTTPResponseReadWriteAcker {
 	return &HTTPResponseReadWriteAcker{
 		r:   req.Body,
 		w:   w,
+		f:   flusher,
 		req: req,
 	}
 }
@@ -174,7 +175,11 @@ func (h *HTTPResponseReadWriteAcker) Read(p []byte) (int, error) {
 }
 
 func (h *HTTPResponseReadWriteAcker) Write(p []byte) (int, error) {
-	return h.w.Write(p)
+	n, err := h.w.Write(p)
+	if n > 0 {
+		h.f.Flush()
+	}
+	return n, err
 }
 
 // AckConnection acks an HTTP connection by sending a switch protocols status code that enables the caller to

connection/connection_test.go

@@ -130,7 +130,8 @@ func wsEchoEndpoint(w ResponseWriter, r *http.Request) error {
 	}
 	wsCtx, cancel := context.WithCancel(r.Context())
 	readPipe, writePipe := io.Pipe()
-	wsConn := websocket.NewConn(wsCtx, NewHTTPResponseReadWriterAcker(w, r), &log)
+
+	wsConn := websocket.NewConn(wsCtx, NewHTTPResponseReadWriterAcker(w, w.(http.Flusher), r), &log)
 	go func() {
 		select {
 		case <-wsCtx.Done():
@@ -175,7 +176,7 @@ func wsFlakyEndpoint(w ResponseWriter, r *http.Request) error {
 	}
 	wsCtx, cancel := context.WithCancel(r.Context())
 
-	wsConn := websocket.NewConn(wsCtx, NewHTTPResponseReadWriterAcker(w, r), &log)
+	wsConn := websocket.NewConn(wsCtx, NewHTTPResponseReadWriterAcker(w, w.(http.Flusher), r), &log)
 
 	closedAfter := time.Millisecond * time.Duration(rand.Intn(50))
 	originConn := &flakyConn{closeAt: time.Now().Add(closedAfter)}

connection/http2.go

@@ -142,7 +142,7 @@ func (c *HTTP2Connection) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			break
 		}
 
-		rws := NewHTTPResponseReadWriterAcker(respWriter, r)
+		rws := NewHTTPResponseReadWriterAcker(respWriter, respWriter, r)
 		requestErr = originProxy.ProxyTCP(r.Context(), rws, &TCPRequest{
 			Dest:      host,
 			CFRay:     FindCfRayHeader(r),
@@ -289,6 +289,10 @@ func (rp *http2RespWriter) Header() http.Header {
 	return rp.respHeaders
 }
 
+func (rp *http2RespWriter) Flush() {
+	rp.flusher.Flush()
+}
+
 func (rp *http2RespWriter) WriteHeader(status int) {
 	if rp.hijacked() {
 		rp.log.Warn().Msg("WriteHeader after hijack")

connection/http2_test.go

@@ -6,7 +6,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net"
 	"net/http"
 	"net/http/httptest"
@@ -84,7 +83,7 @@ func TestHTTP2ConfigurationSet(t *testing.T) {
 	resp, err := edgeHTTP2Conn.RoundTrip(req)
 	require.NoError(t, err)
 	require.Equal(t, http.StatusOK, resp.StatusCode)
-	bdy, err := ioutil.ReadAll(resp.Body)
+	bdy, err := io.ReadAll(resp.Body)
 	require.NoError(t, err)
 	assert.Equal(t, `{"lastAppliedVersion":2,"err":null}`, string(bdy))
 	cancel()
@@ -149,7 +148,7 @@ func TestServeHTTP(t *testing.T) {
 		require.NoError(t, err)
 		require.Equal(t, test.expectedStatus, resp.StatusCode)
 		if test.expectedBody != nil {
-			respBody, err := ioutil.ReadAll(resp.Body)
+			respBody, err := io.ReadAll(resp.Body)
 			require.NoError(t, err)
 			require.Equal(t, test.expectedBody, respBody)
 		}
@@ -546,7 +545,7 @@ func benchmarkServeHTTP(b *testing.B, test testRequest) {
 		require.NoError(b, err)
 		require.Equal(b, test.expectedStatus, resp.StatusCode)
 		if test.expectedBody != nil {
-			respBody, err := ioutil.ReadAll(resp.Body)
+			respBody, err := io.ReadAll(resp.Body)
 			require.NoError(b, err)
 			require.Equal(b, test.expectedBody, respBody)
 		}

connection/quic.go

@@ -63,6 +63,8 @@ type QUICConnection struct {
 	controlStreamHandler ControlStreamHandler
 	connOptions          *tunnelpogs.ConnectionOptions
 	connIndex            uint8
+
+	udpUnregisterTimeout time.Duration
 }
 
 // NewQUICConnection returns a new instance of QUICConnection.
@@ -78,6 +80,7 @@ func NewQUICConnection(
 	controlStreamHandler ControlStreamHandler,
 	logger *zerolog.Logger,
 	packetRouterConfig *ingress.GlobalRouterConfig,
+	udpUnregisterTimeout time.Duration,
 ) (*QUICConnection, error) {
 	udpConn, err := createUDPConnForConnIndex(connIndex, localAddr, logger)
 	if err != nil {
@@ -100,7 +103,7 @@ func NewQUICConnection(
 	sessionDemuxChan := make(chan *packet.Session, demuxChanCapacity)
 	datagramMuxer := quicpogs.NewDatagramMuxerV2(session, logger, sessionDemuxChan)
 	sessionManager := datagramsession.NewManager(logger, datagramMuxer.SendToSession, sessionDemuxChan)
-	packetRouter := ingress.NewPacketRouter(packetRouterConfig, datagramMuxer, logger, orchestrator.WarpRoutingEnabled)
+	packetRouter := ingress.NewPacketRouter(packetRouterConfig, datagramMuxer, logger)
 
 	return &QUICConnection{
 		session:              session,
@@ -112,6 +115,7 @@ func NewQUICConnection(
 		controlStreamHandler: controlStreamHandler,
 		connOptions:          connOptions,
 		connIndex:            connIndex,
+		udpUnregisterTimeout: udpUnregisterTimeout,
 	}, nil
 }
 
@@ -370,7 +374,7 @@ func (q *QUICConnection) closeUDPSession(ctx context.Context, sessionID uuid.UUI
 
 	stream := quicpogs.NewSafeStreamCloser(quicStream)
 	defer stream.Close()
-	rpcClientStream, err := quicpogs.NewRPCClientStream(ctx, stream, q.logger)
+	rpcClientStream, err := quicpogs.NewRPCClientStream(ctx, stream, q.udpUnregisterTimeout, q.logger)
 	if err != nil {
 		// Log this at debug because this is not an error if session was closed due to lost connection
 		// with edge
@@ -445,10 +449,21 @@ func (hrw *httpResponseAdapter) WriteRespHeaders(status int, header http.Header)
 	return hrw.WriteConnectResponseData(nil, metadata...)
 }
 
+func (hrw *httpResponseAdapter) Write(p []byte) (int, error) {
+	// Make sure to send WriteHeader response if not called yet
+	if !hrw.connectResponseSent {
+		hrw.WriteRespHeaders(http.StatusOK, hrw.headers)
+	}
+	return hrw.RequestServerStream.Write(p)
+}
+
 func (hrw *httpResponseAdapter) Header() http.Header {
 	return hrw.headers
 }
 
+// This is a no-op Flush because this adapter is over a quic.Stream and we don't need Flush here.
+func (hrw *httpResponseAdapter) Flush() {}
+
 func (hrw *httpResponseAdapter) WriteHeader(status int) {
 	hrw.WriteRespHeaders(status, hrw.headers)
 }

connection/quic_test.go

@@ -725,6 +725,7 @@ func testQUICConnection(udpListenerAddr net.Addr, t *testing.T, index uint8) *QU
 		fakeControlStream{},
 		&log,
 		nil,
+		5*time.Second,
 	)
 	require.NoError(t, err)
 	return qc

datagramsession/manager.go

@@ -92,7 +92,10 @@ func (m *manager) shutdownSessions(err error) {
 		byRemote: true,
 	}
 	for _, s := range m.sessions {
-		s.close(closeSessionErr)
+		m.unregisterSession(&unregisterSessionEvent{
+			sessionID: s.ID,
+			err:       closeSessionErr,
+		})
 	}
 }
 
@@ -118,6 +121,7 @@ func (m *manager) registerSession(ctx context.Context, registration *registerSes
 	session := m.newSession(registration.sessionID, registration.originProxy)
 	m.sessions[registration.sessionID] = session
 	registration.resultChan <- session
+	incrementUDPSessions()
 }
 
 func (m *manager) newSession(id uuid.UUID, dstConn io.ReadWriteCloser) *Session {
@@ -163,6 +167,7 @@ func (m *manager) unregisterSession(unregistration *unregisterSessionEvent) {
 	if ok {
 		delete(m.sessions, unregistration.sessionID)
 		session.close(unregistration.err)
+		decrementUDPActiveSessions()
 	}
 }
 

datagramsession/metrics.go

@@ -0,0 +1,40 @@
+package datagramsession
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const (
+	namespace = "cloudflared"
+)
+
+var (
+	activeUDPSessions = prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace: namespace,
+		Subsystem: "udp",
+		Name:      "active_sessions",
+		Help:      "Concurrent count of UDP sessions that are being proxied to any origin",
+	})
+	totalUDPSessions = prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace: namespace,
+		Subsystem: "udp",
+		Name:      "total_sessions",
+		Help:      "Total count of UDP sessions that have been proxied to any origin",
+	})
+)
+
+func init() {
+	prometheus.MustRegister(
+		activeUDPSessions,
+		totalUDPSessions,
+	)
+}
+
+func incrementUDPSessions() {
+	totalUDPSessions.Inc()
+	activeUDPSessions.Inc()
+}
+
+func decrementUDPActiveSessions() {
+	activeUDPSessions.Dec()
+}

dev.Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.19 as builder
+FROM golang:1.20.6 as builder
 ENV GO111MODULE=on \
     CGO_ENABLED=0
 WORKDIR /go/src/github.com/cloudflare/cloudflared/

features/features.go

@@ -28,3 +28,22 @@ func Contains(feature string) bool {
 	}
 	return false
 }
+
+// Remove any duplicates from the slice
+func Dedup(slice []string) []string {
+
+	// Convert the slice into a set
+	set := make(map[string]bool, 0)
+	for _, str := range slice {
+		set[str] = true
+	}
+
+	// Convert the set back into a slice
+	keys := make([]string, len(set))
+	i := 0
+	for str := range set {
+		keys[i] = str
+		i++
+	}
+	return keys
+}

features/selector.go

@@ -0,0 +1,164 @@
+package features
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"hash/fnv"
+	"net"
+	"sync"
+	"time"
+
+	"github.com/rs/zerolog"
+)
+
+const (
+	featureSelectorHostname = "cfd-features.argotunnel.com"
+	defaultRefreshFreq      = time.Hour * 6
+	lookupTimeout           = time.Second * 10
+)
+
+type PostQuantumMode uint8
+
+const (
+	PostQuantumDisabled PostQuantumMode = iota
+	// Prefer post quantum, but fallback if connection cannot be established
+	PostQuantumPrefer
+	// If the user passes the --post-quantum flag, we override
+	// CurvePreferences to only support hybrid post-quantum key agreements.
+	PostQuantumStrict
+)
+
+// If the TXT record adds other fields, the umarshal logic will ignore those keys
+// If the TXT record is missing a key, the field will unmarshal to the default Go value
+type featuresRecord struct {
+	PostQuantumPercentage int32 `json:"pq"`
+}
+
+func NewFeatureSelector(ctx context.Context, accountTag string, staticFeatures StaticFeatures, logger *zerolog.Logger) (*FeatureSelector, error) {
+	return newFeatureSelector(ctx, accountTag, logger, newDNSResolver(), staticFeatures, defaultRefreshFreq)
+}
+
+// FeatureSelector determines if this account will try new features. It preiodically queries a DNS TXT record
+// to see which features are turned on
+type FeatureSelector struct {
+	accountHash int32
+	logger      *zerolog.Logger
+	resolver    resolver
+
+	staticFeatures StaticFeatures
+
+	// lock protects concurrent access to dynamic features
+	lock     sync.RWMutex
+	features featuresRecord
+}
+
+// Features set by user provided flags
+type StaticFeatures struct {
+	PostQuantumMode *PostQuantumMode
+}
+
+func newFeatureSelector(ctx context.Context, accountTag string, logger *zerolog.Logger, resolver resolver, staticFeatures StaticFeatures, refreshFreq time.Duration) (*FeatureSelector, error) {
+	selector := &FeatureSelector{
+		accountHash:    switchThreshold(accountTag),
+		logger:         logger,
+		resolver:       resolver,
+		staticFeatures: staticFeatures,
+	}
+
+	if err := selector.refresh(ctx); err != nil {
+		logger.Err(err).Msg("Failed to fetch features, default to disable")
+	}
+
+	go selector.refreshLoop(ctx, refreshFreq)
+
+	return selector, nil
+}
+
+func (fs *FeatureSelector) PostQuantumMode() PostQuantumMode {
+	if fs.staticFeatures.PostQuantumMode != nil {
+		return *fs.staticFeatures.PostQuantumMode
+	}
+
+	fs.lock.RLock()
+	defer fs.lock.RUnlock()
+
+	if fs.features.PostQuantumPercentage > fs.accountHash {
+		return PostQuantumPrefer
+	}
+	return PostQuantumDisabled
+}
+
+func (fs *FeatureSelector) refreshLoop(ctx context.Context, refreshFreq time.Duration) {
+	ticker := time.NewTicker(refreshFreq)
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			err := fs.refresh(ctx)
+			if err != nil {
+				fs.logger.Err(err).Msg("Failed to refresh feature selector")
+			}
+		}
+	}
+}
+
+func (fs *FeatureSelector) refresh(ctx context.Context) error {
+	record, err := fs.resolver.lookupRecord(ctx)
+	if err != nil {
+		return err
+	}
+
+	var features featuresRecord
+	if err := json.Unmarshal(record, &features); err != nil {
+		return err
+	}
+
+	pq_enabled := features.PostQuantumPercentage > fs.accountHash
+	fs.logger.Debug().Int32("account_hash", fs.accountHash).Int32("pq_perct", features.PostQuantumPercentage).Bool("pq_enabled", pq_enabled).Msg("Refreshed feature")
+
+	fs.lock.Lock()
+	defer fs.lock.Unlock()
+
+	fs.features = features
+
+	return nil
+}
+
+// resolver represents an object that can look up featuresRecord
+type resolver interface {
+	lookupRecord(ctx context.Context) ([]byte, error)
+}
+
+type dnsResolver struct {
+	resolver *net.Resolver
+}
+
+func newDNSResolver() *dnsResolver {
+	return &dnsResolver{
+		resolver: net.DefaultResolver,
+	}
+}
+
+func (dr *dnsResolver) lookupRecord(ctx context.Context) ([]byte, error) {
+	ctx, cancel := context.WithTimeout(ctx, lookupTimeout)
+	defer cancel()
+
+	records, err := dr.resolver.LookupTXT(ctx, featureSelectorHostname)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(records) == 0 {
+		return nil, fmt.Errorf("No TXT record found for %s to determine which features to opt-in", featureSelectorHostname)
+	}
+
+	return []byte(records[0]), nil
+}
+
+func switchThreshold(accountTag string) int32 {
+	h := fnv.New32a()
+	_, _ = h.Write([]byte(accountTag))
+	return int32(h.Sum32() % 100)
+}

features/selector_test.go

@@ -0,0 +1,128 @@
+package features
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/rs/zerolog"
+	"github.com/stretchr/testify/require"
+)
+
+func TestUnmarshalFeaturesRecord(t *testing.T) {
+	tests := []struct {
+		record             []byte
+		expectedPercentage int32
+		expectedErr        bool
+	}{
+		{
+			record:             []byte(`{"pq":0}`),
+			expectedPercentage: 0,
+		},
+		{
+			record:             []byte(`{"pq":39}`),
+			expectedPercentage: 39,
+		},
+		{
+			record:             []byte(`{"pq":100}`),
+			expectedPercentage: 100,
+		},
+		{
+			record:             []byte(`{}`),
+			expectedPercentage: 0, // Unmarshal to default struct if key is not present
+		},
+		{
+			record:             []byte(`{"kyber":768}`),
+			expectedPercentage: 0, // Unmarshal to default struct if key is not present
+		},
+		{
+			record:      []byte(`{"pq":"kyber768"}`),
+			expectedErr: true,
+		},
+	}
+
+	for _, test := range tests {
+		var features featuresRecord
+		err := json.Unmarshal(test.record, &features)
+		if test.expectedErr {
+			require.Error(t, err, test)
+		} else {
+			require.NoError(t, err)
+			require.Equal(t, test.expectedPercentage, features.PostQuantumPercentage, test)
+		}
+	}
+}
+
+func TestRefreshFeaturesRecord(t *testing.T) {
+	// The hash of the accountTag is 82
+	accountTag := t.Name()
+	threshold := switchThreshold(accountTag)
+
+	percentages := []int32{0, 10, 80, 83, 100}
+	refreshFreq := time.Millisecond * 10
+	selector := newTestSelector(t, percentages, nil, refreshFreq)
+
+	for _, percentage := range percentages {
+		if percentage > threshold {
+			require.Equal(t, PostQuantumPrefer, selector.PostQuantumMode())
+		} else {
+			require.Equal(t, PostQuantumDisabled, selector.PostQuantumMode())
+		}
+
+		time.Sleep(refreshFreq + time.Millisecond)
+	}
+
+	// Make sure error doesn't override the last fetched features
+	require.Equal(t, PostQuantumPrefer, selector.PostQuantumMode())
+}
+
+func TestStaticFeatures(t *testing.T) {
+	percentages := []int32{0}
+	pqMode := PostQuantumStrict
+	selector := newTestSelector(t, percentages, &pqMode, time.Millisecond*10)
+	require.Equal(t, PostQuantumStrict, selector.PostQuantumMode())
+}
+
+// Verify that if the first refresh fails, the selector will use default features
+func TestFailedRefreshInitToDefault(t *testing.T) {
+	selector := newTestSelector(t, []int32{}, nil, time.Second)
+	require.Equal(t, featuresRecord{}, selector.features)
+	require.Equal(t, PostQuantumDisabled, selector.PostQuantumMode())
+}
+
+func newTestSelector(t *testing.T, percentages []int32, pqMode *PostQuantumMode, refreshFreq time.Duration) *FeatureSelector {
+	accountTag := t.Name()
+	logger := zerolog.Nop()
+
+	resolver := &mockResolver{
+		percentages: percentages,
+	}
+
+	staticFeatures := StaticFeatures{
+		PostQuantumMode: pqMode,
+	}
+	selector, err := newFeatureSelector(context.Background(), accountTag, &logger, resolver, staticFeatures, refreshFreq)
+	require.NoError(t, err)
+
+	return selector
+}
+
+type mockResolver struct {
+	nextIndex   int
+	percentages []int32
+}
+
+func (mr *mockResolver) lookupRecord(ctx context.Context) ([]byte, error) {
+	if mr.nextIndex >= len(mr.percentages) {
+		return nil, fmt.Errorf("no more record to lookup")
+	}
+
+	record, err := json.Marshal(featuresRecord{
+		PostQuantumPercentage: mr.percentages[mr.nextIndex],
+	})
+	mr.nextIndex++
+
+	return record, err
+}

go.mod

@@ -1,18 +1,17 @@
 module github.com/cloudflare/cloudflared
 
-go 1.19
+go 1.20
 
 require (
-	github.com/cloudflare/brotli-go v0.0.0-20191101163834-d34379f7ff93
 	github.com/cloudflare/golibs v0.0.0-20170913112048-333127dbecfc
 	github.com/coredns/coredns v1.10.0
-	github.com/coreos/go-oidc/v3 v3.4.0
-	github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf
+	github.com/coreos/go-oidc/v3 v3.6.0
+	github.com/coreos/go-systemd/v22 v22.5.0
 	github.com/facebookgo/grace v0.0.0-20180706040059-75cf19382434
 	github.com/fsnotify/fsnotify v1.4.9
-	github.com/getsentry/raven-go v0.2.0
 	github.com/getsentry/sentry-go v0.16.0
 	github.com/go-chi/chi/v5 v5.0.8
+	github.com/go-chi/cors v1.2.1
 	github.com/go-jose/go-jose/v3 v3.0.0
 	github.com/gobwas/ws v1.0.4
 	github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3
@@ -37,15 +36,13 @@ require (
 	go.opentelemetry.io/otel/trace v1.6.3
 	go.opentelemetry.io/proto/otlp v0.15.0
 	go.uber.org/automaxprocs v1.4.0
-	golang.org/x/crypto v0.8.0
-	golang.org/x/net v0.9.0
+	golang.org/x/crypto v0.11.0
+	golang.org/x/net v0.12.0
 	golang.org/x/sync v0.1.0
-	golang.org/x/sys v0.7.0
-	golang.org/x/term v0.7.0
+	golang.org/x/sys v0.10.0
+	golang.org/x/term v0.10.0
 	google.golang.org/protobuf v1.28.1
-	gopkg.in/coreos/go-oidc.v2 v2.2.1
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
-	gopkg.in/square/go-jose.v2 v2.6.0
 	gopkg.in/yaml.v3 v3.0.1
 	nhooyr.io/websocket v1.8.7
 	zombiezen.com/go/capnproto2 v2.18.0+incompatible
@@ -55,7 +52,6 @@ require (
 	github.com/BurntSushi/toml v1.2.0 // indirect
 	github.com/apparentlymart/go-cidr v1.1.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/cloudflare/circl v1.2.1-0.20220809205628-0a9554f37a47 // indirect
 	github.com/coredns/caddy v1.1.1 // indirect
@@ -66,7 +62,6 @@ require (
 	github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
 	github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870 // indirect
 	github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 // indirect
-	github.com/go-chi/cors v1.2.1 // indirect
 	github.com/go-logr/logr v1.2.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
@@ -88,7 +83,6 @@ require (
 	github.com/onsi/gomega v1.23.0 // indirect
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 // indirect
 	github.com/prometheus/common v0.37.0 // indirect
 	github.com/prometheus/procfs v0.8.0 // indirect
 	github.com/quic-go/qtls-go1-19 v0.3.2 // indirect
@@ -96,8 +90,8 @@ require (
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	golang.org/x/exp v0.0.0-20221205204356-47842c84f3db // indirect
 	golang.org/x/mod v0.8.0 // indirect
-	golang.org/x/oauth2 v0.4.0 // indirect
-	golang.org/x/text v0.9.0 // indirect
+	golang.org/x/oauth2 v0.6.0 // indirect
+	golang.org/x/text v0.11.0 // indirect
 	golang.org/x/tools v0.6.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd // indirect
@@ -116,8 +110,5 @@ replace gopkg.in/yaml.v3 => gopkg.in/yaml.v3 v3.0.1
 replace github.com/quic-go/quic-go => github.com/devincarr/quic-go v0.0.0-20230502200822-d1f4edacbee7
 
 // Post-quantum tunnel RTG-1339
-replace (
-	// Branches go1.19 go1.20 on github.com/cloudflare/qtls-pq
-	github.com/quic-go/qtls-go1-19 => github.com/cloudflare/qtls-pq v0.0.0-20230320123031-3faac1a945b2
-	github.com/quic-go/qtls-go1-20 => github.com/cloudflare/qtls-pq v0.0.0-20230320122459-4ed280d0d633
-)
+// Branches go1.20 on github.com/cloudflare/qtls-pq
+replace github.com/quic-go/qtls-go1-20 => github.com/cloudflare/qtls-pq v0.0.0-20230320122459-4ed280d0d633

go.sum

@@ -13,36 +13,14 @@ cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKV
 cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
 cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
 cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
-cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI=
-cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk=
-cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg=
-cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8=
-cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0=
-cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY=
-cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM=
-cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY=
-cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ=
-cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI=
-cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4=
-cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc=
-cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA=
-cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A=
-cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+nc=
 cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
 cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
 cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
 cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
 cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
 cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
-cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow=
-cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM=
-cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M=
-cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s=
-cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU=
-cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U=
 cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
 cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
-cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY=
 cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
 cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
 cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
@@ -52,7 +30,6 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo
 cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
 cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
-cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/toml v1.2.0 h1:Rt8g24XnyGTyglgET/PRUNlrUeu9F5L+7FilkXfZgs0=
@@ -74,8 +51,6 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r
 github.com/bwesterb/go-ristretto v1.2.2/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
 github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
-github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d h1:S2NE3iHSwP0XV47EEXL8mWmRdEfGscSJ+7EgePNgt0s=
-github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
@@ -84,34 +59,28 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
-github.com/cloudflare/brotli-go v0.0.0-20191101163834-d34379f7ff93 h1:QrGfkZDnMxcWHaYDdB7CmqS9i26OAnUj/xcus/abYkY=
-github.com/cloudflare/brotli-go v0.0.0-20191101163834-d34379f7ff93/go.mod h1:QiTe66jFdP7cUKMCCf/WrvDyYdtdmdZfVcdoLbzaKVY=
 github.com/cloudflare/circl v1.2.1-0.20220809205628-0a9554f37a47 h1:YzpECHxZ9TzO7LpnKmPxItSd79lLgrR5heIlnqU4dTU=
 github.com/cloudflare/circl v1.2.1-0.20220809205628-0a9554f37a47/go.mod h1:qhx8gBILsYlbam7h09SvHDSkjpe3TfLA7b/z4rxJvkE=
 github.com/cloudflare/golibs v0.0.0-20170913112048-333127dbecfc h1:Dvk3ySBsOm5EviLx6VCyILnafPcQinXGP5jbTdHUJgE=
 github.com/cloudflare/golibs v0.0.0-20170913112048-333127dbecfc/go.mod h1:HlgKKR8V5a1wroIDDIz3/A+T+9Janfq+7n1P5sEFdi0=
 github.com/cloudflare/qtls-pq v0.0.0-20230320122459-4ed280d0d633 h1:ZTub2XMOBpxyBiJf6Q+UKqAi07yt1rZmFitriHvFd8M=
 github.com/cloudflare/qtls-pq v0.0.0-20230320122459-4ed280d0d633/go.mod h1:j/igSUc4PgBMayIsBGjAFu2i7g663rm6kZrKy4htb7E=
-github.com/cloudflare/qtls-pq v0.0.0-20230320123031-3faac1a945b2 h1:0/KuLjh9lBMiXlooAdwoo+FbLVD5DABtquB0ImEFOK0=
-github.com/cloudflare/qtls-pq v0.0.0-20230320123031-3faac1a945b2/go.mod h1:XzuZIjv4mF5cM205RHHW1d60PQtWGwMR6jx38YKuYHs=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
-github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI=
 github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
-github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/coredns/caddy v1.1.1 h1:2eYKZT7i6yxIfGP3qLJoJ7HAsDJqYB+X68g4NYjSrE0=
 github.com/coredns/caddy v1.1.1/go.mod h1:A6ntJQlAWuQfFlsd9hvigKbo2WS0VUs2l1e2F+BawD4=
 github.com/coredns/coredns v1.10.0 h1:jCfuWsBjTs0dapkkhISfPCzn5LqvSRtrFtaf/Tjj4DI=
 github.com/coredns/coredns v1.10.0/go.mod h1:CIfRU5TgpuoIiJBJ4XrofQzfFQpPFh32ERpUevrSlaw=
-github.com/coreos/go-oidc/v3 v3.4.0 h1:xz7elHb/LDwm/ERpwHd+5nb7wFHL32rsr6bBOgaeu6g=
-github.com/coreos/go-oidc/v3 v3.4.0/go.mod h1:eHUXhZtXPQLgEaDrOVTgwbgmz1xGOkJNye6h3zkD2Pw=
+github.com/coreos/go-oidc/v3 v3.6.0 h1:AKVxfYw1Gmkn/w96z0DbT/B/xFnzTd3MkZvWLjF4n/o=
+github.com/coreos/go-oidc/v3 v3.6.0/go.mod h1:ZpHUsHBucTUj6WOkrP4E20UPynbLZzhTQ1XKCXkxyPc=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
-github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU=
-github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs=
+github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM=
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
@@ -124,12 +93,9 @@ github.com/devincarr/quic-go v0.0.0-20230502200822-d1f4edacbee7/go.mod h1:+4CVgV
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
-github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
 github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
-github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
 github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ=
 github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
-github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 h1:0JZ+dUmQeA8IIVUMzysrX4/AKuQwWhV2dYQuPZdvdSQ=
 github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64=
@@ -145,8 +111,6 @@ github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjr
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
-github.com/getsentry/raven-go v0.2.0 h1:no+xWJRb5ZI7eE8TWgIq1jLulQiIoLG0IfYxv5JYMGs=
-github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ=
 github.com/getsentry/sentry-go v0.16.0 h1:owk+S+5XcgJLlGR/3+3s6N4d+uKwqYvh/eS0AIMjPWo=
 github.com/getsentry/sentry-go v0.16.0/go.mod h1:ZXCloQLj0pG7mja5NK6NPf2V4A88YJ4pNlc2mOHwh6Y=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
@@ -197,6 +161,7 @@ github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/E
 github.com/gobwas/ws v1.0.4 h1:5eXU1CZhpQdq5kXbKb+sECH5Ia5KiO6CYzIzdlVx6Bs=
 github.com/gobwas/ws v1.0.4/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=
 github.com/goccy/go-json v0.9.11 h1:/pAaQDLHEoCq/5FFmSKBswWmK6H0e8g4159Kc/X/nqk=
+github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3 h1:zN2lZNZRflqFyxVaTIU61KNKQ9C0055u9CAfpmqUvo4=
 github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3/go.mod h1:nPpo7qLxd6XL3hWJG/O60sR8ZKfMCiIoNap5GvD12KU=
@@ -213,7 +178,6 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt
 github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
 github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
 github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
-github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8=
 github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=
 github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -231,10 +195,8 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
-github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM=
 github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
-github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
@@ -244,21 +206,16 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
-github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8=
 github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
-github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
-github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
@@ -266,12 +223,6 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf
 github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec=
 github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
@@ -279,15 +230,8 @@ github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
-github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0=
-github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM=
-github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM=
-github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM=
-github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c=
-github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4=
 github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
@@ -366,8 +310,6 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 h1:J9b7z+QKAmPf4YLrFg6oQUotqHQeUNWwkvo7jZp1GLU=
-github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
 github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
@@ -393,6 +335,8 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1
 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
 github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo=
 github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4=
+github.com/quic-go/qtls-go1-19 v0.3.2 h1:tFxjCFcTQzK+oMxG6Zcvp4Dq8dx4yD3dDiIiyc86Z5U=
+github.com/quic-go/qtls-go1-19 v0.3.2/go.mod h1:ySOI96ew8lnoKPtSqx2BlI5wCpUVPT05RMAlajtnyOI=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
@@ -428,15 +372,12 @@ github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
-go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
 go.opentelemetry.io/contrib/propagators v0.22.0 h1:KGdv58M2//veiYLIhb31mofaI2LgkIPXXAZVeYVyfd8=
 go.opentelemetry.io/contrib/propagators v0.22.0/go.mod h1:xGOuXr6lLIF9BXipA4pm6UuOSI0M98U6tsI3khbOiwU=
 go.opentelemetry.io/otel v1.0.0-RC2/go.mod h1:w1thVQ7qbAy8MHb0IFj8a5Q2QU0l2ksf8u/CN8m3NOM=
@@ -463,8 +404,8 @@ golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.8.0 h1:pd9TJtTueMTVQXzk8E2XESSMQDj/U7OUu0PqJqPXQjQ=
-golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE=
+golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA=
+golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -489,8 +430,6 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl
 golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
 golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
-golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
-golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
 golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
 golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
@@ -499,8 +438,6 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB
 golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
@@ -532,51 +469,24 @@ golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
-golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
-golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
-golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
-golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
-golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
-golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM=
-golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
+golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50=
+golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
-golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
-golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
-golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE=
-golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg=
-golang.org/x/oauth2 v0.4.0 h1:NF0gk8LVPg1Ml7SSbGyySuoxdsXitj7TvgvuRxIMc/M=
-golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec=
+golang.org/x/oauth2 v0.6.0 h1:Lh8GPgSKBfWSwFvtuWOfeI3aAAnbXTSutYxJiOJFgIw=
+golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -585,10 +495,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -624,63 +532,35 @@ golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220808155132-1c4a2a72c664/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU=
-golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
+golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.7.0 h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ=
-golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
+golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c=
+golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
-golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE=
-golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4=
+golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -725,17 +605,7 @@ golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roY
 golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
-golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE=
-golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
 golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
-golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
-golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
-golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
-golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
@@ -743,9 +613,6 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
-golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
 google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
@@ -762,29 +629,6 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M
 google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
 google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
 google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
-google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg=
-google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE=
-google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8=
-google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU=
-google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94=
-google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo=
-google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4=
-google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw=
-google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU=
-google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k=
-google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE=
-google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE=
-google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI=
-google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I=
-google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo=
-google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g=
-google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA=
-google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8=
-google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs=
-google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA=
-google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw=
-google.golang.org/api v0.80.0/go.mod h1:xY3nI94gbvBrE0J6NHXhxOmW97HG7Khjkku6AFB3Hyg=
-google.golang.org/api v0.84.0/go.mod h1:NTsGnUFJMYROtiquksZHBWtHfeMC7iYthki7Eq3pa8o=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@@ -823,54 +667,7 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc
 google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20210329143202-679c6ae281ee/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A=
-google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A=
-google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A=
-google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
-google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
-google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
-google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24=
-google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k=
-google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k=
-google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48=
-google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48=
-google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w=
-google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
-google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
-google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
-google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
-google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
-google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
 google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20211221195035-429b39de9b1c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI=
-google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI=
-google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI=
-google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI=
-google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E=
-google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
-google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
-google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
-google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
-google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
-google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4=
-google.golang.org/genproto v0.0.0-20220518221133-4f43b3371335/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4=
-google.golang.org/genproto v0.0.0-20220523171625-347a074981d8/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4=
-google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA=
-google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA=
 google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd h1:OjndDrsik+Gt+e6fs45z9AxiewiKyLKYpA45W5Kpkks=
 google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd/go.mod h1:cTsE614GARnxrLsqKREzmNYJACSWWpAWdNMwnD7c2BE=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
@@ -885,29 +682,13 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa
 google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
 google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
 google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
-google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
 google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
-google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
-google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8=
-google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
 google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
-google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
-google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
-google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
-google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
-google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE=
-google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE=
 google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
-google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
 google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
-google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
 google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ=
-google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
-google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
-google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
 google.golang.org/grpc v1.51.0 h1:E1eGv1FTqoLIdnBCZufiSHgKjlqG6fKFf6pPWtMTh8U=
 google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww=
-google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
@@ -930,13 +711,9 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
-gopkg.in/coreos/go-oidc.v2 v2.2.1 h1:MY5SZClJ7vhjKfr64a4nHAOV/c3WH2gB9BMrR64J1Mc=
-gopkg.in/coreos/go-oidc.v2 v2.2.1/go.mod h1:fYaTe2FS96wZZwR17YTDHwG+Mw6fmyqJNxN2eNCGPCI=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
-gopkg.in/square/go-jose.v2 v2.6.0 h1:NGk74WTnPKBNUhNzQX7PYcTLUjoq7mzKk2OKbvwk2iI=
-gopkg.in/square/go-jose.v2 v2.6.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

h2mux/h2_compressor.go

@@ -1,6 +1,3 @@
-//go:build !cgo
-// +build !cgo
-
 package h2mux
 
 import (

h2mux/h2_compressor_brotli.go

@@ -1,22 +0,0 @@
-//go:build cgo
-// +build cgo
-
-package h2mux
-
-import (
-	"io"
-
-	"github.com/cloudflare/brotli-go"
-)
-
-func CompressionIsSupported() bool {
-	return true
-}
-
-func newDecompressor(src io.Reader) *brotli.Reader {
-	return brotli.NewReader(src)
-}
-
-func newCompressor(dst io.Writer, quality, lgwin int) *brotli.Writer {
-	return brotli.NewWriter(dst, brotli.WriterOptions{Quality: quality, LGWin: lgwin})
-}

h2mux/h2mux.go

@@ -135,14 +135,7 @@ func Handshake(
 	m.f.ReadMetaHeaders = hpack.NewDecoder(4096, func(hpack.HeaderField) {})
 	// Initialise the settings to identify this connection and confirm the other end is sane.
 	handshakeSetting := http2.Setting{ID: SettingMuxerMagic, Val: MuxerMagicEdge}
-	compressionSetting := http2.Setting{ID: SettingCompression, Val: config.CompressionQuality.toH2Setting()}
-	if CompressionIsSupported() {
-		config.Log.Debug().Msg("muxer: Compression is supported")
-		m.compressionQuality = config.CompressionQuality.getPreset()
-	} else {
-		config.Log.Debug().Msg("muxer: Compression is not supported")
-		compressionSetting = http2.Setting{ID: SettingCompression, Val: 0}
-	}
+	compressionSetting := http2.Setting{ID: SettingCompression, Val: 0}
 
 	expectedMagic := MuxerMagicOrigin
 	if config.IsClient {

h2mux/h2mux_test.go

@@ -5,7 +5,6 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"math/rand"
 	"net"
 	"os"
@@ -652,7 +651,7 @@ func TestHPACK(t *testing.T) {
 		if stream.Headers[0].Value != "200" {
 			t.Fatalf("expected status 200, got %s", stream.Headers[0].Value)
 		}
-		_, _ = ioutil.ReadAll(stream)
+		_, _ = io.ReadAll(stream)
 		_ = stream.Close()
 	}
 }
@@ -678,157 +677,6 @@ func AssertIfPipeReadable(t *testing.T, pipe io.ReadCloser) {
 	}
 }
 
-func TestMultipleStreamsWithDictionaries(t *testing.T) {
-	l := zerolog.Nop()
-
-	for q := CompressionNone; q <= CompressionMax; q++ {
-		htmlBody := `<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"` +
-			`"http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">` +
-			`<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en">` +
-			`<head>` +
-			`  <title>Your page title here</title>` +
-			`</head>` +
-			`<body>` +
-			`<h1>Your major heading here</h1>` +
-			`<p>` +
-			`This is a regular text paragraph.` +
-			`</p>` +
-			`<ul>` +
-			`  <li>` +
-			`  First bullet of a bullet list.` +
-			`  </li>` +
-			`  <li>` +
-			`  This is the <em>second</em> bullet.` +
-			`  </li>` +
-			`</ul>` +
-			`</body>` +
-			`</html>`
-
-		f := MuxedStreamFunc(func(stream *MuxedStream) error {
-			var contentType string
-			var pathHeader Header
-
-			for _, h := range stream.Headers {
-				if h.Name == ":path" {
-					pathHeader = h
-					break
-				}
-			}
-
-			if pathHeader.Name != ":path" {
-				panic("Couldn't find :path header in test")
-			}
-
-			if strings.Contains(pathHeader.Value, "html") {
-				contentType = "text/html; charset=utf-8"
-			} else if strings.Contains(pathHeader.Value, "js") {
-				contentType = "application/javascript"
-			} else if strings.Contains(pathHeader.Value, "css") {
-				contentType = "text/css"
-			} else {
-				contentType = "img/gif"
-			}
-
-			_ = stream.WriteHeaders([]Header{
-				{Name: "content-type", Value: contentType},
-			})
-			_, _ = stream.Write([]byte(strings.Replace(htmlBody, "paragraph", pathHeader.Value, 1) + stream.Headers[5].Value))
-
-			return nil
-		})
-		muxPair := NewCompressedMuxerPair(t, fmt.Sprintf("%s_%d", t.Name(), q), q, f)
-		muxPair.Serve(t)
-
-		var wg sync.WaitGroup
-
-		paths := []string{
-			"/html1",
-			"/html2?sa:ds",
-			"/html3",
-			"/css1",
-			"/html1",
-			"/html2?sa:ds",
-			"/html3",
-			"/css1",
-			"/css2",
-			"/css3",
-			"/js",
-			"/js",
-			"/js",
-			"/js2",
-			"/img2",
-			"/html1",
-			"/html2?sa:ds",
-			"/html3",
-			"/css1",
-			"/css2",
-			"/css3",
-			"/js",
-			"/js",
-			"/js",
-			"/js2",
-			"/img1",
-		}
-
-		wg.Add(len(paths))
-		errorsC := make(chan error, len(paths))
-
-		for i, s := range paths {
-			go func(index int, path string) {
-				defer wg.Done()
-				stream, err := muxPair.OpenEdgeMuxStream(
-					[]Header{
-						{Name: ":method", Value: "GET"},
-						{Name: ":scheme", Value: "https"},
-						{Name: ":authority", Value: "tunnel.otterlyadorable.co.uk"},
-						{Name: ":path", Value: path},
-						{Name: "cf-ray", Value: "378948953f044408-SFO-DOG"},
-						{Name: "idx", Value: strconv.Itoa(index)},
-						{Name: "accept-encoding", Value: "gzip, br"},
-					},
-					nil,
-				)
-				if err != nil {
-					errorsC <- fmt.Errorf("error in OpenStream: %v", err)
-					return
-				}
-
-				expectBody := strings.Replace(htmlBody, "paragraph", path, 1) + strconv.Itoa(index)
-				responseBody := make([]byte, len(expectBody)*2)
-				n, err := stream.Read(responseBody)
-				if err != nil {
-					errorsC <- fmt.Errorf("stream %d error from (*MuxedStream).Read: %s", stream.streamID, err)
-					return
-				}
-				if n != len(expectBody) {
-					errorsC <- fmt.Errorf("stream %d expected response body to have %d bytes, got %d", stream.streamID, len(expectBody), n)
-					return
-				}
-				if string(responseBody[:n]) != expectBody {
-					errorsC <- fmt.Errorf("stream %d expected response body %s, got %s", stream.streamID, expectBody, responseBody[:n])
-					return
-				}
-			}(i, s)
-		}
-
-		wg.Wait()
-		close(errorsC)
-		testFail := false
-		for err := range errorsC {
-			testFail = true
-			l.Error().Msgf("%s", err)
-		}
-		if testFail {
-			t.Fatalf("TestMultipleStreams failed")
-		}
-
-		originMuxMetrics := muxPair.OriginMux.Metrics()
-		if q > CompressionNone && originMuxMetrics.CompBytesBefore.Value() <= 10*originMuxMetrics.CompBytesAfter.Value() {
-			t.Fatalf("Cross-stream compression is expected to give a better compression ratio")
-		}
-	}
-}
-
 func sampleSiteHandler(files map[string][]byte) MuxedStreamFunc {
 	return func(stream *MuxedStream) error {
 		var contentType string
@@ -905,7 +753,7 @@ func loadSampleFiles(paths []string) (map[string][]byte, error) {
 	files := make(map[string][]byte)
 	for _, path := range paths {
 		if _, ok := files[path]; !ok {
-			expectBody, err := ioutil.ReadFile(path)
+			expectBody, err := os.ReadFile(path)
 			if err != nil {
 				return nil, err
 			}
@@ -915,82 +763,6 @@ func loadSampleFiles(paths []string) (map[string][]byte, error) {
 	return files, nil
 }
 
-func TestSampleSiteWithDictionaries(t *testing.T) {
-	paths := []string{
-		"./sample/index.html",
-		"./sample/index2.html",
-		"./sample/index1.html",
-		"./sample/ghost-url.min.js",
-		"./sample/jquery.fitvids.js",
-		"./sample/index1.html",
-		"./sample/index2.html",
-		"./sample/index.html",
-	}
-	files, err := loadSampleFiles(paths)
-	assert.NoError(t, err)
-
-	for q := CompressionNone; q <= CompressionMax; q++ {
-		muxPair := NewCompressedMuxerPair(t, fmt.Sprintf("%s_%d", t.Name(), q), q, sampleSiteHandler(files))
-		muxPair.Serve(t)
-
-		var wg sync.WaitGroup
-		errC := make(chan error, len(paths))
-
-		wg.Add(len(paths))
-		for _, s := range paths {
-			go func(path string) {
-				defer wg.Done()
-				errC <- sampleSiteTest(muxPair, path, files)
-			}(s)
-		}
-
-		wg.Wait()
-		close(errC)
-
-		for err := range errC {
-			assert.NoError(t, err)
-		}
-
-		originMuxMetrics := muxPair.OriginMux.Metrics()
-		if q > CompressionNone && originMuxMetrics.CompBytesBefore.Value() <= 10*originMuxMetrics.CompBytesAfter.Value() {
-			t.Fatalf("Cross-stream compression is expected to give a better compression ratio")
-		}
-	}
-}
-
-func TestLongSiteWithDictionaries(t *testing.T) {
-	paths := []string{
-		"./sample/index.html",
-		"./sample/index1.html",
-		"./sample/index2.html",
-		"./sample/ghost-url.min.js",
-		"./sample/jquery.fitvids.js",
-	}
-	files, err := loadSampleFiles(paths)
-	assert.NoError(t, err)
-	for q := CompressionNone; q <= CompressionMedium; q++ {
-		muxPair := NewCompressedMuxerPair(t, fmt.Sprintf("%s_%d", t.Name(), q), q, sampleSiteHandler(files))
-		muxPair.Serve(t)
-
-		rand.Seed(time.Now().Unix())
-
-		tstLen := 500
-		errGroup, _ := errgroup.WithContext(context.Background())
-		for i := 0; i < tstLen; i++ {
-			errGroup.Go(func() error {
-				path := paths[rand.Int()%len(paths)]
-				return sampleSiteTest(muxPair, path, files)
-			})
-		}
-		assert.NoError(t, errGroup.Wait())
-
-		originMuxMetrics := muxPair.OriginMux.Metrics()
-		if q > CompressionNone && originMuxMetrics.CompBytesBefore.Value() <= 10*originMuxMetrics.CompBytesAfter.Value() {
-			t.Fatalf("Cross-stream compression is expected to give a better compression ratio")
-		}
-	}
-}
-
 func BenchmarkOpenStream(b *testing.B) {
 	const streams = 5000
 	for i := 0; i < b.N; i++ {

hello/hello.go

@@ -6,7 +6,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"html/template"
-	"io/ioutil"
+	"io"
 	"net"
 	"net/http"
 	"os"
@@ -234,7 +234,7 @@ func rootHandler(serverName string) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		var buffer bytes.Buffer
 		var body string
-		rawBody, err := ioutil.ReadAll(r.Body)
+		rawBody, err := io.ReadAll(r.Body)
 		if err == nil {
 			body = string(rawBody)
 		} else {

ingress/config.go

@@ -44,14 +44,12 @@ const (
 )
 
 type WarpRoutingConfig struct {
-	Enabled        bool                  `yaml:"enabled" json:"enabled"`
 	ConnectTimeout config.CustomDuration `yaml:"connectTimeout" json:"connectTimeout,omitempty"`
 	TCPKeepAlive   config.CustomDuration `yaml:"tcpKeepAlive" json:"tcpKeepAlive,omitempty"`
 }
 
 func NewWarpRoutingConfig(raw *config.WarpRoutingConfig) WarpRoutingConfig {
 	cfg := WarpRoutingConfig{
-		Enabled:        raw.Enabled,
 		ConnectTimeout: defaultWarpRoutingConnectTimeout,
 		TCPKeepAlive:   defaultTCPKeepAlive,
 	}
@@ -65,9 +63,7 @@ func NewWarpRoutingConfig(raw *config.WarpRoutingConfig) WarpRoutingConfig {
 }
 
 func (c *WarpRoutingConfig) RawConfig() config.WarpRoutingConfig {
-	raw := config.WarpRoutingConfig{
-		Enabled: c.Enabled,
-	}
+	raw := config.WarpRoutingConfig{}
 	if c.ConnectTimeout.Duration != defaultWarpRoutingConnectTimeout.Duration {
 		raw.ConnectTimeout = &c.ConnectTimeout
 	}

ingress/origin_connection_test.go

@@ -5,7 +5,6 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net"
 	"net/http"
 	"net/http/httptest"
@@ -118,7 +117,7 @@ func TestSocksStreamWSOverTCPConnection(t *testing.T) {
 	}
 	for _, status := range statusCodes {
 		handler := func(w http.ResponseWriter, r *http.Request) {
-			body, err := ioutil.ReadAll(r.Body)
+			body, err := io.ReadAll(r.Body)
 			require.NoError(t, err)
 			require.Equal(t, []byte(sendMessage), body)
 
@@ -180,7 +179,7 @@ func TestSocksStreamWSOverTCPConnection(t *testing.T) {
 		assert.NoError(t, err)
 		assert.Equal(t, status, resp.StatusCode)
 		require.Equal(t, echoHeaderReturnValue, resp.Header.Get(echoHeaderName))
-		body, err := ioutil.ReadAll(resp.Body)
+		body, err := io.ReadAll(resp.Body)
 		require.NoError(t, err)
 		require.Equal(t, []byte(echoMessage), body)
 

ingress/origin_icmp_proxy_test.go

@@ -390,15 +390,16 @@ func getLocalIPs(t *testing.T, ipv4 bool) []netip.Addr {
 	require.NoError(t, err)
 	localIPs := []netip.Addr{}
 	for _, i := range interfaces {
-		// Skip TUN devices
-		if strings.Contains(i.Name, "tun") {
+		// Skip TUN devices, and Docker Networks
+		if strings.Contains(i.Name, "tun") || strings.Contains(i.Name, "docker") || strings.HasPrefix(i.Name, "br-") {
 			continue
 		}
 		addrs, err := i.Addrs()
 		require.NoError(t, err)
 		for _, addr := range addrs {
 			if ipnet, ok := addr.(*net.IPNet); ok && (ipnet.IP.IsPrivate() || ipnet.IP.IsLoopback()) {
-				if (ipv4 && ipnet.IP.To4() != nil) || (!ipv4 && ipnet.IP.To4() == nil) {
+				// TODO DEVTOOLS-12514: We only run the IPv6 against the loopback interface due to issues on the CI runners.
+				if (ipv4 && ipnet.IP.To4() != nil) || (!ipv4 && ipnet.IP.To4() == nil && ipnet.IP.IsLoopback()) {
 					localIPs = append(localIPs, netip.MustParseAddr(ipnet.IP.String()))
 				}
 			}

ingress/origin_proxy_test.go

@@ -3,7 +3,7 @@ package ingress
 import (
 	"context"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net"
 	"net/http"
 	"net/http/httptest"
@@ -141,7 +141,7 @@ func TestHTTPServiceHostHeaderOverride(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, http.StatusOK, resp.StatusCode)
 
-	respBody, err := ioutil.ReadAll(resp.Body)
+	respBody, err := io.ReadAll(resp.Body)
 	require.NoError(t, err)
 	require.Equal(t, respBody, []byte(originURL.Host))
 }
@@ -180,7 +180,7 @@ func TestHTTPServiceUsesIngressRuleScheme(t *testing.T) {
 		require.NoError(t, err)
 		require.Equal(t, http.StatusOK, resp.StatusCode)
 
-		respBody, err := ioutil.ReadAll(resp.Body)
+		respBody, err := io.ReadAll(resp.Body)
 		require.NoError(t, err)
 		require.Equal(t, respBody, []byte(p))
 	}

ingress/packet_router.go

@@ -23,12 +23,11 @@ type muxer interface {
 
 // PacketRouter routes packets between Upstream and ICMPRouter. Currently it rejects all other type of ICMP packets
 type PacketRouter struct {
-	globalConfig           *GlobalRouterConfig
-	muxer                  muxer
-	logger                 *zerolog.Logger
-	checkRouterEnabledFunc func() bool
-	icmpDecoder            *packet.ICMPDecoder
-	encoder                *packet.Encoder
+	globalConfig *GlobalRouterConfig
+	muxer        muxer
+	logger       *zerolog.Logger
+	icmpDecoder  *packet.ICMPDecoder
+	encoder      *packet.Encoder
 }
 
 // GlobalRouterConfig is the configuration shared by all instance of Router.
@@ -40,14 +39,13 @@ type GlobalRouterConfig struct {
 }
 
 // NewPacketRouter creates a PacketRouter that handles ICMP packets. Packets are read from muxer but dropped if globalConfig is nil.
-func NewPacketRouter(globalConfig *GlobalRouterConfig, muxer muxer, logger *zerolog.Logger, checkRouterEnabledFunc func() bool) *PacketRouter {
+func NewPacketRouter(globalConfig *GlobalRouterConfig, muxer muxer, logger *zerolog.Logger) *PacketRouter {
 	return &PacketRouter{
-		globalConfig:           globalConfig,
-		muxer:                  muxer,
-		logger:                 logger,
-		checkRouterEnabledFunc: checkRouterEnabledFunc,
-		icmpDecoder:            packet.NewICMPDecoder(),
-		encoder:                packet.NewEncoder(),
+		globalConfig: globalConfig,
+		muxer:        muxer,
+		logger:       logger,
+		icmpDecoder:  packet.NewICMPDecoder(),
+		encoder:      packet.NewEncoder(),
 	}
 }
 
@@ -92,10 +90,6 @@ func (r *PacketRouter) handlePacket(ctx context.Context, rawPacket packet.RawPac
 		return
 	}
 
-	if enabled := r.checkRouterEnabledFunc(); !enabled {
-		return
-	}
-
 	icmpPacket, err := r.icmpDecoder.Decode(rawPacket)
 	if err != nil {
 		r.logger.Err(err).Msg("Failed to decode ICMP packet from quic datagram")

ingress/packet_router_test.go

@@ -7,7 +7,6 @@ import (
 	"net/netip"
 	"sync/atomic"
 	"testing"
-	"time"
 
 	"github.com/google/gopacket/layers"
 	"github.com/stretchr/testify/require"
@@ -29,9 +28,7 @@ var (
 
 func TestRouterReturnTTLExceed(t *testing.T) {
 	muxer := newMockMuxer(0)
-	routerEnabled := &routerEnabledChecker{}
-	routerEnabled.set(true)
-	router := NewPacketRouter(packetConfig, muxer, &noopLogger, routerEnabled.isEnabled)
+	router := NewPacketRouter(packetConfig, muxer, &noopLogger)
 	ctx, cancel := context.WithCancel(context.Background())
 	routerStopped := make(chan struct{})
 	go func() {
@@ -80,65 +77,6 @@ func TestRouterReturnTTLExceed(t *testing.T) {
 	<-routerStopped
 }
 
-func TestRouterCheckEnabled(t *testing.T) {
-	muxer := newMockMuxer(0)
-	routerEnabled := &routerEnabledChecker{}
-	router := NewPacketRouter(packetConfig, muxer, &noopLogger, routerEnabled.isEnabled)
-	ctx, cancel := context.WithCancel(context.Background())
-	routerStopped := make(chan struct{})
-	go func() {
-		router.Serve(ctx)
-		close(routerStopped)
-	}()
-
-	pk := packet.ICMP{
-		IP: &packet.IP{
-			Src:      netip.MustParseAddr("192.168.1.1"),
-			Dst:      netip.MustParseAddr("10.0.0.1"),
-			Protocol: layers.IPProtocolICMPv4,
-			TTL:      1,
-		},
-		Message: &icmp.Message{
-			Type: ipv4.ICMPTypeEcho,
-			Code: 0,
-			Body: &icmp.Echo{
-				ID:   12481,
-				Seq:  8036,
-				Data: []byte(t.Name()),
-			},
-		},
-	}
-
-	// router is disabled
-	encoder := packet.NewEncoder()
-	encodedPacket, err := encoder.Encode(&pk)
-	require.NoError(t, err)
-	sendPacket := quicpogs.RawPacket(encodedPacket)
-
-	muxer.edgeToCfd <- sendPacket
-	select {
-	case <-time.After(time.Millisecond * 10):
-	case <-muxer.cfdToEdge:
-		t.Error("Unexpected reply when router is disabled")
-	}
-	routerEnabled.set(true)
-	// router is enabled, expects reply
-	muxer.edgeToCfd <- sendPacket
-	<-muxer.cfdToEdge
-
-	routerEnabled.set(false)
-	// router is disabled
-	muxer.edgeToCfd <- sendPacket
-	select {
-	case <-time.After(time.Millisecond * 10):
-	case <-muxer.cfdToEdge:
-		t.Error("Unexpected reply when router is disabled")
-	}
-
-	cancel()
-	<-routerStopped
-}
-
 func assertTTLExceed(t *testing.T, originalPacket *packet.ICMP, expectedSrc netip.Addr, muxer *mockMuxer) {
 	encoder := packet.NewEncoder()
 	rawPacket, err := encoder.Encode(originalPacket)

jet.yaml

@@ -1,9 +0,0 @@
----
-basename: "cloudflared"
-comment: "Cloudflare Tunnel"
-copyright: "Cloudflare, Inc"
-arch: "x86"
-abi: "64"
-files:
-    - source: cloudflared
-      destination: /var/db/scripts/jet/cloudflared

logger/create.go

@@ -175,7 +175,7 @@ func createFromContext(
 
 	log := newZerolog(loggerConfig)
 	if incompatibleFlagsSet := logFile != "" && logDirectory != ""; incompatibleFlagsSet {
-		log.Error().Msgf("Your config includes values for both %s and %s, but they are incompatible. %s takes precedence.", LogFileFlag, logDirectoryFlagName, LogFileFlag)
+		log.Error().Msgf("Your config includes values for both %s (%s) and %s (%s), but they are incompatible. %s takes precedence.", LogFileFlag, logFile, logDirectoryFlagName, logDirectory, LogFileFlag)
 	}
 	return log
 }

management/service.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"net"
 	"net/http"
+	"net/http/pprof"
 	"os"
 	"sync"
 	"time"
@@ -12,6 +13,7 @@ import (
 	"github.com/go-chi/chi/v5"
 	"github.com/go-chi/cors"
 	"github.com/google/uuid"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/rs/zerolog"
 	"nhooyr.io/websocket"
 )
@@ -29,6 +31,17 @@ const (
 	reasonIdleLimitExceeded                      = "session was idle for too long"
 )
 
+var (
+	// CORS middleware required to allow dash to access management.argotunnel.com requests
+	corsHandler = cors.Handler(cors.Options{
+		// Allows for any subdomain of cloudflare.com
+		AllowedOrigins: []string{"https://*.cloudflare.com"},
+		// Required to present cookies or other authentication across origin boundries
+		AllowCredentials: true,
+		MaxAge:           300, // Maximum value not ignored by any of major browsers
+	})
+)
+
 type ManagementService struct {
 	// The management tunnel hostname
 	Hostname string
@@ -38,6 +51,9 @@ type ManagementService struct {
 	clientID  uuid.UUID
 	label     string
 
+	// Additional Handlers
+	metricsHandler http.Handler
+
 	log    *zerolog.Logger
 	router chi.Router
 
@@ -49,6 +65,7 @@ type ManagementService struct {
 }
 
 func New(managementHostname string,
+	enableDiagServices bool,
 	serviceIP string,
 	clientID uuid.UUID,
 	label string,
@@ -56,29 +73,31 @@ func New(managementHostname string,
 	logger LoggerListener,
 ) *ManagementService {
 	s := &ManagementService{
-		Hostname:  managementHostname,
-		log:       log,
-		logger:    logger,
-		serviceIP: serviceIP,
-		clientID:  clientID,
-		label:     label,
+		Hostname:       managementHostname,
+		log:            log,
+		logger:         logger,
+		serviceIP:      serviceIP,
+		clientID:       clientID,
+		label:          label,
+		metricsHandler: promhttp.Handler(),
 	}
 	r := chi.NewRouter()
 	r.Use(ValidateAccessTokenQueryMiddleware)
-	r.Get("/ping", ping)
-	r.Head("/ping", ping)
+
+	// Default management services
+	r.With(corsHandler).Get("/ping", ping)
+	r.With(corsHandler).Head("/ping", ping)
 	r.Get("/logs", s.logs)
-	r.Route("/host_details", func(r chi.Router) {
-		// CORS middleware required to allow dash to access management.argotunnel.com requests
-		r.Use(cors.Handler(cors.Options{
-			// Allows for any subdomain of cloudflare.com
-			AllowedOrigins: []string{"https://*.cloudflare.com"},
-			// Required to present cookies or other authentication across origin boundries
-			AllowCredentials: true,
-			MaxAge:           300, // Maximum value not ignored by any of major browsers
-		}))
-		r.Get("/", s.getHostDetails)
-	})
+	r.With(corsHandler).Get("/host_details", s.getHostDetails)
+
+	// Diagnostic management services
+	if enableDiagServices {
+		// Prometheus endpoint
+		r.With(corsHandler).Get("/metrics", s.metricsHandler.ServeHTTP)
+		// Supports only heap and goroutine
+		r.With(corsHandler).Get("/debug/pprof/{profile:heap|goroutine}", pprof.Index)
+	}
+
 	s.router = r
 	return s
 }

management/service_test.go

@@ -3,9 +3,13 @@ package management
 import (
 	"context"
 	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
 	"testing"
 	"time"
 
+	"github.com/google/uuid"
 	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -15,9 +19,23 @@ import (
 )
 
 var (
-	noopLogger = zerolog.New(io.Discard)
+	noopLogger         = zerolog.New(io.Discard)
+	managementHostname = "https://management.argotunnel.com"
 )
 
+func TestDisableDiagnosticRoutes(t *testing.T) {
+	mgmt := New("management.argotunnel.com", false, "1.1.1.1:80", uuid.Nil, "", &noopLogger, nil)
+	for _, path := range []string{"/metrics", "/debug/pprof/goroutine", "/debug/pprof/heap"} {
+		t.Run(strings.Replace(path, "/", "_", -1), func(t *testing.T) {
+			req := httptest.NewRequest("GET", managementHostname+path+"?access_token="+validToken, nil)
+			recorder := httptest.NewRecorder()
+			mgmt.ServeHTTP(recorder, req)
+			resp := recorder.Result()
+			require.Equal(t, http.StatusNotFound, resp.StatusCode)
+		})
+	}
+}
+
 func TestReadEventsLoop(t *testing.T) {
 	sentEvent := EventStartStreaming{
 		ClientEvent: ClientEvent{Type: StartStreaming},

management/token_test.go

@@ -7,8 +7,8 @@ import (
 	"errors"
 	"testing"
 
+	"github.com/go-jose/go-jose/v3"
 	"github.com/stretchr/testify/require"
-	"gopkg.in/square/go-jose.v2"
 )
 
 const (

orchestration/config_test.go

@@ -60,7 +60,6 @@ func TestNewLocalConfig_MarshalJSON(t *testing.T) {
 			}
 		],
         "warp-routing": {
-			"enabled": true,
 			"connectTimeout": 1
         }
 	}
@@ -83,7 +82,6 @@ func TestNewLocalConfig_MarshalJSON(t *testing.T) {
 	require.NoError(t, err)
 
 	require.Equal(t, remoteConfig.WarpRouting, ingress.WarpRoutingConfig{
-		Enabled: true,
 		ConnectTimeout: config.CustomDuration{
 			Duration: time.Second,
 		},

orchestration/orchestrator.go

@@ -29,11 +29,10 @@ type Orchestrator struct {
 	// Underlying value is proxy.Proxy, can be read without the lock, but still needs the lock to update
 	proxy atomic.Value
 	// Set of internal ingress rules defined at cloudflared startup (separate from user-defined ingress rules)
-	internalRules      []ingress.Rule
-	warpRoutingEnabled atomic.Bool
-	config             *Config
-	tags               []tunnelpogs.Tag
-	log                *zerolog.Logger
+	internalRules []ingress.Rule
+	config        *Config
+	tags          []tunnelpogs.Tag
+	log           *zerolog.Logger
 
 	// orchestrator must not handle any more updates after shutdownC is closed
 	shutdownC <-chan struct{}
@@ -136,11 +135,6 @@ func (o *Orchestrator) updateIngress(ingressRules ingress.Ingress, warpRouting i
 	o.proxy.Store(proxy)
 	o.config.Ingress = &ingressRules
 	o.config.WarpRouting = warpRouting
-	if warpRouting.Enabled {
-		o.warpRoutingEnabled.Store(true)
-	} else {
-		o.warpRoutingEnabled.Store(false)
-	}
 
 	// If proxyShutdownC is nil, there is no previous running proxy
 	if o.proxyShutdownC != nil {
@@ -209,10 +203,6 @@ func (o *Orchestrator) GetOriginProxy() (connection.OriginProxy, error) {
 	return proxy, nil
 }
 
-func (o *Orchestrator) WarpRoutingEnabled() bool {
-	return o.warpRoutingEnabled.Load()
-}
-
 func (o *Orchestrator) waitToCloseLastProxy() {
 	<-o.shutdownC
 	o.lock.Lock()

orchestration/orchestrator_test.go

@@ -5,7 +5,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net"
 	"net/http"
 	"net/http/httptest"
@@ -51,12 +50,11 @@ func TestUpdateConfiguration(t *testing.T) {
 	initConfig := &Config{
 		Ingress: &ingress.Ingress{},
 	}
-	orchestrator, err := NewOrchestrator(context.Background(), initConfig, testTags, []ingress.Rule{ingress.NewManagementRule(management.New("management.argotunnel.com", "1.1.1.1:80", uuid.Nil, "", &testLogger, nil))}, &testLogger)
+	orchestrator, err := NewOrchestrator(context.Background(), initConfig, testTags, []ingress.Rule{ingress.NewManagementRule(management.New("management.argotunnel.com", false, "1.1.1.1:80", uuid.Nil, "", &testLogger, nil))}, &testLogger)
 	require.NoError(t, err)
 	initOriginProxy, err := orchestrator.GetOriginProxy()
 	require.NoError(t, err)
 	require.Implements(t, (*connection.OriginProxy)(nil), initOriginProxy)
-	require.False(t, orchestrator.WarpRoutingEnabled())
 
 	configJSONV2 := []byte(`
 {
@@ -88,7 +86,6 @@ func TestUpdateConfiguration(t *testing.T) {
         }
     ],
     "warp-routing": {
-        "enabled": true,
         "connectTimeout": 10
     }
 }
@@ -127,8 +124,6 @@ func TestUpdateConfiguration(t *testing.T) {
 	require.Equal(t, config.CustomDuration{Duration: time.Second * 90}, configV2.Ingress.Rules[2].Config.ConnectTimeout)
 	require.Equal(t, false, configV2.Ingress.Rules[2].Config.NoTLSVerify)
 	require.Equal(t, true, configV2.Ingress.Rules[2].Config.NoHappyEyeballs)
-	require.True(t, configV2.WarpRouting.Enabled)
-	require.Equal(t, configV2.WarpRouting.Enabled, orchestrator.WarpRoutingEnabled())
 	require.Equal(t, configV2.WarpRouting.ConnectTimeout.Duration, 10*time.Second)
 
 	originProxyV2, err := orchestrator.GetOriginProxy()
@@ -163,7 +158,6 @@ func TestUpdateConfiguration(t *testing.T) {
         }
     ],
     "warp-routing": {
-        "enabled": false
     }
 }
 `)
@@ -172,8 +166,6 @@ func TestUpdateConfiguration(t *testing.T) {
 	require.Len(t, configV10.Ingress.Rules, 1)
 	require.True(t, configV10.Ingress.Rules[0].Matches("blogs.tunnel.io", "/2022/02/10"))
 	require.Equal(t, ingress.HelloWorldService, configV10.Ingress.Rules[0].Service.String())
-	require.False(t, configV10.WarpRouting.Enabled)
-	require.Equal(t, configV10.WarpRouting.Enabled, orchestrator.WarpRoutingEnabled())
 
 	originProxyV10, err := orchestrator.GetOriginProxy()
 	require.NoError(t, err)
@@ -192,7 +184,6 @@ func TestUpdateConfiguration_FromMigration(t *testing.T) {
 	initOriginProxy, err := orchestrator.GetOriginProxy()
 	require.NoError(t, err)
 	require.Implements(t, (*connection.OriginProxy)(nil), initOriginProxy)
-	require.False(t, orchestrator.WarpRoutingEnabled())
 
 	configJSONV2 := []byte(`
 {
@@ -202,7 +193,6 @@ func TestUpdateConfiguration_FromMigration(t *testing.T) {
         }
     ],
     "warp-routing": {
-        "enabled": true
     }
 }
 `)
@@ -272,7 +262,6 @@ func TestConcurrentUpdateAndRead(t *testing.T) {
         }
     ],
     "warp-routing": {
-        "enabled": true
     }
 }
 `, hostname, httpOrigin.URL, expectedHost))
@@ -284,7 +273,6 @@ func TestConcurrentUpdateAndRead(t *testing.T) {
         }
     ],
     "warp-routing": {
-        "enabled": false
     }
 }
 `)
@@ -297,7 +285,6 @@ func TestConcurrentUpdateAndRead(t *testing.T) {
         }
     ],
     "warp-routing": {
-        "enabled": true
     }
 }
 `)
@@ -338,7 +325,7 @@ func TestConcurrentUpdateAndRead(t *testing.T) {
 			switch resp.StatusCode {
 			// v1 proxy, warp enabled
 			case 200:
-				body, err := ioutil.ReadAll(resp.Body)
+				body, err := io.ReadAll(resp.Body)
 				require.NoError(t, err)
 				require.Equal(t, t.Name(), string(body))
 				warpRoutingDisabled = false
@@ -450,7 +437,7 @@ func proxyTCP(ctx context.Context, originProxy connection.OriginProxy, originAdd
 		CFRay:   "123",
 		LBProbe: false,
 	}
-	rws := connection.NewHTTPResponseReadWriterAcker(respWriter, req)
+	rws := connection.NewHTTPResponseReadWriterAcker(respWriter, w.(http.Flusher), req)
 
 	return originProxy.ProxyTCP(ctx, rws, tcpReq)
 }
@@ -517,7 +504,6 @@ func TestClosePreviousProxies(t *testing.T) {
 		}
     ],
     "warp-routing": {
-        "enabled": true
     }
 }
 `, hostname))
@@ -530,7 +516,6 @@ func TestClosePreviousProxies(t *testing.T) {
 		}
     ],
     "warp-routing": {
-        "enabled": true
     }
 }
 `)
@@ -613,7 +598,6 @@ func TestPersistentConnection(t *testing.T) {
         }
     ],
     "warp-routing": {
-        "enabled": true
     }
 }
 `, wsOrigin.URL))
@@ -680,7 +664,6 @@ func TestPersistentConnection(t *testing.T) {
         }
     ],
     "warp-routing": {
-        "enabled": false
     }
 }
 `)

proxy/metrics.go

@@ -43,6 +43,22 @@ var (
 			Help:      "Count of error proxying to origin",
 		},
 	)
+	activeTCPSessions = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Namespace: connection.MetricsNamespace,
+			Subsystem: "tcp",
+			Name:      "active_sessions",
+			Help:      "Concurrent count of TCP sessions that are being proxied to any origin",
+		},
+	)
+	totalTCPSessions = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Namespace: connection.MetricsNamespace,
+			Subsystem: "tcp",
+			Name:      "total_sessions",
+			Help:      "Total count of TCP sessions that have been proxied to any origin",
+		},
+	)
 )
 
 func init() {
@@ -51,6 +67,8 @@ func init() {
 		concurrentRequests,
 		responseByCode,
 		requestErrors,
+		activeTCPSessions,
+		totalTCPSessions,
 	)
 }
 
@@ -62,3 +80,14 @@ func incrementRequests() {
 func decrementConcurrentRequests() {
 	concurrentRequests.Dec()
 }
+
+func incrementTCPRequests() {
+	incrementRequests()
+	totalTCPSessions.Inc()
+	activeTCPSessions.Inc()
+}
+
+func decrementTCPConcurrentRequests() {
+	decrementConcurrentRequests()
+	activeTCPSessions.Dec()
+}

proxy/proxy.go

@@ -57,10 +57,8 @@ func NewOriginProxy(
 		tags:         tags,
 		log:          log,
 	}
-	if warpRouting.Enabled {
-		proxy.warpRouting = ingress.NewWarpRoutingService(warpRouting)
-		log.Info().Msgf("Warp-routing is enabled")
-	}
+
+	proxy.warpRouting = ingress.NewWarpRoutingService(warpRouting)
 
 	return proxy
 }
@@ -136,8 +134,11 @@ func (p *Proxy) ProxyHTTP(
 		if err != nil {
 			return err
 		}
-
-		rws := connection.NewHTTPResponseReadWriterAcker(w, req)
+		flusher, ok := w.(http.Flusher)
+		if !ok {
+			return fmt.Errorf("response writer is not a flusher")
+		}
+		rws := connection.NewHTTPResponseReadWriterAcker(w, flusher, req)
 		if err := p.proxyStream(tr.ToTracedContext(), rws, dest, originProxy); err != nil {
 			rule, srv := ruleField(p.ingressRules, ruleNum)
 			p.logRequestError(err, cfRay, "", rule, srv)
@@ -158,8 +159,8 @@ func (p *Proxy) ProxyTCP(
 	rwa connection.ReadWriteAcker,
 	req *connection.TCPRequest,
 ) error {
-	incrementRequests()
-	defer decrementConcurrentRequests()
+	incrementTCPRequests()
+	defer decrementTCPConcurrentRequests()
 
 	if p.warpRouting == nil {
 		err := errors.New(`cloudflared received a request from WARP client, but your configuration has disabled ingress from WARP clients. To enable this, set "warp-routing:\n\t enabled: true" in your config.yaml`)

proxy/proxy_posix_test.go

@@ -1,10 +1,8 @@
 //go:build !windows
-// +build !windows
 
 package proxy
 
 import (
-	"io/ioutil"
 	"net"
 	"net/http"
 	"net/http/httptest"
@@ -17,7 +15,7 @@ import (
 )
 
 func TestUnixSocketOrigin(t *testing.T) {
-	file, err := ioutil.TempFile("", "unix.sock")
+	file, err := os.CreateTemp("", "unix.sock")
 	require.NoError(t, err)
 	os.Remove(file.Name()) // remove the file since binding the socket expects to create it
 

proxy/proxy_test.go

@@ -37,7 +37,6 @@ var (
 	testTags        = []tunnelpogs.Tag{{Name: "Name", Value: "value"}}
 	noWarpRouting   = ingress.WarpRoutingConfig{}
 	testWarpRouting = ingress.WarpRoutingConfig{
-		Enabled:        true,
 		ConnectTimeout: config.CustomDuration{Duration: time.Second},
 	}
 )
@@ -698,7 +697,7 @@ func TestConnections(t *testing.T) {
 				}()
 			}
 			if test.args.connectionType == connection.TypeTCP {
-				rwa := connection.NewHTTPResponseReadWriterAcker(respWriter, req)
+				rwa := connection.NewHTTPResponseReadWriterAcker(respWriter, respWriter.(http.Flusher), req)
 				err = proxy.ProxyTCP(ctx, rwa, &connection.TCPRequest{Dest: dest})
 			} else {
 				log := zerolog.Nop()
@@ -834,6 +833,8 @@ func (w *wsRespWriter) WriteRespHeaders(status int, header http.Header) error {
 	return nil
 }
 
+func (w *wsRespWriter) Flush() {}
+
 func (w *wsRespWriter) AddTrailer(trailerName, trailerValue string) {
 	// do nothing
 }
@@ -873,6 +874,8 @@ func (m *mockTCPRespWriter) Write(p []byte) (n int, err error) {
 	return m.w.Write(p)
 }
 
+func (m *mockTCPRespWriter) Flush() {}
+
 func (m *mockTCPRespWriter) AddTrailer(trailerName, trailerValue string) {
 	// do nothing
 }

quic/quic_protocol.go

@@ -39,6 +39,9 @@ const (
 	HandshakeIdleTimeout = 5 * time.Second
 	MaxIdleTimeout       = 5 * time.Second
 	MaxIdlePingPeriod    = 1 * time.Second
+
+	// MaxIncomingStreams is 2^60, which is the maximum supported value by Quic-Go
+	MaxIncomingStreams = 1 << 60
 )
 
 // RequestServerStream is a stream to serve requests
@@ -226,9 +229,12 @@ func writeSignature(stream io.Writer, signature ProtocolSignature) error {
 type RPCClientStream struct {
 	client    tunnelpogs.CloudflaredServer_PogsClient
 	transport rpc.Transport
+
+	// Time we wait for the server to respond to a request before we close the connection.
+	rpcUnregisterUDPSessionDeadline time.Duration
 }
 
-func NewRPCClientStream(ctx context.Context, stream io.ReadWriteCloser, logger *zerolog.Logger) (*RPCClientStream, error) {
+func NewRPCClientStream(ctx context.Context, stream io.ReadWriteCloser, rpcUnregisterUDPSessionDeadline time.Duration, logger *zerolog.Logger) (*RPCClientStream, error) {
 	n, err := stream.Write(RPCStreamProtocolSignature[:])
 	if err != nil {
 		return nil, err
@@ -242,8 +248,9 @@ func NewRPCClientStream(ctx context.Context, stream io.ReadWriteCloser, logger *
 		tunnelrpc.ConnLog(logger),
 	)
 	return &RPCClientStream{
-		client:    tunnelpogs.NewCloudflaredServer_PogsClient(conn.Bootstrap(ctx), conn),
-		transport: transport,
+		client:                          tunnelpogs.NewCloudflaredServer_PogsClient(conn.Bootstrap(ctx), conn),
+		transport:                       transport,
+		rpcUnregisterUDPSessionDeadline: rpcUnregisterUDPSessionDeadline,
 	}, nil
 }
 
@@ -252,6 +259,8 @@ func (rcs *RPCClientStream) RegisterUdpSession(ctx context.Context, sessionID uu
 }
 
 func (rcs *RPCClientStream) UnregisterUdpSession(ctx context.Context, sessionID uuid.UUID, message string) error {
+	ctx, cancel := context.WithTimeout(ctx, rcs.rpcUnregisterUDPSessionDeadline)
+	defer cancel()
 	return rcs.client.UnregisterUdpSession(ctx, sessionID, message)
 }
 

quic/quic_protocol_test.go

@@ -109,6 +109,63 @@ func TestConnectResponseMeta(t *testing.T) {
 	}
 }
 
+func TestUnregisterUdpSession(t *testing.T) {
+	unregisterMessage := "closed by eyeball"
+
+	var tests = []struct {
+		name             string
+		sessionRPCServer mockSessionRPCServer
+		timeout          time.Duration
+	}{
+
+		{
+			name: "UnregisterUdpSessionTimesout if the RPC server does not respond",
+			sessionRPCServer: mockSessionRPCServer{
+				sessionID:         uuid.New(),
+				dstIP:             net.IP{172, 16, 0, 1},
+				dstPort:           8000,
+				closeIdleAfter:    testCloseIdleAfterHint,
+				unregisterMessage: unregisterMessage,
+				traceContext:      "1241ce3ecdefc68854e8514e69ba42ca:b38f1bf5eae406f3:0:1",
+			},
+			// very very low value so we trigger the timeout every time.
+			timeout: time.Nanosecond * 1,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			logger := zerolog.Nop()
+			clientStream, serverStream := newMockRPCStreams()
+			sessionRegisteredChan := make(chan struct{})
+			go func() {
+				protocol, err := DetermineProtocol(serverStream)
+				assert.NoError(t, err)
+				rpcServerStream, err := NewRPCServerStream(serverStream, protocol)
+				assert.NoError(t, err)
+				err = rpcServerStream.Serve(test.sessionRPCServer, nil, &logger)
+				assert.NoError(t, err)
+
+				serverStream.Close()
+				close(sessionRegisteredChan)
+			}()
+
+			rpcClientStream, err := NewRPCClientStream(context.Background(), clientStream, test.timeout, &logger)
+			assert.NoError(t, err)
+
+			reg, err := rpcClientStream.RegisterUdpSession(context.Background(), test.sessionRPCServer.sessionID, test.sessionRPCServer.dstIP, test.sessionRPCServer.dstPort, testCloseIdleAfterHint, test.sessionRPCServer.traceContext)
+			assert.NoError(t, err)
+			assert.NoError(t, reg.Err)
+
+			assert.Error(t, rpcClientStream.UnregisterUdpSession(context.Background(), test.sessionRPCServer.sessionID, unregisterMessage))
+
+			rpcClientStream.Close()
+			<-sessionRegisteredChan
+		})
+	}
+
+}
+
 func TestRegisterUdpSession(t *testing.T) {
 	unregisterMessage := "closed by eyeball"
 
@@ -157,7 +214,7 @@ func TestRegisterUdpSession(t *testing.T) {
 				close(sessionRegisteredChan)
 			}()
 
-			rpcClientStream, err := NewRPCClientStream(context.Background(), clientStream, &logger)
+			rpcClientStream, err := NewRPCClientStream(context.Background(), clientStream, 5*time.Second, &logger)
 			assert.NoError(t, err)
 
 			reg, err := rpcClientStream.RegisterUdpSession(context.Background(), test.sessionRPCServer.sessionID, test.sessionRPCServer.dstIP, test.sessionRPCServer.dstPort, testCloseIdleAfterHint, test.sessionRPCServer.traceContext)
@@ -208,7 +265,7 @@ func TestManageConfiguration(t *testing.T) {
 
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
 	defer cancel()
-	rpcClientStream, err := NewRPCClientStream(ctx, clientStream, &logger)
+	rpcClientStream, err := NewRPCClientStream(ctx, clientStream, 5*time.Second, &logger)
 	assert.NoError(t, err)
 
 	result, err := rpcClientStream.UpdateConfiguration(ctx, version, config)

release_pkgs.py

@@ -13,7 +13,6 @@ import base64
 import logging
 import os
 import shutil
-from hashlib import sha256
 from pathlib import Path
 from subprocess import Popen, PIPE
 
@@ -35,7 +34,6 @@ class PkgUploader:
 
     def upload_pkg_to_r2(self, filename, upload_file_path):
         endpoint_url = f"https://{self.account_id}.r2.cloudflarestorage.com"
-        token_secret_hash = sha256(self.client_secret.encode()).hexdigest()
 
         config = Config(
             region_name='auto',
@@ -48,7 +46,7 @@ class PkgUploader:
             "s3",
             endpoint_url=endpoint_url,
             aws_access_key_id=self.client_id,
-            aws_secret_access_key=token_secret_hash,
+            aws_secret_access_key=self.client_secret,
             config=config,
         )
 

retry/backoffhandler.go

@@ -91,10 +91,12 @@ func (b *BackoffHandler) Backoff(ctx context.Context) bool {
 
 // Sets a grace period within which the the backoff timer is maintained. After the grace
 // period expires, the number of retries & backoff duration is reset.
-func (b *BackoffHandler) SetGracePeriod() {
+func (b *BackoffHandler) SetGracePeriod() time.Duration {
 	maxTimeToWait := b.GetBaseTime() * 2 << (b.retries + 1)
 	timeToWait := time.Duration(rand.Int63n(maxTimeToWait.Nanoseconds()))
 	b.resetDeadline = Clock.Now().Add(timeToWait)
+
+	return timeToWait
 }
 
 func (b BackoffHandler) GetBaseTime() time.Duration {

retry/backoffhandler_test.go

@@ -57,9 +57,9 @@ func TestBackoffGracePeriod(t *testing.T) {
 		t.Fatalf("backoff failed immediately")
 	}
 	// the next call to Backoff would fail unless it's after the grace period
-	backoff.SetGracePeriod()
-	// advance time to after the grace period (~4 seconds) and see what happens
-	currentTime = currentTime.Add(time.Second * 5)
+	gracePeriod := backoff.SetGracePeriod()
+	// advance time to after the grace period, which at most will be 8 seconds, but we will advance +1 second.
+	currentTime = currentTime.Add(gracePeriod + time.Second)
 	if !backoff.Backoff(ctx) {
 		t.Fatalf("backoff failed after the grace period expired")
 	}

socks/connection_handler_test.go

@@ -2,7 +2,7 @@ package socks
 
 import (
 	"encoding/json"
-	"io/ioutil"
+	"io"
 	"net"
 	"net/http"
 	"testing"
@@ -32,7 +32,7 @@ func sendSocksRequest(t *testing.T) []byte {
 	assert.NoError(t, err)
 	defer resp.Body.Close()
 
-	b, err := ioutil.ReadAll(resp.Body)
+	b, err := io.ReadAll(resp.Body)
 	assert.NoError(t, err)
 
 	return b

sshgen/sshgen.go

@@ -10,15 +10,15 @@ import (
 	"encoding/pem"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"net/url"
+	"os"
 	"time"
 
+	"github.com/go-jose/go-jose/v3/jwt"
 	homedir "github.com/mitchellh/go-homedir"
 	"github.com/pkg/errors"
 	gossh "golang.org/x/crypto/ssh"
-	"gopkg.in/square/go-jose.v2/jwt"
 
 	"github.com/cloudflare/cloudflared/config"
 	cfpath "github.com/cloudflare/cloudflared/token"
@@ -148,7 +148,7 @@ func generateKeyPair(fullName string) ([]byte, error) {
 		return nil, err
 	}
 	if exist {
-		return ioutil.ReadFile(pubKeyName)
+		return os.ReadFile(pubKeyName)
 	}
 
 	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
@@ -187,5 +187,5 @@ func writeKey(filename string, data []byte) error {
 		return err
 	}
 
-	return ioutil.WriteFile(filepath, data, 0600)
+	return os.WriteFile(filepath, data, 0600)
 }

sshgen/sshgen_test.go

@@ -1,5 +1,4 @@
 //go:build !windows
-// +build !windows
 
 package sshgen
 
@@ -7,7 +6,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
@@ -16,9 +14,9 @@ import (
 	"testing"
 	"time"
 
+	"github.com/go-jose/go-jose/v3"
+	"github.com/go-jose/go-jose/v3/jwt"
 	"github.com/stretchr/testify/assert"
-	"gopkg.in/square/go-jose.v2"
-	"gopkg.in/square/go-jose.v2/jwt"
 
 	"github.com/cloudflare/cloudflared/config"
 	cfpath "github.com/cloudflare/cloudflared/token"
@@ -64,7 +62,7 @@ func TestCertGenSuccess(t *testing.T) {
 	mockRequest = func(url, contentType string, body io.Reader) (*http.Response, error) {
 		assert.Contains(t, "/cdn-cgi/access/cert_sign", url)
 		assert.Equal(t, "application/json", contentType)
-		buf, err := ioutil.ReadAll(body)
+		buf, err := io.ReadAll(body)
 		assert.NoError(t, err)
 		assert.NotEmpty(t, buf)
 		return w.Result(), nil

stream/debug.go

@@ -0,0 +1,64 @@
+package stream
+
+import (
+	"io"
+	"sync/atomic"
+
+	"github.com/rs/zerolog"
+)
+
+// DebugStream will tee each read and write to the output logger as a debug message
+type DebugStream struct {
+	reader io.Reader
+	writer io.Writer
+	log    *zerolog.Logger
+	max    uint64
+	count  atomic.Uint64
+}
+
+func NewDebugStream(stream io.ReadWriter, logger *zerolog.Logger, max uint64) *DebugStream {
+	return &DebugStream{
+		reader: stream,
+		writer: stream,
+		log:    logger,
+		max:    max,
+	}
+}
+
+func (d *DebugStream) Read(p []byte) (n int, err error) {
+	n, err = d.reader.Read(p)
+	if n > 0 && d.max > d.count.Load() {
+		d.count.Add(1)
+		if err != nil {
+			d.log.Err(err).
+				Str("dir", "r").
+				Int("count", n).
+				Msgf("%+q", p[:n])
+		} else {
+			d.log.Debug().
+				Str("dir", "r").
+				Int("count", n).
+				Msgf("%+q", p[:n])
+		}
+	}
+	return
+}
+
+func (d *DebugStream) Write(p []byte) (n int, err error) {
+	n, err = d.writer.Write(p)
+	if n > 0 && d.max > d.count.Load() {
+		d.count.Add(1)
+		if err != nil {
+			d.log.Err(err).
+				Str("dir", "w").
+				Int("count", n).
+				Msgf("%+q", p[:n])
+		} else {
+			d.log.Debug().
+				Str("dir", "w").
+				Int("count", n).
+				Msgf("%+q", p[:n])
+		}
+	}
+	return
+}

stream/stream.go

@@ -8,13 +8,43 @@ import (
 	"sync/atomic"
 	"time"
 
-	"github.com/getsentry/raven-go"
+	"github.com/getsentry/sentry-go"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
 
 	"github.com/cloudflare/cloudflared/cfio"
 )
 
+type Stream interface {
+	Reader
+	WriterCloser
+}
+
+type Reader interface {
+	io.Reader
+}
+
+type WriterCloser interface {
+	io.Writer
+	WriteCloser
+}
+
+type WriteCloser interface {
+	CloseWrite() error
+}
+
+type nopCloseWriterAdapter struct {
+	io.ReadWriter
+}
+
+func NopCloseWriterAdapter(stream io.ReadWriter) *nopCloseWriterAdapter {
+	return &nopCloseWriterAdapter{stream}
+}
+
+func (n *nopCloseWriterAdapter) CloseWrite() error {
+	return nil
+}
+
 type bidirectionalStreamStatus struct {
 	doneChan chan struct{}
 	anyDone  uint32
@@ -32,8 +62,24 @@ func (s *bidirectionalStreamStatus) markUniStreamDone() {
 	s.doneChan <- struct{}{}
 }
 
-func (s *bidirectionalStreamStatus) waitAnyDone() {
+func (s *bidirectionalStreamStatus) wait(maxWaitForSecondStream time.Duration) error {
 	<-s.doneChan
+
+	// Only wait for second stream to finish if maxWait is greater than zero
+	if maxWaitForSecondStream > 0 {
+
+		timer := time.NewTimer(maxWaitForSecondStream)
+		defer timer.Stop()
+
+		select {
+		case <-timer.C:
+			return fmt.Errorf("timeout waiting for second stream to finish")
+		case <-s.doneChan:
+			return nil
+		}
+	}
+
+	return nil
 }
 func (s *bidirectionalStreamStatus) isAnyDone() bool {
 	return atomic.LoadUint32(&s.anyDone) > 0
@@ -41,16 +87,28 @@ func (s *bidirectionalStreamStatus) isAnyDone() bool {
 
 // Pipe copies copy data to & from provided io.ReadWriters.
 func Pipe(tunnelConn, originConn io.ReadWriter, log *zerolog.Logger) {
-	status := newBiStreamStatus()
-
-	go unidirectionalStream(tunnelConn, originConn, "origin->tunnel", status, log)
-	go unidirectionalStream(originConn, tunnelConn, "tunnel->origin", status, log)
-
-	// If one side is done, we are done.
-	status.waitAnyDone()
+	PipeBidirectional(NopCloseWriterAdapter(tunnelConn), NopCloseWriterAdapter(originConn), 0, log)
 }
 
-func unidirectionalStream(dst io.Writer, src io.Reader, dir string, status *bidirectionalStreamStatus, log *zerolog.Logger) {
+// PipeBidirectional copies data two BidirectionStreams. It is a special case of Pipe where it receives a concept that allows for Read and Write side to be closed independently.
+// The main difference is that when piping data from a reader to a writer, if EOF is read, then this implementation propagates the EOF signal to the destination/writer by closing the write side of the
+// Bidirectional Stream.
+// Finally, depending on once EOF is ready from one of the provided streams, the other direction of streaming data will have a configured time period to also finish, otherwise,
+// the method will return immediately  with a timeout error. It is however, the responsability of the caller to close the associated streams in both ends in order to free all the resources/go-routines.
+func PipeBidirectional(downstream, upstream Stream, maxWaitForSecondStream time.Duration, log *zerolog.Logger) error {
+	status := newBiStreamStatus()
+
+	go unidirectionalStream(downstream, upstream, "upstream->downstream", status, log)
+	go unidirectionalStream(upstream, downstream, "downstream->upstream", status, log)
+
+	if err := status.wait(maxWaitForSecondStream); err != nil {
+		return errors.Wrap(err, "unable to wait for both streams while proxying")
+	}
+
+	return nil
+}
+
+func unidirectionalStream(dst WriterCloser, src Reader, dir string, status *bidirectionalStreamStatus, log *zerolog.Logger) {
 	defer func() {
 		// The bidirectional streaming spawns 2 goroutines to stream each direction.
 		// If any ends, the callstack returns, meaning the Tunnel request/stream (depending on http2 vs quic) will
@@ -58,28 +116,21 @@ func unidirectionalStream(dst io.Writer, src io.Reader, dir string, status *bidi
 		// server/origin listens forever until closure), it may read/write from the underlying ReadWriter (backed by
 		// the Edge<->cloudflared transport) in an unexpected state.
 		// Because of this, we set this recover() logic.
-		if r := recover(); r != nil {
+		if err := recover(); err != nil {
 			if status.isAnyDone() {
 				// We handle such unexpected errors only when we detect that one side of the streaming is done.
-				log.Debug().Msgf("Gracefully handled error %v in Streaming for %s, error %s", r, dir, debug.Stack())
+				log.Debug().Msgf("recovered from panic in stream.Pipe for %s, error %s, %s", dir, err, debug.Stack())
 			} else {
 				// Otherwise, this is unexpected, but we prevent the program from crashing anyway.
-				log.Warn().Msgf("Gracefully handled unexpected error %v in Streaming for %s, error %s", r, dir, debug.Stack())
-
-				tags := make(map[string]string)
-				tags["root"] = "websocket.stream"
-				tags["dir"] = dir
-				switch rval := r.(type) {
-				case error:
-					raven.CaptureError(rval, tags)
-				default:
-					rvalStr := fmt.Sprint(rval)
-					raven.CaptureMessage(rvalStr, tags)
-				}
+				log.Warn().Msgf("recovered from panic in stream.Pipe for %s, error %s, %s", dir, err, debug.Stack())
+				sentry.CurrentHub().Recover(err)
+				sentry.Flush(time.Second * 5)
 			}
 		}
 	}()
 
+	defer dst.CloseWrite()
+
 	_, err := copyData(dst, src, dir)
 	if err != nil {
 		log.Debug().Msgf("%s copy: %v", dir, err)

stream/stream_test.go

@@ -0,0 +1,122 @@
+package stream
+
+import (
+	"fmt"
+	"io"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/rs/zerolog"
+	"github.com/stretchr/testify/require"
+)
+
+func TestPipeBidirectionalFinishBothSides(t *testing.T) {
+	fun := func(upstream, downstream *mockedStream) {
+		downstream.closeReader()
+		upstream.closeReader()
+	}
+
+	testPipeBidirectionalUnblocking(t, fun, time.Millisecond*200, false)
+}
+
+func TestPipeBidirectionalFinishOneSideTimeout(t *testing.T) {
+	fun := func(upstream, downstream *mockedStream) {
+		downstream.closeReader()
+	}
+
+	testPipeBidirectionalUnblocking(t, fun, time.Millisecond*200, true)
+}
+
+func TestPipeBidirectionalClosingWriteBothSidesAlsoExists(t *testing.T) {
+	fun := func(upstream, downstream *mockedStream) {
+		downstream.CloseWrite()
+		upstream.CloseWrite()
+
+		downstream.writeToReader("abc")
+		upstream.writeToReader("abc")
+	}
+
+	testPipeBidirectionalUnblocking(t, fun, time.Millisecond*200, false)
+}
+
+func TestPipeBidirectionalClosingWriteSingleSideAlsoExists(t *testing.T) {
+	fun := func(upstream, downstream *mockedStream) {
+		downstream.CloseWrite()
+
+		downstream.writeToReader("abc")
+		upstream.writeToReader("abc")
+	}
+
+	testPipeBidirectionalUnblocking(t, fun, time.Millisecond*200, true)
+}
+
+func testPipeBidirectionalUnblocking(t *testing.T, afterFun func(*mockedStream, *mockedStream), timeout time.Duration, expectTimeout bool) {
+	logger := zerolog.Nop()
+
+	downstream := newMockedStream()
+	upstream := newMockedStream()
+
+	resultCh := make(chan error)
+	go func() {
+		resultCh <- PipeBidirectional(downstream, upstream, timeout, &logger)
+	}()
+
+	afterFun(upstream, downstream)
+
+	select {
+	case err := <-resultCh:
+		if expectTimeout {
+			require.NotNil(t, err)
+		} else {
+			require.Nil(t, err)
+		}
+
+	case <-time.After(timeout * 2):
+		require.Fail(t, "test timeout")
+	}
+}
+
+func newMockedStream() *mockedStream {
+	return &mockedStream{
+		readCh:  make(chan *string),
+		writeCh: make(chan struct{}),
+	}
+}
+
+type mockedStream struct {
+	readCh  chan *string
+	writeCh chan struct{}
+
+	writeCloseOnce sync.Once
+}
+
+func (m *mockedStream) Read(p []byte) (n int, err error) {
+	result := <-m.readCh
+	if result == nil {
+		return 0, io.EOF
+	}
+
+	return len(*result), nil
+}
+
+func (m *mockedStream) Write(p []byte) (n int, err error) {
+	<-m.writeCh
+
+	return 0, fmt.Errorf("closed")
+}
+
+func (m *mockedStream) CloseWrite() error {
+	m.writeCloseOnce.Do(func() {
+		close(m.writeCh)
+	})
+
+	return nil
+}
+
+func (m *mockedStream) closeReader() {
+	close(m.readCh)
+}
+func (m *mockedStream) writeToReader(content string) {
+	m.readCh <- &content
+}

supervisor/cloudflare_status_page.go

@@ -2,7 +2,7 @@ package supervisor
 
 import (
 	"encoding/json"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"strings"
 	"time"
@@ -72,7 +72,7 @@ func fetchActiveIncidents() (incidents []Incident) {
 		return
 	}
 	defer resp.Body.Close()
-	body, err := ioutil.ReadAll(resp.Body)
+	body, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return
 	}

supervisor/pqtunnels.go

@@ -4,24 +4,23 @@ import (
 	"bytes"
 	"crypto/tls"
 	"encoding/json"
+	"fmt"
 	"net/http"
 	"sync"
+
+	"github.com/cloudflare/cloudflared/features"
 )
 
 // When experimental post-quantum tunnels are enabled, and we're hitting an
 // issue creating the tunnel, we'll report the first error
 // to https://pqtunnels.cloudflareresearch.com.
 
-var (
-	PQKexes = [...]tls.CurveID{
-		tls.CurveID(0xfe30), // X25519Kyber512Draft00
-		tls.CurveID(0xfe31), // X25519Kyber768Draft00
-	}
-	PQKexNames map[tls.CurveID]string = map[tls.CurveID]string{
-		tls.CurveID(0xfe30): "X25519Kyber512Draft00",
-		tls.CurveID(0xfe31): "X25519Kyber768Draft00",
-	}
+const (
+	PQKex     = tls.CurveID(0xfe31) // X25519Kyber768Draft00
+	PQKexName = "X25519Kyber768Draft00"
+)
 
+var (
 	pqtMux       sync.Mutex // protects pqtSubmitted and pqtWaitForMessage
 	pqtSubmitted bool       // whether an error has already been submitted
 
@@ -70,7 +69,7 @@ func submitPQTunnelError(rep error, config *TunnelConfig) {
 		Message string `json:"m"`
 		Version string `json:"v"`
 	}{
-		Group:   int(PQKexes[config.PQKexIdx]),
+		Group:   int(PQKex),
 		Message: rep.Error(),
 		Version: config.ReportedVersion,
 	})
@@ -98,3 +97,32 @@ func submitPQTunnelError(rep error, config *TunnelConfig) {
 	}
 	resp.Body.Close()
 }
+
+func curvePreference(pqMode features.PostQuantumMode, currentCurve []tls.CurveID) ([]tls.CurveID, error) {
+	switch pqMode {
+	case features.PostQuantumStrict:
+		// If the user passes the -post-quantum flag, we override
+		// CurvePreferences to only support hybrid post-quantum key agreements.
+		return []tls.CurveID{PQKex}, nil
+	case features.PostQuantumPrefer:
+		if len(currentCurve) == 0 {
+			return []tls.CurveID{PQKex}, nil
+		}
+
+		if currentCurve[0] != PQKex {
+			return append([]tls.CurveID{PQKex}, currentCurve...), nil
+		}
+		return currentCurve, nil
+	case features.PostQuantumDisabled:
+		curvePref := currentCurve
+		// Remove PQ from curve preference
+		for i, curve := range currentCurve {
+			if curve == PQKex {
+				curvePref = append(curvePref[:i], curvePref[i+1:]...)
+			}
+		}
+		return curvePref, nil
+	default:
+		return nil, fmt.Errorf("Unexpected post quantum mode")
+	}
+}

supervisor/supervisor.go

@@ -2,6 +2,7 @@ package supervisor
 
 import (
 	"context"
+	"crypto/tls"
 	"errors"
 	"net"
 	"strings"
@@ -10,6 +11,8 @@ import (
 	"github.com/quic-go/quic-go"
 	"github.com/rs/zerolog"
 
+	qtls120 "github.com/quic-go/qtls-go1-20"
+
 	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/edgediscovery"
 	"github.com/cloudflare/cloudflared/orchestration"
@@ -78,6 +81,8 @@ func NewSupervisor(config *TunnelConfig, orchestrator *orchestration.Orchestrato
 
 	reconnectCredentialManager := newReconnectCredentialManager(connection.MetricsNamespace, connection.TunnelSubsystem, config.HAConnections)
 
+	registerTLSEventLogger(config.Log)
+
 	tracker := tunnelstate.NewConnTracker(config.Log)
 	log := NewConnAwareLogger(config.Log, tracker, config.Observer)
 
@@ -336,3 +341,26 @@ func (s *Supervisor) waitForNextTunnel(index int) bool {
 func (s *Supervisor) unusedIPs() bool {
 	return s.edgeIPs.AvailableAddrs() > s.config.HAConnections
 }
+
+func registerTLSEventLogger(logger *zerolog.Logger) {
+	qtls120.SetCFEventHandler(func(ev qtls120.CFEvent) {
+		logger.Debug().Bool("handshake", ev.IsHandshake()).Str("handshake_duration", ev.Duration().String()).Str("curve", tlsCurveName(ev.KEX())).Msg("QUIC TLS event")
+	})
+}
+
+func tlsCurveName(curve tls.CurveID) string {
+	switch curve {
+	case tls.CurveP256:
+		return "p256"
+	case tls.CurveP384:
+		return "p384"
+	case tls.CurveP521:
+		return "p521"
+	case tls.X25519:
+		return "X25519"
+	case PQKex:
+		return PQKexName
+	default:
+		return "unknown"
+	}
+}

supervisor/tunnel.go

@@ -61,13 +61,16 @@ type TunnelConfig struct {
 
 	NeedPQ bool
 
-	// Index into PQKexes of post-quantum kex to use if NeedPQ is set.
-	PQKexIdx int
-
 	NamedTunnel      *connection.NamedTunnelProperties
 	ProtocolSelector connection.ProtocolSelector
 	EdgeTLSConfigs   map[connection.Protocol]*tls.Config
 	PacketConfig     *ingress.GlobalRouterConfig
+
+	UDPUnregisterSessionTimeout time.Duration
+
+	DisableQUICPathMTUDiscovery bool
+
+	FeatureSelector *features.FeatureSelector
 }
 
 func (c *TunnelConfig) registrationOptions(connectionID uint8, OriginLocalIP string, uuid uuid.UUID) *tunnelpogs.RegistrationOptions {
@@ -350,7 +353,7 @@ func selectNextProtocol(
 				"Cloudflare Network with `quic` protocol, then most likely your machine/network is getting its egress " +
 				"UDP to port 7844 (or others) blocked or dropped. Make sure to allow egress connectivity as per " +
 				"https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/configuration/ports-and-ips/\n" +
-				"If you are using private routing to this Tunnel, then UDP (and Private DNS Resolution) will not work " +
+				"If you are using private routing to this Tunnel, then ICMP, UDP (and Private DNS Resolution) will not work " +
 				"unless your cloudflared can connect with Cloudflare Network with `quic`.")
 		}
 
@@ -535,7 +538,8 @@ func (e *EdgeTunnelServer) serveHTTP2(
 	controlStreamHandler connection.ControlStreamHandler,
 	connIndex uint8,
 ) error {
-	if e.config.NeedPQ {
+	pqMode := e.config.FeatureSelector.PostQuantumMode()
+	if pqMode == features.PostQuantumStrict {
 		return unrecoverableError{errors.New("HTTP/2 transport does not support post-quantum")}
 	}
 
@@ -578,30 +582,28 @@ func (e *EdgeTunnelServer) serveQUIC(
 ) (err error, recoverable bool) {
 	tlsConfig := e.config.EdgeTLSConfigs[connection.QUIC]
 
-	if e.config.NeedPQ {
-		// If the user passes the -post-quantum flag, we override
-		// CurvePreferences to only support hybrid post-quantum key agreements.
-		cs := make([]tls.CurveID, len(PQKexes))
-		copy(cs, PQKexes[:])
-
-		// It is unclear whether Kyber512 or Kyber768 will become the standard.
-		// Kyber768 is a bit bigger (and doesn't fit in one initial
-		// datagram anymore). We're enabling both, but pick randomly which
-		// one to put first. (TLS will use the first one in the list
-		// and allows a fallback to the second.)
-		cs[0], cs[e.config.PQKexIdx] = cs[e.config.PQKexIdx], cs[0]
-		tlsConfig.CurvePreferences = cs
+	pqMode := e.config.FeatureSelector.PostQuantumMode()
+	if pqMode == features.PostQuantumStrict || pqMode == features.PostQuantumPrefer {
+		connOptions.Client.Features = features.Dedup(append(connOptions.Client.Features, features.FeaturePostQuantum))
 	}
 
+	curvePref, err := curvePreference(pqMode, tlsConfig.CurvePreferences)
+	if err != nil {
+		return err, true
+	}
+
+	tlsConfig.CurvePreferences = curvePref
+
 	quicConfig := &quic.Config{
-		HandshakeIdleTimeout:  quicpogs.HandshakeIdleTimeout,
-		MaxIdleTimeout:        quicpogs.MaxIdleTimeout,
-		KeepAlivePeriod:       quicpogs.MaxIdlePingPeriod,
-		MaxIncomingStreams:    connection.MaxConcurrentStreams,
-		MaxIncomingUniStreams: connection.MaxConcurrentStreams,
-		EnableDatagrams:       true,
-		MaxDatagramFrameSize:  quicpogs.MaxDatagramFrameSize,
-		Tracer:                quicpogs.NewClientTracer(connLogger.Logger(), connIndex),
+		HandshakeIdleTimeout:    quicpogs.HandshakeIdleTimeout,
+		MaxIdleTimeout:          quicpogs.MaxIdleTimeout,
+		KeepAlivePeriod:         quicpogs.MaxIdlePingPeriod,
+		MaxIncomingStreams:      quicpogs.MaxIncomingStreams,
+		MaxIncomingUniStreams:   quicpogs.MaxIncomingStreams,
+		EnableDatagrams:         true,
+		MaxDatagramFrameSize:    quicpogs.MaxDatagramFrameSize,
+		Tracer:                  quicpogs.NewClientTracer(connLogger.Logger(), connIndex),
+		DisablePathMTUDiscovery: e.config.DisableQUICPathMTUDiscovery,
 	}
 
 	quicConn, err := connection.NewQUICConnection(
@@ -615,9 +617,11 @@ func (e *EdgeTunnelServer) serveQUIC(
 		connOptions,
 		controlStreamHandler,
 		connLogger.Logger(),
-		e.config.PacketConfig)
+		e.config.PacketConfig,
+		e.config.UDPUnregisterSessionTimeout,
+	)
 	if err != nil {
-		if e.config.NeedPQ {
+		if pqMode == features.PostQuantumStrict || pqMode == features.PostQuantumPrefer {
 			handlePQTunnelError(err, e.config)
 		}
 

tlsconfig/certreloader.go

@@ -4,7 +4,7 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"fmt"
-	"io/ioutil"
+	"os"
 	"runtime"
 	"sync"
 
@@ -77,7 +77,7 @@ func LoadOriginCA(originCAPoolFilename string, log *zerolog.Logger) (*x509.CertP
 
 	if originCAPoolFilename != "" {
 		var err error
-		originCustomCAPool, err = ioutil.ReadFile(originCAPoolFilename)
+		originCustomCAPool, err = os.ReadFile(originCAPoolFilename)
 		if err != nil {
 			return nil, errors.Wrap(err, fmt.Sprintf("unable to read the file %s for --%s", originCAPoolFilename, OriginCAPoolFlag))
 		}
@@ -116,7 +116,7 @@ func LoadCustomOriginCA(originCAFilename string) (*x509.CertPool, error) {
 		return certPool, nil
 	}
 
-	customOriginCA, err := ioutil.ReadFile(originCAFilename)
+	customOriginCA, err := os.ReadFile(originCAFilename)
 	if err != nil {
 		return nil, errors.Wrap(err, fmt.Sprintf("unable to read the file %s", originCAFilename))
 	}