Release 2023.3.1

The previous logic of var == x86 never fired for 386 arch windows
systems causing us to set ProgramFiles64Folder for the older windows
versions causing downloads to default to a different location. This
change fixes that.

.docker-images

@@ -2,8 +2,7 @@ images:
   - name: cloudflared
     dockerfile: Dockerfile.$ARCH
     context: .
-    versions: 
-      - latest
+    version_file: versions
     registries:
     - name: docker.io/cloudflare
       user: env:DOCKER_USER

.github/workflows/check.yaml

@@ -9,10 +9,10 @@ jobs:
     runs-on: ${{ matrix.os }}
     steps:
     - name: Install Go
-      uses: actions/setup-go@v2
+      uses: actions/setup-go@v3
       with:
         go-version: ${{ matrix.go-version }}
     - name: Checkout code
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
     - name: Test
       run: make test

.teamcity/build-macos.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set -exo pipefail
+
 if [[ "$(uname)" != "Darwin" ]] ; then
     echo "This should be run on macOS"
     exit 1
@@ -33,7 +35,9 @@ if [[ ! -z "$CFD_CODE_SIGN_KEY" ]]; then
   if [[ ! -z "$CFD_CODE_SIGN_PASS" ]]; then
     # write private key to disk and then import it keychain
     echo -n -e ${CFD_CODE_SIGN_KEY} | base64 -D > ${CODE_SIGN_PRIV}
-    out=$(security import ${CODE_SIGN_PRIV} -A -P "${CFD_CODE_SIGN_PASS}" 2>&1)
+    # we set || true  here and for every `security import invoke` because the  "duplicate SecKeychainItemImport" error 
+    # will cause set -e to exit 1. It is okay we do this because we deliberately handle this error in the lines below.
+    out=$(security import ${CODE_SIGN_PRIV} -A -P "${CFD_CODE_SIGN_PASS}" 2>&1) || true 
     exitcode=$?
     if [ -n "$out" ]; then
       if [ $exitcode -eq 0 ]; then
@@ -53,7 +57,7 @@ fi
 if [[ ! -z "$CFD_CODE_SIGN_CERT" ]]; then
   # write certificate to disk and then import it keychain
   echo -n -e ${CFD_CODE_SIGN_CERT} | base64 -D > ${CODE_SIGN_CERT}
-  out1=$(security import ${CODE_SIGN_CERT} -A 2>&1)
+  out1=$(security import ${CODE_SIGN_CERT} -A 2>&1) || true
   exitcode1=$?
   if [ -n "$out1" ]; then
     if [ $exitcode1 -eq 0 ]; then
@@ -75,7 +79,7 @@ if [[ ! -z "$CFD_INSTALLER_KEY" ]]; then
   if [[ ! -z "$CFD_INSTALLER_PASS" ]]; then
     # write private key to disk and then import it into the keychain
     echo -n -e ${CFD_INSTALLER_KEY} | base64 -D > ${INSTALLER_PRIV}
-    out2=$(security import ${INSTALLER_PRIV} -A -P "${CFD_INSTALLER_PASS}" 2>&1)
+    out2=$(security import ${INSTALLER_PRIV} -A -P "${CFD_INSTALLER_PASS}" 2>&1) || true
     exitcode2=$?
     if [ -n "$out2" ]; then
       if [ $exitcode2 -eq 0 ]; then
@@ -95,7 +99,7 @@ fi
 if [[ ! -z "$CFD_INSTALLER_CERT" ]]; then
   # write certificate to disk and then import it keychain
   echo -n -e ${CFD_INSTALLER_CERT} | base64 -D > ${INSTALLER_CERT}
-  out3=$(security import ${INSTALLER_CERT} -A 2>&1)
+  out3=$(security import ${INSTALLER_CERT} -A 2>&1) || true
   exitcode3=$?
   if [ -n "$out3" ]; then
     if [ $exitcode3 -eq 0 ]; then
@@ -138,14 +142,10 @@ fi
 if [[ ! -z "$CODE_SIGN_NAME" ]]; then
   codesign -s "${CODE_SIGN_NAME}" -f -v --timestamp --options runtime ${BINARY_NAME}
   
-  # notarize the binary
-  if [[ ! -z "$CFD_NOTE_PASSWORD" ]]; then
-    zip "${BINARY_NAME}.zip" ${BINARY_NAME} 
-    xcrun altool --notarize-app -f "${BINARY_NAME}.zip" -t osx -u ${CFD_NOTE_USERNAME} -p ${CFD_NOTE_PASSWORD} --primary-bundle-id ${BUNDLE_ID}
-  fi
+ # notarize the binary
+ # TODO: https://jira.cfdata.org/browse/TUN-5789
 fi
 
-
 # creating build directory
 rm -rf $TARGET_DIRECTORY
 mkdir "${TARGET_DIRECTORY}"
@@ -169,10 +169,7 @@ if [[ ! -z "$PKG_SIGN_NAME" ]]; then
       ${PKGNAME}
 
       # notarize the package
-      if [[ ! -z "$CFD_NOTE_PASSWORD" ]]; then
-        xcrun altool --notarize-app -f ${PKGNAME} -t osx -u ${CFD_NOTE_USERNAME} -p ${CFD_NOTE_PASSWORD} --primary-bundle-id ${BUNDLE_ID}
-        xcrun stapler staple ${PKGNAME}
-      fi
+      # TODO: https://jira.cfdata.org/browse/TUN-5789
 else
     pkgbuild --identifier com.cloudflare.${PRODUCT} \
       --version ${VERSION} \

.teamcity/update-homebrew.sh

@@ -3,7 +3,6 @@
 set -euo pipefail
 
 FILENAME="${PWD}/artifacts/cloudflared-darwin-amd64.tgz"
-
 if ! VERSION="$(git describe --tags --exact-match 2>/dev/null)" ; then
     echo "Skipping public release for an untagged commit."
     echo "##teamcity[buildStatus status='SUCCESS' text='Skipped due to lack of tag']"
@@ -15,24 +14,24 @@ if [[ ! -f "$FILENAME" ]] ; then
     exit 1
 fi
 
-if [[ "${GITHUB_PRIVATE_KEY:-}" == "" ]] ; then
-    echo "Missing GITHUB_PRIVATE_KEY"
+if [[ "${GITHUB_PRIVATE_KEY_B64:-}" == "" ]] ; then
+    echo "Missing GITHUB_PRIVATE_KEY_B64"
     exit 1
 fi
 
 # upload to s3 bucket for use by Homebrew formula
 s3cmd \
-    --acl-public --signature-v2 --access_key="$AWS_ACCESS_KEY_ID" --secret_key="$AWS_SECRET_ACCESS_KEY" --host-bucket="%(bucket)s.s3.cfdata.org" \
+    --acl-public --access_key="$AWS_ACCESS_KEY_ID" --secret_key="$AWS_SECRET_ACCESS_KEY" --host-bucket="%(bucket)s.s3.cfdata.org" \
     put "$FILENAME" "s3://cftunnel-docs/dl/cloudflared-$VERSION-darwin-amd64.tgz"
 s3cmd \
-    --acl-public --signature-v2 --access_key="$AWS_ACCESS_KEY_ID" --secret_key="$AWS_SECRET_ACCESS_KEY" --host-bucket="%(bucket)s.s3.cfdata.org" \
+    --acl-public --access_key="$AWS_ACCESS_KEY_ID" --secret_key="$AWS_SECRET_ACCESS_KEY" --host-bucket="%(bucket)s.s3.cfdata.org" \
     cp "s3://cftunnel-docs/dl/cloudflared-$VERSION-darwin-amd64.tgz" "s3://cftunnel-docs/dl/cloudflared-stable-darwin-amd64.tgz"
 SHA256=$(sha256sum "$FILENAME" | cut -b1-64)
 
 # set up git (note that UserKnownHostsFile is an absolute path so we can cd wherever)
 mkdir -p tmp
 ssh-keyscan -t rsa github.com > tmp/github.txt
-echo "$GITHUB_PRIVATE_KEY" > tmp/private.key
+echo "$GITHUB_PRIVATE_KEY_B64" | base64 --decode > tmp/private.key
 chmod 0400 tmp/private.key
 export GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=$PWD/tmp/github.txt -i $PWD/tmp/private.key -o IdentitiesOnly=yes"
 

CHANGES.md

@@ -1,3 +1,24 @@
+## 2023.3.1
+### Breaking Change
+- Running a tunnel without ingress rules defined in configuration file nor from the CLI flags will no longer provide a default ingress rule to localhost:8080 and instead will return HTTP response code 503 for all incoming HTTP requests.
+
+## 2023.2.2
+### Notices
+- Legacy tunnels were officially deprecated on December 1, 2022. Starting with this version, cloudflared no longer supports connecting legacy tunnels.
+- h2mux tunnel connection protocol is no longer supported. Any tunnels still configured to use this protocol will alert and use http2 tunnel protocol instead. We recommend using quic protocol for all tunnels going forward.
+
+## 2023.2.1
+### Bug fixes
+- Fixed a bug in TCP connection proxy that could result in the connection being closed before all data was written.
+- cloudflared now correctly aborts body write if connection to origin service fails after response headers were sent already.
+- Fixed a bug introduced in the previous release where debug endpoints were removed.
+
+## 2022.12.0
+### Improvements
+- cloudflared now attempts to try other edge addresses before falling back to a lower protoocol.
+- cloudflared tunnel no longer spins up a quick tunnel. The call has to be explicit and provide a --url flag.
+- cloudflared will now randomly pick the first or second region to connect to instead of always connecting to region2 first.
+
 ## 2022.9.0
 ### New Features
 - cloudflared now rejects ingress rules with invalid http status codes for http_status.

Dockerfile

@@ -7,8 +7,6 @@ ENV GO111MODULE=on \
     TARGET_GOOS=${TARGET_GOOS} \
     TARGET_GOARCH=${TARGET_GOARCH}
     
-LABEL org.opencontainers.image.source="https://github.com/cloudflare/cloudflared"
-
 WORKDIR /go/src/github.com/cloudflare/cloudflared/
 
 # copy our sources into the builder image
@@ -20,6 +18,8 @@ RUN make cloudflared
 # use a distroless base image with glibc
 FROM gcr.io/distroless/base-debian11:nonroot
 
+LABEL org.opencontainers.image.source="https://github.com/cloudflare/cloudflared"
+
 # copy our compiled binary
 COPY --from=builder --chown=nonroot /go/src/github.com/cloudflare/cloudflared/cloudflared /usr/local/bin/
 

Dockerfile.amd64

@@ -3,8 +3,6 @@ FROM golang:1.19 as builder
 ENV GO111MODULE=on \
     CGO_ENABLED=0 
     
-LABEL org.opencontainers.image.source="https://github.com/cloudflare/cloudflared"
-
 WORKDIR /go/src/github.com/cloudflare/cloudflared/
 
 # copy our sources into the builder image
@@ -16,6 +14,8 @@ RUN GOOS=linux GOARCH=amd64 make cloudflared
 # use a distroless base image with glibc
 FROM gcr.io/distroless/base-debian11:nonroot
 
+LABEL org.opencontainers.image.source="https://github.com/cloudflare/cloudflared"
+
 # copy our compiled binary
 COPY --from=builder --chown=nonroot /go/src/github.com/cloudflare/cloudflared/cloudflared /usr/local/bin/
 

Dockerfile.arm64

@@ -3,8 +3,6 @@ FROM golang:1.19 as builder
 ENV GO111MODULE=on \
     CGO_ENABLED=0 
     
-LABEL org.opencontainers.image.source="https://github.com/cloudflare/cloudflared"
-
 WORKDIR /go/src/github.com/cloudflare/cloudflared/
 
 # copy our sources into the builder image
@@ -16,6 +14,8 @@ RUN GOOS=linux GOARCH=arm64 make cloudflared
 # use a distroless base image with glibc
 FROM gcr.io/distroless/base-debian11:nonroot-arm64
 
+LABEL org.opencontainers.image.source="https://github.com/cloudflare/cloudflared"
+
 # copy our compiled binary
 COPY --from=builder --chown=nonroot /go/src/github.com/cloudflare/cloudflared/cloudflared /usr/local/bin/
 

Makefile

@@ -108,6 +108,9 @@ else
 	PACKAGE_ARCH := $(TARGET_ARCH)
 endif
 
+#for FIPS compliance, FPM defaults to MD5.
+RPM_DIGEST := --rpm-digest sha256
+
 .PHONY: all
 all: cloudflared test
 
@@ -131,6 +134,10 @@ endif
 container:
 	docker build --build-arg=TARGET_ARCH=$(TARGET_ARCH) --build-arg=TARGET_OS=$(TARGET_OS) -t cloudflare/cloudflared-$(TARGET_OS)-$(TARGET_ARCH):"$(VERSION)" .
 
+.PHONY: generate-docker-version
+generate-docker-version:
+	echo latest $(VERSION) > versions
+
 .PHONY: test
 test: vet
 ifndef CI
@@ -146,7 +153,7 @@ test-ssh-server:
 	docker-compose -f ssh_server_tests/docker-compose.yml up
 
 cloudflared.1: cloudflared_man_template
-	cat cloudflared_man_template | sed -e 's/\$${VERSION}/$(VERSION)/; s/\$${DATE}/$(DATE)/' > cloudflared.1
+	sed -e 's/\$${VERSION}/$(VERSION)/; s/\$${DATE}/$(DATE)/' cloudflared_man_template > cloudflared.1
 
 install: cloudflared cloudflared.1
 	mkdir -p $(DESTDIR)$(INSTALL_BINDIR) $(DESTDIR)$(INSTALL_MANDIR)
@@ -159,13 +166,13 @@ define build_package
 	mkdir -p $(PACKAGE_DIR)
 	cp cloudflared $(PACKAGE_DIR)/cloudflared
 	cp cloudflared.1 $(PACKAGE_DIR)/cloudflared.1
-	fakeroot fpm -C $(PACKAGE_DIR) -s dir -t $(1) \
+	fpm -C $(PACKAGE_DIR) -s dir -t $(1) \
 		--description 'Cloudflare Tunnel daemon' \
 		--vendor 'Cloudflare' \
 		--license 'Apache License Version 2.0' \
 		--url 'https://github.com/cloudflare/cloudflared' \
 		-m 'Cloudflare <support@cloudflare.com>' \
-	    -a $(PACKAGE_ARCH) -v $(VERSION) -n $(DEB_PACKAGE_NAME) $(NIGHTLY_FLAGS) --after-install postinst.sh --after-remove postrm.sh \
+	    -a $(PACKAGE_ARCH) -v $(VERSION) -n $(DEB_PACKAGE_NAME) $(RPM_DIGEST) $(NIGHTLY_FLAGS) --after-install postinst.sh --after-remove postrm.sh \
 		cloudflared=$(INSTALL_BINDIR) cloudflared.1=$(INSTALL_MANDIR)
 endef
 
@@ -293,8 +300,8 @@ quic-deps:
 
 .PHONY: vet
 vet:
-	go vet -v -mod=vendor ./...
+	go vet -v -mod=vendor github.com/cloudflare/cloudflared/...
 
-.PHONY: goimports
-goimports:
-	for d in $$(go list -mod=readonly -f '{{.Dir}}' -a ./... | fgrep -v tunnelrpc) ; do goimports -format-only -local github.com/cloudflare/cloudflared -w $$d ; done
+.PHONY: fmt
+fmt:
+	goimports -l -w -local github.com/cloudflare/cloudflared $$(go list -mod=vendor -f '{{.Dir}}' -a ./... | fgrep -v tunnelrpc)

README.md

@@ -25,12 +25,13 @@ routing), but for legacy reasons this requirement is still necessary:
 
 ## Installing `cloudflared`
 
-Downloads are available as standalone binaries, a Docker image, and Debian, RPM, and Homebrew packages. You can also find releases here on the `cloudflared` GitHub repository.
+Downloads are available as standalone binaries, a Docker image, and Debian, RPM, and Homebrew packages. You can also find releases [here](https://github.com/cloudflare/cloudflared/releases) on the `cloudflared` GitHub repository.
 
 * You can [install on macOS](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/install-and-setup/installation#macos) via Homebrew or by downloading the [latest Darwin amd64 release](https://github.com/cloudflare/cloudflared/releases)
 * Binaries, Debian, and RPM packages for Linux [can be found here](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/install-and-setup/installation#linux)
 * A Docker image of `cloudflared` is [available on DockerHub](https://hub.docker.com/r/cloudflare/cloudflared)
 * You can install on Windows machines with the [steps here](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/install-and-setup/installation#windows)
+* Build from source with the [instructions here](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/install-and-setup/installation#build-from-source)
 
 User documentation for Cloudflare Tunnel can be found at https://developers.cloudflare.com/cloudflare-one/connections/connect-apps
 

RELEASE_NOTES

@@ -1,3 +1,108 @@
+2023.3.1
+- 2023-03-13 TUN-7271: Return 503 status code when no ingress rules configured
+- 2023-03-10 TUN-7272: Fix cloudflared returning non supported status service which breaks configuration migration
+- 2023-03-09 TUN-7259: Add warning for missing ingress rules
+- 2023-03-09 TUN-7268: Default to Program Files as location for win32
+- 2023-03-07 TUN-7252: Remove h2mux connection
+- 2023-03-07 TUN-7253: Adopt http.ResponseWriter for connection.ResponseWriter
+- 2023-03-06 TUN-7245: Add bastion flag to origin service check
+- 2023-03-06 EDGESTORE-108: Remove deprecated s3v2 signature
+- 2023-03-02 TUN-7226: Fixed a missed rename
+
+2023.3.0
+- 2023-03-01 GH-352: Add Tunnel CLI option "edge-bind-address" (#870)
+- 2023-03-01 Fixed WIX template to allow MSI upgrades (#838)
+- 2023-02-28 TUN-7213: Decode Base64 encoded key before writing it
+- 2023-02-28 check.yaml: update actions to v3 (#876)
+- 2023-02-27 TUN-7213: Debug homebrew-cloudflare  build
+- 2023-02-15 RTG-2476 Add qtls override for Go 1.20
+
+2023.2.2
+- 2023-02-22 TUN-7197: Add connIndex tag to debug messages of incoming requests
+- 2023-02-08 TUN-7167: Respect protocol overrides with --token
+- 2023-02-06 TUN-7065: Remove classic tunnel creation
+- 2023-02-06 TUN-6938: Force h2mux protocol to http2 for named tunnels
+- 2023-02-06 TUN-6938: Provide QUIC as first in protocol list
+- 2023-02-03 TUN-7158: Correct TCP tracing propagation
+- 2023-02-01 TUN-7151: Update changes file with latest release notices
+
+2023.2.1
+- 2023-02-01 TUN-7065: Revert Ingress Rule check for named tunnel configurations
+- 2023-02-01 Revert "TUN-7065: Revert Ingress Rule check for named tunnel configurations"
+- 2023-02-01 Revert "TUN-7065: Remove classic tunnel creation"
+2023.1.0
+- 2023-01-10 TUN-7064: RPM digests are now sha256 instead of md5sum
+- 2023-01-04 RTG-2418 Update qtls
+- 2022-12-24 TUN-7057: Remove dependency github.com/gorilla/mux
+- 2022-12-24 TUN-6724: Migrate to sentry-go from raven-go
+
+2022.12.1
+- 2022-12-20 TUN-7021: Fix proxy-dns not starting when cloudflared tunnel is run
+- 2022-12-15 TUN-7010: Changelog for release 2022.12.0
+
+2022.12.0
+- 2022-12-14 TUN-6999: cloudflared should attempt other edge addresses before falling back on protocol
+- 2022-12-13 TUN-7004: Dont show local config dirs for remotely configured tuns
+- 2022-12-12 TUN-7003: Tempoarily disable erroneous notarize-app
+- 2022-12-12 TUN-7003: Add back a missing fi
+- 2022-12-07 TUN-7000: Reduce metric cardinality of closedConnections metric by removing error as tag
+- 2022-12-07 TUN-6994: Improve logging config file not found
+- 2022-12-07 TUN-7002: Randomise first region selection
+- 2022-12-07 TUN-6995: Disable quick-tunnels spin up by default
+- 2022-12-05 TUN-6984: Add bash set x to improve visibility during builds
+- 2022-12-05 TUN-6984: [CI] Ignore security import errors for code_sigining
+- 2022-12-05 TUN-6984: [CI] Don't fail on unset.
+- 2022-11-30 TUN-6984: Set euo pipefile for homebrew builds
+
+2022.11.1
+- 2022-11-29 TUN-6981: We should close UDP socket if failed to connecto to edge
+- 2022-11-25 CUSTESC-23757: Fix a bug where a wildcard ingress rule would match an host without starting with a dot
+- 2022-11-24 TUN-6970: Print newline when printing tunnel token
+- 2022-11-22 TUN-6963: Refactor Metrics service setup
+2022.11.0
+- 2022-11-16 Revert "TUN-6935: Cloudflared should use APIToken instead of serviceKey"
+- 2022-11-16 TUN-6929: Use same protocol for other connections as first one
+- 2022-11-14 TUN-6941: Reduce log level to debug when failing to proxy ICMP reply
+- 2022-11-14 TUN-6935: Cloudflared should use APIToken instead of serviceKey
+- 2022-11-14 TUN-6935: Cloudflared should use APIToken instead of serviceKey
+- 2022-11-11 TUN-6937: Bump golang.org/x/* packages to new release tags
+- 2022-11-10 ZTC-234: macOS tests
+- 2022-11-09 TUN-6927: Refactor validate access configuration to allow empty audTags only
+- 2022-11-08 ZTC-234: Replace ICMP funnels when ingress connection changes
+- 2022-11-04 TUN-6917: Bump go to 1.19.3
+- 2022-11-02 Issue #574: Better ssh config for short-lived cert (#763)
+- 2022-10-28 TUN-6898: Fix bug handling IPv6 based ingresses with missing port
+- 2022-10-28 TUN-6898: Refactor addPortIfMissing
+2022.10.3
+- 2022-10-24 TUN-6871: Add default feature to cloudflared to support EOF on QUIC connections
+- 2022-10-19 TUN-6876: Fix flaky TestTraceICMPRouterEcho by taking account request span can return before reply
+- 2022-10-18 TUN-6867: Clear spans right after they are serialized to avoid returning duplicate spans
+2022.10.2
+- 2022-10-18 TUN-6869: Fix Makefile complaining about missing GO packages
+- 2022-10-18 TUN-6864: Don't reuse port in quic unit tests
+- 2022-10-18 TUN-6868: Return left padded tracing ID when tracing identity is converted to string
+
+2022.10.1
+- 2022-10-16 TUN-6861: Trace ICMP on Windows
+- 2022-10-15 TUN-6860: Send access configuration keys to the edge
+- 2022-10-14 TUN-6858: Trace ICMP reply
+- 2022-10-13 TUN-6855: Add DatagramV2Type for IP packet with trace and tracing spans
+- 2022-10-13 TUN-6856: Refactor to lay foundation for tracing ICMP
+- 2022-10-13 TUN-6604: Trace icmp echo request on Linux and Darwin
+- 2022-10-12 Fix log message (#591)
+- 2022-10-12 TUN-6853: Reuse source port when connecting to the edge for quic connections
+- 2022-10-11 TUN-6829: Allow user of datagramsession to control logging level of errors
+- 2022-10-10 RTG-2276 Update qtls and go mod tidy
+- 2022-10-05 Add post-quantum flag to quick tunnel
+- 2022-10-05 TUN-6823: Update github release message to pull from KV
+- 2022-10-04 TUN-6825: Fix cloudflared:version images require arch hyphens
+- 2022-10-03 TUN-6806: Add ingress rule number to log when filtering due to middlware handler
+- 2022-08-17 Label correct container
+- 2022-08-16 Fix typo in help text for `cloudflared tunnel route lb`
+- 2022-07-18 drop usage of cat when sed is invoked to generate the manpage
+- 2021-03-15 update-build-readme
+- 2021-03-15 fix link
+
 2022.10.0
 - 2022-09-30 TUN-6755: Remove unused publish functions
 - 2022-09-30 TUN-6813: Only proxy ICMP packets when warp-routing is enabled

carrier/carrier.go

@@ -1,6 +1,6 @@
-//Package carrier provides a WebSocket proxy to carry or proxy a connection
-//from the local client to the edge. See it as a wrapper around any protocol
-//that it packages up in a WebSocket connection to the edge.
+// Package carrier provides a WebSocket proxy to carry or proxy a connection
+// from the local client to the edge. See it as a wrapper around any protocol
+// that it packages up in a WebSocket connection to the edge.
 package carrier
 
 import (

carrier/websocket.go

@@ -9,6 +9,7 @@ import (
 	"github.com/gorilla/websocket"
 	"github.com/rs/zerolog"
 
+	"github.com/cloudflare/cloudflared/stream"
 	"github.com/cloudflare/cloudflared/token"
 	cfwebsocket "github.com/cloudflare/cloudflared/websocket"
 )
@@ -37,7 +38,7 @@ func (ws *Websocket) ServeStream(options *StartOptions, conn io.ReadWriter) erro
 	}
 	defer wsConn.Close()
 
-	cfwebsocket.Stream(wsConn, conn, ws.log)
+	stream.Pipe(wsConn, conn, ws.log)
 	return nil
 }
 

certutil/certutil.go

@@ -1,25 +1,21 @@
 package certutil
 
 import (
-	"crypto/x509"
 	"encoding/json"
 	"encoding/pem"
 	"fmt"
-	"strings"
 )
 
 type namedTunnelToken struct {
-	ZoneID     string `json:"zoneID"`
-	AccountID  string `json:"accountID"`
-	ServiceKey string `json:"serviceKey"`
+	ZoneID    string `json:"zoneID"`
+	AccountID string `json:"accountID"`
+	APIToken  string `json:"apiToken"`
 }
 
 type OriginCert struct {
-	PrivateKey interface{}
-	Cert       *x509.Certificate
-	ZoneID     string
-	ServiceKey string
-	AccountID  string
+	ZoneID    string
+	APIToken  string
+	AccountID string
 }
 
 func DecodeOriginCert(blocks []byte) (*OriginCert, error) {
@@ -33,29 +29,11 @@ func DecodeOriginCert(blocks []byte) (*OriginCert, error) {
 			break
 		}
 		switch block.Type {
-		case "PRIVATE KEY":
-			if originCert.PrivateKey != nil {
-				return nil, fmt.Errorf("Found multiple private key in the certificate")
-			}
-			// RSA private key
-			privateKey, err := x509.ParsePKCS8PrivateKey(block.Bytes)
-			if err != nil {
-				return nil, fmt.Errorf("Cannot parse private key")
-			}
-			originCert.PrivateKey = privateKey
-		case "CERTIFICATE":
-			if originCert.Cert != nil {
-				return nil, fmt.Errorf("Found multiple certificates in the certificate")
-			}
-			cert, err := x509.ParseCertificates(block.Bytes)
-			if err != nil {
-				return nil, fmt.Errorf("Cannot parse certificate")
-			} else if len(cert) > 1 {
-				return nil, fmt.Errorf("Found multiple certificates in the certificate")
-			}
-			originCert.Cert = cert[0]
-		case "WARP TOKEN", "ARGO TUNNEL TOKEN":
-			if originCert.ZoneID != "" || originCert.ServiceKey != "" {
+		case "PRIVATE KEY", "CERTIFICATE":
+			// this is for legacy purposes.
+			break
+		case "ARGO TUNNEL TOKEN":
+			if originCert.ZoneID != "" || originCert.APIToken != "" {
 				return nil, fmt.Errorf("Found multiple tokens in the certificate")
 			}
 			// The token is a string,
@@ -63,18 +41,8 @@ func DecodeOriginCert(blocks []byte) (*OriginCert, error) {
 			ntt := namedTunnelToken{}
 			if err := json.Unmarshal(block.Bytes, &ntt); err == nil {
 				originCert.ZoneID = ntt.ZoneID
-				originCert.ServiceKey = ntt.ServiceKey
+				originCert.APIToken = ntt.APIToken
 				originCert.AccountID = ntt.AccountID
-			} else {
-				// Try the older format, where the zoneID and service key are separated by
-				// a new line character
-				token := string(block.Bytes)
-				s := strings.Split(token, "\n")
-				if len(s) != 2 {
-					return nil, fmt.Errorf("Cannot parse token")
-				}
-				originCert.ZoneID = s[0]
-				originCert.ServiceKey = s[1]
 			}
 		default:
 			return nil, fmt.Errorf("Unknown block %s in the certificate", block.Type)
@@ -82,11 +50,7 @@ func DecodeOriginCert(blocks []byte) (*OriginCert, error) {
 		block, rest = pem.Decode(rest)
 	}
 
-	if originCert.PrivateKey == nil {
-		return nil, fmt.Errorf("Missing private key in the certificate")
-	} else if originCert.Cert == nil {
-		return nil, fmt.Errorf("Missing certificate in the certificate")
-	} else if originCert.ZoneID == "" || originCert.ServiceKey == "" {
+	if originCert.ZoneID == "" || originCert.APIToken == "" {
 		return nil, fmt.Errorf("Missing token in the certificate")
 	}
 

certutil/certutil_test.go

@@ -13,49 +13,33 @@ func TestLoadOriginCert(t *testing.T) {
 	assert.Equal(t, fmt.Errorf("Cannot decode empty certificate"), err)
 	assert.Nil(t, cert)
 
-	blocks, err := ioutil.ReadFile("test-cert-no-key.pem")
-	assert.Nil(t, err)
-	cert, err = DecodeOriginCert(blocks)
-	assert.Equal(t, fmt.Errorf("Missing private key in the certificate"), err)
-	assert.Nil(t, cert)
-
-	blocks, err = ioutil.ReadFile("test-cert-two-certificates.pem")
-	assert.Nil(t, err)
-	cert, err = DecodeOriginCert(blocks)
-	assert.Equal(t, fmt.Errorf("Found multiple certificates in the certificate"), err)
-	assert.Nil(t, cert)
-
-	blocks, err = ioutil.ReadFile("test-cert-unknown-block.pem")
+	blocks, err := ioutil.ReadFile("test-cert-unknown-block.pem")
 	assert.Nil(t, err)
 	cert, err = DecodeOriginCert(blocks)
 	assert.Equal(t, fmt.Errorf("Unknown block RSA PRIVATE KEY in the certificate"), err)
 	assert.Nil(t, cert)
-
-	blocks, err = ioutil.ReadFile("test-cert.pem")
-	assert.Nil(t, err)
-	cert, err = DecodeOriginCert(blocks)
-	assert.Nil(t, err)
-	assert.NotNil(t, cert)
-	assert.Equal(t, "7b0a4d77dfb881c1a3b7d61ea9443e19", cert.ZoneID)
-	key := "v1.0-58bd4f9e28f7b3c28e05a35ff3e80ab4fd9644ef3fece537eb0d12e2e9258217-183442fbb0bbdb3e571558fec9b5589ebd77aafc87498ee3f09f64a4ad79ffe8791edbae08b36c1d8f1d70a8670de56922dff92b15d214a524f4ebfa1958859e-7ce80f79921312a6022c5d25e2d380f82ceaefe3fbdc43dd13b080e3ef1e26f7"
-	assert.Equal(t, key, cert.ServiceKey)
 }
 
-func TestNewlineArgoTunnelToken(t *testing.T) {
-	ArgoTunnelTokenTest(t, "test-argo-tunnel-cert.pem")
+func TestJSONArgoTunnelTokenEmpty(t *testing.T) {
+	cert, err := DecodeOriginCert([]byte{})
+	blocks, err := ioutil.ReadFile("test-cert-no-token.pem")
+	assert.Nil(t, err)
+	cert, err = DecodeOriginCert(blocks)
+	assert.Equal(t, fmt.Errorf("Missing token in the certificate"), err)
+	assert.Nil(t, cert)
 }
 
 func TestJSONArgoTunnelToken(t *testing.T) {
 	// The given cert's Argo Tunnel Token was generated by base64 encoding this JSON:
 	// {
 	// "zoneID": "7b0a4d77dfb881c1a3b7d61ea9443e19",
-	// "serviceKey": "test-service-key",
+	// "apiToken": "test-service-key",
 	// "accountID": "abcdabcdabcdabcd1234567890abcdef"
 	// }
-	ArgoTunnelTokenTest(t, "test-argo-tunnel-cert-json.pem")
+	CloudflareTunnelTokenTest(t, "test-cloudflare-tunnel-cert-json.pem")
 }
 
-func ArgoTunnelTokenTest(t *testing.T, path string) {
+func CloudflareTunnelTokenTest(t *testing.T, path string) {
 	blocks, err := ioutil.ReadFile(path)
 	assert.Nil(t, err)
 	cert, err := DecodeOriginCert(blocks)
@@ -63,5 +47,5 @@ func ArgoTunnelTokenTest(t *testing.T, path string) {
 	assert.NotNil(t, cert)
 	assert.Equal(t, "7b0a4d77dfb881c1a3b7d61ea9443e19", cert.ZoneID)
 	key := "test-service-key"
-	assert.Equal(t, key, cert.ServiceKey)
+	assert.Equal(t, key, cert.APIToken)
 }

certutil/test-cert-no-key.pem

@@ -1,33 +0,0 @@
------BEGIN CERTIFICATE-----
-MIID+jCCA6CgAwIBAgIUJhFxUKEGvTRc3CjCok6dbPGH/P4wCgYIKoZIzj0EAwIw
-gagxCzAJBgNVBAYTAlVTMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMTgwNgYD
-VQQLEy9DbG91ZEZsYXJlIE9yaWdpbiBTU0wgRUNDIENlcnRpZmljYXRlIEF1dGhv
-cml0eTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzETMBEGA1UECBMKQ2FsaWZvcm5p
-YTEXMBUGA1UEAxMOKGRldiB1c2Ugb25seSkwHhcNMTcxMDEzMTM1OTAwWhcNMzIx
-MDA5MTM1OTAwWjBiMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMR0wGwYDVQQL
-ExRDbG91ZEZsYXJlIE9yaWdpbiBDQTEmMCQGA1UEAxMdQ2xvdWRGbGFyZSBPcmln
-aW4gQ2VydGlmaWNhdGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCf
-GswL16Fz9Ei3sAg5AmBizoN2nZdyXHP8T57UxUMcrlJXEEXCVS5RR4m9l+EmK0ng
-6yHR1H5oX1Lg1WKyXgWwr0whwmdTD+qWFJW2M8HyefyBKLrsGPuxw4CVYT0h72bx
-tG0uyrXYh7Mtz0lHjGV90qrFpq5o0jx0sLbDlDvpFPbIO58uYzKG4Sn2VTC4rOyX
-PE6SuDvMHIeX6Ekw4wSVQ9eTbksLQqTyxSqM3zp2ygc56SjGjy1nGQT8ZBGFzSbZ
-AzNOxVKrUsySx7LzZVl+zCGCPlQwaYLKObKXadZJmrqSFmErC5jcbVgBz7oJQOgl
-HJ2n0sMcZ+Ja1Y649mPVAgMBAAGjggEgMIIBHDAOBgNVHQ8BAf8EBAMCBaAwEwYD
-VR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUzA6f2Ajq
-zhX67c6piY2a1uTiUkwwHwYDVR0jBBgwFoAU2qfBlqxKMZnf0QeTeYiMelfqJfgw
-RAYIKwYBBQUHAQEEODA2MDQGCCsGAQUFBzABhihodHRwOi8vb2NzcC5jbG91ZGZs
-YXJlLmNvbS9vcmlnaW5fZWNjX2NhMCMGA1UdEQQcMBqCDCouYXJub2xkLmNvbYIK
-YXJub2xkLmNvbTA8BgNVHR8ENTAzMDGgL6AthitodHRwOi8vY3JsLmNsb3VkZmxh
-cmUuY29tL29yaWdpbl9lY2NfY2EuY3JsMAoGCCqGSM49BAMCA0gAMEUCIDV7HoMj
-K5rShE/l+90YAOzHC89OH/wUz3I5KYOFuehoAiEA8e92aIf9XBkr0K6EvFCiSsD+
-x+Yo/cL8fGfVpPt4UM8=
------END CERTIFICATE-----
------BEGIN WARP TOKEN-----
-N2IwYTRkNzdkZmI4ODFjMWEzYjdkNjFlYTk0NDNlMTkKdjEuMC01OGJkNGY5ZTI4
-ZjdiM2MyOGUwNWEzNWZmM2U4MGFiNGZkOTY0NGVmM2ZlY2U1MzdlYjBkMTJlMmU5
-MjU4MjE3LTE4MzQ0MmZiYjBiYmRiM2U1NzE1NThmZWM5YjU1ODllYmQ3N2FhZmM4
-NzQ5OGVlM2YwOWY2NGE0YWQ3OWZmZTg3OTFlZGJhZTA4YjM2YzFkOGYxZDcwYTg2
-NzBkZTU2OTIyZGZmOTJiMTVkMjE0YTUyNGY0ZWJmYTE5NTg4NTllLTdjZTgwZjc5
-OTIxMzEyYTYwMjJjNWQyNWUyZDM4MGY4MmNlYWVmZTNmYmRjNDNkZDEzYjA4MGUz
-ZWYxZTI2Zjc=
------END WARP TOKEN-----

certutil/test-cert-no-token.pem

@@ -51,7 +51,6 @@ K5rShE/l+90YAOzHC89OH/wUz3I5KYOFuehoAiEA8e92aIf9XBkr0K6EvFCiSsD+
 x+Yo/cL8fGfVpPt4UM8=
 -----END CERTIFICATE-----
 -----BEGIN ARGO TUNNEL TOKEN-----
-eyJ6b25lSUQiOiAiN2IwYTRkNzdkZmI4ODFjMWEzYjdkNjFlYTk0NDNlMTkiLCAi
-c2VydmljZUtleSI6ICJ0ZXN0LXNlcnZpY2Uta2V5IiwgImFjY291bnRJRCI6ICJh
-YmNkYWJjZGFiY2RhYmNkMTIzNDU2Nzg5MGFiY2RlZiJ9
+eyJ6b25lSUQiOiAiN2IwYTRkNzdkZmI4ODFjMWEzYjdkNjFlYTk0NDNlMTkiLCAiYWNjb3VudElE
+IjogImFiY2RhYmNkYWJjZGFiY2QxMjM0NTY3ODkwYWJjZGVmIn0=
 -----END ARGO TUNNEL TOKEN-----

certutil/test-cert-two-certificates.pem

@@ -1,85 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQCfGswL16Fz9Ei3
-sAg5AmBizoN2nZdyXHP8T57UxUMcrlJXEEXCVS5RR4m9l+EmK0ng6yHR1H5oX1Lg
-1WKyXgWwr0whwmdTD+qWFJW2M8HyefyBKLrsGPuxw4CVYT0h72bxtG0uyrXYh7Mt
-z0lHjGV90qrFpq5o0jx0sLbDlDvpFPbIO58uYzKG4Sn2VTC4rOyXPE6SuDvMHIeX
-6Ekw4wSVQ9eTbksLQqTyxSqM3zp2ygc56SjGjy1nGQT8ZBGFzSbZAzNOxVKrUsyS
-x7LzZVl+zCGCPlQwaYLKObKXadZJmrqSFmErC5jcbVgBz7oJQOglHJ2n0sMcZ+Ja
-1Y649mPVAgMBAAECggEAEbPF0ah9fH0IzTU/CPbIeh3flyY8GDuMpR1HvwUurSWB
-IFI9bLyVAXKb8vYP1TMaTnXi5qmFof+/JShgyZc3+1tZtWTfoaiC8Y1bRfE2yk+D
-xmwddhDmijYGG7i8uEaeddSdFEh2GKAqkbV/QgBvN2Nl4EVmIOAJXXNe9l5LFyjy
-sR10aNVJRYV1FahrCTwZ3SovHP4d4AUvHh/3FFZDukHc37CFA0+CcR4uehp5yedi
-2UdqaszXqunFo/3h+Tn9dW2C7gTTZx4+mfyaws3p3YOmdYArXvpejxHIc0FGwLBm
-sb9K7wGVUiF0Bt0ch+C1mdYrCaFNHnPuDswjmm3FwQKBgQDYtxOwwSLA6ZyppozX
-Doyx9a7PhiMHCFKSdVB4l8rpK545a+AmpG6LRScTtBsMTHBhT3IQ3QPWlVm1AhjF
-AvXMa1rOeaGbCbDn1xqEoEVPtj4tys8eTfyWmtU73jWTFauOt4/xpf/urEpg91xj
-m+Gl/8qgBrpm5rQxV5Y4MysRlQKBgQC78jzzlhocXGNvw0wT/K2NsknyeoZXqpIE
-QYL60FMl4geZn6w9hwxaL1r+g/tUjTnpBPQtS1r2Ed2gXby5zspN1g/PW8U3t3to
-P7zHIJ/sLBXrCh5RJko3hUgGhDNOOCIQj4IaKUfvHYvEIbIxlyI0vdsXsgXgMuQ8
-pb9Yifn5QQKBgQCmGu0EtYQlyOlDP10EGSrN3Dm45l9CrKZdi326cN4eCkikSoLs
-G2x/YumouItiydP5QiNzuXOPrbmse4bwumwb2s0nJSMw6iSmDsFMlmuJxW2zO5e0
-6qGH7fUyhgcaTanJIfk6hrm7/mKkH/S4hGpYCc8NCRsmc/35M+D4AoAoYQKBgQC0
-LWpZaxDlF30MbAHHN3l6We2iU+vup0sMYXGb2ZOcwa/fir+ozIr++l8VmJmdWTan
-OWSM96zgMghx8Os4hhJTxF+rvqK242OfcVsc2x31X94zUaP2z+peh5uhA6Pb3Nxr
-W+iyA9k+Vujiwhr+h5D3VvtvH++aG6/KpGtoCf5nAQKBgQDXX2+d7bd5CLNLLFNd
-M2i4QoOFcSKIG+v4SuvgEJHgG8vGvxh2qlSxnMWuPV+7/1P5ATLqDj1PlKms+BNR
-y7sc5AT9PclkL3Y9MNzOu0LXyBkGYcl8M0EQfLv9VPbWT+NXiMg/O2CHiT02pAAz
-uQicoQq3yzeQh20wtrtaXzTNmA==
------END PRIVATE KEY-----
------BEGIN CERTIFICATE-----
-MIID+jCCA6CgAwIBAgIUJhFxUKEGvTRc3CjCok6dbPGH/P4wCgYIKoZIzj0EAwIw
-gagxCzAJBgNVBAYTAlVTMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMTgwNgYD
-VQQLEy9DbG91ZEZsYXJlIE9yaWdpbiBTU0wgRUNDIENlcnRpZmljYXRlIEF1dGhv
-cml0eTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzETMBEGA1UECBMKQ2FsaWZvcm5p
-YTEXMBUGA1UEAxMOKGRldiB1c2Ugb25seSkwHhcNMTcxMDEzMTM1OTAwWhcNMzIx
-MDA5MTM1OTAwWjBiMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMR0wGwYDVQQL
-ExRDbG91ZEZsYXJlIE9yaWdpbiBDQTEmMCQGA1UEAxMdQ2xvdWRGbGFyZSBPcmln
-aW4gQ2VydGlmaWNhdGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCf
-GswL16Fz9Ei3sAg5AmBizoN2nZdyXHP8T57UxUMcrlJXEEXCVS5RR4m9l+EmK0ng
-6yHR1H5oX1Lg1WKyXgWwr0whwmdTD+qWFJW2M8HyefyBKLrsGPuxw4CVYT0h72bx
-tG0uyrXYh7Mtz0lHjGV90qrFpq5o0jx0sLbDlDvpFPbIO58uYzKG4Sn2VTC4rOyX
-PE6SuDvMHIeX6Ekw4wSVQ9eTbksLQqTyxSqM3zp2ygc56SjGjy1nGQT8ZBGFzSbZ
-AzNOxVKrUsySx7LzZVl+zCGCPlQwaYLKObKXadZJmrqSFmErC5jcbVgBz7oJQOgl
-HJ2n0sMcZ+Ja1Y649mPVAgMBAAGjggEgMIIBHDAOBgNVHQ8BAf8EBAMCBaAwEwYD
-VR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUzA6f2Ajq
-zhX67c6piY2a1uTiUkwwHwYDVR0jBBgwFoAU2qfBlqxKMZnf0QeTeYiMelfqJfgw
-RAYIKwYBBQUHAQEEODA2MDQGCCsGAQUFBzABhihodHRwOi8vb2NzcC5jbG91ZGZs
-YXJlLmNvbS9vcmlnaW5fZWNjX2NhMCMGA1UdEQQcMBqCDCouYXJub2xkLmNvbYIK
-YXJub2xkLmNvbTA8BgNVHR8ENTAzMDGgL6AthitodHRwOi8vY3JsLmNsb3VkZmxh
-cmUuY29tL29yaWdpbl9lY2NfY2EuY3JsMAoGCCqGSM49BAMCA0gAMEUCIDV7HoMj
-K5rShE/l+90YAOzHC89OH/wUz3I5KYOFuehoAiEA8e92aIf9XBkr0K6EvFCiSsD+
-x+Yo/cL8fGfVpPt4UM8=
------END CERTIFICATE-----
------BEGIN CERTIFICATE-----
-MIID+jCCA6CgAwIBAgIUJhFxUKEGvTRc3CjCok6dbPGH/P4wCgYIKoZIzj0EAwIw
-gagxCzAJBgNVBAYTAlVTMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMTgwNgYD
-VQQLEy9DbG91ZEZsYXJlIE9yaWdpbiBTU0wgRUNDIENlcnRpZmljYXRlIEF1dGhv
-cml0eTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzETMBEGA1UECBMKQ2FsaWZvcm5p
-YTEXMBUGA1UEAxMOKGRldiB1c2Ugb25seSkwHhcNMTcxMDEzMTM1OTAwWhcNMzIx
-MDA5MTM1OTAwWjBiMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMR0wGwYDVQQL
-ExRDbG91ZEZsYXJlIE9yaWdpbiBDQTEmMCQGA1UEAxMdQ2xvdWRGbGFyZSBPcmln
-aW4gQ2VydGlmaWNhdGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCf
-GswL16Fz9Ei3sAg5AmBizoN2nZdyXHP8T57UxUMcrlJXEEXCVS5RR4m9l+EmK0ng
-6yHR1H5oX1Lg1WKyXgWwr0whwmdTD+qWFJW2M8HyefyBKLrsGPuxw4CVYT0h72bx
-tG0uyrXYh7Mtz0lHjGV90qrFpq5o0jx0sLbDlDvpFPbIO58uYzKG4Sn2VTC4rOyX
-PE6SuDvMHIeX6Ekw4wSVQ9eTbksLQqTyxSqM3zp2ygc56SjGjy1nGQT8ZBGFzSbZ
-AzNOxVKrUsySx7LzZVl+zCGCPlQwaYLKObKXadZJmrqSFmErC5jcbVgBz7oJQOgl
-HJ2n0sMcZ+Ja1Y649mPVAgMBAAGjggEgMIIBHDAOBgNVHQ8BAf8EBAMCBaAwEwYD
-VR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUzA6f2Ajq
-zhX67c6piY2a1uTiUkwwHwYDVR0jBBgwFoAU2qfBlqxKMZnf0QeTeYiMelfqJfgw
-RAYIKwYBBQUHAQEEODA2MDQGCCsGAQUFBzABhihodHRwOi8vb2NzcC5jbG91ZGZs
-YXJlLmNvbS9vcmlnaW5fZWNjX2NhMCMGA1UdEQQcMBqCDCouYXJub2xkLmNvbYIK
-YXJub2xkLmNvbTA8BgNVHR8ENTAzMDGgL6AthitodHRwOi8vY3JsLmNsb3VkZmxh
-cmUuY29tL29yaWdpbl9lY2NfY2EuY3JsMAoGCCqGSM49BAMCA0gAMEUCIDV7HoMj
-K5rShE/l+90YAOzHC89OH/wUz3I5KYOFuehoAiEA8e92aIf9XBkr0K6EvFCiSsD+
-x+Yo/cL8fGfVpPt4UM8=
------END CERTIFICATE-----
------BEGIN WARP TOKEN-----
-N2IwYTRkNzdkZmI4ODFjMWEzYjdkNjFlYTk0NDNlMTkKdjEuMC01OGJkNGY5ZTI4
-ZjdiM2MyOGUwNWEzNWZmM2U4MGFiNGZkOTY0NGVmM2ZlY2U1MzdlYjBkMTJlMmU5
-MjU4MjE3LTE4MzQ0MmZiYjBiYmRiM2U1NzE1NThmZWM5YjU1ODllYmQ3N2FhZmM4
-NzQ5OGVlM2YwOWY2NGE0YWQ3OWZmZTg3OTFlZGJhZTA4YjM2YzFkOGYxZDcwYTg2
-NzBkZTU2OTIyZGZmOTJiMTVkMjE0YTUyNGY0ZWJmYTE5NTg4NTllLTdjZTgwZjc5
-OTIxMzEyYTYwMjJjNWQyNWUyZDM4MGY4MmNlYWVmZTNmYmRjNDNkZDEzYjA4MGUz
-ZWYxZTI2Zjc=
------END WARP TOKEN-----

certutil/test-cert-unknown-block.pem

@@ -50,7 +50,7 @@ cmUuY29tL29yaWdpbl9lY2NfY2EuY3JsMAoGCCqGSM49BAMCA0gAMEUCIDV7HoMj
 K5rShE/l+90YAOzHC89OH/wUz3I5KYOFuehoAiEA8e92aIf9XBkr0K6EvFCiSsD+
 x+Yo/cL8fGfVpPt4UM8=
 -----END CERTIFICATE-----
------BEGIN WARP TOKEN-----
+-----BEGIN ARGO TUNNEL TOKEN-----
 N2IwYTRkNzdkZmI4ODFjMWEzYjdkNjFlYTk0NDNlMTkKdjEuMC01OGJkNGY5ZTI4
 ZjdiM2MyOGUwNWEzNWZmM2U4MGFiNGZkOTY0NGVmM2ZlY2U1MzdlYjBkMTJlMmU5
 MjU4MjE3LTE4MzQ0MmZiYjBiYmRiM2U1NzE1NThmZWM5YjU1ODllYmQ3N2FhZmM4
@@ -58,7 +58,7 @@ NzQ5OGVlM2YwOWY2NGE0YWQ3OWZmZTg3OTFlZGJhZTA4YjM2YzFkOGYxZDcwYTg2
 NzBkZTU2OTIyZGZmOTJiMTVkMjE0YTUyNGY0ZWJmYTE5NTg4NTllLTdjZTgwZjc5
 OTIxMzEyYTYwMjJjNWQyNWUyZDM4MGY4MmNlYWVmZTNmYmRjNDNkZDEzYjA4MGUz
 ZWYxZTI2Zjc=
------END WARP TOKEN-----
+-----END ARGO TUNNEL TOKEN-----
 -----BEGIN RSA PRIVATE KEY-----
 MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQCfGswL16Fz9Ei3
 sAg5AmBizoN2nZdyXHP8T57UxUMcrlJXEEXCVS5RR4m9l+EmK0ng6yHR1H5oX1Lg

certutil/test-cert.pem

@@ -1,61 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQCfGswL16Fz9Ei3
-sAg5AmBizoN2nZdyXHP8T57UxUMcrlJXEEXCVS5RR4m9l+EmK0ng6yHR1H5oX1Lg
-1WKyXgWwr0whwmdTD+qWFJW2M8HyefyBKLrsGPuxw4CVYT0h72bxtG0uyrXYh7Mt
-z0lHjGV90qrFpq5o0jx0sLbDlDvpFPbIO58uYzKG4Sn2VTC4rOyXPE6SuDvMHIeX
-6Ekw4wSVQ9eTbksLQqTyxSqM3zp2ygc56SjGjy1nGQT8ZBGFzSbZAzNOxVKrUsyS
-x7LzZVl+zCGCPlQwaYLKObKXadZJmrqSFmErC5jcbVgBz7oJQOglHJ2n0sMcZ+Ja
-1Y649mPVAgMBAAECggEAEbPF0ah9fH0IzTU/CPbIeh3flyY8GDuMpR1HvwUurSWB
-IFI9bLyVAXKb8vYP1TMaTnXi5qmFof+/JShgyZc3+1tZtWTfoaiC8Y1bRfE2yk+D
-xmwddhDmijYGG7i8uEaeddSdFEh2GKAqkbV/QgBvN2Nl4EVmIOAJXXNe9l5LFyjy
-sR10aNVJRYV1FahrCTwZ3SovHP4d4AUvHh/3FFZDukHc37CFA0+CcR4uehp5yedi
-2UdqaszXqunFo/3h+Tn9dW2C7gTTZx4+mfyaws3p3YOmdYArXvpejxHIc0FGwLBm
-sb9K7wGVUiF0Bt0ch+C1mdYrCaFNHnPuDswjmm3FwQKBgQDYtxOwwSLA6ZyppozX
-Doyx9a7PhiMHCFKSdVB4l8rpK545a+AmpG6LRScTtBsMTHBhT3IQ3QPWlVm1AhjF
-AvXMa1rOeaGbCbDn1xqEoEVPtj4tys8eTfyWmtU73jWTFauOt4/xpf/urEpg91xj
-m+Gl/8qgBrpm5rQxV5Y4MysRlQKBgQC78jzzlhocXGNvw0wT/K2NsknyeoZXqpIE
-QYL60FMl4geZn6w9hwxaL1r+g/tUjTnpBPQtS1r2Ed2gXby5zspN1g/PW8U3t3to
-P7zHIJ/sLBXrCh5RJko3hUgGhDNOOCIQj4IaKUfvHYvEIbIxlyI0vdsXsgXgMuQ8
-pb9Yifn5QQKBgQCmGu0EtYQlyOlDP10EGSrN3Dm45l9CrKZdi326cN4eCkikSoLs
-G2x/YumouItiydP5QiNzuXOPrbmse4bwumwb2s0nJSMw6iSmDsFMlmuJxW2zO5e0
-6qGH7fUyhgcaTanJIfk6hrm7/mKkH/S4hGpYCc8NCRsmc/35M+D4AoAoYQKBgQC0
-LWpZaxDlF30MbAHHN3l6We2iU+vup0sMYXGb2ZOcwa/fir+ozIr++l8VmJmdWTan
-OWSM96zgMghx8Os4hhJTxF+rvqK242OfcVsc2x31X94zUaP2z+peh5uhA6Pb3Nxr
-W+iyA9k+Vujiwhr+h5D3VvtvH++aG6/KpGtoCf5nAQKBgQDXX2+d7bd5CLNLLFNd
-M2i4QoOFcSKIG+v4SuvgEJHgG8vGvxh2qlSxnMWuPV+7/1P5ATLqDj1PlKms+BNR
-y7sc5AT9PclkL3Y9MNzOu0LXyBkGYcl8M0EQfLv9VPbWT+NXiMg/O2CHiT02pAAz
-uQicoQq3yzeQh20wtrtaXzTNmA==
------END PRIVATE KEY-----
------BEGIN CERTIFICATE-----
-MIID+jCCA6CgAwIBAgIUJhFxUKEGvTRc3CjCok6dbPGH/P4wCgYIKoZIzj0EAwIw
-gagxCzAJBgNVBAYTAlVTMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMTgwNgYD
-VQQLEy9DbG91ZEZsYXJlIE9yaWdpbiBTU0wgRUNDIENlcnRpZmljYXRlIEF1dGhv
-cml0eTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzETMBEGA1UECBMKQ2FsaWZvcm5p
-YTEXMBUGA1UEAxMOKGRldiB1c2Ugb25seSkwHhcNMTcxMDEzMTM1OTAwWhcNMzIx
-MDA5MTM1OTAwWjBiMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMR0wGwYDVQQL
-ExRDbG91ZEZsYXJlIE9yaWdpbiBDQTEmMCQGA1UEAxMdQ2xvdWRGbGFyZSBPcmln
-aW4gQ2VydGlmaWNhdGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCf
-GswL16Fz9Ei3sAg5AmBizoN2nZdyXHP8T57UxUMcrlJXEEXCVS5RR4m9l+EmK0ng
-6yHR1H5oX1Lg1WKyXgWwr0whwmdTD+qWFJW2M8HyefyBKLrsGPuxw4CVYT0h72bx
-tG0uyrXYh7Mtz0lHjGV90qrFpq5o0jx0sLbDlDvpFPbIO58uYzKG4Sn2VTC4rOyX
-PE6SuDvMHIeX6Ekw4wSVQ9eTbksLQqTyxSqM3zp2ygc56SjGjy1nGQT8ZBGFzSbZ
-AzNOxVKrUsySx7LzZVl+zCGCPlQwaYLKObKXadZJmrqSFmErC5jcbVgBz7oJQOgl
-HJ2n0sMcZ+Ja1Y649mPVAgMBAAGjggEgMIIBHDAOBgNVHQ8BAf8EBAMCBaAwEwYD
-VR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUzA6f2Ajq
-zhX67c6piY2a1uTiUkwwHwYDVR0jBBgwFoAU2qfBlqxKMZnf0QeTeYiMelfqJfgw
-RAYIKwYBBQUHAQEEODA2MDQGCCsGAQUFBzABhihodHRwOi8vb2NzcC5jbG91ZGZs
-YXJlLmNvbS9vcmlnaW5fZWNjX2NhMCMGA1UdEQQcMBqCDCouYXJub2xkLmNvbYIK
-YXJub2xkLmNvbTA8BgNVHR8ENTAzMDGgL6AthitodHRwOi8vY3JsLmNsb3VkZmxh
-cmUuY29tL29yaWdpbl9lY2NfY2EuY3JsMAoGCCqGSM49BAMCA0gAMEUCIDV7HoMj
-K5rShE/l+90YAOzHC89OH/wUz3I5KYOFuehoAiEA8e92aIf9XBkr0K6EvFCiSsD+
-x+Yo/cL8fGfVpPt4UM8=
------END CERTIFICATE-----
------BEGIN WARP TOKEN-----
-N2IwYTRkNzdkZmI4ODFjMWEzYjdkNjFlYTk0NDNlMTkKdjEuMC01OGJkNGY5ZTI4
-ZjdiM2MyOGUwNWEzNWZmM2U4MGFiNGZkOTY0NGVmM2ZlY2U1MzdlYjBkMTJlMmU5
-MjU4MjE3LTE4MzQ0MmZiYjBiYmRiM2U1NzE1NThmZWM5YjU1ODllYmQ3N2FhZmM4
-NzQ5OGVlM2YwOWY2NGE0YWQ3OWZmZTg3OTFlZGJhZTA4YjM2YzFkOGYxZDcwYTg2
-NzBkZTU2OTIyZGZmOTJiMTVkMjE0YTUyNGY0ZWJmYTE5NTg4NTllLTdjZTgwZjc5
-OTIxMzEyYTYwMjJjNWQyNWUyZDM4MGY4MmNlYWVmZTNmYmRjNDNkZDEzYjA4MGUz
-ZWYxZTI2Zjc=
------END WARP TOKEN-----

certutil/test-cloudflare-tunnel-cert-json.pem

@@ -51,6 +51,7 @@ K5rShE/l+90YAOzHC89OH/wUz3I5KYOFuehoAiEA8e92aIf9XBkr0K6EvFCiSsD+
 x+Yo/cL8fGfVpPt4UM8=
 -----END CERTIFICATE-----
 -----BEGIN ARGO TUNNEL TOKEN-----
-N2IwYTRkNzdkZmI4ODFjMWEzYjdkNjFlYTk0NDNlMTkKdGVzdC1zZXJ2aWNlLWtl
-eQ==
+eyJ6b25lSUQiOiAiN2IwYTRkNzdkZmI4ODFjMWEzYjdkNjFlYTk0NDNlMTkiLCAiYXBpVG9rZW4i
+OiAidGVzdC1zZXJ2aWNlLWtleSIsICJhY2NvdW50SUQiOiAiYWJjZGFiY2RhYmNkYWJjZDEyMzQ1
+Njc4OTBhYmNkZWYifQ==
 -----END ARGO TUNNEL TOKEN-----

cfapi/base_client.go

@@ -104,7 +104,7 @@ func (r *RESTClient) sendRequest(method string, url url.URL, body interface{}) (
 	if bodyReader != nil {
 		req.Header.Set("Content-Type", jsonContentType)
 	}
-	req.Header.Add("X-Auth-User-Service-Key", r.authToken)
+	req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", r.authToken))
 	req.Header.Add("Accept", "application/json;version=1")
 	return r.client.Do(req)
 }

cfapi/client.go

@@ -28,7 +28,7 @@ type IPRouteClient interface {
 type VnetClient interface {
 	CreateVirtualNetwork(newVnet NewVirtualNetwork) (VirtualNetwork, error)
 	ListVirtualNetworks(filter *VnetFilter) ([]*VirtualNetwork, error)
-	DeleteVirtualNetwork(id uuid.UUID) error
+	DeleteVirtualNetwork(id uuid.UUID, force bool) error
 	UpdateVirtualNetwork(id uuid.UUID, updates UpdateVirtualNetwork) error
 }
 

cfapi/virtual_network.go

@@ -80,9 +80,16 @@ func (r *RESTClient) ListVirtualNetworks(filter *VnetFilter) ([]*VirtualNetwork,
 	return nil, r.statusCodeToError("list virtual networks", resp)
 }
 
-func (r *RESTClient) DeleteVirtualNetwork(id uuid.UUID) error {
+func (r *RESTClient) DeleteVirtualNetwork(id uuid.UUID, force bool) error {
 	endpoint := r.baseEndpoints.accountVnets
 	endpoint.Path = path.Join(endpoint.Path, url.PathEscape(id.String()))
+
+	queryParams := url.Values{}
+	if force {
+		queryParams.Set("force", strconv.FormatBool(force))
+	}
+	endpoint.RawQuery = queryParams.Encode()
+
 	resp, err := r.sendRequest("DELETE", endpoint, nil)
 	if err != nil {
 		return errors.Wrap(err, "REST request failed")

cfsetup.yaml

@@ -1,23 +1,29 @@
-pinned_go: &pinned_go go=1.18.6-1
-pinned_go_fips: &pinned_go_fips go-boring=1.18.6-1
+pinned_go: &pinned_go go=1.19.3-1
+pinned_go_fips: &pinned_go_fips go-boring=1.19.3-1
 
 build_dir: &build_dir /cfsetup_build
 default-flavor: bullseye
 stretch: &stretch
   build:
     build_dir: *build_dir
-    builddeps:
+    builddeps: &build_deps
       - *pinned_go
       - build-essential
+      - gotest-to-teamcity
+    pre-cache: &build_pre_cache
+      - export GOCACHE=/cfsetup_build/.cache/go-build
+      - go install golang.org/x/tools/cmd/goimports@latest
     post-cache:
       - export GOOS=linux
       - export GOARCH=amd64
       - make cloudflared
   build-fips:
     build_dir: *build_dir
-    builddeps:
+    builddeps: &build_deps_fips
       - *pinned_go_fips
       - build-essential
+      - gotest-to-teamcity
+    pre-cache: *build_pre_cache
     post-cache:
       - export GOOS=linux
       - export GOARCH=amd64
@@ -78,6 +84,13 @@ stretch: &stretch
       # same logic as above, but for FIPS packages only
       - ./build-packages-fips.sh
       - make github-release-built-pkgs
+  generate-versions-file:
+    build_dir: *build_dir
+    builddeps: 
+      - *pinned_go
+      - build-essential
+    post-cache:
+      - make generate-docker-version
   build-deb:
     build_dir: *build_dir
     builddeps: &build_deb_deps
@@ -156,12 +169,8 @@ stretch: &stretch
       - make github-windows-upload
   test:
     build_dir: *build_dir
-    builddeps:
-      - *pinned_go
-      - build-essential
-      - gotest-to-teamcity
-    pre-cache: &test_pre_cache
-      - go install golang.org/x/tools/cmd/goimports@latest
+    builddeps: *build_deps
+    pre-cache: *build_pre_cache
     post-cache:
       - export GOOS=linux
       - export GOARCH=amd64
@@ -170,11 +179,8 @@ stretch: &stretch
       - make test | gotest-to-teamcity
   test-fips:
     build_dir: *build_dir
-    builddeps:
-      - *pinned_go_fips
-      - build-essential
-      - gotest-to-teamcity
-    pre-cache: *test_pre_cache
+    builddeps: *build_deps_fips
+    pre-cache: *build_pre_cache
     post-cache:
       - export GOOS=linux
       - export GOARCH=amd64

cloudflared.wxs

@@ -1,62 +1,64 @@
 <?xml version="1.0"?>
 
-<?if $(var.Platform)="x86"?>
-    <?define Program_Files="ProgramFilesFolder"?>
-<?else?>
+<?if $(var.Platform)="x64" ?>
     <?define Program_Files="ProgramFiles64Folder"?>
-<?endif?>
+<?else ?>
+    <?define Program_Files="ProgramFilesFolder"?>
+<?endif ?>
 <?ifndef var.Version?>
     <?error Undefined Version variable?>
-<?endif?>
+<?endif ?>
 <?ifndef var.Path?>
     <?error Undefined Path variable?>
-<?endif?>
+<?endif ?>
 
 <Wix xmlns="http://schemas.microsoft.com/wix/2006/wi">
-    <Product Id="35e5e858-9372-4449-bf73-1cd6f7267128" 
+    <Product Id="*"
         UpgradeCode="23f90fdd-9328-47ea-ab52-5380855a4b12"
         Name="cloudflared"
         Version="$(var.Version)"
         Manufacturer="cloudflare"
         Language="1033">
 
-      <Package InstallerVersion="200" Compressed="yes" Comments="Windows Installer Package" InstallScope="perMachine"/>
+        <Package InstallerVersion="200" Compressed="yes" Comments="Windows Installer Package" InstallScope="perMachine" />
 
-      <Media Id="1" Cabinet="product.cab" EmbedCab="yes"/>
+        <Media Id="1" Cabinet="product.cab" EmbedCab="yes" />
 
-      <Upgrade Id="23f90fdd-9328-47ea-ab52-5380855a4b12">
-         <UpgradeVersion Minimum="$(var.Version)" OnlyDetect="yes" Property="NEWERVERSIONDETECTED"/>
-         <UpgradeVersion Minimum="2020.8.0" Maximum="$(var.Version)" IncludeMinimum="yes" IncludeMaximum="no"
-                         Property="OLDERVERSIONBEINGUPGRADED"/>
-      </Upgrade>
-      <Condition Message="A newer version of this software is already installed.">NOT NEWERVERSIONDETECTED</Condition>
+        <MajorUpgrade DowngradeErrorMessage="A later version of [ProductName] is already installed. Setup will now exit." />
 
-    <Directory Id="TARGETDIR" Name="SourceDir">
-        <!--This specifies where the cloudflared.exe is moved to in the windows Operation System-->
-        <Directory Id="$(var.Program_Files)">
-            <Directory Id="INSTALLDIR" Name="cloudflared">
-                <Component Id="ApplicationFiles" Guid="35e5e858-9372-4449-bf73-1cd6f7267128">
-                    <File Id="ApplicationFile0" Source="$(var.Path)"/>
-                </Component>
+        <Upgrade Id="23f90fdd-9328-47ea-ab52-5380855a4b12">
+            <UpgradeVersion Minimum="$(var.Version)" OnlyDetect="yes" Property="NEWERVERSIONDETECTED" />
+            <UpgradeVersion Minimum="2020.8.0" Maximum="$(var.Version)" IncludeMinimum="yes" IncludeMaximum="no"
+                Property="OLDERVERSIONBEINGUPGRADED" />
+        </Upgrade>
+        <Condition Message="A newer version of this software is already installed.">NOT NEWERVERSIONDETECTED</Condition>
+
+        <Directory Id="TARGETDIR" Name="SourceDir">
+            <!--This specifies where the cloudflared.exe is moved to in the windows Operation System-->
+            <Directory Id="$(var.Program_Files)">
+                <Directory Id="INSTALLDIR" Name="cloudflared">
+                    <Component Id="ApplicationFiles" Guid="35e5e858-9372-4449-bf73-1cd6f7267128">
+                        <File Id="ApplicationFile0" Source="$(var.Path)" />
+                    </Component>
+                </Directory>
             </Directory>
+            <Component Id="ENVS" Guid="6bb74449-d10d-4f4a-933e-6fc9fa006eae">
+                <!--Set the cloudflared bin location to the Path Environment Variable-->
+                <Environment Id="ENV0"
+                    Name="PATH"
+                    Value="[INSTALLDIR]."
+                    Permanent="no"
+                    Part="last"
+                    Action="create"
+                    System="yes" />
+            </Component>
         </Directory>
-        <Component Id="ENVS" Guid="6bb74449-d10d-4f4a-933e-6fc9fa006eae">
-        <!--Set the cloudflared bin location to the Path Environment Variable-->
-         <Environment Id="ENV0"
-           Name="PATH"
-           Value="[INSTALLDIR]."
-           Permanent="no"
-           Part="last"
-           Action="create"
-           System="yes" />
-        </Component>
-    </Directory>
 
 
-    <Feature Id='Complete' Level='1'>
-      <ComponentRef Id="ENVS"/>
-      <ComponentRef Id='ApplicationFiles' />
-    </Feature>
+        <Feature Id='Complete' Level='1'>
+            <ComponentRef Id="ENVS" />
+            <ComponentRef Id='ApplicationFiles' />
+        </Feature>
 
     </Product>
 </Wix>

cmd/cloudflared/access/cmd.go

@@ -11,7 +11,7 @@ import (
 	"text/template"
 	"time"
 
-	"github.com/getsentry/raven-go"
+	"github.com/getsentry/sentry-go"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
 	"github.com/urfave/cli/v2"
@@ -37,16 +37,13 @@ const (
 	sshConfigTemplate  = `
 Add to your {{.Home}}/.ssh/config:
 
-Host {{.Hostname}}
 {{- if .ShortLivedCerts}}
-  ProxyCommand bash -c '{{.Cloudflared}} access ssh-gen --hostname %h; ssh -tt %r@cfpipe-{{.Hostname}} >&2 <&1'
-
-Host cfpipe-{{.Hostname}}
-  HostName {{.Hostname}}
+Match host {{.Hostname}} exec "{{.Cloudflared}} access ssh-gen --hostname %h"
   ProxyCommand {{.Cloudflared}} access ssh --hostname %h
-  IdentityFile ~/.cloudflared/{{.Hostname}}-cf_key
-  CertificateFile ~/.cloudflared/{{.Hostname}}-cf_key-cert.pub
+  IdentityFile ~/.cloudflared/%h-cf_key
+  CertificateFile ~/.cloudflared/%h-cf_key-cert.pub
 {{- else}}
+Host {{.Hostname}}
   ProxyCommand {{.Cloudflared}} access ssh --hostname %h
 {{end}}
 `
@@ -205,7 +202,11 @@ func Commands() []*cli.Command {
 
 // login pops up the browser window to do the actual login and JWT generation
 func login(c *cli.Context) error {
-	if err := raven.SetDSN(sentryDSN); err != nil {
+	err := sentry.Init(sentry.ClientOptions{
+		Dsn:     sentryDSN,
+		Release: c.App.Version,
+	})
+	if err != nil {
 		return err
 	}
 
@@ -254,7 +255,11 @@ func ensureURLScheme(url string) string {
 
 // curl provides a wrapper around curl, passing Access JWT along in request
 func curl(c *cli.Context) error {
-	if err := raven.SetDSN(sentryDSN); err != nil {
+	err := sentry.Init(sentry.ClientOptions{
+		Dsn:     sentryDSN,
+		Release: c.App.Version,
+	})
+	if err != nil {
 		return err
 	}
 	log := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
@@ -317,7 +322,11 @@ func run(cmd string, args ...string) error {
 
 // token dumps provided token to stdout
 func generateToken(c *cli.Context) error {
-	if err := raven.SetDSN(sentryDSN); err != nil {
+	err := sentry.Init(sentry.ClientOptions{
+		Dsn:     sentryDSN,
+		Release: c.App.Version,
+	})
+	if err != nil {
 		return err
 	}
 	appURL, err := url.Parse(ensureURLScheme(c.String("app")))

cmd/cloudflared/main.go

@@ -6,7 +6,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/getsentry/raven-go"
+	"github.com/getsentry/sentry-go"
 	homedir "github.com/mitchellh/go-homedir"
 	"github.com/pkg/errors"
 	"github.com/urfave/cli/v2"
@@ -50,7 +50,6 @@ var (
 func main() {
 	rand.Seed(time.Now().UnixNano())
 	metrics.RegisterBuildInfo(BuildType, BuildTime, Version)
-	raven.SetRelease(Version)
 	maxprocs.Set()
 	bInfo := cliutil.GetBuildInfo(BuildType, Version)
 
@@ -156,10 +155,10 @@ func action(graceShutdownC chan struct{}) cli.ActionFunc {
 		if isEmptyInvocation(c) {
 			return handleServiceMode(c, graceShutdownC)
 		}
-		tags := make(map[string]string)
-		tags["hostname"] = c.String("hostname")
-		raven.SetTagsContext(tags)
-		raven.CapturePanic(func() { err = tunnel.TunnelCommand(c) }, nil)
+		func() {
+			defer sentry.Recover()
+			err = tunnel.TunnelCommand(c)
+		}()
 		if err != nil {
 			captureError(err)
 		}
@@ -187,7 +186,7 @@ func captureError(err error) {
 			return
 		}
 	}
-	raven.CaptureError(err, nil)
+	sentry.CaptureException(err)
 }
 
 // cloudflared was started without any flags

cmd/cloudflared/proxydns/cmd.go

@@ -1,6 +1,7 @@
 package proxydns
 
 import (
+	"context"
 	"net"
 	"os"
 	"os/signal"
@@ -73,7 +74,7 @@ func Run(c *cli.Context) error {
 		log.Fatal().Err(err).Msg("Failed to open the metrics listener")
 	}
 
-	go metrics.ServeMetrics(metricsListener, nil, nil, "", nil, log)
+	go metrics.ServeMetrics(metricsListener, context.Background(), metrics.Config{}, log)
 
 	listener, err := tunneldns.CreateListener(
 		c.String("address"),

cmd/cloudflared/tunnel/cmd.go

@@ -14,7 +14,7 @@ import (
 
 	"github.com/coreos/go-systemd/daemon"
 	"github.com/facebookgo/grace/gracenet"
-	"github.com/getsentry/raven-go"
+	"github.com/getsentry/sentry-go"
 	"github.com/google/uuid"
 	homedir "github.com/mitchellh/go-homedir"
 	"github.com/pkg/errors"
@@ -36,6 +36,7 @@ import (
 	"github.com/cloudflare/cloudflared/supervisor"
 	"github.com/cloudflare/cloudflared/tlsconfig"
 	"github.com/cloudflare/cloudflared/tunneldns"
+	"github.com/cloudflare/cloudflared/validation"
 )
 
 const (
@@ -82,6 +83,15 @@ const (
 	LogFieldPIDPathname         = "pidPathname"
 	LogFieldTmpTraceFilename    = "tmpTraceFilename"
 	LogFieldTraceOutputFilepath = "traceOutputFilepath"
+
+	tunnelCmdErrorMessage = `You did not specify any valid additional argument to the cloudflared tunnel command.
+
+If you are trying to run a Quick Tunnel then you need to explicitly pass the --url flag.
+Eg. cloudflared tunnel --url localhost:8080/.
+
+Please note that Quick Tunnels are meant to be ephemeral and should only be used for testing purposes.
+For production usage, we recommend creating Named Tunnels. (https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/install-and-setup/tunnel-guide/)
+`
 )
 
 var (
@@ -91,6 +101,7 @@ var (
 	routeFailMsg = fmt.Sprintf("failed to provision routing, please create it manually via Cloudflare dashboard or UI; "+
 		"most likely you already have a conflicting record there. You can also rerun this command with --%s to overwrite "+
 		"any existing DNS records for this hostname.", overwriteDNSFlag)
+	deprecatedClassicTunnelErr = fmt.Errorf("Classic tunnels have been deprecated, please use Named Tunnels. (https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/install-and-setup/tunnel-guide/)")
 )
 
 func Flags() []cli.Flag {
@@ -166,21 +177,54 @@ func TunnelCommand(c *cli.Context) error {
 	if err != nil {
 		return err
 	}
-	if name := c.String("name"); name != "" { // Start a named tunnel
+
+	// Run a adhoc named tunnel
+	// Allows for the creation, routing (optional), and startup of a tunnel in one command
+	// --name required
+	// --url or --hello-world required
+	// --hostname optional
+	if name := c.String("name"); name != "" {
+		hostname, err := validation.ValidateHostname(c.String("hostname"))
+		if err != nil {
+			return errors.Wrap(err, "Invalid hostname provided")
+		}
+		url := c.String("url")
+		if url == hostname && url != "" && hostname != "" {
+			return fmt.Errorf("hostname and url shouldn't match. See --help for more information")
+		}
+
 		return runAdhocNamedTunnel(sc, name, c.String(CredFileFlag))
 	}
+
+	// Run a quick tunnel
+	// A unauthenticated named tunnel hosted on <random>.<quick-tunnels-service>.com
+	// We don't support running proxy-dns and a quick tunnel at the same time as the same process
+	shouldRunQuickTunnel := c.IsSet("url") || c.IsSet(ingress.HelloWorldFlag)
+	if !c.IsSet("proxy-dns") && c.String("quick-service") != "" && shouldRunQuickTunnel {
+		return RunQuickTunnel(sc)
+	}
+
+	// If user provides a config, check to see if they meant to use `tunnel run` instead
 	if ref := config.GetConfiguration().TunnelID; ref != "" {
 		return fmt.Errorf("Use `cloudflared tunnel run` to start tunnel %s", ref)
 	}
 
-	// Unauthenticated named tunnel on <random>.<quick-tunnels-service>.com
-	// For now, default to legacy setup unless quick-service is specified
-	if !dnsProxyStandAlone(c, nil) && c.String("hostname") == "" && c.String("quick-service") != "" {
-		return RunQuickTunnel(sc)
+	// Classic tunnel usage is no longer supported
+	if c.String("hostname") != "" {
+		return deprecatedClassicTunnelErr
 	}
 
-	// Start a classic tunnel
-	return runClassicTunnel(sc)
+	if c.IsSet("proxy-dns") {
+		if shouldRunQuickTunnel {
+			return fmt.Errorf("running a quick tunnel with `proxy-dns` is not supported")
+		}
+		// NamedTunnelProperties are nil since proxy dns server does not need it.
+		// This is supported for legacy reasons: dns proxy server is not a tunnel and ideally should
+		// not run as part of cloudflared tunnel.
+		return StartServer(sc.c, buildInfo, nil, sc.log)
+	}
+
+	return errors.New(tunnelCmdErrorMessage)
 }
 
 func Init(info *cliutil.BuildInfo, gracefulShutdown chan struct{}) {
@@ -215,11 +259,6 @@ func runAdhocNamedTunnel(sc *subcommandContext, name, credentialsOutputPath stri
 	return nil
 }
 
-// runClassicTunnel creates a "classic" non-named tunnel
-func runClassicTunnel(sc *subcommandContext) error {
-	return StartServer(sc.c, buildInfo, nil, sc.log)
-}
-
 func routeFromFlag(c *cli.Context) (route cfapi.HostnameRoute, ok bool) {
 	if hostname := c.String("hostname"); hostname != "" {
 		if lbPool := c.String("lb-pool"); lbPool != "" {
@@ -236,12 +275,19 @@ func StartServer(
 	namedTunnel *connection.NamedTunnelProperties,
 	log *zerolog.Logger,
 ) error {
-	_ = raven.SetDSN(sentryDSN)
+	err := sentry.Init(sentry.ClientOptions{
+		Dsn:     sentryDSN,
+		Release: c.App.Version,
+	})
+	if err != nil {
+		return err
+	}
 	var wg sync.WaitGroup
 	listeners := gracenet.Net{}
 	errC := make(chan error)
 
-	if config.GetConfiguration().Source() == "" {
+	// Only log for locally configured tunnels (Token is blank).
+	if config.GetConfiguration().Source() == "" && c.String(TunnelTokenFlag) == "" {
 		log.Info().Msg(config.ErrNoConfigFile.Error())
 	}
 
@@ -314,21 +360,13 @@ func StartServer(
 		errC <- autoupdater.Run(ctx)
 	}()
 
-	// Serve DNS proxy stand-alone if no hostname or tag or app is going to run
+	// Serve DNS proxy stand-alone if no tunnel type (quick, adhoc, named) is going to run
 	if dnsProxyStandAlone(c, namedTunnel) {
 		connectedSignal.Notify()
 		// no grace period, handle SIGINT/SIGTERM immediately
 		return waitToShutdown(&wg, cancel, errC, graceShutdownC, 0, log)
 	}
 
-	url := c.String("url")
-	hostname := c.String("hostname")
-	if url == hostname && url != "" && hostname != "" {
-		errText := "hostname and url shouldn't match. See --help for more information"
-		log.Error().Msg(errText)
-		return fmt.Errorf(errText)
-	}
-
 	logTransport := logger.CreateTransportLoggerFromContext(c, logger.EnableTerminalLog)
 
 	observer := connection.NewObserver(log, logTransport)
@@ -372,7 +410,12 @@ func StartServer(
 		defer wg.Done()
 		readinessServer := metrics.NewReadyServer(log, clientID)
 		observer.RegisterSink(readinessServer)
-		errC <- metrics.ServeMetrics(metricsListener, ctx.Done(), readinessServer, quickTunnelURL, orchestrator, log)
+		metricsConfig := metrics.Config{
+			ReadyServer:         readinessServer,
+			QuickTunnelHostname: quickTunnelURL,
+			Orchestrator:        orchestrator,
+		}
+		errC <- metrics.ServeMetrics(metricsListener, ctx, metricsConfig, log)
 	}()
 
 	reconnectCh := make(chan supervisor.ReconnectSignal, c.Int("ha-connections"))
@@ -511,11 +554,17 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 		}),
 		altsrc.NewStringFlag(&cli.StringFlag{
 			Name:    "edge-ip-version",
-			Usage:   "Cloudflare Edge ip address version to connect with. {4, 6, auto}",
+			Usage:   "Cloudflare Edge IP address version to connect with. {4, 6, auto}",
 			EnvVars: []string{"TUNNEL_EDGE_IP_VERSION"},
 			Value:   "4",
 			Hidden:  false,
 		}),
+		altsrc.NewStringFlag(&cli.StringFlag{
+			Name:    "edge-bind-address",
+			Usage:   "Bind to IP address for outgoing connections to Cloudflare Edge.",
+			EnvVars: []string{"TUNNEL_EDGE_BIND_ADDRESS"},
+			Hidden:  false,
+		}),
 		altsrc.NewStringFlag(&cli.StringFlag{
 			Name:    tlsconfig.CaCertFlag,
 			Usage:   "Certificate Authority authenticating connections with Cloudflare's edge network.",
@@ -591,6 +640,12 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			Value:  5,
 			Hidden: true,
 		}),
+		altsrc.NewIntFlag(&cli.IntFlag{
+			Name:   "max-edge-addr-retries",
+			Usage:  "Maximum number of times to retry on edge addrs before falling back to a lower protocol",
+			Value:  8,
+			Hidden: true,
+		}),
 		// Note TUN-3758 , we use Int because UInt is not supported with altsrc
 		altsrc.NewIntFlag(&cli.IntFlag{
 			Name:    "retries",
@@ -665,6 +720,13 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			EnvVars: []string{"TUNNEL_MAX_FETCH_SIZE"},
 			Hidden:  true,
 		}),
+		altsrc.NewBoolFlag(&cli.BoolFlag{
+			Name:    "post-quantum",
+			Usage:   "When given creates an experimental post-quantum secure tunnel",
+			Aliases: []string{"pq"},
+			EnvVars: []string{"TUNNEL_POST_QUANTUM"},
+			Hidden:  FipsEnabled,
+		}),
 		selectProtocolFlag,
 		overwriteDNSFlag,
 	}...)
@@ -727,7 +789,7 @@ func configureProxyFlags(shouldHide bool) []cli.Flag {
 			Hidden:  shouldHide,
 		}),
 		altsrc.NewBoolFlag(&cli.BoolFlag{
-			Name:    "hello-world",
+			Name:    ingress.HelloWorldFlag,
 			Value:   false,
 			Usage:   "Run Hello World Server",
 			EnvVars: []string{"TUNNEL_HELLO_WORLD"},

cmd/cloudflared/tunnel/configuration.go

@@ -21,19 +21,15 @@ import (
 	"golang.org/x/crypto/ssh/terminal"
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
-	"github.com/cloudflare/cloudflared/edgediscovery/allregions"
-	"github.com/cloudflare/cloudflared/packet"
-
 	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/edgediscovery"
-	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/edgediscovery/allregions"
 	"github.com/cloudflare/cloudflared/ingress"
 	"github.com/cloudflare/cloudflared/orchestration"
 	"github.com/cloudflare/cloudflared/supervisor"
 	"github.com/cloudflare/cloudflared/tlsconfig"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
-	"github.com/cloudflare/cloudflared/validation"
 )
 
 const LogFieldOriginCertPath = "originCertPath"
@@ -44,12 +40,10 @@ var (
 	serviceUrl      = developerPortal + "/reference/service/"
 	argumentsUrl    = developerPortal + "/reference/arguments/"
 
-	LogFieldHostname = "hostname"
-
 	secretFlags     = [2]*altsrc.StringFlag{credentialsContentsFlag, tunnelTokenFlag}
-	defaultFeatures = []string{supervisor.FeatureAllowRemoteConfig, supervisor.FeatureSerializedHeaders, supervisor.FeatureDatagramV2}
+	defaultFeatures = []string{supervisor.FeatureAllowRemoteConfig, supervisor.FeatureSerializedHeaders, supervisor.FeatureDatagramV2, supervisor.FeatureQUICSupportEOF}
 
-	configFlags = []string{"autoupdate-freq", "no-autoupdate", "retries", "protocol", "loglevel", "transport-loglevel", "origincert", "metrics", "metrics-update-freq", "edge-ip-version"}
+	configFlags = []string{"autoupdate-freq", "no-autoupdate", "retries", "protocol", "loglevel", "transport-loglevel", "origincert", "metrics", "metrics-update-freq", "edge-ip-version", "edge-bind-address"}
 )
 
 // returns the first path that contains a cert.pem file. If none of the DefaultConfigSearchDirectories
@@ -128,7 +122,10 @@ func isSecretEnvVar(key string) bool {
 }
 
 func dnsProxyStandAlone(c *cli.Context, namedTunnel *connection.NamedTunnelProperties) bool {
-	return c.IsSet("proxy-dns") && (!c.IsSet("hostname") && !c.IsSet("tag") && !c.IsSet("hello-world") && namedTunnel == nil)
+	return c.IsSet("proxy-dns") &&
+		!(c.IsSet("name") || // adhoc-named tunnel
+			c.IsSet(ingress.HelloWorldFlag) || // quick or named tunnel
+			namedTunnel != nil) // named tunnel
 }
 
 func findOriginCert(originCertPath string, log *zerolog.Logger) (string, error) {
@@ -194,37 +191,19 @@ func prepareTunnelConfig(
 	observer *connection.Observer,
 	namedTunnel *connection.NamedTunnelProperties,
 ) (*supervisor.TunnelConfig, *orchestration.Config, error) {
-	isNamedTunnel := namedTunnel != nil
-
-	configHostname := c.String("hostname")
-	hostname, err := validation.ValidateHostname(configHostname)
+	clientID, err := uuid.NewRandom()
 	if err != nil {
-		log.Err(err).Str(LogFieldHostname, configHostname).Msg("Invalid hostname")
-		return nil, nil, errors.Wrap(err, "Invalid hostname")
+		return nil, nil, errors.Wrap(err, "can't generate connector UUID")
 	}
-	clientID := c.String("id")
-	if !c.IsSet("id") {
-		clientID, err = generateRandomClientID(log)
-		if err != nil {
-			return nil, nil, err
-		}
-	}
-
+	log.Info().Msgf("Generated Connector ID: %s", clientID)
 	tags, err := NewTagSliceFromCLI(c.StringSlice("tag"))
 	if err != nil {
 		log.Err(err).Msg("Tag parse failure")
 		return nil, nil, errors.Wrap(err, "Tag parse failure")
 	}
-
-	tags = append(tags, tunnelpogs.Tag{Name: "ID", Value: clientID})
-
-	var (
-		ingressRules  ingress.Ingress
-		classicTunnel *connection.ClassicTunnelProperties
-	)
+	tags = append(tags, tunnelpogs.Tag{Name: "ID", Value: clientID.String()})
 
 	transportProtocol := c.String("protocol")
-
 	needPQ := c.Bool("post-quantum")
 	if needPQ {
 		if FipsEnabled {
@@ -237,81 +216,23 @@ func prepareTunnelConfig(
 		transportProtocol = connection.QUIC.String()
 	}
 
-	protocolFetcher := edgediscovery.ProtocolPercentage
-
+	features := dedup(append(c.StringSlice("features"), defaultFeatures...))
+	if needPQ {
+		features = append(features, supervisor.FeaturePostQuantum)
+	}
+	namedTunnel.Client = tunnelpogs.ClientInfo{
+		ClientID: clientID[:],
+		Features: features,
+		Version:  info.Version(),
+		Arch:     info.OSArch(),
+	}
 	cfg := config.GetConfiguration()
-	if isNamedTunnel {
-		clientUUID, err := uuid.NewRandom()
-		if err != nil {
-			return nil, nil, errors.Wrap(err, "can't generate connector UUID")
-		}
-		log.Info().Msgf("Generated Connector ID: %s", clientUUID)
-		features := append(c.StringSlice("features"), defaultFeatures...)
-		if needPQ {
-			features = append(features, supervisor.FeaturePostQuantum)
-		}
-		if c.IsSet(TunnelTokenFlag) {
-			if transportProtocol == connection.AutoSelectFlag {
-				protocolFetcher = func() (edgediscovery.ProtocolPercents, error) {
-					// If the Tunnel is remotely managed and no protocol is set, we prefer QUIC, but still allow fall-back.
-					preferQuic := []edgediscovery.ProtocolPercent{
-						{
-							Protocol:   connection.QUIC.String(),
-							Percentage: 100,
-						},
-						{
-							Protocol:   connection.HTTP2.String(),
-							Percentage: 100,
-						},
-					}
-					return preferQuic, nil
-				}
-			}
-			log.Info().Msg("Will be fetching remotely managed configuration from Cloudflare API. Defaulting to protocol: quic")
-		}
-		namedTunnel.Client = tunnelpogs.ClientInfo{
-			ClientID: clientUUID[:],
-			Features: dedup(features),
-			Version:  info.Version(),
-			Arch:     info.OSArch(),
-		}
-		ingressRules, err = ingress.ParseIngress(cfg)
-		if err != nil && err != ingress.ErrNoIngressRules {
-			return nil, nil, err
-		}
-		if !ingressRules.IsEmpty() && c.IsSet("url") {
-			return nil, nil, ingress.ErrURLIncompatibleWithIngress
-		}
-	} else {
-
-		originCertPath := c.String("origincert")
-		originCertLog := log.With().
-			Str(LogFieldOriginCertPath, originCertPath).
-			Logger()
-
-		originCert, err := getOriginCert(originCertPath, &originCertLog)
-		if err != nil {
-			return nil, nil, errors.Wrap(err, "Error getting origin cert")
-		}
-
-		classicTunnel = &connection.ClassicTunnelProperties{
-			Hostname:   hostname,
-			OriginCert: originCert,
-			// turn off use of reconnect token and auth refresh when using named tunnels
-			UseReconnectToken: !isNamedTunnel && c.Bool("use-reconnect-token"),
-		}
+	ingressRules, err := ingress.ParseIngressFromConfigAndCLI(cfg, c, log)
+	if err != nil {
+		return nil, nil, err
 	}
 
-	// Convert single-origin configuration into multi-origin configuration.
-	if ingressRules.IsEmpty() {
-		ingressRules, err = ingress.NewSingleOrigin(c, !isNamedTunnel)
-		if err != nil {
-			return nil, nil, err
-		}
-	}
-
-	warpRoutingEnabled := isWarpRoutingEnabled(cfg.WarpRouting, isNamedTunnel)
-	protocolSelector, err := connection.NewProtocolSelector(transportProtocol, warpRoutingEnabled, namedTunnel, protocolFetcher, supervisor.ResolveTTL, log, c.Bool("post-quantum"))
+	protocolSelector, err := connection.NewProtocolSelector(transportProtocol, namedTunnel.Credentials.AccountTag, c.IsSet(TunnelTokenFlag), c.Bool("post-quantum"), edgediscovery.ProtocolPercentage, connection.ResolveTTL, log)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -337,18 +258,22 @@ func prepareTunnelConfig(
 	if err != nil {
 		return nil, nil, err
 	}
-	muxerConfig := &connection.MuxerConfig{
-		HeartbeatInterval: c.Duration("heartbeat-interval"),
-		// Note TUN-3758 , we use Int because UInt is not supported with altsrc
-		MaxHeartbeats: uint64(c.Int("heartbeat-count")),
-		// Note TUN-3758 , we use Int because UInt is not supported with altsrc
-		CompressionSetting: h2mux.CompressionSetting(uint64(c.Int("compression-quality"))),
-		MetricsUpdateFreq:  c.Duration("metrics-update-freq"),
-	}
 	edgeIPVersion, err := parseConfigIPVersion(c.String("edge-ip-version"))
 	if err != nil {
 		return nil, nil, err
 	}
+	edgeBindAddr, err := parseConfigBindAddress(c.String("edge-bind-address"))
+	if err != nil {
+		return nil, nil, err
+	}
+	if err := testIPBindable(edgeBindAddr); err != nil {
+		return nil, nil, fmt.Errorf("invalid edge-bind-address %s: %v", edgeBindAddr, err)
+	}
+	edgeIPVersion, err = adjustIPVersionByBindAddress(edgeIPVersion, edgeBindAddr)
+	if err != nil {
+		// This is not a fatal error, we just overrode edgeIPVersion
+		log.Warn().Str("edgeIPVersion", edgeIPVersion.String()).Err(err).Msg("Overriding edge-ip-version")
+	}
 
 	var pqKexIdx int
 	if needPQ {
@@ -363,10 +288,11 @@ func prepareTunnelConfig(
 		GracePeriod:     gracePeriod,
 		ReplaceExisting: c.Bool("force"),
 		OSArch:          info.OSArch(),
-		ClientID:        clientID,
+		ClientID:        clientID.String(),
 		EdgeAddrs:       c.StringSlice("edge"),
 		Region:          c.String("region"),
 		EdgeIPVersion:   edgeIPVersion,
+		EdgeBindAddr:    edgeBindAddr,
 		HAConnections:   c.Int("ha-connections"),
 		IncidentLookup:  supervisor.NewIncidentLookup(),
 		IsAutoupdated:   c.Bool("is-autoupdated"),
@@ -377,15 +303,14 @@ func prepareTunnelConfig(
 		Observer:        observer,
 		ReportedVersion: info.Version(),
 		// Note TUN-3758 , we use Int because UInt is not supported with altsrc
-		Retries:          uint(c.Int("retries")),
-		RunFromTerminal:  isRunningFromTerminal(),
-		NamedTunnel:      namedTunnel,
-		ClassicTunnel:    classicTunnel,
-		MuxerConfig:      muxerConfig,
-		ProtocolSelector: protocolSelector,
-		EdgeTLSConfigs:   edgeTLSConfigs,
-		NeedPQ:           needPQ,
-		PQKexIdx:         pqKexIdx,
+		Retries:            uint(c.Int("retries")),
+		RunFromTerminal:    isRunningFromTerminal(),
+		NamedTunnel:        namedTunnel,
+		ProtocolSelector:   protocolSelector,
+		EdgeTLSConfigs:     edgeTLSConfigs,
+		NeedPQ:             needPQ,
+		PQKexIdx:           pqKexIdx,
+		MaxEdgeAddrRetries: uint8(c.Int("max-edge-addr-retries")),
 	}
 	packetConfig, err := newPacketConfig(c, log)
 	if err != nil {
@@ -421,10 +346,6 @@ func gracePeriod(c *cli.Context) (time.Duration, error) {
 	return period, nil
 }
 
-func isWarpRoutingEnabled(warpConfig config.WarpRoutingConfig, isNamedTunnel bool) bool {
-	return warpConfig.Enabled && isNamedTunnel
-}
-
 func isRunningFromTerminal() bool {
 	return terminal.IsTerminal(int(os.Stdout.Fd()))
 }
@@ -463,7 +384,52 @@ func parseConfigIPVersion(version string) (v allregions.ConfigIPVersion, err err
 	return
 }
 
-func newPacketConfig(c *cli.Context, logger *zerolog.Logger) (*packet.GlobalRouterConfig, error) {
+func parseConfigBindAddress(ipstr string) (net.IP, error) {
+	// Unspecified - it's fine
+	if ipstr == "" {
+		return nil, nil
+	}
+	ip := net.ParseIP(ipstr)
+	if ip == nil {
+		return nil, fmt.Errorf("invalid value for edge-bind-address: %s", ipstr)
+	}
+	return ip, nil
+}
+
+func testIPBindable(ip net.IP) error {
+	// "Unspecified" = let OS choose, so always bindable
+	if ip == nil {
+		return nil
+	}
+
+	addr := &net.UDPAddr{IP: ip, Port: 0}
+	listener, err := net.ListenUDP("udp", addr)
+	if err != nil {
+		return err
+	}
+	listener.Close()
+	return nil
+}
+
+func adjustIPVersionByBindAddress(ipVersion allregions.ConfigIPVersion, ip net.IP) (allregions.ConfigIPVersion, error) {
+	if ip == nil {
+		return ipVersion, nil
+	}
+	// https://pkg.go.dev/net#IP.To4: "If ip is not an IPv4 address, To4 returns nil."
+	if ip.To4() != nil {
+		if ipVersion == allregions.IPv6Only {
+			return allregions.IPv4Only, fmt.Errorf("IPv4 bind address is specified, but edge-ip-version is IPv6")
+		}
+		return allregions.IPv4Only, nil
+	} else {
+		if ipVersion == allregions.IPv4Only {
+			return allregions.IPv6Only, fmt.Errorf("IPv6 bind address is specified, but edge-ip-version is IPv4")
+		}
+		return allregions.IPv6Only, nil
+	}
+}
+
+func newPacketConfig(c *cli.Context, logger *zerolog.Logger) (*ingress.GlobalRouterConfig, error) {
 	ipv4Src, err := determineICMPv4Src(c.String("icmpv4-src"), logger)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to determine IPv4 source address for ICMP proxy")
@@ -484,7 +450,7 @@ func newPacketConfig(c *cli.Context, logger *zerolog.Logger) (*packet.GlobalRout
 	if err != nil {
 		return nil, err
 	}
-	return &packet.GlobalRouterConfig{
+	return &ingress.GlobalRouterConfig{
 		ICMPRouter: icmpRouter,
 		IPv4Src:    ipv4Src,
 		IPv6Src:    ipv6Src,

cmd/cloudflared/tunnel/configuration_test.go

@@ -9,6 +9,7 @@ import (
 	"crypto/x509"
 	"crypto/x509/pkix"
 	"encoding/asn1"
+	"net"
 	"os"
 	"testing"
 
@@ -214,3 +215,23 @@ func getCertPoolSubjects(certPool *x509.CertPool) ([]*pkix.Name, error) {
 func isUnrecoverableError(err error) bool {
 	return err != nil && err.Error() != "crypto/x509: system root pool is not available on Windows"
 }
+
+func TestTestIPBindable(t *testing.T) {
+	assert.Nil(t, testIPBindable(nil))
+
+	// Public services - if one of these IPs is on the machine, the test environment is too weird
+	assert.NotNil(t, testIPBindable(net.ParseIP("8.8.8.8")))
+	assert.NotNil(t, testIPBindable(net.ParseIP("1.1.1.1")))
+
+	addrs, err := net.InterfaceAddrs()
+	if err != nil {
+		t.Fatal(err)
+	}
+	for i, addr := range addrs {
+		if i >= 3 {
+			break
+		}
+		ip := addr.(*net.IPNet).IP
+		assert.Nil(t, testIPBindable(ip))
+	}
+}

cmd/cloudflared/tunnel/subcommand_context.go

@@ -74,7 +74,7 @@ func (sc *subcommandContext) client() (cfapi.Client, error) {
 		sc.c.String("api-url"),
 		credential.cert.AccountID,
 		credential.cert.ZoneID,
-		credential.cert.ServiceKey,
+		credential.cert.APIToken,
 		userAgent,
 		sc.log,
 	)

cmd/cloudflared/tunnel/subcommand_context_vnets.go

@@ -23,12 +23,12 @@ func (sc *subcommandContext) listVirtualNetworks(filter *cfapi.VnetFilter) ([]*c
 	return client.ListVirtualNetworks(filter)
 }
 
-func (sc *subcommandContext) deleteVirtualNetwork(vnetId uuid.UUID) error {
+func (sc *subcommandContext) deleteVirtualNetwork(vnetId uuid.UUID, force bool) error {
 	client, err := sc.client()
 	if err != nil {
 		return errors.Wrap(err, noClientMsg)
 	}
-	return client.DeleteVirtualNetwork(vnetId)
+	return client.DeleteVirtualNetwork(vnetId, force)
 }
 
 func (sc *subcommandContext) updateVirtualNetwork(vnetId uuid.UUID, updates cfapi.UpdateVirtualNetwork) error {

cmd/cloudflared/tunnel/subcommands.go

@@ -783,7 +783,7 @@ func tokenCommand(c *cli.Context) error {
 		return err
 	}
 
-	fmt.Printf("%s", encodedToken)
+	fmt.Println(encodedToken)
 	return nil
 }
 
@@ -821,7 +821,7 @@ Further information about managing Cloudflare WARP traffic to your tunnel is ava
 				Name:        "lb",
 				Action:      cliutil.ConfiguredAction(routeLbCommand),
 				Usage:       "Use this tunnel as a load balancer origin, creating pool and load balancer if necessary",
-				UsageText:   "cloudflared tunnel route dns [TUNNEL] [HOSTNAME] [LB-POOL]",
+				UsageText:   "cloudflared tunnel route lb [TUNNEL] [HOSTNAME] [LB-POOL-NAME]",
 				Description: `Creates Load Balancer with an origin pool that points to the tunnel.`,
 			},
 			buildRouteIPSubcommand(),

cmd/cloudflared/tunnel/vnets_subcommands.go

@@ -33,6 +33,12 @@ var (
 		Aliases: []string{"c"},
 		Usage:   "A new comment describing the purpose of the virtual network.",
 	}
+	vnetForceDeleteFlag = &cli.BoolFlag{
+		Name:    "force",
+		Aliases: []string{"f"},
+		Usage: "Force the deletion of the virtual network even if it is being relied upon by other resources. Those" +
+			"resources will either be deleted (e.g. IP Routes) or moved to the current default virutal network.",
+	}
 )
 
 func buildVirtualNetworkSubcommand(hidden bool) *cli.Command {
@@ -82,6 +88,7 @@ be the current default.`,
 				UsageText: "cloudflared tunnel [--config FILEPATH] network delete VIRTUAL_NETWORK",
 				Description: `Deletes the virtual network (given its ID or name). This is only possible if that virtual network is unused. 
 A virtual network may be used by IP routes or by WARP devices.`,
+				Flags:  []cli.Flag{vnetForceDeleteFlag},
 				Hidden: hidden,
 			},
 			{
@@ -188,7 +195,7 @@ func deleteVirtualNetworkCommand(c *cli.Context) error {
 	if err != nil {
 		return err
 	}
-	if c.NArg() != 1 {
+	if c.NArg() < 1 {
 		return errors.New("You must supply exactly one argument, either the ID or name of the virtual network to delete")
 	}
 
@@ -198,7 +205,12 @@ func deleteVirtualNetworkCommand(c *cli.Context) error {
 		return err
 	}
 
-	if err := sc.deleteVirtualNetwork(vnetId); err != nil {
+	forceDelete := false
+	if c.IsSet(vnetForceDeleteFlag.Name) {
+		forceDelete = c.Bool(vnetForceDeleteFlag.Name)
+	}
+
+	if err := sc.deleteVirtualNetwork(vnetId, forceDelete); err != nil {
 		return errors.Wrap(err, "API error")
 	}
 	fmt.Printf("Successfully deleted virtual network '%s'\n", input)

component-tests/config.py

@@ -30,6 +30,7 @@ class NamedTunnelBaseConfig(BaseConfig):
     tunnel: str = None
     credentials_file: str = None
     ingress: list = None
+    hostname: str = None
 
     def __post_init__(self):
         if self.tunnel is None:
@@ -41,8 +42,10 @@ class NamedTunnelBaseConfig(BaseConfig):
 
     def merge_config(self, additional):
         config = super(NamedTunnelBaseConfig, self).merge_config(additional)
-        config['tunnel'] = self.tunnel
-        config['credentials-file'] = self.credentials_file
+        if 'tunnel' not in config:
+            config['tunnel'] = self.tunnel
+        if 'credentials-file' not in config:
+            config['credentials-file'] = self.credentials_file
         # In some cases we want to override default ingress, such as in config tests
         if 'ingress' not in config:
             config['ingress'] = self.ingress
@@ -61,7 +64,7 @@ class NamedTunnelConfig(NamedTunnelBaseConfig):
                            self.merge_config(additional_config))
 
     def get_url(self):
-        return "https://" + self.ingress[0]['hostname']
+        return "https://" + self.hostname
 
     def base_config(self):
         config = self.full_config.copy()
@@ -84,28 +87,9 @@ class NamedTunnelConfig(NamedTunnelBaseConfig):
     def get_credentials_json(self):
         with open(self.credentials_file) as json_file:
             return json.load(json_file)
-
-
+        
 @dataclass(frozen=True)
-class ClassicTunnelBaseConfig(BaseConfig):
-    hostname: str = None
-    origincert: str = None
-
-    def __post_init__(self):
-        if self.hostname is None:
-            raise TypeError("Field tunnel is not set")
-        if self.origincert is None:
-            raise TypeError("Field credentials_file is not set")
-
-    def merge_config(self, additional):
-        config = super(ClassicTunnelBaseConfig, self).merge_config(additional)
-        config['hostname'] = self.hostname
-        config['origincert'] = self.origincert
-        return config
-
-
-@dataclass(frozen=True)
-class ClassicTunnelConfig(ClassicTunnelBaseConfig):
+class QuickTunnelConfig(BaseConfig):
     full_config: dict = None
     additional_config: InitVar[dict] = {}
 
@@ -115,10 +99,6 @@ class ClassicTunnelConfig(ClassicTunnelBaseConfig):
         object.__setattr__(self, 'full_config',
                            self.merge_config(additional_config))
 
-    def get_url(self):
-        return "https://" + self.hostname
-
-
 @dataclass(frozen=True)
 class ProxyDnsConfig(BaseConfig):
     full_config = {

component-tests/conftest.py

@@ -5,14 +5,14 @@ from time import sleep
 import pytest
 import yaml
 
-from config import NamedTunnelConfig, ClassicTunnelConfig, ProxyDnsConfig
+from config import NamedTunnelConfig, ProxyDnsConfig, QuickTunnelConfig
 from constants import BACKOFF_SECS, PROXY_DNS_PORT
 from util import LOGGER
 
 
 class CfdModes(Enum):
     NAMED = auto()
-    CLASSIC = auto()
+    QUICK = auto()
     PROXY_DNS = auto()
 
 
@@ -26,7 +26,7 @@ def component_tests_config():
         config = yaml.safe_load(stream)
         LOGGER.info(f"component tests base config {config}")
 
-        def _component_tests_config(additional_config={}, cfd_mode=CfdModes.NAMED, run_proxy_dns=True):
+        def _component_tests_config(additional_config={}, cfd_mode=CfdModes.NAMED, run_proxy_dns=True, provide_ingress=True):
             if run_proxy_dns:
                 # Regression test for TUN-4177, running with proxy-dns should not prevent tunnels from running.
                 # So we run all tests with it.
@@ -36,18 +36,25 @@ def component_tests_config():
                 additional_config.pop("proxy-dns", None)
                 additional_config.pop("proxy-dns-port", None)
 
+            # Allows the ingress rules to be omitted from the provided config
+            ingress = []
+            if provide_ingress:
+                ingress = config['ingress']
+
+            # Provide the hostname to allow routing to the tunnel even if the ingress rule isn't defined in the config
+            hostname = config['ingress'][0]['hostname']
+
             if cfd_mode is CfdModes.NAMED:
                 return NamedTunnelConfig(additional_config=additional_config,
                                          cloudflared_binary=config['cloudflared_binary'],
                                          tunnel=config['tunnel'],
                                          credentials_file=config['credentials_file'],
-                                         ingress=config['ingress'])
-            elif cfd_mode is CfdModes.CLASSIC:
-                return ClassicTunnelConfig(
-                    additional_config=additional_config, cloudflared_binary=config['cloudflared_binary'],
-                    hostname=config['classic_hostname'], origincert=config['origincert'])
+                                         ingress=ingress,
+                                         hostname=hostname)
             elif cfd_mode is CfdModes.PROXY_DNS:
                 return ProxyDnsConfig(cloudflared_binary=config['cloudflared_binary'])
+            elif cfd_mode is CfdModes.QUICK:
+                return QuickTunnelConfig(additional_config=additional_config, cloudflared_binary=config['cloudflared_binary'])
             else:
                 raise Exception(f"Unknown cloudflared mode {cfd_mode}")
 

component-tests/constants.py

@@ -7,4 +7,4 @@ PROXY_DNS_PORT = 9053
 
 
 def protocols():
-    return ["h2mux", "http2", "quic"]
+    return ["http2", "quic"]

component-tests/test_proxy_dns.py

@@ -12,18 +12,12 @@ class TestProxyDns:
     def test_proxy_dns_with_named_tunnel(self, tmp_path, component_tests_config):
         run_test_scenario(tmp_path, component_tests_config, CfdModes.NAMED, run_proxy_dns=True)
 
-    def test_proxy_dns_with_classic_tunnel(self, tmp_path, component_tests_config):
-        run_test_scenario(tmp_path, component_tests_config, CfdModes.CLASSIC, run_proxy_dns=True)
-
     def test_proxy_dns_alone(self, tmp_path, component_tests_config):
         run_test_scenario(tmp_path, component_tests_config, CfdModes.PROXY_DNS, run_proxy_dns=True)
 
     def test_named_tunnel_alone(self, tmp_path, component_tests_config):
         run_test_scenario(tmp_path, component_tests_config, CfdModes.NAMED, run_proxy_dns=False)
 
-    def test_classic_tunnel_alone(self, tmp_path, component_tests_config):
-        run_test_scenario(tmp_path, component_tests_config, CfdModes.CLASSIC, run_proxy_dns=False)
-
 
 def run_test_scenario(tmp_path, component_tests_config, cfd_mode, run_proxy_dns):
     expect_proxy_dns = run_proxy_dns
@@ -33,10 +27,6 @@ def run_test_scenario(tmp_path, component_tests_config, cfd_mode, run_proxy_dns)
         expect_tunnel = True
         pre_args = ["tunnel"]
         args = ["run"]
-    elif cfd_mode == CfdModes.CLASSIC:
-        expect_tunnel = True
-        pre_args = []
-        args = []
     elif cfd_mode == CfdModes.PROXY_DNS:
         expect_proxy_dns = True
         pre_args = []

component-tests/test_quicktunnels.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python
+from conftest import CfdModes
+from constants import METRICS_PORT
+from util import LOGGER, start_cloudflared, wait_tunnel_ready, get_quicktunnel_url, send_requests
+
+class TestQuickTunnels:
+    def test_quick_tunnel(self, tmp_path, component_tests_config):
+        config = component_tests_config(cfd_mode=CfdModes.QUICK, run_proxy_dns=False)
+        LOGGER.debug(config)
+        with start_cloudflared(tmp_path, config, cfd_args=["--hello-world"], new_process=True):
+            wait_tunnel_ready(require_min_connections=4)
+            url = get_quicktunnel_url()
+            send_requests(url, 3, True)
+    
+    def test_quick_tunnel_url(self, tmp_path, component_tests_config):
+        config = component_tests_config(cfd_mode=CfdModes.QUICK, run_proxy_dns=False)
+        LOGGER.debug(config)
+        with start_cloudflared(tmp_path, config, cfd_args=["--url", f"http://localhost:{METRICS_PORT}/"], new_process=True):
+            wait_tunnel_ready()
+            url = get_quicktunnel_url()
+            send_requests(url+"/ready", 3, True)
+
+    def test_quick_tunnel_proxy_dns_url(self, tmp_path, component_tests_config):
+        config = component_tests_config(cfd_mode=CfdModes.QUICK, run_proxy_dns=True)
+        LOGGER.debug(config)
+        failed_start = start_cloudflared(tmp_path, config, cfd_args=["--url", f"http://localhost:{METRICS_PORT}/"], expect_success=False)
+        assert failed_start.returncode == 1, "Expected cloudflared to fail to run with `proxy-dns` and `hello-world`"
+    
+    def test_quick_tunnel_proxy_dns_hello_world(self, tmp_path, component_tests_config):
+        config = component_tests_config(cfd_mode=CfdModes.QUICK, run_proxy_dns=True)
+        LOGGER.debug(config)
+        failed_start = start_cloudflared(tmp_path, config, cfd_args=["--hello-world"], expect_success=False)
+        assert failed_start.returncode == 1, "Expected cloudflared to fail to run with `proxy-dns` and `url`"

component-tests/test_reconnect.py

@@ -33,13 +33,6 @@ class TestReconnect:
             # Repeat the test multiple times because some issues only occur after multiple reconnects
             self.assert_reconnect(config, cloudflared, 5)
 
-    def test_classic_reconnect(self, tmp_path, component_tests_config):
-        extra_config = copy.copy(self.extra_config)
-        extra_config["hello-world"] = True
-        config = component_tests_config(additional_config=extra_config, cfd_mode=CfdModes.CLASSIC)
-        with start_cloudflared(tmp_path, config, cfd_args=[], new_process=True, allow_input=True, capture_output=False) as cloudflared:
-            self.assert_reconnect(config, cloudflared, 1)
-
     def send_reconnect(self, cloudflared, secs):
         # Although it is recommended to use the Popen.communicate method, we cannot
         # use it because it blocks on reading stdout and stderr until EOF is reached

component-tests/test_tunnel.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python
+import requests
+from conftest import CfdModes
+from constants import METRICS_PORT, MAX_RETRIES, BACKOFF_SECS
+from retrying import retry
+from util import LOGGER, start_cloudflared, wait_tunnel_ready, send_requests
+
+class TestTunnel:
+    '''Test tunnels with no ingress rules from config.yaml but ingress rules from CLI only'''
+
+    def test_tunnel_hello_world(self, tmp_path, component_tests_config):
+        config = component_tests_config(cfd_mode=CfdModes.NAMED, run_proxy_dns=False, provide_ingress=False)
+        LOGGER.debug(config)
+        with start_cloudflared(tmp_path, config, cfd_args=["run", "--hello-world"], new_process=True):
+            wait_tunnel_ready(tunnel_url=config.get_url(),
+                              require_min_connections=4)
+    
+    def test_tunnel_url(self, tmp_path, component_tests_config):
+        config = component_tests_config(cfd_mode=CfdModes.NAMED, run_proxy_dns=False, provide_ingress=False)
+        LOGGER.debug(config)
+        with start_cloudflared(tmp_path, config, cfd_args=["run", "--url", f"http://localhost:{METRICS_PORT}/"], new_process=True):
+            wait_tunnel_ready(require_min_connections=4)
+            send_requests(config.get_url()+"/ready", 3, True)
+
+    def test_tunnel_no_ingress(self, tmp_path, component_tests_config):
+        '''
+        Running a tunnel with no ingress rules provided from either config.yaml or CLI will still work but return 503
+        for all incoming requests.
+        '''
+        config = component_tests_config(cfd_mode=CfdModes.NAMED, run_proxy_dns=False, provide_ingress=False)
+        LOGGER.debug(config)
+        with start_cloudflared(tmp_path, config, cfd_args=["run"], new_process=True):
+            wait_tunnel_ready(require_min_connections=4)
+            resp = send_request(config.get_url()+"/")
+            assert resp.status_code == 503, "Expected cloudflared to return 503 for all requests with no ingress defined"
+            resp = send_request(config.get_url()+"/test")
+            assert resp.status_code == 503, "Expected cloudflared to return 503 for all requests with no ingress defined"
+
+
+@retry(stop_max_attempt_number=MAX_RETRIES, wait_fixed=BACKOFF_SECS * 1000)
+def send_request(url):
+    with requests.Session() as s:
+        return s.get(url, timeout=BACKOFF_SECS)

component-tests/util.py

@@ -35,7 +35,7 @@ def write_config(directory, config):
 
 
 def start_cloudflared(directory, config, cfd_args=["run"], cfd_pre_args=["tunnel"], new_process=False,
-                      allow_input=False, capture_output=True, root=False, skip_config_flag=False):
+                      allow_input=False, capture_output=True, root=False, skip_config_flag=False, expect_success=True):
 
     config_path = None
     if not skip_config_flag:
@@ -46,7 +46,7 @@ def start_cloudflared(directory, config, cfd_args=["run"], cfd_pre_args=["tunnel
     if new_process:
         return run_cloudflared_background(cmd, allow_input, capture_output)
     # By setting check=True, it will raise an exception if the process exits with non-zero exit code
-    return subprocess.run(cmd, check=True, capture_output=capture_output)
+    return subprocess.run(cmd, check=expect_success, capture_output=capture_output)
 
 
 def cloudflared_cmd(config, config_path, cfd_args, cfd_pre_args, root):
@@ -77,7 +77,18 @@ def run_cloudflared_background(cmd, allow_input, capture_output):
             cfd.terminate()
             if capture_output:
                 LOGGER.info(f"cloudflared log: {cfd.stderr.read()}")
+    
 
+def get_quicktunnel_url():
+    quicktunnel_url = f'http://localhost:{METRICS_PORT}/quicktunnel'
+    with requests.Session() as s:
+        resp = send_request(s, quicktunnel_url, True)
+
+        hostname = resp.json()["hostname"]
+        assert hostname, \
+            f"Quicktunnel endpoint returned {hostname} but we expected a url"
+
+        return f"https://{hostname}"
 
 def wait_tunnel_ready(tunnel_url=None, require_min_connections=1, cfd_logs=None):
     try:

config/configuration.go

@@ -391,7 +391,8 @@ func ReadConfigFile(c *cli.Context, log *zerolog.Logger) (settings *configFileSe
 	log.Debug().Msgf("Loading configuration from %s", configFile)
 	file, err := os.Open(configFile)
 	if err != nil {
-		if os.IsNotExist(err) {
+		// If does not exist and config file was not specificly specified then return ErrNoConfigFile found.
+		if os.IsNotExist(err) && !c.IsSet("config") {
 			err = ErrNoConfigFile
 		}
 		return nil, "", err

connection/connection.go

@@ -142,6 +142,7 @@ type TCPRequest struct {
 	LBProbe   bool
 	FlowID    string
 	CfTraceID string
+	ConnIndex uint8
 }
 
 // ReadWriteAcker is a readwriter with the ability to Acknowledge to the downstream (edge) that the origin has
@@ -199,6 +200,7 @@ func (h *HTTPResponseReadWriteAcker) AckConnection(tracePropagation string) erro
 type ResponseWriter interface {
 	WriteRespHeaders(status int, header http.Header) error
 	AddTrailer(trailerName, trailerValue string)
+	http.ResponseWriter
 	io.Writer
 }
 

connection/connection_test.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/rs/zerolog"
 
+	"github.com/cloudflare/cloudflared/stream"
 	"github.com/cloudflare/cloudflared/tracing"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 	"github.com/cloudflare/cloudflared/websocket"
@@ -140,7 +141,7 @@ func wsEchoEndpoint(w ResponseWriter, r *http.Request) error {
 	}()
 
 	originConn := &echoPipe{reader: readPipe, writer: writePipe}
-	websocket.Stream(wsConn, originConn, &log)
+	stream.Pipe(wsConn, originConn, &log)
 	cancel()
 	wsConn.Close()
 	return nil
@@ -178,7 +179,7 @@ func wsFlakyEndpoint(w ResponseWriter, r *http.Request) error {
 
 	closedAfter := time.Millisecond * time.Duration(rand.Intn(50))
 	originConn := &flakyConn{closeAt: time.Now().Add(closedAfter)}
-	websocket.Stream(wsConn, originConn, &log)
+	stream.Pipe(wsConn, originConn, &log)
 	cancel()
 	wsConn.Close()
 	return nil

connection/control.go

@@ -85,7 +85,7 @@ func (c *controlStream) ServeControlStream(
 		return err
 	}
 
-	c.observer.logServerInfo(c.connIndex, registrationDetails.Location, c.edgeAddress, fmt.Sprintf("Connection %s registered", registrationDetails.UUID))
+	c.observer.logServerInfo(c.connIndex, registrationDetails.Location, c.edgeAddress, fmt.Sprintf("Connection %s registered with protocol: %s", registrationDetails.UUID, c.protocol))
 	c.observer.sendConnectedEvent(c.connIndex, c.protocol, registrationDetails.Location)
 	c.connectedFuse.Connected()
 

connection/h2mux.go

@@ -1,46 +1,17 @@
 package connection
 
 import (
-	"context"
-	"io"
-	"net"
-	"net/http"
 	"time"
 
-	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
-	"golang.org/x/sync/errgroup"
 
 	"github.com/cloudflare/cloudflared/h2mux"
-	"github.com/cloudflare/cloudflared/tracing"
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
-	"github.com/cloudflare/cloudflared/websocket"
 )
 
 const (
-	muxerTimeout      = 5 * time.Second
-	openStreamTimeout = 30 * time.Second
+	muxerTimeout = 5 * time.Second
 )
 
-type h2muxConnection struct {
-	orchestrator Orchestrator
-	gracePeriod  time.Duration
-	muxerConfig  *MuxerConfig
-	muxer        *h2mux.Muxer
-	// connectionID is only used by metrics, and prometheus requires labels to be string
-	connIndexStr string
-	connIndex    uint8
-
-	observer          *Observer
-	gracefulShutdownC <-chan struct{}
-	stoppedGracefully bool
-
-	log *zerolog.Logger
-
-	// newRPCClientFunc allows us to mock RPCs during testing
-	newRPCClientFunc func(context.Context, io.ReadWriteCloser, *zerolog.Logger) NamedTunnelRPCClient
-}
-
 type MuxerConfig struct {
 	HeartbeatInterval  time.Duration
 	MaxHeartbeats      uint64
@@ -59,220 +30,3 @@ func (mc *MuxerConfig) H2MuxerConfig(h h2mux.MuxedStreamHandler, log *zerolog.Lo
 		CompressionQuality: mc.CompressionSetting,
 	}
 }
-
-// NewTunnelHandler returns a TunnelHandler, origin LAN IP and error
-func NewH2muxConnection(
-	orchestrator Orchestrator,
-	gracePeriod time.Duration,
-	muxerConfig *MuxerConfig,
-	edgeConn net.Conn,
-	connIndex uint8,
-	observer *Observer,
-	gracefulShutdownC <-chan struct{},
-	log *zerolog.Logger,
-) (*h2muxConnection, error, bool) {
-	h := &h2muxConnection{
-		orchestrator:      orchestrator,
-		gracePeriod:       gracePeriod,
-		muxerConfig:       muxerConfig,
-		connIndexStr:      uint8ToString(connIndex),
-		connIndex:         connIndex,
-		observer:          observer,
-		gracefulShutdownC: gracefulShutdownC,
-		newRPCClientFunc:  newRegistrationRPCClient,
-		log:               log,
-	}
-
-	// Establish a muxed connection with the edge
-	// Client mux handshake with agent server
-	muxer, err := h2mux.Handshake(edgeConn, edgeConn, *muxerConfig.H2MuxerConfig(h, observer.logTransport), h2mux.ActiveStreams)
-	if err != nil {
-		recoverable := isHandshakeErrRecoverable(err, connIndex, observer)
-		return nil, err, recoverable
-	}
-	h.muxer = muxer
-	return h, nil, false
-}
-
-func (h *h2muxConnection) ServeNamedTunnel(ctx context.Context, namedTunnel *NamedTunnelProperties, connOptions *tunnelpogs.ConnectionOptions, connectedFuse ConnectedFuse) error {
-	errGroup, serveCtx := errgroup.WithContext(ctx)
-	errGroup.Go(func() error {
-		return h.serveMuxer(serveCtx)
-	})
-
-	errGroup.Go(func() error {
-		if err := h.registerNamedTunnel(serveCtx, namedTunnel, connOptions); err != nil {
-			return err
-		}
-		connectedFuse.Connected()
-		return nil
-	})
-
-	errGroup.Go(func() error {
-		h.controlLoop(serveCtx, connectedFuse, true)
-		return nil
-	})
-
-	err := errGroup.Wait()
-	if err == errMuxerStopped {
-		if h.stoppedGracefully {
-			return nil
-		}
-		h.observer.log.Info().Uint8(LogFieldConnIndex, h.connIndex).Msg("Unexpected muxer shutdown")
-	}
-	return err
-}
-
-func (h *h2muxConnection) ServeClassicTunnel(ctx context.Context, classicTunnel *ClassicTunnelProperties, credentialManager CredentialManager, registrationOptions *tunnelpogs.RegistrationOptions, connectedFuse ConnectedFuse) error {
-	errGroup, serveCtx := errgroup.WithContext(ctx)
-	errGroup.Go(func() error {
-		return h.serveMuxer(serveCtx)
-	})
-
-	errGroup.Go(func() (err error) {
-		defer func() {
-			if err == nil {
-				connectedFuse.Connected()
-			}
-		}()
-		if classicTunnel.UseReconnectToken && connectedFuse.IsConnected() {
-			err := h.reconnectTunnel(ctx, credentialManager, classicTunnel, registrationOptions)
-			if err == nil {
-				return nil
-			}
-			// log errors and proceed to RegisterTunnel
-			h.observer.log.Err(err).
-				Uint8(LogFieldConnIndex, h.connIndex).
-				Msg("Couldn't reconnect connection. Re-registering it instead.")
-		}
-		return h.registerTunnel(ctx, credentialManager, classicTunnel, registrationOptions)
-	})
-
-	errGroup.Go(func() error {
-		h.controlLoop(serveCtx, connectedFuse, false)
-		return nil
-	})
-
-	err := errGroup.Wait()
-	if err == errMuxerStopped {
-		if h.stoppedGracefully {
-			return nil
-		}
-		h.observer.log.Info().Uint8(LogFieldConnIndex, h.connIndex).Msg("Unexpected muxer shutdown")
-	}
-	return err
-}
-
-func (h *h2muxConnection) serveMuxer(ctx context.Context) error {
-	// All routines should stop when muxer finish serving. When muxer is shutdown
-	// gracefully, it doesn't return an error, so we need to return errMuxerShutdown
-	// here to notify other routines to stop
-	err := h.muxer.Serve(ctx)
-	if err == nil {
-		return errMuxerStopped
-	}
-	return err
-}
-
-func (h *h2muxConnection) controlLoop(ctx context.Context, connectedFuse ConnectedFuse, isNamedTunnel bool) {
-	updateMetricsTicker := time.NewTicker(h.muxerConfig.MetricsUpdateFreq)
-	defer updateMetricsTicker.Stop()
-	var shutdownCompleted <-chan struct{}
-	for {
-		select {
-		case <-h.gracefulShutdownC:
-			if connectedFuse.IsConnected() {
-				h.unregister(isNamedTunnel)
-			}
-			h.stoppedGracefully = true
-			h.gracefulShutdownC = nil
-			shutdownCompleted = h.muxer.Shutdown()
-
-		case <-shutdownCompleted:
-			return
-
-		case <-ctx.Done():
-			// UnregisterTunnel blocks until the RPC call returns
-			if !h.stoppedGracefully && connectedFuse.IsConnected() {
-				h.unregister(isNamedTunnel)
-			}
-			h.muxer.Shutdown()
-			// don't wait for shutdown to finish when context is closed, this is the hard termination path
-			return
-
-		case <-updateMetricsTicker.C:
-			h.observer.metrics.updateMuxerMetrics(h.connIndexStr, h.muxer.Metrics())
-		}
-	}
-}
-
-func (h *h2muxConnection) newRPCStream(ctx context.Context, rpcName rpcName) (*h2mux.MuxedStream, error) {
-	openStreamCtx, openStreamCancel := context.WithTimeout(ctx, openStreamTimeout)
-	defer openStreamCancel()
-	stream, err := h.muxer.OpenRPCStream(openStreamCtx)
-	if err != nil {
-		return nil, err
-	}
-	return stream, nil
-}
-
-func (h *h2muxConnection) ServeStream(stream *h2mux.MuxedStream) error {
-	respWriter := &h2muxRespWriter{stream}
-
-	req, reqErr := h.newRequest(stream)
-	if reqErr != nil {
-		respWriter.WriteErrorResponse()
-		return reqErr
-	}
-
-	var sourceConnectionType = TypeHTTP
-	if websocket.IsWebSocketUpgrade(req) {
-		sourceConnectionType = TypeWebsocket
-	}
-
-	originProxy, err := h.orchestrator.GetOriginProxy()
-	if err != nil {
-		respWriter.WriteErrorResponse()
-		return err
-	}
-
-	err = originProxy.ProxyHTTP(respWriter, tracing.NewTracedHTTPRequest(req, h.log), sourceConnectionType == TypeWebsocket)
-	if err != nil {
-		respWriter.WriteErrorResponse()
-	}
-	return err
-}
-
-func (h *h2muxConnection) newRequest(stream *h2mux.MuxedStream) (*http.Request, error) {
-	req, err := http.NewRequest("GET", "http://localhost:8080", h2mux.MuxedStreamReader{MuxedStream: stream})
-	if err != nil {
-		return nil, errors.Wrap(err, "Unexpected error from http.NewRequest")
-	}
-	err = H2RequestHeadersToH1Request(stream.Headers, req)
-	if err != nil {
-		return nil, errors.Wrap(err, "invalid request received")
-	}
-	return req, nil
-}
-
-type h2muxRespWriter struct {
-	*h2mux.MuxedStream
-}
-
-func (rp *h2muxRespWriter) AddTrailer(trailerName, trailerValue string) {
-	// do nothing. we don't support trailers over h2mux
-}
-
-func (rp *h2muxRespWriter) WriteRespHeaders(status int, header http.Header) error {
-	headers := H1ResponseToH2ResponseHeaders(status, header)
-	headers = append(headers, h2mux.Header{Name: ResponseMetaHeader, Value: responseMetaHeaderOrigin})
-	return rp.WriteHeaders(headers)
-}
-
-func (rp *h2muxRespWriter) WriteErrorResponse() {
-	_ = rp.WriteHeaders([]h2mux.Header{
-		{Name: ":status", Value: "502"},
-		{Name: ResponseMetaHeader, Value: responseMetaHeaderCfd},
-	})
-	_, _ = rp.Write([]byte("502 Bad Gateway"))
-}

connection/h2mux_test.go

@@ -1,301 +0,0 @@
-package connection
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"net"
-	"net/http"
-	"strconv"
-	"sync"
-	"testing"
-	"time"
-
-	"github.com/gobwas/ws/wsutil"
-	"github.com/rs/zerolog"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-
-	"github.com/cloudflare/cloudflared/h2mux"
-)
-
-var (
-	testMuxerConfig = &MuxerConfig{
-		HeartbeatInterval:  time.Second * 5,
-		MaxHeartbeats:      5,
-		CompressionSetting: 0,
-		MetricsUpdateFreq:  time.Second * 5,
-	}
-)
-
-func newH2MuxConnection(t require.TestingT) (*h2muxConnection, *h2mux.Muxer) {
-	edgeConn, originConn := net.Pipe()
-	edgeMuxChan := make(chan *h2mux.Muxer)
-	go func() {
-		edgeMuxConfig := h2mux.MuxerConfig{
-			Log: &log,
-			Handler: h2mux.MuxedStreamFunc(func(stream *h2mux.MuxedStream) error {
-				// we only expect RPC traffic in client->edge direction, provide minimal support for mocking
-				require.True(t, stream.IsRPCStream())
-				return stream.WriteHeaders([]h2mux.Header{
-					{Name: ":status", Value: "200"},
-				})
-			}),
-		}
-		edgeMux, err := h2mux.Handshake(edgeConn, edgeConn, edgeMuxConfig, h2mux.ActiveStreams)
-		require.NoError(t, err)
-		edgeMuxChan <- edgeMux
-	}()
-	var connIndex = uint8(0)
-	testObserver := NewObserver(&log, &log)
-	h2muxConn, err, _ := NewH2muxConnection(testOrchestrator, testGracePeriod, testMuxerConfig, originConn, connIndex, testObserver, nil, &log)
-	require.NoError(t, err)
-	return h2muxConn, <-edgeMuxChan
-}
-
-func TestServeStreamHTTP(t *testing.T) {
-	tests := []testRequest{
-		{
-			name:           "ok",
-			endpoint:       "/ok",
-			expectedStatus: http.StatusOK,
-			expectedBody:   []byte(http.StatusText(http.StatusOK)),
-		},
-		{
-			name:           "large_file",
-			endpoint:       "/large_file",
-			expectedStatus: http.StatusOK,
-			expectedBody:   testLargeResp,
-		},
-		{
-			name:           "Bad request",
-			endpoint:       "/400",
-			expectedStatus: http.StatusBadRequest,
-			expectedBody:   []byte(http.StatusText(http.StatusBadRequest)),
-		},
-		{
-			name:           "Internal server error",
-			endpoint:       "/500",
-			expectedStatus: http.StatusInternalServerError,
-			expectedBody:   []byte(http.StatusText(http.StatusInternalServerError)),
-		},
-		{
-			name:           "Proxy error",
-			endpoint:       "/error",
-			expectedStatus: http.StatusBadGateway,
-			expectedBody:   nil,
-			isProxyError:   true,
-		},
-	}
-
-	ctx, cancel := context.WithCancel(context.Background())
-	h2muxConn, edgeMux := newH2MuxConnection(t)
-
-	var wg sync.WaitGroup
-	wg.Add(2)
-	go func() {
-		defer wg.Done()
-		_ = edgeMux.Serve(ctx)
-	}()
-	go func() {
-		defer wg.Done()
-		err := h2muxConn.serveMuxer(ctx)
-		require.Error(t, err)
-	}()
-
-	for _, test := range tests {
-		headers := []h2mux.Header{
-			{
-				Name:  ":path",
-				Value: test.endpoint,
-			},
-		}
-		stream, err := edgeMux.OpenStream(ctx, headers, nil)
-		require.NoError(t, err)
-		require.True(t, hasHeader(stream, ":status", strconv.Itoa(test.expectedStatus)))
-
-		if test.isProxyError {
-			assert.True(t, hasHeader(stream, ResponseMetaHeader, responseMetaHeaderCfd))
-		} else {
-			assert.True(t, hasHeader(stream, ResponseMetaHeader, responseMetaHeaderOrigin))
-			body := make([]byte, len(test.expectedBody))
-			_, err = stream.Read(body)
-			require.NoError(t, err)
-			require.Equal(t, test.expectedBody, body)
-		}
-	}
-	cancel()
-	wg.Wait()
-}
-
-func TestServeStreamWS(t *testing.T) {
-	ctx, cancel := context.WithCancel(context.Background())
-	h2muxConn, edgeMux := newH2MuxConnection(t)
-
-	var wg sync.WaitGroup
-	wg.Add(2)
-	go func() {
-		defer wg.Done()
-		edgeMux.Serve(ctx)
-	}()
-	go func() {
-		defer wg.Done()
-		err := h2muxConn.serveMuxer(ctx)
-		require.Error(t, err)
-	}()
-
-	headers := []h2mux.Header{
-		{
-			Name:  ":path",
-			Value: "/ws/echo",
-		},
-		{
-			Name:  "connection",
-			Value: "upgrade",
-		},
-		{
-			Name:  "upgrade",
-			Value: "websocket",
-		},
-	}
-
-	readPipe, writePipe := io.Pipe()
-	stream, err := edgeMux.OpenStream(ctx, headers, readPipe)
-	require.NoError(t, err)
-
-	require.True(t, hasHeader(stream, ":status", strconv.Itoa(http.StatusSwitchingProtocols)))
-	assert.True(t, hasHeader(stream, ResponseMetaHeader, responseMetaHeaderOrigin))
-
-	data := []byte("test websocket")
-	err = wsutil.WriteClientBinary(writePipe, data)
-	require.NoError(t, err)
-
-	respBody, err := wsutil.ReadServerBinary(stream)
-	require.NoError(t, err)
-	require.Equal(t, data, respBody, fmt.Sprintf("Expect %s, got %s", string(data), string(respBody)))
-
-	cancel()
-	wg.Wait()
-}
-
-func TestGracefulShutdownH2Mux(t *testing.T) {
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	h2muxConn, edgeMux := newH2MuxConnection(t)
-
-	shutdownC := make(chan struct{})
-	unregisteredC := make(chan struct{})
-	h2muxConn.gracefulShutdownC = shutdownC
-	h2muxConn.newRPCClientFunc = func(_ context.Context, _ io.ReadWriteCloser, _ *zerolog.Logger) NamedTunnelRPCClient {
-		return &mockNamedTunnelRPCClient{
-			registered:   nil,
-			unregistered: unregisteredC,
-		}
-	}
-
-	var wg sync.WaitGroup
-	wg.Add(3)
-	go func() {
-		defer wg.Done()
-		_ = edgeMux.Serve(ctx)
-	}()
-	go func() {
-		defer wg.Done()
-		_ = h2muxConn.serveMuxer(ctx)
-	}()
-
-	go func() {
-		defer wg.Done()
-		h2muxConn.controlLoop(ctx, &mockConnectedFuse{}, true)
-	}()
-
-	time.Sleep(100 * time.Millisecond)
-	close(shutdownC)
-
-	select {
-	case <-unregisteredC:
-		break // ok
-	case <-time.Tick(time.Second):
-		assert.Fail(t, "timed out waiting for control loop to unregister")
-	}
-
-	cancel()
-	wg.Wait()
-
-	assert.True(t, h2muxConn.stoppedGracefully)
-	assert.Nil(t, h2muxConn.gracefulShutdownC)
-}
-
-func hasHeader(stream *h2mux.MuxedStream, name, val string) bool {
-	for _, header := range stream.Headers {
-		if header.Name == name && header.Value == val {
-			return true
-		}
-	}
-	return false
-}
-
-func benchmarkServeStreamHTTPSimple(b *testing.B, test testRequest) {
-	ctx, cancel := context.WithCancel(context.Background())
-	h2muxConn, edgeMux := newH2MuxConnection(b)
-
-	var wg sync.WaitGroup
-	wg.Add(2)
-	go func() {
-		defer wg.Done()
-		edgeMux.Serve(ctx)
-	}()
-	go func() {
-		defer wg.Done()
-		err := h2muxConn.serveMuxer(ctx)
-		require.Error(b, err)
-	}()
-
-	headers := []h2mux.Header{
-		{
-			Name:  ":path",
-			Value: test.endpoint,
-		},
-	}
-
-	body := make([]byte, len(test.expectedBody))
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		b.StartTimer()
-		stream, openstreamErr := edgeMux.OpenStream(ctx, headers, nil)
-		_, readBodyErr := stream.Read(body)
-		b.StopTimer()
-
-		require.NoError(b, openstreamErr)
-		assert.True(b, hasHeader(stream, ResponseMetaHeader, responseMetaHeaderOrigin))
-		require.True(b, hasHeader(stream, ":status", strconv.Itoa(http.StatusOK)))
-		require.NoError(b, readBodyErr)
-		require.Equal(b, test.expectedBody, body)
-	}
-
-	cancel()
-	wg.Wait()
-}
-
-func BenchmarkServeStreamHTTPSimple(b *testing.B) {
-	test := testRequest{
-		name:           "ok",
-		endpoint:       "/ok",
-		expectedStatus: http.StatusOK,
-		expectedBody:   []byte(http.StatusText(http.StatusOK)),
-	}
-
-	benchmarkServeStreamHTTPSimple(b, test)
-}
-
-func BenchmarkServeStreamHTTPLargeFile(b *testing.B) {
-	test := testRequest{
-		name:           "large_file",
-		endpoint:       "/large_file",
-		expectedStatus: http.StatusOK,
-		expectedBody:   testLargeResp,
-	}
-
-	benchmarkServeStreamHTTPSimple(b, test)
-}

connection/http2.go

@@ -115,52 +115,53 @@ func (c *HTTP2Connection) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	var requestErr error
 	switch connType {
 	case TypeControlStream:
-		if err := c.controlStreamHandler.ServeControlStream(r.Context(), respWriter, c.connOptions, c.orchestrator); err != nil {
-			c.controlStreamErr = err
-			c.log.Error().Err(err)
-			respWriter.WriteErrorResponse()
+		requestErr = c.controlStreamHandler.ServeControlStream(r.Context(), respWriter, c.connOptions, c.orchestrator)
+		if requestErr != nil {
+			c.controlStreamErr = requestErr
 		}
 
 	case TypeConfiguration:
-		if err := c.handleConfigurationUpdate(respWriter, r); err != nil {
-			c.log.Error().Err(err)
-			respWriter.WriteErrorResponse()
-		}
+		requestErr = c.handleConfigurationUpdate(respWriter, r)
 
 	case TypeWebsocket, TypeHTTP:
 		stripWebsocketUpgradeHeader(r)
 		// Check for tracing on request
-		tr := tracing.NewTracedHTTPRequest(r, c.log)
+		tr := tracing.NewTracedHTTPRequest(r, c.connIndex, c.log)
 		if err := originProxy.ProxyHTTP(respWriter, tr, connType == TypeWebsocket); err != nil {
-			err := fmt.Errorf("Failed to proxy HTTP: %w", err)
-			c.log.Error().Err(err)
-			respWriter.WriteErrorResponse()
+			requestErr = fmt.Errorf("Failed to proxy HTTP: %w", err)
 		}
 
 	case TypeTCP:
 		host, err := getRequestHost(r)
 		if err != nil {
-			err := fmt.Errorf(`cloudflared received a warp-routing request with an empty host value: %w`, err)
-			c.log.Error().Err(err)
-			respWriter.WriteErrorResponse()
+			requestErr = fmt.Errorf(`cloudflared received a warp-routing request with an empty host value: %w`, err)
+			break
 		}
 
 		rws := NewHTTPResponseReadWriterAcker(respWriter, r)
-		if err := originProxy.ProxyTCP(r.Context(), rws, &TCPRequest{
+		requestErr = originProxy.ProxyTCP(r.Context(), rws, &TCPRequest{
 			Dest:      host,
 			CFRay:     FindCfRayHeader(r),
 			LBProbe:   IsLBProbeRequest(r),
 			CfTraceID: r.Header.Get(tracing.TracerContextName),
-		}); err != nil {
-			respWriter.WriteErrorResponse()
-		}
+			ConnIndex: c.connIndex,
+		})
 
 	default:
-		err := fmt.Errorf("Received unknown connection type: %s", connType)
-		c.log.Error().Err(err)
-		respWriter.WriteErrorResponse()
+		requestErr = fmt.Errorf("Received unknown connection type: %s", connType)
+	}
+
+	if requestErr != nil {
+		c.log.Error().Err(requestErr).Msg("failed to serve incoming request")
+
+		// WriteErrorResponse will return false if status was already written. we need to abort handler.
+		if !respWriter.WriteErrorResponse() {
+			c.log.Debug().Msg("Handler aborted due to failure to write error response after status already sent")
+			panic(http.ErrAbortHandler)
+		}
 	}
 }
 
@@ -196,6 +197,7 @@ type http2RespWriter struct {
 	flusher       http.Flusher
 	shouldFlush   bool
 	statusWritten bool
+	respHeaders   http.Header
 	log           *zerolog.Logger
 }
 
@@ -275,9 +277,24 @@ func (rp *http2RespWriter) WriteRespHeaders(status int, header http.Header) erro
 	return nil
 }
 
-func (rp *http2RespWriter) WriteErrorResponse() {
+func (rp *http2RespWriter) Header() http.Header {
+	return rp.respHeaders
+}
+
+func (rp *http2RespWriter) WriteHeader(status int) {
+	rp.WriteRespHeaders(status, rp.respHeaders)
+}
+
+func (rp *http2RespWriter) WriteErrorResponse() bool {
+	if rp.statusWritten {
+		return false
+	}
+
 	rp.setResponseMetaHeader(responseMetaHeaderCfd)
 	rp.w.WriteHeader(http.StatusBadGateway)
+	rp.statusWritten = true
+
+	return true
 }
 
 func (rp *http2RespWriter) setResponseMetaHeader(value string) {

connection/protocol.go

@@ -1,7 +1,6 @@
 package connection
 
 import (
-	"errors"
 	"fmt"
 	"hash/fnv"
 	"sync"
@@ -13,7 +12,7 @@ import (
 )
 
 const (
-	AvailableProtocolFlagMessage = "Available protocols: 'auto' - automatically chooses the best protocol over time (the default; and also the recommended one); 'quic' - based on QUIC, relying on UDP egress to Cloudflare edge; 'http2' - using Go's HTTP2 library, relying on TCP egress to Cloudflare edge; 'h2mux' - Cloudflare's implementation of HTTP/2, deprecated"
+	AvailableProtocolFlagMessage = "Available protocols: 'auto' - automatically chooses the best protocol over time (the default; and also the recommended one); 'quic' - based on QUIC, relying on UDP egress to Cloudflare edge; 'http2' - using Go's HTTP2 library, relying on TCP egress to Cloudflare edge"
 	// edgeH2muxTLSServerName is the server name to establish h2mux connection with edge
 	edgeH2muxTLSServerName = "cftunnel.com"
 	// edgeH2TLSServerName is the server name to establish http2 connection with edge
@@ -21,43 +20,32 @@ const (
 	// edgeQUICServerName is the server name to establish quic connection with edge.
 	edgeQUICServerName = "quic.cftunnel.com"
 	AutoSelectFlag     = "auto"
+	// SRV and TXT record resolution TTL
+	ResolveTTL = time.Hour
 )
 
 var (
-	// ProtocolList represents a list of supported protocols for communication with the edge.
-	ProtocolList = []Protocol{H2mux, HTTP2, HTTP2Warp, QUIC, QUICWarp}
+	// ProtocolList represents a list of supported protocols for communication with the edge
+	// in order of precedence for remote percentage fetcher.
+	ProtocolList = []Protocol{QUIC, HTTP2}
 )
 
 type Protocol int64
 
 const (
-	// H2mux protocol can be used both with Classic and Named Tunnels. .
-	H2mux Protocol = iota
-	// HTTP2 is used only with named tunnels. It's more efficient than H2mux for L4 proxying.
-	HTTP2
-	// QUIC is used only with named tunnels.
+	// HTTP2 using golang HTTP2 library for edge connections.
+	HTTP2 Protocol = iota
+	// QUIC using quic-go for edge connections.
 	QUIC
-	// HTTP2Warp is used only with named tunnels. It's useful for warp-routing where we don't want to fallback to
-	// H2mux on HTTP2 failure to connect.
-	HTTP2Warp
-	//QUICWarp is used only with named tunnels. It's useful for warp-routing where we want to fallback to HTTP2 but
-	// don't want HTTP2 to fallback to H2mux
-	QUICWarp
 )
 
 // Fallback returns the fallback protocol and whether the protocol has a fallback
 func (p Protocol) fallback() (Protocol, bool) {
 	switch p {
-	case H2mux:
-		return 0, false
 	case HTTP2:
-		return H2mux, true
-	case HTTP2Warp:
 		return 0, false
 	case QUIC:
 		return HTTP2, true
-	case QUICWarp:
-		return HTTP2Warp, true
 	default:
 		return 0, false
 	}
@@ -65,11 +53,9 @@ func (p Protocol) fallback() (Protocol, bool) {
 
 func (p Protocol) String() string {
 	switch p {
-	case H2mux:
-		return "h2mux"
-	case HTTP2, HTTP2Warp:
+	case HTTP2:
 		return "http2"
-	case QUIC, QUICWarp:
+	case QUIC:
 		return "quic"
 	default:
 		return fmt.Sprintf("unknown protocol")
@@ -78,15 +64,11 @@ func (p Protocol) String() string {
 
 func (p Protocol) TLSSettings() *TLSSettings {
 	switch p {
-	case H2mux:
-		return &TLSSettings{
-			ServerName: edgeH2muxTLSServerName,
-		}
-	case HTTP2, HTTP2Warp:
+	case HTTP2:
 		return &TLSSettings{
 			ServerName: edgeH2TLSServerName,
 		}
-	case QUIC, QUICWarp:
+	case QUIC:
 		return &TLSSettings{
 			ServerName: edgeQUICServerName,
 			NextProtos: []string{"argotunnel"},
@@ -106,6 +88,7 @@ type ProtocolSelector interface {
 	Fallback() (Protocol, bool)
 }
 
+// staticProtocolSelector will not provide a different protocol for Fallback
 type staticProtocolSelector struct {
 	current Protocol
 }
@@ -115,10 +98,11 @@ func (s *staticProtocolSelector) Current() Protocol {
 }
 
 func (s *staticProtocolSelector) Fallback() (Protocol, bool) {
-	return 0, false
+	return s.current, false
 }
 
-type autoProtocolSelector struct {
+// remoteProtocolSelector will fetch a list of remote protocols to provide for edge discovery
+type remoteProtocolSelector struct {
 	lock sync.RWMutex
 
 	current Protocol
@@ -127,23 +111,21 @@ type autoProtocolSelector struct {
 	protocolPool []Protocol
 
 	switchThreshold int32
-	fetchFunc       PercentageFetcher
+	fetchFunc       edgediscovery.PercentageFetcher
 	refreshAfter    time.Time
 	ttl             time.Duration
 	log             *zerolog.Logger
-	needPQ          bool
 }
 
-func newAutoProtocolSelector(
+func newRemoteProtocolSelector(
 	current Protocol,
 	protocolPool []Protocol,
 	switchThreshold int32,
-	fetchFunc PercentageFetcher,
+	fetchFunc edgediscovery.PercentageFetcher,
 	ttl time.Duration,
 	log *zerolog.Logger,
-	needPQ bool,
-) *autoProtocolSelector {
-	return &autoProtocolSelector{
+) *remoteProtocolSelector {
+	return &remoteProtocolSelector{
 		current:         current,
 		protocolPool:    protocolPool,
 		switchThreshold: switchThreshold,
@@ -151,11 +133,10 @@ func newAutoProtocolSelector(
 		refreshAfter:    time.Now().Add(ttl),
 		ttl:             ttl,
 		log:             log,
-		needPQ:          needPQ,
 	}
 }
 
-func (s *autoProtocolSelector) Current() Protocol {
+func (s *remoteProtocolSelector) Current() Protocol {
 	s.lock.Lock()
 	defer s.lock.Unlock()
 	if time.Now().Before(s.refreshAfter) {
@@ -173,7 +154,13 @@ func (s *autoProtocolSelector) Current() Protocol {
 	return s.current
 }
 
-func getProtocol(protocolPool []Protocol, fetchFunc PercentageFetcher, switchThreshold int32) (Protocol, error) {
+func (s *remoteProtocolSelector) Fallback() (Protocol, bool) {
+	s.lock.RLock()
+	defer s.lock.RUnlock()
+	return s.current.fallback()
+}
+
+func getProtocol(protocolPool []Protocol, fetchFunc edgediscovery.PercentageFetcher, switchThreshold int32) (Protocol, error) {
 	protocolPercentages, err := fetchFunc()
 	if err != nil {
 		return 0, err
@@ -185,112 +172,78 @@ func getProtocol(protocolPool []Protocol, fetchFunc PercentageFetcher, switchThr
 		}
 	}
 
-	return protocolPool[len(protocolPool)-1], nil
+	// Default to first index in protocolPool list
+	return protocolPool[0], nil
 }
 
-func (s *autoProtocolSelector) Fallback() (Protocol, bool) {
+// defaultProtocolSelector will allow for a protocol to have a fallback
+type defaultProtocolSelector struct {
+	lock    sync.RWMutex
+	current Protocol
+}
+
+func newDefaultProtocolSelector(
+	current Protocol,
+) *defaultProtocolSelector {
+	return &defaultProtocolSelector{
+		current: current,
+	}
+}
+
+func (s *defaultProtocolSelector) Current() Protocol {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+	return s.current
+}
+
+func (s *defaultProtocolSelector) Fallback() (Protocol, bool) {
 	s.lock.RLock()
 	defer s.lock.RUnlock()
-	if s.needPQ {
-		return 0, false
-	}
 	return s.current.fallback()
 }
 
-type PercentageFetcher func() (edgediscovery.ProtocolPercents, error)
-
 func NewProtocolSelector(
 	protocolFlag string,
-	warpRoutingEnabled bool,
-	namedTunnel *NamedTunnelProperties,
-	fetchFunc PercentageFetcher,
-	ttl time.Duration,
-	log *zerolog.Logger,
+	accountTag string,
+	tunnelTokenProvided bool,
 	needPQ bool,
+	protocolFetcher edgediscovery.PercentageFetcher,
+	resolveTTL time.Duration,
+	log *zerolog.Logger,
 ) (ProtocolSelector, error) {
-	// Classic tunnel is only supported with h2mux
-	if namedTunnel == nil {
-		if needPQ {
-			return nil, errors.New("Classic tunnel does not support post-quantum")
-		}
-
+	// With --post-quantum, we force quic
+	if needPQ {
 		return &staticProtocolSelector{
-			current: H2mux,
+			current: QUIC,
 		}, nil
 	}
 
-	threshold := switchThreshold(namedTunnel.Credentials.AccountTag)
-	fetchedProtocol, err := getProtocol([]Protocol{QUIC, HTTP2}, fetchFunc, threshold)
-	if err != nil && protocolFlag == "auto" {
-		log.Err(err).Msg("Unable to lookup protocol. Defaulting to `http2`. If this fails, you can attempt `--protocol quic` instead.")
-		if needPQ {
-			return nil, errors.New("http2 does not support post-quantum")
-		}
-		return &staticProtocolSelector{
-			current: HTTP2,
-		}, nil
-	}
-	if warpRoutingEnabled {
-		if protocolFlag == H2mux.String() || fetchedProtocol == H2mux {
-			log.Warn().Msg("Warp routing is not supported in h2mux protocol. Upgrading to http2 to allow it.")
-			protocolFlag = HTTP2.String()
-			fetchedProtocol = HTTP2Warp
-		}
-		return selectWarpRoutingProtocols(protocolFlag, fetchFunc, ttl, log, threshold, fetchedProtocol, needPQ)
+	threshold := switchThreshold(accountTag)
+	fetchedProtocol, err := getProtocol(ProtocolList, protocolFetcher, threshold)
+	log.Debug().Msgf("Fetched protocol: %s", fetchedProtocol)
+	if err != nil {
+		log.Warn().Msg("Unable to lookup protocol percentage.")
+		// Falling through here since 'auto' is handled in the switch and failing
+		// to do the protocol lookup isn't a failure since it can be triggered again
+		// after the TTL.
 	}
 
-	return selectNamedTunnelProtocols(protocolFlag, fetchFunc, ttl, log, threshold, fetchedProtocol, needPQ)
-}
-
-func selectNamedTunnelProtocols(
-	protocolFlag string,
-	fetchFunc PercentageFetcher,
-	ttl time.Duration,
-	log *zerolog.Logger,
-	threshold int32,
-	protocol Protocol,
-	needPQ bool,
-) (ProtocolSelector, error) {
 	// If the user picks a protocol, then we stick to it no matter what.
 	switch protocolFlag {
-	case H2mux.String():
-		return &staticProtocolSelector{current: H2mux}, nil
+	case "h2mux":
+		// Any users still requesting h2mux will be upgraded to http2 instead
+		log.Warn().Msg("h2mux is no longer a supported protocol: upgrading edge connection to http2. Please remove '--protocol h2mux' from runtime arguments to remove this warning.")
+		return &staticProtocolSelector{current: HTTP2}, nil
 	case QUIC.String():
 		return &staticProtocolSelector{current: QUIC}, nil
 	case HTTP2.String():
 		return &staticProtocolSelector{current: HTTP2}, nil
-	}
-
-	// If the user does not pick (hopefully the majority) then we use the one derived from the TXT DNS record and
-	// fallback on failures.
-	if protocolFlag == AutoSelectFlag {
-		return newAutoProtocolSelector(protocol, []Protocol{QUIC, HTTP2, H2mux}, threshold, fetchFunc, ttl, log, needPQ), nil
-	}
-
-	return nil, fmt.Errorf("Unknown protocol %s, %s", protocolFlag, AvailableProtocolFlagMessage)
-}
-
-func selectWarpRoutingProtocols(
-	protocolFlag string,
-	fetchFunc PercentageFetcher,
-	ttl time.Duration,
-	log *zerolog.Logger,
-	threshold int32,
-	protocol Protocol,
-	needPQ bool,
-) (ProtocolSelector, error) {
-	// If the user picks a protocol, then we stick to it no matter what.
-	switch protocolFlag {
-	case QUIC.String():
-		return &staticProtocolSelector{current: QUICWarp}, nil
-	case HTTP2.String():
-		return &staticProtocolSelector{current: HTTP2Warp}, nil
-	}
-
-	// If the user does not pick (hopefully the majority) then we use the one derived from the TXT DNS record and
-	// fallback on failures.
-	if protocolFlag == AutoSelectFlag {
-		return newAutoProtocolSelector(protocol, []Protocol{QUICWarp, HTTP2Warp}, threshold, fetchFunc, ttl, log, needPQ), nil
+	case AutoSelectFlag:
+		// When a --token is provided, we want to start with QUIC but have fallback to HTTP2
+		if tunnelTokenProvided {
+			return newDefaultProtocolSelector(QUIC), nil
+		}
+		return newRemoteProtocolSelector(fetchedProtocol, ProtocolList, threshold, protocolFetcher, resolveTTL, log), nil
 	}
 
 	return nil, fmt.Errorf("Unknown protocol %s, %s", protocolFlag, AvailableProtocolFlagMessage)

connection/protocol_test.go

@@ -3,7 +3,6 @@ package connection
 import (
 	"fmt"
 	"testing"
-	"time"
 
 	"github.com/stretchr/testify/assert"
 
@@ -11,19 +10,11 @@ import (
 )
 
 const (
-	testNoTTL            = 0
-	noWarpRoutingEnabled = false
+	testNoTTL      = 0
+	testAccountTag = "testAccountTag"
 )
 
-var (
-	testNamedTunnelProperties = &NamedTunnelProperties{
-		Credentials: Credentials{
-			AccountTag: "testAccountTag",
-		},
-	}
-)
-
-func mockFetcher(getError bool, protocolPercent ...edgediscovery.ProtocolPercent) PercentageFetcher {
+func mockFetcher(getError bool, protocolPercent ...edgediscovery.ProtocolPercent) edgediscovery.PercentageFetcher {
 	return func() (edgediscovery.ProtocolPercents, error) {
 		if getError {
 			return nil, fmt.Errorf("failed to fetch percentage")
@@ -37,7 +28,7 @@ type dynamicMockFetcher struct {
 	err              error
 }
 
-func (dmf *dynamicMockFetcher) fetch() PercentageFetcher {
+func (dmf *dynamicMockFetcher) fetch() edgediscovery.PercentageFetcher {
 	return func() (edgediscovery.ProtocolPercents, error) {
 		return dmf.protocolPercents, dmf.err
 	}
@@ -45,181 +36,58 @@ func (dmf *dynamicMockFetcher) fetch() PercentageFetcher {
 
 func TestNewProtocolSelector(t *testing.T) {
 	tests := []struct {
-		name               string
-		protocol           string
-		expectedProtocol   Protocol
-		hasFallback        bool
-		expectedFallback   Protocol
-		warpRoutingEnabled bool
-		namedTunnelConfig  *NamedTunnelProperties
-		fetchFunc          PercentageFetcher
-		wantErr            bool
+		name                string
+		protocol            string
+		tunnelTokenProvided bool
+		needPQ              bool
+		expectedProtocol    Protocol
+		hasFallback         bool
+		expectedFallback    Protocol
+		wantErr             bool
 	}{
 		{
-			name:              "classic tunnel",
-			protocol:          "h2mux",
-			expectedProtocol:  H2mux,
-			namedTunnelConfig: nil,
+			name:     "named tunnel with unknown protocol",
+			protocol: "unknown",
+			wantErr:  true,
 		},
 		{
-			name:              "named tunnel over h2mux",
-			protocol:          "h2mux",
-			expectedProtocol:  H2mux,
-			fetchFunc:         func() (edgediscovery.ProtocolPercents, error) { return nil, nil },
-			namedTunnelConfig: testNamedTunnelProperties,
-		},
-		{
-			name:              "named tunnel over http2",
-			protocol:          "http2",
-			expectedProtocol:  HTTP2,
-			fetchFunc:         mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 0}),
-			namedTunnelConfig: testNamedTunnelProperties,
-		},
-		{
-			name:              "named tunnel http2 disabled still gets http2 because it is manually picked",
-			protocol:          "http2",
-			expectedProtocol:  HTTP2,
-			fetchFunc:         mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: -1}),
-			namedTunnelConfig: testNamedTunnelProperties,
-		},
-		{
-			name:              "named tunnel quic disabled still gets quic because it is manually picked",
-			protocol:          "quic",
-			expectedProtocol:  QUIC,
-			fetchFunc:         mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 100}, edgediscovery.ProtocolPercent{Protocol: "quic", Percentage: -1}),
-			namedTunnelConfig: testNamedTunnelProperties,
-		},
-		{
-			name:              "named tunnel quic and http2 disabled",
-			protocol:          AutoSelectFlag,
-			expectedProtocol:  H2mux,
-			fetchFunc:         mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: -1}, edgediscovery.ProtocolPercent{Protocol: "quic", Percentage: -1}),
-			namedTunnelConfig: testNamedTunnelProperties,
-		},
-		{
-			name:             "named tunnel quic disabled",
-			protocol:         AutoSelectFlag,
+			name:             "named tunnel with h2mux: force to http2",
+			protocol:         "h2mux",
 			expectedProtocol: HTTP2,
-			// Hasfallback true is because if http2 fails, then we further fallback to h2mux.
-			hasFallback:       true,
-			expectedFallback:  H2mux,
-			fetchFunc:         mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 100}, edgediscovery.ProtocolPercent{Protocol: "quic", Percentage: -1}),
-			namedTunnelConfig: testNamedTunnelProperties,
 		},
 		{
-			name:              "named tunnel auto all http2 disabled",
-			protocol:          AutoSelectFlag,
-			expectedProtocol:  H2mux,
-			fetchFunc:         mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: -1}),
-			namedTunnelConfig: testNamedTunnelProperties,
+			name:             "named tunnel with http2: no fallback",
+			protocol:         "http2",
+			expectedProtocol: HTTP2,
 		},
 		{
-			name:              "named tunnel auto to h2mux",
-			protocol:          AutoSelectFlag,
-			expectedProtocol:  H2mux,
-			fetchFunc:         mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 0}),
-			namedTunnelConfig: testNamedTunnelProperties,
+			name:             "named tunnel with auto: quic",
+			protocol:         AutoSelectFlag,
+			expectedProtocol: QUIC,
+			hasFallback:      true,
+			expectedFallback: HTTP2,
 		},
 		{
-			name:              "named tunnel auto to http2",
-			protocol:          AutoSelectFlag,
-			expectedProtocol:  HTTP2,
-			hasFallback:       true,
-			expectedFallback:  H2mux,
-			fetchFunc:         mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 100}),
-			namedTunnelConfig: testNamedTunnelProperties,
+			name:             "named tunnel (post quantum)",
+			protocol:         AutoSelectFlag,
+			needPQ:           true,
+			expectedProtocol: QUIC,
 		},
 		{
-			name:              "named tunnel auto to quic",
-			protocol:          AutoSelectFlag,
-			expectedProtocol:  QUIC,
-			hasFallback:       true,
-			expectedFallback:  HTTP2,
-			fetchFunc:         mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "quic", Percentage: 100}),
-			namedTunnelConfig: testNamedTunnelProperties,
-		},
-		{
-			name:               "warp routing requesting h2mux",
-			protocol:           "h2mux",
-			expectedProtocol:   HTTP2Warp,
-			hasFallback:        false,
-			fetchFunc:          mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 100}),
-			warpRoutingEnabled: true,
-			namedTunnelConfig:  testNamedTunnelProperties,
-		},
-		{
-			name:               "warp routing requesting h2mux picks HTTP2 even if http2 percent is -1",
-			protocol:           "h2mux",
-			expectedProtocol:   HTTP2Warp,
-			hasFallback:        false,
-			fetchFunc:          mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: -1}),
-			warpRoutingEnabled: true,
-			namedTunnelConfig:  testNamedTunnelProperties,
-		},
-		{
-			name:               "warp routing http2",
-			protocol:           "http2",
-			expectedProtocol:   HTTP2Warp,
-			hasFallback:        false,
-			fetchFunc:          mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 100}),
-			warpRoutingEnabled: true,
-			namedTunnelConfig:  testNamedTunnelProperties,
-		},
-		{
-			name:               "warp routing quic",
-			protocol:           AutoSelectFlag,
-			expectedProtocol:   QUICWarp,
-			hasFallback:        true,
-			expectedFallback:   HTTP2Warp,
-			fetchFunc:          mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "quic", Percentage: 100}),
-			warpRoutingEnabled: true,
-			namedTunnelConfig:  testNamedTunnelProperties,
-		},
-		{
-			name:               "warp routing auto",
-			protocol:           AutoSelectFlag,
-			expectedProtocol:   HTTP2Warp,
-			hasFallback:        false,
-			fetchFunc:          mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 100}),
-			warpRoutingEnabled: true,
-			namedTunnelConfig:  testNamedTunnelProperties,
-		},
-		{
-			name:               "warp routing auto- quic",
-			protocol:           AutoSelectFlag,
-			expectedProtocol:   QUICWarp,
-			hasFallback:        true,
-			expectedFallback:   HTTP2Warp,
-			fetchFunc:          mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 100}, edgediscovery.ProtocolPercent{Protocol: "quic", Percentage: 100}),
-			warpRoutingEnabled: true,
-			namedTunnelConfig:  testNamedTunnelProperties,
-		},
-		{
-			// None named tunnel can only use h2mux, so specifying an unknown protocol is not an error
-			name:             "classic tunnel unknown protocol",
-			protocol:         "unknown",
-			expectedProtocol: H2mux,
-		},
-		{
-			name:              "named tunnel unknown protocol",
-			protocol:          "unknown",
-			fetchFunc:         mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 100}),
-			namedTunnelConfig: testNamedTunnelProperties,
-			wantErr:           true,
-		},
-		{
-			name:              "named tunnel fetch error",
-			protocol:          AutoSelectFlag,
-			fetchFunc:         mockFetcher(true),
-			namedTunnelConfig: testNamedTunnelProperties,
-			expectedProtocol:  HTTP2,
-			wantErr:           false,
+			name:             "named tunnel (post quantum) w/http2",
+			protocol:         "http2",
+			needPQ:           true,
+			expectedProtocol: QUIC,
 		},
 	}
 
+	fetcher := dynamicMockFetcher{
+		protocolPercents: edgediscovery.ProtocolPercents{},
+	}
+
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			selector, err := NewProtocolSelector(test.protocol, test.warpRoutingEnabled, test.namedTunnelConfig, test.fetchFunc, testNoTTL, &log, false)
+			selector, err := NewProtocolSelector(test.protocol, testAccountTag, test.tunnelTokenProvided, test.needPQ, fetcher.fetch(), ResolveTTL, &log)
 			if test.wantErr {
 				assert.Error(t, err, fmt.Sprintf("test %s failed", test.name))
 			} else {
@@ -237,15 +105,15 @@ func TestNewProtocolSelector(t *testing.T) {
 
 func TestAutoProtocolSelectorRefresh(t *testing.T) {
 	fetcher := dynamicMockFetcher{}
-	selector, err := NewProtocolSelector(AutoSelectFlag, noWarpRoutingEnabled, testNamedTunnelProperties, fetcher.fetch(), testNoTTL, &log, false)
+	selector, err := NewProtocolSelector(AutoSelectFlag, testAccountTag, false, false, fetcher.fetch(), testNoTTL, &log)
 	assert.NoError(t, err)
-	assert.Equal(t, H2mux, selector.Current())
+	assert.Equal(t, QUIC, selector.Current())
 
 	fetcher.protocolPercents = edgediscovery.ProtocolPercents{edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 100}}
 	assert.Equal(t, HTTP2, selector.Current())
 
 	fetcher.protocolPercents = edgediscovery.ProtocolPercents{edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 0}}
-	assert.Equal(t, H2mux, selector.Current())
+	assert.Equal(t, QUIC, selector.Current())
 
 	fetcher.protocolPercents = edgediscovery.ProtocolPercents{edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 100}}
 	assert.Equal(t, HTTP2, selector.Current())
@@ -255,10 +123,10 @@ func TestAutoProtocolSelectorRefresh(t *testing.T) {
 
 	fetcher.protocolPercents = edgediscovery.ProtocolPercents{edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: -1}}
 	fetcher.err = nil
-	assert.Equal(t, H2mux, selector.Current())
+	assert.Equal(t, QUIC, selector.Current())
 
 	fetcher.protocolPercents = edgediscovery.ProtocolPercents{edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 0}}
-	assert.Equal(t, H2mux, selector.Current())
+	assert.Equal(t, QUIC, selector.Current())
 
 	fetcher.protocolPercents = edgediscovery.ProtocolPercents{edgediscovery.ProtocolPercent{Protocol: "quic", Percentage: 100}}
 	assert.Equal(t, QUIC, selector.Current())
@@ -267,7 +135,7 @@ func TestAutoProtocolSelectorRefresh(t *testing.T) {
 func TestHTTP2ProtocolSelectorRefresh(t *testing.T) {
 	fetcher := dynamicMockFetcher{}
 	// Since the user chooses http2 on purpose, we always stick to it.
-	selector, err := NewProtocolSelector("http2", noWarpRoutingEnabled, testNamedTunnelProperties, fetcher.fetch(), testNoTTL, &log, false)
+	selector, err := NewProtocolSelector(HTTP2.String(), testAccountTag, false, false, fetcher.fetch(), testNoTTL, &log)
 	assert.NoError(t, err)
 	assert.Equal(t, HTTP2, selector.Current())
 
@@ -294,13 +162,12 @@ func TestHTTP2ProtocolSelectorRefresh(t *testing.T) {
 	assert.Equal(t, HTTP2, selector.Current())
 }
 
-func TestProtocolSelectorRefreshTTL(t *testing.T) {
+func TestAutoProtocolSelectorNoRefreshWithToken(t *testing.T) {
 	fetcher := dynamicMockFetcher{}
-	fetcher.protocolPercents = edgediscovery.ProtocolPercents{edgediscovery.ProtocolPercent{Protocol: "quic", Percentage: 100}}
-	selector, err := NewProtocolSelector(AutoSelectFlag, noWarpRoutingEnabled, testNamedTunnelProperties, fetcher.fetch(), time.Hour, &log, false)
+	selector, err := NewProtocolSelector(AutoSelectFlag, testAccountTag, true, false, fetcher.fetch(), testNoTTL, &log)
 	assert.NoError(t, err)
 	assert.Equal(t, QUIC, selector.Current())
 
-	fetcher.protocolPercents = edgediscovery.ProtocolPercents{edgediscovery.ProtocolPercent{Protocol: "quic", Percentage: 0}}
+	fetcher.protocolPercents = edgediscovery.ProtocolPercents{edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 100}}
 	assert.Equal(t, QUIC, selector.Current())
 }

connection/quic.go

@@ -10,6 +10,7 @@ import (
 	"net/netip"
 	"strconv"
 	"strings"
+	"sync"
 	"sync/atomic"
 	"time"
 
@@ -42,6 +43,11 @@ const (
 	demuxChanCapacity = 16
 )
 
+var (
+	portForConnIndex = make(map[uint8]int, 0)
+	portMapMutex     sync.Mutex
+)
+
 // QUICConnection represents the type that facilitates Proxying via QUIC streams.
 type QUICConnection struct {
 	session      quic.Connection
@@ -51,31 +57,47 @@ type QUICConnection struct {
 	sessionManager datagramsession.Manager
 	// datagramMuxer mux/demux datagrams from quic connection
 	datagramMuxer        *quicpogs.DatagramMuxerV2
-	packetRouter         *packet.Router
+	packetRouter         *ingress.PacketRouter
 	controlStreamHandler ControlStreamHandler
 	connOptions          *tunnelpogs.ConnectionOptions
+	connIndex            uint8
 }
 
 // NewQUICConnection returns a new instance of QUICConnection.
 func NewQUICConnection(
 	quicConfig *quic.Config,
 	edgeAddr net.Addr,
+	localAddr net.IP,
+	connIndex uint8,
 	tlsConfig *tls.Config,
 	orchestrator Orchestrator,
 	connOptions *tunnelpogs.ConnectionOptions,
 	controlStreamHandler ControlStreamHandler,
 	logger *zerolog.Logger,
-	packetRouterConfig *packet.GlobalRouterConfig,
+	packetRouterConfig *ingress.GlobalRouterConfig,
 ) (*QUICConnection, error) {
-	session, err := quic.DialAddr(edgeAddr.String(), tlsConfig, quicConfig)
+	udpConn, err := createUDPConnForConnIndex(connIndex, localAddr, logger)
 	if err != nil {
+		return nil, err
+	}
+
+	session, err := quic.Dial(udpConn, edgeAddr, edgeAddr.String(), tlsConfig, quicConfig)
+	if err != nil {
+		// close the udp server socket in case of error connecting to the edge
+		udpConn.Close()
 		return nil, &EdgeQuicDialError{Cause: err}
 	}
 
+	// wrap the session, so that the UDPConn is closed after session is closed.
+	session = &wrapCloseableConnQuicConnection{
+		session,
+		udpConn,
+	}
+
 	sessionDemuxChan := make(chan *packet.Session, demuxChanCapacity)
 	datagramMuxer := quicpogs.NewDatagramMuxerV2(session, logger, sessionDemuxChan)
 	sessionManager := datagramsession.NewManager(logger, datagramMuxer.SendToSession, sessionDemuxChan)
-	packetRouter := packet.NewRouter(packetRouterConfig, datagramMuxer, &returnPipe{muxer: datagramMuxer}, logger, orchestrator.WarpRoutingEnabled)
+	packetRouter := ingress.NewPacketRouter(packetRouterConfig, datagramMuxer, logger, orchestrator.WarpRoutingEnabled)
 
 	return &QUICConnection{
 		session:              session,
@@ -86,6 +108,7 @@ func NewQUICConnection(
 		packetRouter:         packetRouter,
 		controlStreamHandler: controlStreamHandler,
 		connOptions:          connOptions,
+		connIndex:            connIndex,
 	}, nil
 }
 
@@ -173,7 +196,11 @@ func (q *QUICConnection) runStream(quicStream quic.Stream) {
 	// A call to close will simulate a close to the read-side, which will fail subsequent reads.
 	noCloseStream := &nopCloserReadWriter{ReadWriteCloser: stream}
 	if err := q.handleStream(ctx, noCloseStream); err != nil {
-		q.logger.Err(err).Msg("Failed to handle QUIC stream")
+		q.logger.Debug().Err(err).Msg("Failed to handle QUIC stream")
+
+		// if we received an error at this level, then close write side of stream with an error, which will result in
+		// RST_STREAM frame.
+		quicStream.CancelWrite(0)
 	}
 }
 
@@ -206,43 +233,61 @@ func (q *QUICConnection) handleDataStream(ctx context.Context, stream *quicpogs.
 		return err
 	}
 
-	if err := q.dispatchRequest(ctx, stream, err, request); err != nil {
-		_ = stream.WriteConnectResponseData(err)
+	if err, connectResponseSent := q.dispatchRequest(ctx, stream, err, request); err != nil {
 		q.logger.Err(err).Str("type", request.Type.String()).Str("dest", request.Dest).Msg("Request failed")
+
+		// if the connectResponse was already sent and we had an error, we need to propagate it up, so that the stream is
+		// closed with an RST_STREAM frame
+		if connectResponseSent {
+			return err
+		}
+
+		if writeRespErr := stream.WriteConnectResponseData(err); writeRespErr != nil {
+			return writeRespErr
+		}
 	}
 
 	return nil
 }
 
-func (q *QUICConnection) dispatchRequest(ctx context.Context, stream *quicpogs.RequestServerStream, err error, request *quicpogs.ConnectRequest) error {
+// dispatchRequest will dispatch the request depending on the type and returns an error if it occurs.
+// More importantly, it also tells if the during processing of the request the ConnectResponse metadata was sent downstream.
+// This is important since it informs
+func (q *QUICConnection) dispatchRequest(ctx context.Context, stream *quicpogs.RequestServerStream, err error, request *quicpogs.ConnectRequest) (error, bool) {
 	originProxy, err := q.orchestrator.GetOriginProxy()
 	if err != nil {
-		return err
+		return err, false
 	}
 
 	switch request.Type {
 	case quicpogs.ConnectionTypeHTTP, quicpogs.ConnectionTypeWebsocket:
-		tracedReq, err := buildHTTPRequest(ctx, request, stream, q.logger)
+		tracedReq, err := buildHTTPRequest(ctx, request, stream, q.connIndex, q.logger)
 		if err != nil {
-			return err
+			return err, false
 		}
 		w := newHTTPResponseAdapter(stream)
-		return originProxy.ProxyHTTP(w, tracedReq, request.Type == quicpogs.ConnectionTypeWebsocket)
+		return originProxy.ProxyHTTP(&w, tracedReq, request.Type == quicpogs.ConnectionTypeWebsocket), w.connectResponseSent
 
 	case quicpogs.ConnectionTypeTCP:
-		rwa := &streamReadWriteAcker{stream}
+		rwa := &streamReadWriteAcker{RequestServerStream: stream}
 		metadata := request.MetadataMap()
 		return originProxy.ProxyTCP(ctx, rwa, &TCPRequest{
 			Dest:      request.Dest,
 			FlowID:    metadata[QUICMetadataFlowID],
 			CfTraceID: metadata[tracing.TracerContextName],
-		})
+			ConnIndex: q.connIndex,
+		}), rwa.connectResponseSent
+	default:
+		return errors.Errorf("unsupported error type: %s", request.Type), false
 	}
-	return nil
 }
 
 func (q *QUICConnection) handleRPCStream(rpcStream *quicpogs.RPCServerStream) error {
-	return rpcStream.Serve(q, q, q.logger)
+	if err := rpcStream.Serve(q, q, q.logger); err != nil {
+		q.logger.Err(err).Msg("failed handling RPC stream")
+	}
+
+	return nil
 }
 
 // RegisterUdpSession is the RPC method invoked by edge to register and run a session
@@ -337,31 +382,39 @@ func (q *QUICConnection) UpdateConfiguration(ctx context.Context, version int32,
 // the client.
 type streamReadWriteAcker struct {
 	*quicpogs.RequestServerStream
+	connectResponseSent bool
 }
 
 // AckConnection acks response back to the proxy.
 func (s *streamReadWriteAcker) AckConnection(tracePropagation string) error {
-	metadata := quicpogs.Metadata{
-		Key: tracing.CanonicalCloudflaredTracingHeader,
-		Val: tracePropagation,
+	metadata := []quicpogs.Metadata{}
+	// Only add tracing if provided by origintunneld
+	if tracePropagation != "" {
+		metadata = append(metadata, quicpogs.Metadata{
+			Key: tracing.CanonicalCloudflaredTracingHeader,
+			Val: tracePropagation,
+		})
 	}
-	return s.WriteConnectResponseData(nil, metadata)
+	s.connectResponseSent = true
+	return s.WriteConnectResponseData(nil, metadata...)
 }
 
 // httpResponseAdapter translates responses written by the HTTP Proxy into ones that can be used in QUIC.
 type httpResponseAdapter struct {
 	*quicpogs.RequestServerStream
+	headers             http.Header
+	connectResponseSent bool
 }
 
 func newHTTPResponseAdapter(s *quicpogs.RequestServerStream) httpResponseAdapter {
-	return httpResponseAdapter{s}
+	return httpResponseAdapter{RequestServerStream: s}
 }
 
-func (hrw httpResponseAdapter) AddTrailer(trailerName, trailerValue string) {
+func (hrw *httpResponseAdapter) AddTrailer(trailerName, trailerValue string) {
 	// we do not support trailers over QUIC
 }
 
-func (hrw httpResponseAdapter) WriteRespHeaders(status int, header http.Header) error {
+func (hrw *httpResponseAdapter) WriteRespHeaders(status int, header http.Header) error {
 	metadata := make([]quicpogs.Metadata, 0)
 	metadata = append(metadata, quicpogs.Metadata{Key: "HttpStatus", Val: strconv.Itoa(status)})
 	for k, vv := range header {
@@ -370,17 +423,32 @@ func (hrw httpResponseAdapter) WriteRespHeaders(status int, header http.Header)
 			metadata = append(metadata, quicpogs.Metadata{Key: httpHeaderKey, Val: v})
 		}
 	}
+
 	return hrw.WriteConnectResponseData(nil, metadata...)
 }
 
-func (hrw httpResponseAdapter) WriteErrorResponse(err error) {
+func (hrw *httpResponseAdapter) Header() http.Header {
+	return hrw.headers
+}
+
+func (hrw *httpResponseAdapter) WriteHeader(status int) {
+	hrw.WriteRespHeaders(status, hrw.headers)
+}
+
+func (hrw *httpResponseAdapter) WriteErrorResponse(err error) {
 	hrw.WriteConnectResponseData(err, quicpogs.Metadata{Key: "HttpStatus", Val: strconv.Itoa(http.StatusBadGateway)})
 }
 
+func (hrw *httpResponseAdapter) WriteConnectResponseData(respErr error, metadata ...quicpogs.Metadata) error {
+	hrw.connectResponseSent = true
+	return hrw.RequestServerStream.WriteConnectResponseData(respErr, metadata...)
+}
+
 func buildHTTPRequest(
 	ctx context.Context,
 	connectRequest *quicpogs.ConnectRequest,
 	body io.ReadCloser,
+	connIndex uint8,
 	log *zerolog.Logger,
 ) (*tracing.TracedHTTPRequest, error) {
 	metadata := connectRequest.MetadataMap()
@@ -424,7 +492,7 @@ func buildHTTPRequest(
 	stripWebsocketUpgradeHeader(req)
 
 	// Check for tracing on request
-	tracedReq := tracing.NewTracedHTTPRequest(req, log)
+	tracedReq := tracing.NewTracedHTTPRequest(req, connIndex, log)
 	return tracedReq, err
 }
 
@@ -480,15 +548,73 @@ func (np *nopCloserReadWriter) Close() error {
 	return nil
 }
 
-// returnPipe wraps DatagramMuxerV2 to satisfy the packet.FunnelUniPipe interface
-type returnPipe struct {
+// muxerWrapper wraps DatagramMuxerV2 to satisfy the packet.FunnelUniPipe interface
+type muxerWrapper struct {
 	muxer *quicpogs.DatagramMuxerV2
 }
 
-func (rp *returnPipe) SendPacket(dst netip.Addr, pk packet.RawPacket) error {
-	return rp.muxer.SendPacket(pk)
+func (rp *muxerWrapper) SendPacket(dst netip.Addr, pk packet.RawPacket) error {
+	return rp.muxer.SendPacket(quicpogs.RawPacket(pk))
 }
 
-func (rp *returnPipe) Close() error {
+func (rp *muxerWrapper) ReceivePacket(ctx context.Context) (packet.RawPacket, error) {
+	pk, err := rp.muxer.ReceivePacket(ctx)
+	if err != nil {
+		return packet.RawPacket{}, err
+	}
+	rawPacket, ok := pk.(quicpogs.RawPacket)
+	if ok {
+		return packet.RawPacket(rawPacket), nil
+	}
+	return packet.RawPacket{}, fmt.Errorf("unexpected packet type %+v", pk)
+}
+
+func (rp *muxerWrapper) Close() error {
 	return nil
 }
+
+func createUDPConnForConnIndex(connIndex uint8, localIP net.IP, logger *zerolog.Logger) (*net.UDPConn, error) {
+	portMapMutex.Lock()
+	defer portMapMutex.Unlock()
+
+	if localIP == nil {
+		localIP = net.IPv4zero
+	}
+
+	// if port was not set yet, it will be zero, so bind will randomly allocate one.
+	if port, ok := portForConnIndex[connIndex]; ok {
+		udpConn, err := net.ListenUDP("udp", &net.UDPAddr{IP: localIP, Port: port})
+		// if there wasn't an error, or if port was 0 (independently of error or not, just return)
+		if err == nil {
+			return udpConn, nil
+		} else {
+			logger.Debug().Err(err).Msgf("Unable to reuse port %d for connIndex %d. Falling back to random allocation.", port, connIndex)
+		}
+	}
+
+	// if we reached here, then there was an error or port as not been allocated it.
+	udpConn, err := net.ListenUDP("udp", &net.UDPAddr{IP: localIP, Port: 0})
+	if err == nil {
+		udpAddr, ok := (udpConn.LocalAddr()).(*net.UDPAddr)
+		if !ok {
+			return nil, fmt.Errorf("unable to cast to udpConn")
+		}
+		portForConnIndex[connIndex] = udpAddr.Port
+	} else {
+		delete(portForConnIndex, connIndex)
+	}
+
+	return udpConn, err
+}
+
+type wrapCloseableConnQuicConnection struct {
+	quic.Connection
+	udpConn *net.UDPConn
+}
+
+func (w *wrapCloseableConnQuicConnection) CloseWithError(errorCode quic.ApplicationErrorCode, reason string) error {
+	err := w.Connection.CloseWithError(errorCode, reason)
+	w.udpConn.Close()
+
+	return err
+}

connection/quic_test.go

@@ -11,7 +11,6 @@ import (
 	"net/url"
 	"os"
 	"strings"
-	"sync"
 	"testing"
 	"time"
 
@@ -142,24 +141,33 @@ func TestQUICServer(t *testing.T) {
 		},
 	}
 
-	for _, test := range tests {
+	for i, test := range tests {
+		test := test // capture range variable
 		t.Run(test.desc, func(t *testing.T) {
 			ctx, cancel := context.WithCancel(context.Background())
-			var wg sync.WaitGroup
-			wg.Add(1)
+
+			quicListener, err := quic.Listen(udpListener, testTLSServerConfig, testQUICConfig)
+			require.NoError(t, err)
+
+			serverDone := make(chan struct{})
 			go func() {
-				defer wg.Done()
 				quicServer(
-					t, udpListener, testTLSServerConfig, testQUICConfig,
-					test.dest, test.connectionType, test.metadata, test.message, test.expectedResponse,
+					ctx, t, quicListener, test.dest, test.connectionType, test.metadata, test.message, test.expectedResponse,
 				)
+				close(serverDone)
 			}()
 
-			qc := testQUICConnection(udpListener.LocalAddr(), t)
-			go qc.Serve(ctx)
+			qc := testQUICConnection(udpListener.LocalAddr(), t, uint8(i))
 
-			wg.Wait()
+			connDone := make(chan struct{})
+			go func() {
+				qc.Serve(ctx)
+				close(connDone)
+			}()
+
+			<-serverDone
 			cancel()
+			<-connDone
 		})
 	}
 }
@@ -177,23 +185,16 @@ func (fakeControlStream) IsStopped() bool {
 }
 
 func quicServer(
+	ctx context.Context,
 	t *testing.T,
-	conn net.PacketConn,
-	tlsConf *tls.Config,
-	config *quic.Config,
+	listener quic.Listener,
 	dest string,
 	connectionType quicpogs.ConnectionType,
 	metadata []quicpogs.Metadata,
 	message []byte,
 	expectedResponse []byte,
 ) {
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	earlyListener, err := quic.Listen(conn, tlsConf, config)
-	require.NoError(t, err)
-
-	session, err := earlyListener.Accept(ctx)
+	session, err := listener.Accept(ctx)
 	require.NoError(t, err)
 
 	quicStream, err := session.OpenStreamSync(context.Background())
@@ -482,8 +483,9 @@ func TestBuildHTTPRequest(t *testing.T) {
 
 	log := zerolog.Nop()
 	for _, test := range tests {
+		test := test // capture range variable
 		t.Run(test.name, func(t *testing.T) {
-			req, err := buildHTTPRequest(context.Background(), test.connectRequest, test.body, &log)
+			req, err := buildHTTPRequest(context.Background(), test.connectRequest, test.body, 0, &log)
 			assert.NoError(t, err)
 			test.req = test.req.WithContext(req.Context())
 			assert.Equal(t, test.req, req.Request)
@@ -519,7 +521,8 @@ func TestServeUDPSession(t *testing.T) {
 		edgeQUICSessionChan <- edgeQUICSession
 	}()
 
-	qc := testQUICConnection(val, t)
+	// Random index to avoid reusing port
+	qc := testQUICConnection(val, t, 28)
 	go qc.Serve(ctx)
 
 	edgeQUICSession := <-edgeQUICSessionChan
@@ -567,6 +570,37 @@ func TestNopCloserReadWriterCloseAfterEOF(t *testing.T) {
 	require.Equal(t, err, io.EOF)
 }
 
+func TestCreateUDPConnReuseSourcePort(t *testing.T) {
+	logger := zerolog.Nop()
+	conn, err := createUDPConnForConnIndex(0, nil, &logger)
+	require.NoError(t, err)
+
+	getPortFunc := func(conn *net.UDPConn) int {
+		addr := conn.LocalAddr().(*net.UDPAddr)
+		return addr.Port
+	}
+
+	initialPort := getPortFunc(conn)
+
+	// close conn
+	conn.Close()
+
+	// should get the same port as before.
+	conn, err = createUDPConnForConnIndex(0, nil, &logger)
+	require.NoError(t, err)
+	require.Equal(t, initialPort, getPortFunc(conn))
+
+	// new index, should get a different port
+	conn1, err := createUDPConnForConnIndex(1, nil, &logger)
+	require.NoError(t, err)
+	require.NotEqual(t, initialPort, getPortFunc(conn1))
+
+	// not closing the conn and trying to obtain a new conn for same index should give a different random port
+	conn, err = createUDPConnForConnIndex(0, nil, &logger)
+	require.NoError(t, err)
+	require.NotEqual(t, initialPort, getPortFunc(conn))
+}
+
 func serveSession(ctx context.Context, qc *QUICConnection, edgeQUICSession quic.Connection, closeType closeReason, expectedReason string, t *testing.T) {
 	var (
 		payload = []byte(t.Name())
@@ -672,7 +706,7 @@ func (s mockSessionRPCServer) UnregisterUdpSession(ctx context.Context, sessionI
 	return nil
 }
 
-func testQUICConnection(udpListenerAddr net.Addr, t *testing.T) *QUICConnection {
+func testQUICConnection(udpListenerAddr net.Addr, t *testing.T, index uint8) *QUICConnection {
 	tlsClientConfig := &tls.Config{
 		InsecureSkipVerify: true,
 		NextProtos:         []string{"argotunnel"},
@@ -682,6 +716,8 @@ func testQUICConnection(udpListenerAddr net.Addr, t *testing.T) *QUICConnection
 	qc, err := NewQUICConnection(
 		testQUICConfig,
 		udpListenerAddr,
+		nil,
+		index,
 		tlsClientConfig,
 		&mockOrchestrator{originProxy: &mockOriginProxyWithRequest{}},
 		&tunnelpogs.ConnectionOptions{},

connection/rpc.go

@@ -2,7 +2,6 @@ package connection
 
 import (
 	"context"
-	"fmt"
 	"io"
 	"net"
 	"time"
@@ -152,178 +151,3 @@ const (
 	unregister   rpcName = "unregister"
 	authenticate rpcName = " authenticate"
 )
-
-func (h *h2muxConnection) registerTunnel(ctx context.Context, credentialSetter CredentialManager, classicTunnel *ClassicTunnelProperties, registrationOptions *tunnelpogs.RegistrationOptions) error {
-	h.observer.sendRegisteringEvent(registrationOptions.ConnectionID)
-
-	stream, err := h.newRPCStream(ctx, register)
-	if err != nil {
-		return err
-	}
-	rpcClient := NewTunnelServerClient(ctx, stream, h.observer.log)
-	defer rpcClient.Close()
-
-	_ = h.logServerInfo(ctx, rpcClient)
-	registration := rpcClient.client.RegisterTunnel(
-		ctx,
-		classicTunnel.OriginCert,
-		classicTunnel.Hostname,
-		registrationOptions,
-	)
-	if registrationErr := registration.DeserializeError(); registrationErr != nil {
-		// RegisterTunnel RPC failure
-		return h.processRegisterTunnelError(registrationErr, register)
-	}
-
-	credentialSetter.SetEventDigest(h.connIndex, registration.EventDigest)
-	return h.processRegistrationSuccess(registration, register, credentialSetter, classicTunnel)
-}
-
-type CredentialManager interface {
-	ReconnectToken() ([]byte, error)
-	EventDigest(connID uint8) ([]byte, error)
-	SetEventDigest(connID uint8, digest []byte)
-	ConnDigest(connID uint8) ([]byte, error)
-	SetConnDigest(connID uint8, digest []byte)
-}
-
-func (h *h2muxConnection) processRegistrationSuccess(
-	registration *tunnelpogs.TunnelRegistration,
-	name rpcName,
-	credentialManager CredentialManager, classicTunnel *ClassicTunnelProperties,
-) error {
-	for _, logLine := range registration.LogLines {
-		h.observer.log.Info().Msg(logLine)
-	}
-
-	if registration.TunnelID != "" {
-		h.observer.metrics.tunnelsHA.AddTunnelID(h.connIndex, registration.TunnelID)
-		h.observer.log.Info().Msgf("Each HA connection's tunnel IDs: %v", h.observer.metrics.tunnelsHA.String())
-	}
-
-	credentialManager.SetConnDigest(h.connIndex, registration.ConnDigest)
-	h.observer.metrics.userHostnamesCounts.WithLabelValues(registration.Url).Inc()
-
-	h.observer.log.Info().Msgf("Route propagating, it may take up to 1 minute for your new route to become functional")
-	h.observer.metrics.regSuccess.WithLabelValues(string(name)).Inc()
-	return nil
-}
-
-func (h *h2muxConnection) processRegisterTunnelError(err tunnelpogs.TunnelRegistrationError, name rpcName) error {
-	if err.Error() == DuplicateConnectionError {
-		h.observer.metrics.regFail.WithLabelValues("dup_edge_conn", string(name)).Inc()
-		return errDuplicationConnection
-	}
-	h.observer.metrics.regFail.WithLabelValues("server_error", string(name)).Inc()
-	return ServerRegisterTunnelError{
-		Cause:     err,
-		Permanent: err.IsPermanent(),
-	}
-}
-
-func (h *h2muxConnection) reconnectTunnel(ctx context.Context, credentialManager CredentialManager, classicTunnel *ClassicTunnelProperties, registrationOptions *tunnelpogs.RegistrationOptions) error {
-	token, err := credentialManager.ReconnectToken()
-	if err != nil {
-		return err
-	}
-	eventDigest, err := credentialManager.EventDigest(h.connIndex)
-	if err != nil {
-		return err
-	}
-	connDigest, err := credentialManager.ConnDigest(h.connIndex)
-	if err != nil {
-		return err
-	}
-
-	h.observer.log.Debug().Msg("initiating RPC stream to reconnect")
-	stream, err := h.newRPCStream(ctx, register)
-	if err != nil {
-		return err
-	}
-	rpcClient := NewTunnelServerClient(ctx, stream, h.observer.log)
-	defer rpcClient.Close()
-
-	_ = h.logServerInfo(ctx, rpcClient)
-	registration := rpcClient.client.ReconnectTunnel(
-		ctx,
-		token,
-		eventDigest,
-		connDigest,
-		classicTunnel.Hostname,
-		registrationOptions,
-	)
-	if registrationErr := registration.DeserializeError(); registrationErr != nil {
-		// ReconnectTunnel RPC failure
-		return h.processRegisterTunnelError(registrationErr, reconnect)
-	}
-	return h.processRegistrationSuccess(registration, reconnect, credentialManager, classicTunnel)
-}
-
-func (h *h2muxConnection) logServerInfo(ctx context.Context, rpcClient *tunnelServerClient) error {
-	// Request server info without blocking tunnel registration; must use capnp library directly.
-	serverInfoPromise := tunnelrpc.TunnelServer{Client: rpcClient.client.Client}.GetServerInfo(ctx, func(tunnelrpc.TunnelServer_getServerInfo_Params) error {
-		return nil
-	})
-	serverInfoMessage, err := serverInfoPromise.Result().Struct()
-	if err != nil {
-		h.observer.log.Err(err).Msg("Failed to retrieve server information")
-		return err
-	}
-	serverInfo, err := tunnelpogs.UnmarshalServerInfo(serverInfoMessage)
-	if err != nil {
-		h.observer.log.Err(err).Msg("Failed to retrieve server information")
-		return err
-	}
-	h.observer.logServerInfo(h.connIndex, serverInfo.LocationName, net.IP{}, "Connection established")
-	return nil
-}
-
-func (h *h2muxConnection) registerNamedTunnel(
-	ctx context.Context,
-	namedTunnel *NamedTunnelProperties,
-	connOptions *tunnelpogs.ConnectionOptions,
-) error {
-	stream, err := h.newRPCStream(ctx, register)
-	if err != nil {
-		return err
-	}
-	rpcClient := h.newRPCClientFunc(ctx, stream, h.observer.log)
-	defer rpcClient.Close()
-
-	registrationDetails, err := rpcClient.RegisterConnection(ctx, namedTunnel, connOptions, h.connIndex, nil, h.observer)
-	if err != nil {
-		return err
-	}
-	h.observer.logServerInfo(h.connIndex, registrationDetails.Location, nil, fmt.Sprintf("Connection %s registered", registrationDetails.UUID))
-	h.observer.sendConnectedEvent(h.connIndex, H2mux, registrationDetails.Location)
-
-	return nil
-}
-
-func (h *h2muxConnection) unregister(isNamedTunnel bool) {
-	h.observer.sendUnregisteringEvent(h.connIndex)
-
-	unregisterCtx, cancel := context.WithTimeout(context.Background(), h.gracePeriod)
-	defer cancel()
-
-	stream, err := h.newRPCStream(unregisterCtx, unregister)
-	if err != nil {
-		return
-	}
-	defer stream.Close()
-
-	if isNamedTunnel {
-		rpcClient := h.newRPCClientFunc(unregisterCtx, stream, h.observer.log)
-		defer rpcClient.Close()
-
-		rpcClient.GracefulShutdown(unregisterCtx, h.gracePeriod)
-	} else {
-		rpcClient := NewTunnelServerClient(unregisterCtx, stream, h.observer.log)
-		defer rpcClient.Close()
-
-		// gracePeriod is encoded in int64 using capnproto
-		_ = rpcClient.client.UnregisterTunnel(unregisterCtx, h.gracePeriod.Nanoseconds())
-	}
-
-	h.observer.log.Info().Uint8(LogFieldConnIndex, h.connIndex).Msg("Unregistered tunnel connection")
-}

datagramsession/session.go

@@ -24,6 +24,13 @@ func SessionIdleErr(timeout time.Duration) error {
 
 type transportSender func(session *packet.Session) error
 
+// ErrVithVariableSeverity are errors that have variable severity
+type ErrVithVariableSeverity interface {
+	error
+	// LogLevel return the severity of this error
+	LogLevel() zerolog.Level
+}
+
 // Session is a bidirectional pipe of datagrams between transport and dstConn
 // Destination can be a connection with origin or with eyeball
 // When the destination is origin:
@@ -53,7 +60,11 @@ func (s *Session) Serve(ctx context.Context, closeAfterIdle time.Duration) (clos
 				if errors.Is(err, net.ErrClosed) {
 					s.log.Debug().Msg("Destination connection closed")
 				} else {
-					s.log.Error().Err(err).Msg("Failed to send session payload from destination to transport")
+					level := zerolog.ErrorLevel
+					if variableErr, ok := err.(ErrVithVariableSeverity); ok {
+						level = variableErr.LogLevel()
+					}
+					s.log.WithLevel(level).Err(err).Msg("Failed to send session payload from destination to transport")
 				}
 				if closeSession {
 					s.closeChan <- err

dev.Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.18 as builder
+FROM golang:1.19 as builder
 ENV GO111MODULE=on \
     CGO_ENABLED=0
 WORKDIR /go/src/github.com/cloudflare/cloudflared/

edgediscovery/allregions/discovery.go

@@ -41,6 +41,19 @@ const (
 	IPv6Only ConfigIPVersion = 6
 )
 
+func (c ConfigIPVersion) String() string {
+	switch c {
+	case Auto:
+		return "auto"
+	case IPv4Only:
+		return "4"
+	case IPv6Only:
+		return "6"
+	default:
+		return ""
+	}
+}
+
 // IPVersion is the IP version of an EdgeAddr
 type EdgeIPVersion int8
 
@@ -71,11 +84,14 @@ type EdgeAddr struct {
 // If the call to net.LookupSRV fails, try to fall back to DoT from Cloudflare directly.
 //
 // Note: Instead of DoT, we could also have used DoH. Either of these:
-//     - directly via the JSON API (https://1.1.1.1/dns-query?ct=application/dns-json&name=_origintunneld._tcp.argotunnel.com&type=srv)
-//     - indirectly via `tunneldns.NewUpstreamHTTPS()`
+//   - directly via the JSON API (https://1.1.1.1/dns-query?ct=application/dns-json&name=_origintunneld._tcp.argotunnel.com&type=srv)
+//   - indirectly via `tunneldns.NewUpstreamHTTPS()`
+//
 // But both of these cases miss out on a key feature from the stdlib:
-//     "The returned records are sorted by priority and randomized by weight within a priority."
-//     (https://golang.org/pkg/net/#Resolver.LookupSRV)
+//
+//	"The returned records are sorted by priority and randomized by weight within a priority."
+//	(https://golang.org/pkg/net/#Resolver.LookupSRV)
+//
 // Does this matter? I don't know. It may someday. Let's use DoT so we don't need to worry about it.
 // See also: Go feature request for stdlib-supported DoH: https://github.com/golang/go/issues/27552
 var fallbackLookupSRV = lookupSRVWithDOT

edgediscovery/allregions/regions.go

@@ -2,6 +2,7 @@ package allregions
 
 import (
 	"fmt"
+	"math/rand"
 
 	"github.com/rs/zerolog"
 )
@@ -85,6 +86,15 @@ func (rs *Regions) AddrUsedBy(connID int) *EdgeAddr {
 // GetUnusedAddr gets an unused addr from the edge, excluding the given addr. Prefer to use addresses
 // evenly across both regions.
 func (rs *Regions) GetUnusedAddr(excluding *EdgeAddr, connID int) *EdgeAddr {
+	// If both regions have the same number of available addrs, lets randomise which one
+	// we pick. The rest of this algorithm will continue to make sure we always use addresses
+	// evenly across both regions.
+	if rs.region1.AvailableAddrs() == rs.region2.AvailableAddrs() {
+		regions := []Region{rs.region1, rs.region2}
+		firstChoice := rand.Intn(2)
+		return getAddrs(excluding, connID, &regions[firstChoice], &regions[1-firstChoice])
+	}
+
 	if rs.region1.AvailableAddrs() > rs.region2.AvailableAddrs() {
 		return getAddrs(excluding, connID, &rs.region1, &rs.region2)
 	}

edgediscovery/dial.go

@@ -15,12 +15,16 @@ func DialEdge(
 	timeout time.Duration,
 	tlsConfig *tls.Config,
 	edgeTCPAddr *net.TCPAddr,
+	localIP net.IP,
 ) (net.Conn, error) {
 	// Inherit from parent context so we can cancel (Ctrl-C) while dialing
 	dialCtx, dialCancel := context.WithTimeout(ctx, timeout)
 	defer dialCancel()
 
 	dialer := net.Dialer{}
+	if localIP != nil {
+		dialer.LocalAddr = &net.TCPAddr{IP: localIP, Port: 0}
+	}
 	edgeConn, err := dialer.DialContext(dialCtx, "tcp", edgeTCPAddr.String())
 	if err != nil {
 		return nil, newDialError(err, "DialContext error")

edgediscovery/protocol.go

@@ -15,6 +15,8 @@ var (
 	errNoProtocolRecord = fmt.Errorf("No TXT record found for %s to determine connection protocol", protocolRecord)
 )
 
+type PercentageFetcher func() (ProtocolPercents, error)
+
 // ProtocolPercent represents a single Protocol Percentage combination.
 type ProtocolPercent struct {
 	Protocol   string `json:"protocol"`

fmt-check.sh

@@ -2,12 +2,12 @@
 
 set -e -o pipefail
 
-OUTPUT=$(for d in $(go list -mod=vendor -f '{{.Dir}}' -a ./... | fgrep -v tunnelrpc) ; do goimports -format-only -local github.com/cloudflare/cloudflared -d $d ; done)
+OUTPUT=$(goimports -l -d -local github.com/cloudflare/cloudflared $(go list -mod=vendor -f '{{.Dir}}' -a ./... | fgrep -v tunnelrpc))
 
 if [ -n "$OUTPUT" ] ; then
   PAGER=$(which colordiff || echo cat)
   echo
-  echo "Code formatting issues found, use 'goimports -format-only -local github.com/cloudflare/cloudflared' to correct them"
+  echo "Code formatting issues found, use 'make fmt' to correct them"
   echo
   echo "$OUTPUT" | $PAGER
   exit 1

github_message.py

@@ -1,29 +1,56 @@
 #!/usr/bin/python3
 """
-Creates Github Releases Notes with content hashes
+Create Github Releases Notes with binary checksums from Workers KV
 """
 
 import argparse
 import logging
 import os
-import hashlib
-import glob
+import requests
 
-from github import Github, GithubException, UnknownObjectException
+from github import Github, UnknownObjectException
 
 FORMAT = "%(levelname)s - %(asctime)s: %(message)s"
-logging.basicConfig(format=FORMAT)
+logging.basicConfig(format=FORMAT, level=logging.INFO)
 
 CLOUDFLARED_REPO = os.environ.get("GITHUB_REPO", "cloudflare/cloudflared")
 GITHUB_CONFLICT_CODE = "already_exists"
+BASE_KV_URL = 'https://api.cloudflare.com/client/v4/accounts/'
 
-def get_sha256(filename):
-    """ get the sha256 of a file """
-    sha256_hash = hashlib.sha256()
-    with open(filename,"rb") as f:
-        for byte_block in iter(lambda: f.read(4096),b""):
-            sha256_hash.update(byte_block)
-        return sha256_hash.hexdigest()
+
+def kv_get_keys(prefix, account, namespace, api_token):
+    """ get the KV keys for a given prefix """
+    response = requests.get(
+        BASE_KV_URL + account + "/storage/kv/namespaces/" +
+        namespace + "/keys" + "?prefix=" + prefix,
+        headers={
+            "Content-Type": "application/json",
+            "Authorization": "Bearer " + api_token,
+        },
+    )
+    if response.status_code != 200:
+        jsonResponse = response.json()
+        errors = jsonResponse["errors"]
+        if len(errors) > 0:
+            raise Exception("failed to get checksums: {0}", errors[0])
+    return response.json()["result"]
+
+
+def kv_get_value(key, account, namespace, api_token):
+    """ get the KV value for a provided key """
+    response = requests.get(
+        BASE_KV_URL + account + "/storage/kv/namespaces/" + namespace + "/values/" + key,
+        headers={
+            "Content-Type": "application/json",
+            "Authorization": "Bearer " + api_token,
+        },
+    )
+    if response.status_code != 200:
+        jsonResponse = response.json()
+        errors = jsonResponse["errors"]
+        if len(errors) > 0:
+            raise Exception("failed to get checksums: {0}", errors[0])
+    return response.text
 
 
 def update_or_add_message(msg, name, sha):
@@ -42,6 +69,7 @@ def update_or_add_message(msg, name, sha):
         return '{0}{1}```'.format(msg[:back], new_text)
     return '{0} \n### SHA256 Checksums:\n```\n{1}```'.format(msg, new_text)
 
+
 def get_release(repo, version):
     """ Get a Github Release matching the version tag. """
     try:
@@ -55,11 +83,20 @@ def get_release(repo, version):
 def parse_args():
     """ Parse and validate args """
     parser = argparse.ArgumentParser(
-        description="Creates Github Releases and uploads assets."
+        description="Updates a Github Release with checksums from KV"
     )
     parser.add_argument(
         "--api-key", default=os.environ.get("API_KEY"), help="Github API key"
     )
+    parser.add_argument(
+        "--kv-namespace-id", default=os.environ.get("KV_NAMESPACE"), help="workers KV namespace id"
+    )
+    parser.add_argument(
+        "--kv-account-id", default=os.environ.get("KV_ACCOUNT"), help="workers KV account id"
+    )
+    parser.add_argument(
+        "--kv-api-token", default=os.environ.get("KV_API_TOKEN"), help="workers KV API Token"
+    )
     parser.add_argument(
         "--release-version",
         metavar="version",
@@ -80,6 +117,18 @@ def parse_args():
         logging.error("Missing API key")
         is_valid = False
 
+    if not args.kv_namespace_id:
+        logging.error("Missing KV namespace id")
+        is_valid = False
+
+    if not args.kv_account_id:
+        logging.error("Missing KV account id")
+        is_valid = False
+
+    if not args.kv_api_token:
+        logging.error("Missing KV API token")
+        is_valid = False
+
     if is_valid:
         return args
 
@@ -95,16 +144,20 @@ def main():
         repo = client.get_repo(CLOUDFLARED_REPO)
         release = get_release(repo, args.release_version)
 
-        msg = release.body
+        msg = ""
 
-        for filepath in glob.glob("artifacts/*"):
-            pkg_hash = get_sha256(filepath)
-            # add the sha256 of the new artifact to the release message body
-            name = os.path.basename(filepath)
-            msg = update_or_add_message(msg, name, pkg_hash)
+        prefix = f"update_{args.release_version}_"
+        keys = kv_get_keys(prefix, args.kv_account_id,
+                           args.kv_namespace_id, args.kv_api_token)
+        for key in [k["name"] for k in keys]:
+            checksum = kv_get_value(
+                key, args.kv_account_id, args.kv_namespace_id, args.kv_api_token)
+            binary_name = key[len(prefix):]
+            msg = update_or_add_message(msg, binary_name, checksum)
 
         if args.dry_run:
-            logging.info("Skipping asset upload because of dry-run")
+            logging.info("Skipping release message update because of dry-run")
+            logging.info(f"Github message:\n{msg}")
             return
 
         # update the release body text
@@ -115,4 +168,4 @@ def main():
         exit(1)
 
 
-main()
+main()

go.mod

@@ -1,31 +1,32 @@
 module github.com/cloudflare/cloudflared
 
-go 1.18
+go 1.19
 
 require (
 	github.com/cloudflare/brotli-go v0.0.0-20191101163834-d34379f7ff93
 	github.com/cloudflare/golibs v0.0.0-20170913112048-333127dbecfc
-	github.com/coredns/coredns v1.8.7
+	github.com/coredns/coredns v1.10.0
+	github.com/coreos/go-oidc/v3 v3.4.0
 	github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf
 	github.com/facebookgo/grace v0.0.0-20180706040059-75cf19382434
 	github.com/fsnotify/fsnotify v1.4.9
-	github.com/getsentry/raven-go v0.0.0-20180517221441-ed7bcb39ff10
+	github.com/getsentry/raven-go v0.2.0
+	github.com/getsentry/sentry-go v0.16.0
 	github.com/gobwas/ws v1.0.4
 	github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3
 	github.com/google/gopacket v1.1.19
 	github.com/google/uuid v1.3.0
-	github.com/gorilla/mux v1.8.0
 	github.com/gorilla/websocket v1.4.2
 	github.com/json-iterator/go v1.1.12
 	github.com/lucas-clemente/quic-go v0.28.1
-	github.com/mattn/go-colorable v0.1.8
-	github.com/miekg/dns v1.1.45
+	github.com/mattn/go-colorable v0.1.13
+	github.com/miekg/dns v1.1.50
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/pkg/errors v0.9.1
-	github.com/prometheus/client_golang v1.12.1
+	github.com/prometheus/client_golang v1.13.0
 	github.com/prometheus/client_model v0.2.0
 	github.com/rs/zerolog v1.20.0
-	github.com/stretchr/testify v1.7.1
+	github.com/stretchr/testify v1.8.1
 	github.com/urfave/cli/v2 v2.3.0
 	go.opentelemetry.io/contrib/propagators v0.22.0
 	go.opentelemetry.io/otel v1.6.3
@@ -34,12 +35,12 @@ require (
 	go.opentelemetry.io/otel/trace v1.6.3
 	go.opentelemetry.io/proto/otlp v0.15.0
 	go.uber.org/automaxprocs v1.4.0
-	golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa
-	golang.org/x/net v0.0.0-20220909164309-bea034e7d591
-	golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f
-	golang.org/x/sys v0.0.0-20220808155132-1c4a2a72c664
-	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211
-	google.golang.org/protobuf v1.28.0
+	golang.org/x/crypto v0.5.0
+	golang.org/x/net v0.5.0
+	golang.org/x/sync v0.1.0
+	golang.org/x/sys v0.4.0
+	golang.org/x/term v0.4.0
+	google.golang.org/protobuf v1.28.1
 	gopkg.in/coreos/go-oidc.v2 v2.2.1
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
 	gopkg.in/square/go-jose.v2 v2.6.0
@@ -48,15 +49,14 @@ require (
 )
 
 require (
-	github.com/BurntSushi/toml v0.3.1 // indirect
+	github.com/BurntSushi/toml v1.2.0 // indirect
 	github.com/apparentlymart/go-cidr v1.1.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894 // indirect
+	github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/cheekybits/genny v1.0.0 // indirect
 	github.com/cloudflare/circl v1.2.1-0.20220809205628-0a9554f37a47 // indirect
 	github.com/coredns/caddy v1.1.1 // indirect
-	github.com/coreos/go-oidc/v3 v3.4.0 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 // indirect
@@ -72,31 +72,33 @@ require (
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect
 	github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 // indirect
+	github.com/kr/text v0.2.0 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/marten-seemann/qtls-go1-16 v0.1.5 // indirect
 	github.com/marten-seemann/qtls-go1-17 v0.1.2 // indirect
 	github.com/marten-seemann/qtls-go1-18 v0.1.2 // indirect
 	github.com/marten-seemann/qtls-go1-19 v0.1.0-beta.1 // indirect
-	github.com/mattn/go-isatty v0.0.12 // indirect
+	github.com/mattn/go-isatty v0.0.16 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/nxadm/tail v1.4.8 // indirect
 	github.com/onsi/ginkgo v1.16.5 // indirect
+	github.com/onsi/gomega v1.23.0 // indirect
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 // indirect
-	github.com/prometheus/common v0.32.1 // indirect
-	github.com/prometheus/procfs v0.7.3 // indirect
+	github.com/prometheus/common v0.37.0 // indirect
+	github.com/prometheus/procfs v0.8.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
-	golang.org/x/mod v0.4.2 // indirect
-	golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094 // indirect
-	golang.org/x/text v0.3.7 // indirect
-	golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2 // indirect
-	golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect
+	golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
+	golang.org/x/oauth2 v0.4.0 // indirect
+	golang.org/x/text v0.6.0 // indirect
+	golang.org/x/tools v0.1.12 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90 // indirect
-	google.golang.org/grpc v1.47.0 // indirect
+	google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd // indirect
+	google.golang.org/grpc v1.51.0 // indirect
+	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 )
@@ -112,9 +114,11 @@ replace gopkg.in/yaml.v3 => gopkg.in/yaml.v3 v3.0.1
 
 // Post-quantum tunnel RTG-1339
 replace (
-	// branch go1.18
-	github.com/marten-seemann/qtls-go1-18 => github.com/cloudflare/qtls-pq v0.0.0-20220824105406-fb955667e0af
-
-	// branch go1.19
-	github.com/marten-seemann/qtls-go1-19 => github.com/cloudflare/qtls-pq v0.0.0-20220824104809-96561a41e0af
+	// Branches go1.18 go1.19 go1.20 on github.com/cloudflare/qtls-pq
+	github.com/marten-seemann/qtls-go1-18 => github.com/cloudflare/qtls-pq v0.0.0-20230103171413-e7a2fb559a0e
+	github.com/marten-seemann/qtls-go1-19 => github.com/cloudflare/qtls-pq v0.0.0-20230103171656-05e84f90909e
+	github.com/marten-seemann/qtls-go1-20 => github.com/cloudflare/qtls-pq v0.0.0-20230215110727-8b4e1699c2a8
+	github.com/quic-go/qtls-go1-18 => github.com/cloudflare/qtls-pq v0.0.0-20230103171413-e7a2fb559a0e
+	github.com/quic-go/qtls-go1-19 => github.com/cloudflare/qtls-pq v0.0.0-20230103171656-05e84f90909e
+	github.com/quic-go/qtls-go1-20 => github.com/cloudflare/qtls-pq v0.0.0-20230215110727-8b4e1699c2a8
 )

go.sum

@@ -61,37 +61,11 @@ dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBr
 dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4=
 dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU=
 git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg=
-github.com/Azure/azure-sdk-for-go v61.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
-github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
-github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA=
-github.com/Azure/go-autorest/autorest v0.11.19/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA=
-github.com/Azure/go-autorest/autorest v0.11.23/go.mod h1:BAWYUWGPEtKPzjVkp0Q6an0MJcJDsoh5Z1BFAEFs4Xs=
-github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A=
-github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M=
-github.com/Azure/go-autorest/autorest/adal v0.9.14/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M=
-github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ=
-github.com/Azure/go-autorest/autorest/azure/auth v0.5.10/go.mod h1:zQXYYNX9kXzRMrJNVXWUfNy38oPMF5/2TeZ4Wylc9fE=
-github.com/Azure/go-autorest/autorest/azure/cli v0.4.2/go.mod h1:7qkJkT+j6b+hIpzMOwPChJhTqS8VbsqqgULzMNRugoM=
-github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74=
-github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
-github.com/Azure/go-autorest/autorest/to v0.2.0/go.mod h1:GunWKJp1AEqgMaGLV+iocmRAJWqST1wQYhyyjXJ3SJc=
-github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
-github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
-github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/toml v1.2.0 h1:Rt8g24XnyGTyglgET/PRUNlrUeu9F5L+7FilkXfZgs0=
+github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
-github.com/DataDog/datadog-go v4.4.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
-github.com/DataDog/gostackparse v0.5.0/go.mod h1:lTfqcJKqS9KnXQGnyQMCugq3u1FP6UZMfWR0aitKFMM=
-github.com/DataDog/sketches-go v1.0.0/go.mod h1:O+XkJHWk9w4hDwY2ZUDU31ZC9sNYlYo8DiFsxjYeo1k=
-github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
-github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
-github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
-github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
-github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
-github.com/Shopify/sarama v1.30.0/go.mod h1:zujlQQx1kzHsh4jfV1USnptCQrHAEZ2Hk8fTKCulPVs=
-github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
-github.com/Shopify/toxiproxy/v2 v2.1.6-0.20210914104332-15ea381dcdae/go.mod h1:/cvHQkZ1fst0EmZnA5dFtiQdWCNCFYzb+uE2vqVgvx0=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
@@ -101,8 +75,6 @@ github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYU
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/apparentlymart/go-cidr v1.1.0 h1:2mAhrMoF+nhXqxTzSZMUzDHkLjmIHC+Zzn4tdgBZjnU=
 github.com/apparentlymart/go-cidr v1.1.0/go.mod h1:EBcsNrHc3zQeuaeCeCtQruQm+n9/YjEn/vI25Lg7Gwc=
-github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
-github.com/aws/aws-sdk-go v1.42.30/go.mod h1:OGr6lGMAKGlG9CVrYnWYDKIyb829c6EVBRjxqjmPepc=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -112,8 +84,8 @@ github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7
 github.com/bwesterb/go-ristretto v1.2.2/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
 github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
-github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894 h1:JLaf/iINcLyjwbtTsCJjc6rtlASgHeIJPrB6QmwURnA=
-github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA=
+github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d h1:S2NE3iHSwP0XV47EEXL8mWmRdEfGscSJ+7EgePNgt0s=
+github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
@@ -132,10 +104,10 @@ github.com/cloudflare/circl v1.2.1-0.20220809205628-0a9554f37a47 h1:YzpECHxZ9TzO
 github.com/cloudflare/circl v1.2.1-0.20220809205628-0a9554f37a47/go.mod h1:qhx8gBILsYlbam7h09SvHDSkjpe3TfLA7b/z4rxJvkE=
 github.com/cloudflare/golibs v0.0.0-20170913112048-333127dbecfc h1:Dvk3ySBsOm5EviLx6VCyILnafPcQinXGP5jbTdHUJgE=
 github.com/cloudflare/golibs v0.0.0-20170913112048-333127dbecfc/go.mod h1:HlgKKR8V5a1wroIDDIz3/A+T+9Janfq+7n1P5sEFdi0=
-github.com/cloudflare/qtls-pq v0.0.0-20220824104809-96561a41e0af h1:JMpOQAaXjRRBkUX73fTNe9mConJLFl6FsIp9fHdLm7Y=
-github.com/cloudflare/qtls-pq v0.0.0-20220824104809-96561a41e0af/go.mod h1:aIsWqC0WXyUiUxBl/RfxAjDyWE9CCLqvSMnCMTd/+bc=
-github.com/cloudflare/qtls-pq v0.0.0-20220824105406-fb955667e0af h1:bhCmedjwrOSyzLtHVeQ+KhimcNTSfs0P5T7kbRQS+gA=
-github.com/cloudflare/qtls-pq v0.0.0-20220824105406-fb955667e0af/go.mod h1:mW0BgKFFDAiSmOdUwoORtjo0V2vqw5QzVYRtKQqw/Jg=
+github.com/cloudflare/qtls-pq v0.0.0-20230103171413-e7a2fb559a0e h1:frfo+L0qloEb6Vj+qjS4pbAYSJQZAlUnKZu0uJoErac=
+github.com/cloudflare/qtls-pq v0.0.0-20230103171413-e7a2fb559a0e/go.mod h1:mW0BgKFFDAiSmOdUwoORtjo0V2vqw5QzVYRtKQqw/Jg=
+github.com/cloudflare/qtls-pq v0.0.0-20230103171656-05e84f90909e h1:RtQDXvDi0PK3EonP0v7zkE5/rApK4MsgRATCdD+ughg=
+github.com/cloudflare/qtls-pq v0.0.0-20230103171656-05e84f90909e/go.mod h1:aIsWqC0WXyUiUxBl/RfxAjDyWE9CCLqvSMnCMTd/+bc=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
@@ -147,16 +119,14 @@ github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWH
 github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/coredns/caddy v1.1.1 h1:2eYKZT7i6yxIfGP3qLJoJ7HAsDJqYB+X68g4NYjSrE0=
 github.com/coredns/caddy v1.1.1/go.mod h1:A6ntJQlAWuQfFlsd9hvigKbo2WS0VUs2l1e2F+BawD4=
-github.com/coredns/coredns v1.8.7 h1:wVMjAnyFnY7Mc18AFO+9qbGD6ODPtdVUIlzoWrHr3hk=
-github.com/coredns/coredns v1.8.7/go.mod h1:bFmbgEfeRz5aizL2VsQ5LRlsvJuXWkgG/MWG9zxqjVM=
+github.com/coredns/coredns v1.10.0 h1:jCfuWsBjTs0dapkkhISfPCzn5LqvSRtrFtaf/Tjj4DI=
+github.com/coredns/coredns v1.10.0/go.mod h1:CIfRU5TgpuoIiJBJ4XrofQzfFQpPFh32ERpUevrSlaw=
 github.com/coreos/go-oidc/v3 v3.4.0 h1:xz7elHb/LDwm/ERpwHd+5nb7wFHL32rsr6bBOgaeu6g=
 github.com/coreos/go-oidc/v3 v3.4.0/go.mod h1:eHUXhZtXPQLgEaDrOVTgwbgmz1xGOkJNye6h3zkD2Pw=
-github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU=
 github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
-github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM=
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
@@ -164,18 +134,7 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8=
-github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
-github.com/dnstap/golang-dnstap v0.4.0/go.mod h1:FqsSdH58NAmkAvKcpyxht7i4FoBjKu8E4JUPt8ipSUs=
-github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
-github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
-github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
-github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
-github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
-github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
-github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
-github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
@@ -186,7 +145,6 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.m
 github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
 github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
-github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 h1:0JZ+dUmQeA8IIVUMzysrX4/AKuQwWhV2dYQuPZdvdSQ=
 github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64=
 github.com/facebookgo/freeport v0.0.0-20150612182905-d4adf43b75b9 h1:wWke/RUCl7VRjQhwPlR/v0glZXNYzBHdNUzf/Am2Nmg=
@@ -197,44 +155,36 @@ github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojt
 github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg=
 github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870 h1:E2s37DuLxFhQDg5gKsWoLBOB0n+ZW8s599zru8FJ2/Y=
 github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0=
-github.com/farsightsec/golang-framestream v0.3.0/go.mod h1:eNde4IQyEiA5br02AouhEHCu3p3UzrCdFR4LuQHklMI=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
-github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
-github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
-github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
 github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY=
-github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
-github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg=
-github.com/getsentry/raven-go v0.0.0-20180517221441-ed7bcb39ff10 h1:YO10pIIBftO/kkTFdWhctH96grJ7qiy7bMdiZcIvPKs=
-github.com/getsentry/raven-go v0.0.0-20180517221441-ed7bcb39ff10/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ=
+github.com/getsentry/raven-go v0.2.0 h1:no+xWJRb5ZI7eE8TWgIq1jLulQiIoLG0IfYxv5JYMGs=
+github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ=
+github.com/getsentry/sentry-go v0.16.0 h1:owk+S+5XcgJLlGR/3+3s6N4d+uKwqYvh/eS0AIMjPWo=
+github.com/getsentry/sentry-go v0.16.0/go.mod h1:ZXCloQLj0pG7mja5NK6NPf2V4A88YJ4pNlc2mOHwh6Y=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
 github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
+github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
+github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
-github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
-github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
-github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0=
 github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
-github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
-github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
-github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8=
-github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I=
 github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
@@ -244,21 +194,15 @@ github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og=
 github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
 github.com/gobwas/ws v1.0.4 h1:5eXU1CZhpQdq5kXbKb+sECH5Ia5KiO6CYzIzdlVx6Bs=
 github.com/gobwas/ws v1.0.4/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=
-github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
-github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
-github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
-github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3 h1:zN2lZNZRflqFyxVaTIU61KNKQ9C0055u9CAfpmqUvo4=
 github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3/go.mod h1:nPpo7qLxd6XL3hWJG/O60sR8ZKfMCiIoNap5GvD12KU=
-github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ=
 github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
@@ -288,12 +232,9 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS
 github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM=
 github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
-github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
-github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
-github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
@@ -307,13 +248,11 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
-github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
 github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8=
 github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
@@ -331,7 +270,6 @@ github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLe
 github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/pprof v0.0.0-20210423192551-a2663126120b/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
@@ -350,47 +288,25 @@ github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0
 github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM=
 github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM=
 github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c=
-github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU=
-github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA=
 github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
-github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
-github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
-github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
-github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
-github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
-github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
-github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=
 github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
-github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 h1:BZHcxBETFHIdVyhyEfOvn/RdU/QGdLI4y34qQGjGWO0=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks=
 github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 h1:MJG/KsmcqMwFAkh8mTnAwhyKoB+sTAnY4CACC110tbU=
 github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645/go.mod h1:6iZfnjpejD4L/4DwD7NryNaJyCQdzwWwH2MWhCA90Kw=
-github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
-github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
-github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
-github.com/infobloxopen/go-trees v0.0.0-20200715205103-96a057b8dfb9/go.mod h1:BaIJzjD2ZnHmx2acPF6XfGLPzNCMiBbMRqJr+8/8uRI=
 github.com/ipostelnik/cli/v2 v2.3.1-0.20210324024421-b6ea8234fe3d h1:PRDnysJ9dF1vUMmEzBu6aHQeUluSQy4eWH3RsSSy/vI=
 github.com/ipostelnik/cli/v2 v2.3.1-0.20210324024421-b6ea8234fe3d/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
-github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs=
-github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM=
-github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o=
-github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg=
-github.com/jcmturner/gokrb5/v8 v8.4.2/go.mod h1:sb+Xq/fTY5yktf/VxLsE3wlfPqQjp0aWNYyvBVK62bc=
-github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc=
 github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU=
-github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
-github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
@@ -401,14 +317,11 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
-github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
-github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -419,29 +332,23 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI=
-github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
 github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/marten-seemann/qpack v0.2.1/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc=
 github.com/marten-seemann/qtls-go1-16 v0.1.5 h1:o9JrYPPco/Nukd/HpOHMHZoBDXQqoNtUCmny98/1uqQ=
 github.com/marten-seemann/qtls-go1-16 v0.1.5/go.mod h1:gNpI2Ol+lRS3WwSOtIUUtRwZEQMXjYK+dQSBFbethAk=
 github.com/marten-seemann/qtls-go1-17 v0.1.2 h1:JADBlm0LYiVbuSySCHeY863dNkcpMmDR7s0bLKJeYlQ=
 github.com/marten-seemann/qtls-go1-17 v0.1.2/go.mod h1:C2ekUKcDdz9SDWxec1N/MvcXBpaX9l3Nx67XaR84L5s=
-github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8=
-github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
-github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
-github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
+github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
+github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
+github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ=
+github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4=
-github.com/miekg/dns v1.1.31/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
-github.com/miekg/dns v1.1.45 h1:g5fRIhm9nx7g8osrAvgb16QJfmyMsyOCb+J7LSv+Qzk=
-github.com/miekg/dns v1.1.45/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME=
+github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA=
+github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
-github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
-github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -449,52 +356,34 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
-github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
-github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
 github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo=
 github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM=
-github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
 github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
 github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
-github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
 github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
 github.com/onsi/ginkgo v1.16.2/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E=
 github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
 github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
 github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
-github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
-github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
 github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY=
-github.com/onsi/gomega v1.16.0 h1:6gjqkI8iiRHMvdccRJM8rVKjCWk6ZIm6FTm3ddIe4/c=
-github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
-github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis=
-github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
+github.com/onsi/gomega v1.23.0 h1:/oxKu9c2HVap+F3PfKort2Hw5DEU+HGlW8n+tguWsys=
+github.com/onsi/gomega v1.23.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg=
 github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs=
 github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
-github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxSfWAKL3wpBW7V8scJMt8N8gnaMCS9E/cA=
 github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8=
-github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4=
-github.com/openzipkin/zipkin-go v0.3.0/go.mod h1:4c3sLeE8xjNqehmF5RpAFLPLJxXscc0R4l6Zg0P1tTQ=
-github.com/oschwald/geoip2-golang v1.5.0/go.mod h1:xdvYt5xQzB8ORWFqPnqMwZpCpgNagttWdoZLlJQzg7s=
-github.com/oschwald/maxminddb-golang v1.8.0/go.mod h1:RXZtst0N6+FY/3qCNmZMBApR19cdQj43/NM9VkrNAis=
-github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
 github.com/philhofer/fwd v1.1.1 h1:GdGcTjf5RNAxwS4QLsiMzJYj5KEvPJD3Abr261yRQXQ=
-github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
-github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc=
-github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
+github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 h1:J9b7z+QKAmPf4YLrFg6oQUotqHQeUNWwkvo7jZp1GLU=
@@ -504,8 +393,9 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
 github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
 github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
-github.com/prometheus/client_golang v1.12.1 h1:ZiaPsmm9uiBeaSMRznKsCDNtPCS0T3JVDGF+06gjBzk=
 github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
+github.com/prometheus/client_golang v1.13.0 h1:b71QUfeo5M8gq2+evJdTPfZhYMAU0uKPkyPJ7TPsloU=
+github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
@@ -515,18 +405,17 @@ github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7q
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
 github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
-github.com/prometheus/common v0.32.1 h1:hWIdL3N2HoUx3B8j3YN9mWor0qhY/NlEKZEaXxuIRh4=
 github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
+github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE=
+github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA=
 github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
 github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
-github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU=
 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
-github.com/rabbitmq/amqp091-go v1.1.0/go.mod h1:ogQDLSOACsLPsIq0NpbtiifNZi2YOz0VTJ0kHRghqbM=
-github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
-github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
+github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo=
+github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
@@ -563,42 +452,32 @@ github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5k
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
-github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
-github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE=
 github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
-github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
-github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
-github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8=
-github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA=
 github.com/tinylib/msgp v1.1.2 h1:gWmO7n0Ys2RBEb7GPYB9Ujq8Mk5p2U08lRnmMcGy6BQ=
-github.com/tinylib/msgp v1.1.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
 github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU=
 github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM=
-github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
-github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs=
-github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
-go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
-go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
-go.etcd.io/etcd/client/v3 v3.5.1/go.mod h1:OnjH4M8OnAotwaB2l9bVgZzRFKru7/ZMoS46OtKyd3Q=
 go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
@@ -623,11 +502,8 @@ go.opentelemetry.io/otel/trace v1.6.3/go.mod h1:GNJQusJlUgZl9/TQBPKU/Y/ty+0iVB5f
 go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
 go.opentelemetry.io/proto/otlp v0.15.0 h1:h0bKrvdrT/9sBwEJ6iWUqT/N/xPcS66bL4u3isneJ6w=
 go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
-go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/automaxprocs v1.4.0 h1:CpDZl6aOlLhReez+8S3eEotD7Jx0Os++lemPlMULQP0=
 go.uber.org/automaxprocs v1.4.0/go.mod h1:/mTEdr7LvHhs0v7mjdxDreTz1OG5zdZGqgOnhWiR/+Q=
-go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
-go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
 go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE=
 golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
@@ -639,13 +515,9 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.0.0-20210920023735-84f357641f63/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa h1:zuSxTR4o9y82ebqCUJYNGJbGPo6sKVl54f/TVDObg1c=
 golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.5.0 h1:U/0M97KRkSFvyD/3FSmdP5W5swImpNgle/EHFhOsQPE=
+golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -681,8 +553,9 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -701,8 +574,6 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -730,10 +601,7 @@ golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT
 golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20210917221730-978cfadd31cf/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
@@ -741,8 +609,9 @@ golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su
 golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.0.0-20220909164309-bea034e7d591 h1:D0B/7al0LLrVC8aWF4+oxpv/m8bc7ViFfVS8/gXGdqI=
-golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
+golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
+golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw=
+golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -765,8 +634,9 @@ golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j
 golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
 golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
 golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE=
-golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094 h1:2o1E+E8TpNLklK9nHiPiK1uzIYrIHt+cQx3ynCwq9V8=
 golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg=
+golang.org/x/oauth2 v0.4.0 h1:NF0gk8LVPg1Ml7SSbGyySuoxdsXitj7TvgvuRxIMc/M=
+golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec=
 golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -779,8 +649,9 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f h1:Ax0t5p6N38Ga0dThY21weqDEyz2oklo4IvDkpigvkD8=
 golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -797,17 +668,13 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191224085550-c709ea063b76/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -836,7 +703,6 @@ golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -848,7 +714,6 @@ golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -864,12 +729,14 @@ golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220808155132-1c4a2a72c664 h1:v1W7bwXHsnLLloWYTVEdvGvA7BHMeBYsPcF0GLDxIRs=
 golang.org/x/sys v0.0.0-20220808155132-1c4a2a72c664/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18=
+golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.4.0 h1:O7UWfv5+A2qiuulQk30kVinPoMtoIPeVaKLEgLpVkvg=
+golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -878,13 +745,13 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k=
+golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -908,7 +775,6 @@ golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
@@ -923,11 +789,9 @@ golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjs
 golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
 golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
 golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
@@ -937,22 +801,21 @@ golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4f
 golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
 golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
-golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2 h1:BonxutuHCTL0rBDnZlKjpGIQFTjyUVTexFOdWkB6Fg0=
 golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
-golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f h1:uF6paiQQebLeSXkrTqHqz0MXhXXS1KgF41eUdBNvxK0=
 golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
 google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
 google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
@@ -988,7 +851,6 @@ google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqiv
 google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI=
 google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I=
 google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo=
-google.golang.org/api v0.64.0/go.mod h1:931CdxA8Rm4t6zqTFGSsgwbAEZ2+GMYurbndwSimebM=
 google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g=
 google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA=
 google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8=
@@ -1042,7 +904,6 @@ google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6D
 google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
@@ -1073,7 +934,6 @@ google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ6
 google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
 google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
 google.golang.org/genproto v0.0.0-20211221195035-429b39de9b1c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20211223182754-3ac035c7e7cb/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
 google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
 google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
 google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI=
@@ -1090,16 +950,15 @@ google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP
 google.golang.org/genproto v0.0.0-20220518221133-4f43b3371335/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4=
 google.golang.org/genproto v0.0.0-20220523171625-347a074981d8/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4=
 google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA=
-google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90 h1:4SPz2GL2CXJt28MTF8V6Ap/9ZiVbQlJeGSd9qtA7DLs=
 google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA=
+google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd h1:OjndDrsik+Gt+e6fs45z9AxiewiKyLKYpA45W5Kpkks=
+google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd/go.mod h1:cTsE614GARnxrLsqKREzmNYJACSWWpAWdNMwnD7c2BE=
 google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
 google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio=
 google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
-google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
-google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
 google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
@@ -1123,15 +982,14 @@ google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnD
 google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE=
 google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
 google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
-google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k=
 google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
-google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
 google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
 google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ=
 google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
 google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
-google.golang.org/grpc v1.47.0 h1:9n77onPX5F3qfFCqjy9dhn8PbNQsIKeVU04J9G7umt8=
 google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
+google.golang.org/grpc v1.51.0 h1:E1eGv1FTqoLIdnBCZufiSHgKjlqG6fKFf6pPWtMTh8U=
+google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww=
 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
@@ -1146,14 +1004,13 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw=
 google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
-gopkg.in/DataDog/dd-trace-go.v1 v1.34.0/go.mod h1:HtrC65fyJ6lWazShCC9rlOeiTSZJ0XtZhkwjZM2WpC4=
+google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=
+google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/coreos/go-oidc.v2 v2.2.1 h1:MY5SZClJ7vhjKfr64a4nHAOV/c3WH2gB9BMrR64J1Mc=
@@ -1172,7 +1029,6 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
@@ -1187,24 +1043,9 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-k8s.io/api v0.23.1/go.mod h1:WfXnOnwSqNtG62Y1CdjoMxh7r7u9QXGCkA1u0na2jgo=
-k8s.io/apimachinery v0.23.1/go.mod h1:SADt2Kl8/sttJ62RRsi9MIV4o8f5S3coArm0Iu3fBno=
-k8s.io/client-go v0.23.1/go.mod h1:6QSI8fEuqD4zgFK0xbdwfB/PthBsIxCJMa3s17WlcO0=
-k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
-k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
-k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
-k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
-k8s.io/klog/v2 v2.40.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
-k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk=
-k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
-k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
-sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs=
-sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
-sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4=
-sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
 sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck=
 sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0=
 zombiezen.com/go/capnproto2 v2.18.0+incompatible h1:mwfXZniffG5mXokQGHUJWGnqIBggoPfT/CEwon9Yess=

ingress/config.go

@@ -113,7 +113,7 @@ func (rc *RemoteConfig) UnmarshalJSON(b []byte) error {
 	return nil
 }
 
-func originRequestFromSingeRule(c *cli.Context) OriginRequestConfig {
+func originRequestFromSingleRule(c *cli.Context) OriginRequestConfig {
 	var connectTimeout = defaultHTTPConnectTimeout
 	var tlsTimeout = defaultTLSTimeout
 	var tcpKeepAlive = defaultTCPKeepAlive
@@ -427,12 +427,19 @@ func (defaults *OriginRequestConfig) setHttp2Origin(overrides config.OriginReque
 	}
 }
 
+func (defaults *OriginRequestConfig) setAccess(overrides config.OriginRequestConfig) {
+	if val := overrides.Access; val != nil {
+		defaults.Access = *val
+	}
+}
+
 // SetConfig gets config for the requests that cloudflared sends to origins.
 // Each field has a setter method which sets a value for the field by trying to find:
-//   1. The user config for this rule
-//   2. The user config for the overall ingress config
-//   3. Defaults chosen by the cloudflared team
-//   4. Golang zero values for that type
+//  1. The user config for this rule
+//  2. The user config for the overall ingress config
+//  3. Defaults chosen by the cloudflared team
+//  4. Golang zero values for that type
+//
 // If an earlier option isn't set, it will try the next option down.
 func setConfig(defaults OriginRequestConfig, overrides config.OriginRequestConfig) OriginRequestConfig {
 	cfg := defaults
@@ -453,6 +460,8 @@ func setConfig(defaults OriginRequestConfig, overrides config.OriginRequestConfi
 	cfg.setProxyType(overrides)
 	cfg.setIPRules(overrides)
 	cfg.setHttp2Origin(overrides)
+	cfg.setAccess(overrides)
+
 	return cfg
 }
 
@@ -463,6 +472,7 @@ func ConvertToRawOriginConfig(c OriginRequestConfig) config.OriginRequestConfig
 	var keepAliveConnections *int
 	var keepAliveTimeout *config.CustomDuration
 	var proxyAddress *string
+	var access *config.AccessConfig
 
 	if c.ConnectTimeout != defaultHTTPConnectTimeout {
 		connectTimeout = &c.ConnectTimeout
@@ -482,6 +492,9 @@ func ConvertToRawOriginConfig(c OriginRequestConfig) config.OriginRequestConfig
 	if c.ProxyAddress != defaultProxyAddress {
 		proxyAddress = &c.ProxyAddress
 	}
+	if c.Access.Required {
+		access = &c.Access
+	}
 
 	return config.OriginRequestConfig{
 		ConnectTimeout:         connectTimeout,
@@ -501,6 +514,7 @@ func ConvertToRawOriginConfig(c OriginRequestConfig) config.OriginRequestConfig
 		ProxyType:              emptyStringToNil(c.ProxyType),
 		IPRules:                convertToRawIPRules(c.IPRules),
 		Http2Origin:            defaultBoolToNil(c.Http2Origin),
+		Access:                 access,
 	}
 }
 

ingress/config_test.go

@@ -411,7 +411,7 @@ func TestDefaultConfigFromCLI(t *testing.T) {
 		KeepAliveTimeout:     defaultKeepAliveTimeout,
 		ProxyAddress:         defaultProxyAddress,
 	}
-	actual := originRequestFromSingeRule(c)
+	actual := originRequestFromSingleRule(c)
 	require.Equal(t, expected, actual)
 }
 

ingress/icmp_darwin.go

@@ -11,16 +11,17 @@ import (
 	"context"
 	"fmt"
 	"math"
-	"net"
 	"net/netip"
 	"strconv"
 	"sync"
 	"time"
 
 	"github.com/rs/zerolog"
+	"go.opentelemetry.io/otel/attribute"
 	"golang.org/x/net/icmp"
 
 	"github.com/cloudflare/cloudflared/packet"
+	"github.com/cloudflare/cloudflared/tracing"
 )
 
 type icmpProxy struct {
@@ -129,44 +130,53 @@ func newICMPProxy(listenIP netip.Addr, zone string, logger *zerolog.Logger, idle
 	}, nil
 }
 
-func (ip *icmpProxy) Request(pk *packet.ICMP, responder packet.FunnelUniPipe) error {
-	if pk == nil {
-		return errPacketNil
-	}
+func (ip *icmpProxy) Request(ctx context.Context, pk *packet.ICMP, responder *packetResponder) error {
+	ctx, span := responder.requestSpan(ctx, pk)
+	defer responder.exportSpan()
+
 	originalEcho, err := getICMPEcho(pk.Message)
 	if err != nil {
+		tracing.EndWithErrorStatus(span, err)
 		return err
 	}
+	span.SetAttributes(
+		attribute.Int("originalEchoID", originalEcho.ID),
+		attribute.Int("seq", originalEcho.Seq),
+	)
 	echoIDTrackerKey := flow3Tuple{
 		srcIP:          pk.Src,
 		dstIP:          pk.Dst,
 		originalEchoID: originalEcho.ID,
 	}
-	// TODO: TUN-6744 assign unique flow per (src, echo ID)
 	assignedEchoID, success := ip.echoIDTracker.getOrAssign(echoIDTrackerKey)
 	if !success {
-		return fmt.Errorf("failed to assign unique echo ID")
+		err := fmt.Errorf("failed to assign unique echo ID")
+		tracing.EndWithErrorStatus(span, err)
+		return err
 	}
+	span.SetAttributes(attribute.Int("assignedEchoID", int(assignedEchoID)))
+
+	shouldReplaceFunnelFunc := createShouldReplaceFunnelFunc(ip.logger, responder.datagramMuxer, pk, originalEcho.ID)
 	newFunnelFunc := func() (packet.Funnel, error) {
 		originalEcho, err := getICMPEcho(pk.Message)
 		if err != nil {
 			return nil, err
 		}
-		originSender := originSender{
-			conn:             ip.conn,
-			echoIDTracker:    ip.echoIDTracker,
-			echoIDTrackerKey: echoIDTrackerKey,
-			assignedEchoID:   assignedEchoID,
+		closeCallback := func() error {
+			ip.echoIDTracker.release(echoIDTrackerKey, assignedEchoID)
+			return nil
 		}
-		icmpFlow := newICMPEchoFlow(pk.Src, &originSender, responder, int(assignedEchoID), originalEcho.ID, ip.encoder)
+		icmpFlow := newICMPEchoFlow(pk.Src, closeCallback, ip.conn, responder, int(assignedEchoID), originalEcho.ID, ip.encoder)
 		return icmpFlow, nil
 	}
 	funnelID := echoFunnelID(assignedEchoID)
-	funnel, isNew, err := ip.srcFunnelTracker.GetOrRegister(funnelID, newFunnelFunc)
+	funnel, isNew, err := ip.srcFunnelTracker.GetOrRegister(funnelID, shouldReplaceFunnelFunc, newFunnelFunc)
 	if err != nil {
+		tracing.EndWithErrorStatus(span, err)
 		return err
 	}
 	if isNew {
+		span.SetAttributes(attribute.Bool("newFlow", true))
 		ip.logger.Debug().
 			Str("src", pk.Src.String()).
 			Str("dst", pk.Dst.String()).
@@ -176,9 +186,16 @@ func (ip *icmpProxy) Request(pk *packet.ICMP, responder packet.FunnelUniPipe) er
 	}
 	icmpFlow, err := toICMPEchoFlow(funnel)
 	if err != nil {
+		tracing.EndWithErrorStatus(span, err)
 		return err
 	}
-	return icmpFlow.sendToDst(pk.Dst, pk.Message)
+	err = icmpFlow.sendToDst(pk.Dst, pk.Message)
+	if err != nil {
+		tracing.EndWithErrorStatus(span, err)
+		return err
+	}
+	tracing.End(span)
+	return nil
 }
 
 // Serve listens for responses to the requests until context is done
@@ -202,7 +219,7 @@ func (ip *icmpProxy) Serve(ctx context.Context) error {
 			ip.logger.Debug().Err(err).Str("dst", from.String()).Msg("Failed to parse ICMP reply, continue to parse as full packet")
 			// In unit test, we found out when the listener listens on 0.0.0.0, the socket reads the full packet after
 			// the second reply
-			if err := ip.handleFullPacket(icmpDecoder, buf[:n]); err != nil {
+			if err := ip.handleFullPacket(ctx, icmpDecoder, buf[:n]); err != nil {
 				ip.logger.Debug().Err(err).Str("dst", from.String()).Msg("Failed to parse ICMP reply as full packet")
 			}
 			continue
@@ -211,14 +228,14 @@ func (ip *icmpProxy) Serve(ctx context.Context) error {
 			ip.logger.Debug().Str("dst", from.String()).Msgf("Drop ICMP %s from reply", reply.msg.Type)
 			continue
 		}
-		if err := ip.sendReply(reply); err != nil {
-			ip.logger.Error().Err(err).Str("dst", from.String()).Msg("Failed to send ICMP reply")
+		if err := ip.sendReply(ctx, reply); err != nil {
+			ip.logger.Debug().Err(err).Str("dst", from.String()).Msg("Failed to send ICMP reply")
 			continue
 		}
 	}
 }
 
-func (ip *icmpProxy) handleFullPacket(decoder *packet.ICMPDecoder, rawPacket []byte) error {
+func (ip *icmpProxy) handleFullPacket(ctx context.Context, decoder *packet.ICMPDecoder, rawPacket []byte) error {
 	icmpPacket, err := decoder.Decode(packet.RawPacket{Data: rawPacket})
 	if err != nil {
 		return err
@@ -232,13 +249,13 @@ func (ip *icmpProxy) handleFullPacket(decoder *packet.ICMPDecoder, rawPacket []b
 		msg:  icmpPacket.Message,
 		echo: echo,
 	}
-	if ip.sendReply(&reply); err != nil {
+	if ip.sendReply(ctx, &reply); err != nil {
 		return err
 	}
 	return nil
 }
 
-func (ip *icmpProxy) sendReply(reply *echoReply) error {
+func (ip *icmpProxy) sendReply(ctx context.Context, reply *echoReply) error {
 	funnelID := echoFunnelID(reply.echo.ID)
 	funnel, ok := ip.srcFunnelTracker.Get(funnelID)
 	if !ok {
@@ -248,25 +265,19 @@ func (ip *icmpProxy) sendReply(reply *echoReply) error {
 	if err != nil {
 		return err
 	}
-	return icmpFlow.returnToSrc(reply)
-}
 
-// originSender wraps icmp.PacketConn to implement packet.FunnelUniPipe interface
-type originSender struct {
-	conn             *icmp.PacketConn
-	echoIDTracker    *echoIDTracker
-	echoIDTrackerKey flow3Tuple
-	assignedEchoID   uint16
-}
+	_, span := icmpFlow.responder.replySpan(ctx, ip.logger)
+	defer icmpFlow.responder.exportSpan()
 
-func (os *originSender) SendPacket(dst netip.Addr, pk packet.RawPacket) error {
-	_, err := os.conn.WriteTo(pk.Data, &net.UDPAddr{
-		IP: dst.AsSlice(),
-	})
-	return err
-}
-
-func (os *originSender) Close() error {
-	os.echoIDTracker.release(os.echoIDTrackerKey, os.assignedEchoID)
+	span.SetAttributes(
+		attribute.String("dst", reply.from.String()),
+		attribute.Int("echoID", reply.echo.ID),
+		attribute.Int("seq", reply.echo.Seq),
+		attribute.Int("originalEchoID", icmpFlow.originalEchoID),
+	)
+	if err := icmpFlow.returnToSrc(reply); err != nil {
+		tracing.EndWithErrorStatus(span, err)
+	}
+	tracing.End(span)
 	return nil
 }

ingress/icmp_darwin_test.go

@@ -8,6 +8,8 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/require"
+
+	"github.com/cloudflare/cloudflared/packet"
 )
 
 func TestSingleEchoIDTracker(t *testing.T) {
@@ -119,3 +121,9 @@ func (eit *echoIDTracker) get(key flow3Tuple) (id uint16, exist bool) {
 	id, exists := eit.mapping[key]
 	return id, exists
 }
+
+func getFunnel(t *testing.T, proxy *icmpProxy, tuple flow3Tuple) (packet.Funnel, bool) {
+	assignedEchoID, success := proxy.echoIDTracker.getOrAssign(tuple)
+	require.True(t, success)
+	return proxy.srcFunnelTracker.Get(echoFunnelID(assignedEchoID))
+}

ingress/icmp_generic.go

@@ -18,7 +18,7 @@ var errICMPProxyNotImplemented = fmt.Errorf("ICMP proxy is not implemented on %s
 
 type icmpProxy struct{}
 
-func (ip icmpProxy) Request(pk *packet.ICMP, responder packet.FunnelUniPipe) error {
+func (ip icmpProxy) Request(ctx context.Context, pk *packet.ICMP, responder *packetResponder) error {
 	return errICMPProxyNotImplemented
 }
 

ingress/icmp_linux.go

@@ -19,9 +19,10 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
-	"golang.org/x/net/icmp"
+	"go.opentelemetry.io/otel/attribute"
 
 	"github.com/cloudflare/cloudflared/packet"
+	"github.com/cloudflare/cloudflared/tracing"
 )
 
 const (
@@ -97,29 +98,39 @@ func checkInPingGroup() error {
 	return fmt.Errorf("did not find group range in %s", pingGroupPath)
 }
 
-func (ip *icmpProxy) Request(pk *packet.ICMP, responder packet.FunnelUniPipe) error {
-	if pk == nil {
-		return errPacketNil
-	}
+func (ip *icmpProxy) Request(ctx context.Context, pk *packet.ICMP, responder *packetResponder) error {
+	ctx, span := responder.requestSpan(ctx, pk)
+	defer responder.exportSpan()
+
 	originalEcho, err := getICMPEcho(pk.Message)
 	if err != nil {
+		tracing.EndWithErrorStatus(span, err)
 		return err
 	}
-	newConnChan := make(chan *icmp.PacketConn, 1)
+	span.SetAttributes(
+		attribute.Int("originalEchoID", originalEcho.ID),
+		attribute.Int("seq", originalEcho.Seq),
+	)
+
+	shouldReplaceFunnelFunc := createShouldReplaceFunnelFunc(ip.logger, responder.datagramMuxer, pk, originalEcho.ID)
 	newFunnelFunc := func() (packet.Funnel, error) {
 		conn, err := newICMPConn(ip.listenIP, ip.ipv6Zone)
 		if err != nil {
+			tracing.EndWithErrorStatus(span, err)
 			return nil, errors.Wrap(err, "failed to open ICMP socket")
 		}
 		ip.logger.Debug().Msgf("Opened ICMP socket listen on %s", conn.LocalAddr())
-		newConnChan <- conn
+		closeCallback := func() error {
+			return conn.Close()
+		}
 		localUDPAddr, ok := conn.LocalAddr().(*net.UDPAddr)
 		if !ok {
 			return nil, fmt.Errorf("ICMP listener address %s is not net.UDPAddr", conn.LocalAddr())
 		}
-		originSender := originSender{conn: conn}
+		span.SetAttributes(attribute.Int("port", localUDPAddr.Port))
+
 		echoID := localUDPAddr.Port
-		icmpFlow := newICMPEchoFlow(pk.Src, &originSender, responder, echoID, originalEcho.ID, packet.NewEncoder())
+		icmpFlow := newICMPEchoFlow(pk.Src, closeCallback, conn, responder, echoID, originalEcho.ID, packet.NewEncoder())
 		return icmpFlow, nil
 	}
 	funnelID := flow3Tuple{
@@ -127,24 +138,26 @@ func (ip *icmpProxy) Request(pk *packet.ICMP, responder packet.FunnelUniPipe) er
 		dstIP:          pk.Dst,
 		originalEchoID: originalEcho.ID,
 	}
-	funnel, isNew, err := ip.srcFunnelTracker.GetOrRegister(funnelID, newFunnelFunc)
+	funnel, isNew, err := ip.srcFunnelTracker.GetOrRegister(funnelID, shouldReplaceFunnelFunc, newFunnelFunc)
 	if err != nil {
+		tracing.EndWithErrorStatus(span, err)
 		return err
 	}
 	icmpFlow, err := toICMPEchoFlow(funnel)
 	if err != nil {
+		tracing.EndWithErrorStatus(span, err)
 		return err
 	}
 	if isNew {
+		span.SetAttributes(attribute.Bool("newFlow", true))
 		ip.logger.Debug().
 			Str("src", pk.Src.String()).
 			Str("dst", pk.Dst.String()).
 			Int("originalEchoID", originalEcho.ID).
 			Msg("New flow")
-		conn := <-newConnChan
 		go func() {
 			defer ip.srcFunnelTracker.Unregister(funnelID, icmpFlow)
-			if err := ip.listenResponse(icmpFlow, conn); err != nil {
+			if err := ip.listenResponse(ctx, icmpFlow); err != nil {
 				ip.logger.Debug().Err(err).
 					Str("src", pk.Src.String()).
 					Str("dst", pk.Dst.String()).
@@ -154,8 +167,10 @@ func (ip *icmpProxy) Request(pk *packet.ICMP, responder packet.FunnelUniPipe) er
 		}()
 	}
 	if err := icmpFlow.sendToDst(pk.Dst, pk.Message); err != nil {
+		tracing.EndWithErrorStatus(span, err)
 		return errors.Wrap(err, "failed to send ICMP echo request")
 	}
+	tracing.End(span)
 	return nil
 }
 
@@ -164,43 +179,53 @@ func (ip *icmpProxy) Serve(ctx context.Context) error {
 	return ctx.Err()
 }
 
-func (ip *icmpProxy) listenResponse(flow *icmpEchoFlow, conn *icmp.PacketConn) error {
+func (ip *icmpProxy) listenResponse(ctx context.Context, flow *icmpEchoFlow) error {
 	buf := make([]byte, mtu)
 	for {
-		n, from, err := conn.ReadFrom(buf)
-		if err != nil {
+		retryable, err := ip.handleResponse(ctx, flow, buf)
+		if err != nil && !retryable {
 			return err
 		}
-		reply, err := parseReply(from, buf[:n])
-		if err != nil {
-			ip.logger.Error().Err(err).Str("dst", from.String()).Msg("Failed to parse ICMP reply")
-			continue
-		}
-		if !isEchoReply(reply.msg) {
-			ip.logger.Debug().Str("dst", from.String()).Msgf("Drop ICMP %s from reply", reply.msg.Type)
-			continue
-		}
-		if err := flow.returnToSrc(reply); err != nil {
-			ip.logger.Err(err).Str("dst", from.String()).Msg("Failed to send ICMP reply")
-			continue
-		}
 	}
 }
 
-// originSender wraps icmp.PacketConn to implement packet.FunnelUniPipe interface
-type originSender struct {
-	conn *icmp.PacketConn
-}
+func (ip *icmpProxy) handleResponse(ctx context.Context, flow *icmpEchoFlow, buf []byte) (retryableErr bool, err error) {
+	_, span := flow.responder.replySpan(ctx, ip.logger)
+	defer flow.responder.exportSpan()
 
-func (os *originSender) SendPacket(dst netip.Addr, pk packet.RawPacket) error {
-	_, err := os.conn.WriteTo(pk.Data, &net.UDPAddr{
-		IP: dst.AsSlice(),
-	})
-	return err
-}
+	span.SetAttributes(
+		attribute.Int("originalEchoID", flow.originalEchoID),
+	)
 
-func (os *originSender) Close() error {
-	return os.conn.Close()
+	n, from, err := flow.originConn.ReadFrom(buf)
+	if err != nil {
+		tracing.EndWithErrorStatus(span, err)
+		return false, err
+	}
+	reply, err := parseReply(from, buf[:n])
+	if err != nil {
+		ip.logger.Error().Err(err).Str("dst", from.String()).Msg("Failed to parse ICMP reply")
+		tracing.EndWithErrorStatus(span, err)
+		return true, err
+	}
+	if !isEchoReply(reply.msg) {
+		err := fmt.Errorf("Expect ICMP echo reply, got %s", reply.msg.Type)
+		ip.logger.Debug().Str("dst", from.String()).Msgf("Drop ICMP %s from reply", reply.msg.Type)
+		tracing.EndWithErrorStatus(span, err)
+		return true, err
+	}
+	span.SetAttributes(
+		attribute.String("dst", reply.from.String()),
+		attribute.Int("echoID", reply.echo.ID),
+		attribute.Int("seq", reply.echo.Seq),
+	)
+	if err := flow.returnToSrc(reply); err != nil {
+		ip.logger.Debug().Err(err).Str("dst", from.String()).Msg("Failed to send ICMP reply")
+		tracing.EndWithErrorStatus(span, err)
+		return true, err
+	}
+	tracing.End(span)
+	return true, nil
 }
 
 // Only linux uses flow3Tuple as FunnelID

ingress/icmp_linux_test.go

@@ -0,0 +1,13 @@
+//go:build linux
+
+package ingress
+
+import (
+	"testing"
+
+	"github.com/cloudflare/cloudflared/packet"
+)
+
+func getFunnel(t *testing.T, proxy *icmpProxy, tuple flow3Tuple) (packet.Funnel, bool) {
+	return proxy.srcFunnelTracker.Get(tuple)
+}

ingress/icmp_posix.go

@@ -10,6 +10,7 @@ import (
 	"net/netip"
 
 	"github.com/google/gopacket/layers"
+	"github.com/rs/zerolog"
 	"golang.org/x/net/icmp"
 
 	"github.com/cloudflare/cloudflared/packet"
@@ -43,24 +44,54 @@ type flow3Tuple struct {
 
 // icmpEchoFlow implements the packet.Funnel interface.
 type icmpEchoFlow struct {
-	*packet.RawPacketFunnel
+	*packet.ActivityTracker
+	closeCallback  func() error
+	src            netip.Addr
+	originConn     *icmp.PacketConn
+	responder      *packetResponder
 	assignedEchoID int
 	originalEchoID int
 	// it's up to the user to ensure respEncoder is not used concurrently
 	respEncoder *packet.Encoder
 }
 
-func newICMPEchoFlow(src netip.Addr, sendPipe, returnPipe packet.FunnelUniPipe, assignedEchoID, originalEchoID int, respEncoder *packet.Encoder) *icmpEchoFlow {
+func newICMPEchoFlow(src netip.Addr, closeCallback func() error, originConn *icmp.PacketConn, responder *packetResponder, assignedEchoID, originalEchoID int, respEncoder *packet.Encoder) *icmpEchoFlow {
 	return &icmpEchoFlow{
-		RawPacketFunnel: packet.NewRawPacketFunnel(src, sendPipe, returnPipe),
+		ActivityTracker: packet.NewActivityTracker(),
+		closeCallback:   closeCallback,
+		src:             src,
+		originConn:      originConn,
+		responder:       responder,
 		assignedEchoID:  assignedEchoID,
 		originalEchoID:  originalEchoID,
 		respEncoder:     respEncoder,
 	}
 }
 
+func (ief *icmpEchoFlow) Equal(other packet.Funnel) bool {
+	otherICMPFlow, ok := other.(*icmpEchoFlow)
+	if !ok {
+		return false
+	}
+	if otherICMPFlow.src != ief.src {
+		return false
+	}
+	if otherICMPFlow.originalEchoID != ief.originalEchoID {
+		return false
+	}
+	if otherICMPFlow.assignedEchoID != ief.assignedEchoID {
+		return false
+	}
+	return true
+}
+
+func (ief *icmpEchoFlow) Close() error {
+	return ief.closeCallback()
+}
+
 // sendToDst rewrites the echo ID to the one assigned to this flow
 func (ief *icmpEchoFlow) sendToDst(dst netip.Addr, msg *icmp.Message) error {
+	ief.UpdateLastActive()
 	originalEcho, err := getICMPEcho(msg)
 	if err != nil {
 		return err
@@ -80,17 +111,21 @@ func (ief *icmpEchoFlow) sendToDst(dst netip.Addr, msg *icmp.Message) error {
 	if err != nil {
 		return err
 	}
-	return ief.SendToDst(dst, packet.RawPacket{Data: serializedPacket})
+	_, err = ief.originConn.WriteTo(serializedPacket, &net.UDPAddr{
+		IP: dst.AsSlice(),
+	})
+	return err
 }
 
 // returnToSrc rewrites the echo ID to the original echo ID from the eyeball
 func (ief *icmpEchoFlow) returnToSrc(reply *echoReply) error {
+	ief.UpdateLastActive()
 	reply.echo.ID = ief.originalEchoID
 	reply.msg.Body = reply.echo
 	pk := packet.ICMP{
 		IP: &packet.IP{
 			Src:      reply.from,
-			Dst:      ief.Src,
+			Dst:      ief.src,
 			Protocol: layers.IPProtocol(reply.msg.Type.Protocol()),
 			TTL:      packet.DefaultTTL,
 		},
@@ -100,7 +135,7 @@ func (ief *icmpEchoFlow) returnToSrc(reply *echoReply) error {
 	if err != nil {
 		return err
 	}
-	return ief.ReturnToSrc(serializedPacket)
+	return ief.responder.returnPacket(serializedPacket)
 }
 
 type echoReply struct {
@@ -140,3 +175,30 @@ func toICMPEchoFlow(funnel packet.Funnel) (*icmpEchoFlow, error) {
 	}
 	return icmpFlow, nil
 }
+
+func createShouldReplaceFunnelFunc(logger *zerolog.Logger, muxer muxer, pk *packet.ICMP, originalEchoID int) func(packet.Funnel) bool {
+	return func(existing packet.Funnel) bool {
+		existingFlow, err := toICMPEchoFlow(existing)
+		if err != nil {
+			logger.Err(err).
+				Str("src", pk.Src.String()).
+				Str("dst", pk.Dst.String()).
+				Int("originalEchoID", originalEchoID).
+				Msg("Funnel of wrong type found")
+			return true
+		}
+		// Each quic connection should have a unique muxer.
+		// If the existing flow has a different muxer, there's a new quic connection where return packets should be
+		// routed. Otherwise, return packets will be send to the first observed incoming connection, rather than the
+		// most recently observed connection.
+		if existingFlow.responder.datagramMuxer != muxer {
+			logger.Debug().
+				Str("src", pk.Src.String()).
+				Str("dst", pk.Dst.String()).
+				Int("originalEchoID", originalEchoID).
+				Msg("Replacing funnel with new responder")
+			return true
+		}
+		return false
+	}
+}

ingress/icmp_posix_test.go

@@ -52,24 +52,88 @@ func TestFunnelIdleTimeout(t *testing.T) {
 			},
 		},
 	}
-	responder := echoFlowResponder{
-		decoder:  packet.NewICMPDecoder(),
-		respChan: make(chan []byte),
+	muxer := newMockMuxer(0)
+	responder := packetResponder{
+		datagramMuxer: muxer,
 	}
-	require.NoError(t, proxy.Request(&pk, &responder))
-	responder.validate(t, &pk)
+	require.NoError(t, proxy.Request(ctx, &pk, &responder))
+	validateEchoFlow(t, <-muxer.cfdToEdge, &pk)
 
 	// Send second request, should reuse the funnel
-	require.NoError(t, proxy.Request(&pk, nil))
-	responder.validate(t, &pk)
+	require.NoError(t, proxy.Request(ctx, &pk, &packetResponder{
+		datagramMuxer: muxer,
+	}))
+	validateEchoFlow(t, <-muxer.cfdToEdge, &pk)
 
 	time.Sleep(idleTimeout * 2)
-	newResponder := echoFlowResponder{
-		decoder:  packet.NewICMPDecoder(),
-		respChan: make(chan []byte),
+	newMuxer := newMockMuxer(0)
+	newResponder := packetResponder{
+		datagramMuxer: newMuxer,
 	}
-	require.NoError(t, proxy.Request(&pk, &newResponder))
-	newResponder.validate(t, &pk)
+	require.NoError(t, proxy.Request(ctx, &pk, &newResponder))
+	validateEchoFlow(t, <-newMuxer.cfdToEdge, &pk)
+
+	cancel()
+	<-proxyDone
+}
+
+func TestReuseFunnel(t *testing.T) {
+	const (
+		idleTimeout = time.Second
+		echoID      = 42573
+		startSeq    = 8129
+	)
+	logger := zerolog.New(os.Stderr)
+	proxy, err := newICMPProxy(localhostIP, "", &logger, idleTimeout)
+	require.NoError(t, err)
+
+	ctx, cancel := context.WithCancel(context.Background())
+
+	proxyDone := make(chan struct{})
+	go func() {
+		proxy.Serve(ctx)
+		close(proxyDone)
+	}()
+
+	// Send a packet to register the flow
+	pk := packet.ICMP{
+		IP: &packet.IP{
+			Src:      localhostIP,
+			Dst:      localhostIP,
+			Protocol: layers.IPProtocolICMPv4,
+		},
+		Message: &icmp.Message{
+			Type: ipv4.ICMPTypeEcho,
+			Code: 0,
+			Body: &icmp.Echo{
+				ID:   echoID,
+				Seq:  startSeq,
+				Data: []byte(t.Name()),
+			},
+		},
+	}
+	tuple := flow3Tuple{
+		srcIP:          pk.Src,
+		dstIP:          pk.Dst,
+		originalEchoID: echoID,
+	}
+	muxer := newMockMuxer(0)
+	responder := packetResponder{
+		datagramMuxer: muxer,
+	}
+	require.NoError(t, proxy.Request(ctx, &pk, &responder))
+	validateEchoFlow(t, <-muxer.cfdToEdge, &pk)
+	funnel1, found := getFunnel(t, proxy, tuple)
+	require.True(t, found)
+
+	// Send second request, should reuse the funnel
+	require.NoError(t, proxy.Request(ctx, &pk, &packetResponder{
+		datagramMuxer: muxer,
+	}))
+	validateEchoFlow(t, <-muxer.cfdToEdge, &pk)
+	funnel2, found := getFunnel(t, proxy, tuple)
+	require.True(t, found)
+	require.Equal(t, funnel1, funnel2)
 
 	cancel()
 	<-proxyDone

ingress/icmp_windows.go

@@ -21,11 +21,13 @@ import (
 	"github.com/google/gopacket/layers"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
+	"go.opentelemetry.io/otel/attribute"
 	"golang.org/x/net/icmp"
 	"golang.org/x/net/ipv4"
 	"golang.org/x/net/ipv6"
 
 	"github.com/cloudflare/cloudflared/packet"
+	"github.com/cloudflare/cloudflared/tracing"
 )
 
 const (
@@ -265,34 +267,51 @@ func (ip *icmpProxy) Serve(ctx context.Context) error {
 // Request sends an ICMP echo request and wait for a reply or timeout.
 // The async version of Win32 APIs take a callback whose memory is not garbage collected, so we use the synchronous version.
 // It's possible that a slow request will block other requests, so we set the timeout to only 1s.
-func (ip *icmpProxy) Request(pk *packet.ICMP, responder packet.FunnelUniPipe) error {
-	if pk == nil {
-		return errPacketNil
-	}
+func (ip *icmpProxy) Request(ctx context.Context, pk *packet.ICMP, responder *packetResponder) error {
 	defer func() {
 		if r := recover(); r != nil {
 			ip.logger.Error().Interface("error", r).Msgf("Recover panic from sending icmp request/response, error %s", debug.Stack())
 		}
 	}()
 
+	_, requestSpan := responder.requestSpan(ctx, pk)
+	defer responder.exportSpan()
+
 	echo, err := getICMPEcho(pk.Message)
 	if err != nil {
 		return err
 	}
-	respData, err := ip.icmpEchoRoundtrip(pk.Dst, echo)
+	requestSpan.SetAttributes(
+		attribute.Int("originalEchoID", echo.ID),
+		attribute.Int("seq", echo.Seq),
+	)
+
+	resp, err := ip.icmpEchoRoundtrip(pk.Dst, echo)
 	if err != nil {
 		ip.logger.Err(err).Msg("ICMP echo roundtrip failed")
+		tracing.EndWithErrorStatus(requestSpan, err)
 		return err
 	}
+	tracing.End(requestSpan)
+	responder.exportSpan()
 
-	err = ip.handleEchoReply(pk, echo, respData, responder)
+	_, replySpan := responder.replySpan(ctx, ip.logger)
+	replySpan.SetAttributes(
+		attribute.Int("originalEchoID", echo.ID),
+		attribute.Int("seq", echo.Seq),
+		attribute.Int64("rtt", int64(resp.rtt())),
+		attribute.String("status", resp.status().String()),
+	)
+	err = ip.handleEchoReply(pk, echo, resp, responder)
 	if err != nil {
+		tracing.EndWithErrorStatus(replySpan, err)
 		return errors.Wrap(err, "failed to handle ICMP echo reply")
 	}
+	tracing.End(replySpan)
 	return nil
 }
 
-func (ip *icmpProxy) handleEchoReply(request *packet.ICMP, echoReq *icmp.Echo, data []byte, responder packet.FunnelUniPipe) error {
+func (ip *icmpProxy) handleEchoReply(request *packet.ICMP, echoReq *icmp.Echo, resp echoResp, responder *packetResponder) error {
 	var replyType icmp.Type
 	if request.Dst.Is4() {
 		replyType = ipv4.ICMPTypeEchoReply
@@ -313,7 +332,7 @@ func (ip *icmpProxy) handleEchoReply(request *packet.ICMP, echoReq *icmp.Echo, d
 			Body: &icmp.Echo{
 				ID:   echoReq.ID,
 				Seq:  echoReq.Seq,
-				Data: data,
+				Data: resp.payload(),
 			},
 		},
 	}
@@ -331,19 +350,20 @@ func (ip *icmpProxy) handleEchoReply(request *packet.ICMP, echoReq *icmp.Echo, d
 	if err != nil {
 		return err
 	}
-	return responder.SendPacket(request.Src, serializedPacket)
+	return responder.returnPacket(serializedPacket)
 }
 
-func (ip *icmpProxy) icmpEchoRoundtrip(dst netip.Addr, echo *icmp.Echo) ([]byte, error) {
+func (ip *icmpProxy) icmpEchoRoundtrip(dst netip.Addr, echo *icmp.Echo) (echoResp, error) {
 	if dst.Is6() {
 		if ip.srcSocketAddr == nil {
 			return nil, fmt.Errorf("cannot send ICMPv6 using ICMPv4 proxy")
 		}
 		resp, err := ip.icmp6SendEcho(dst, echo)
 		if err != nil {
+
 			return nil, errors.Wrap(err, "failed to send/receive ICMPv6 echo")
 		}
-		return resp.data, nil
+		return resp, nil
 	}
 	if ip.srcSocketAddr != nil {
 		return nil, fmt.Errorf("cannot send ICMPv4 using ICMPv6 proxy")
@@ -352,26 +372,26 @@ func (ip *icmpProxy) icmpEchoRoundtrip(dst netip.Addr, echo *icmp.Echo) ([]byte,
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to send/receive ICMPv4 echo")
 	}
-	return resp.data, nil
+	return resp, nil
 }
 
 /*
-	Wrapper to call https://docs.microsoft.com/en-us/windows/win32/api/icmpapi/nf-icmpapi-icmpsendecho
-	Parameters:
-	- IcmpHandle: Handle created by IcmpCreateFile
-	- DestinationAddress: IPv4 in the form of https://docs.microsoft.com/en-us/windows/win32/api/inaddr/ns-inaddr-in_addr#syntax
-	- RequestData: A pointer to echo data
-	- RequestSize: Number of bytes in buffer pointed by echo data
-	- RequestOptions: IP header options
-	- ReplyBuffer: A pointer to the buffer for echoReply, options and data
-	- ReplySize: Number of bytes allocated for ReplyBuffer
-	- Timeout: Timeout in milliseconds to wait for a reply
-	Returns:
-	- the number of replies in uint32 https://docs.microsoft.com/en-us/windows/win32/api/icmpapi/nf-icmpapi-icmpsendecho#return-value
-	To retain the reference allocated objects, conversion from pointer to uintptr must happen as arguments to the
-	syscall function
+Wrapper to call https://docs.microsoft.com/en-us/windows/win32/api/icmpapi/nf-icmpapi-icmpsendecho
+Parameters:
+- IcmpHandle: Handle created by IcmpCreateFile
+- DestinationAddress: IPv4 in the form of https://docs.microsoft.com/en-us/windows/win32/api/inaddr/ns-inaddr-in_addr#syntax
+- RequestData: A pointer to echo data
+- RequestSize: Number of bytes in buffer pointed by echo data
+- RequestOptions: IP header options
+- ReplyBuffer: A pointer to the buffer for echoReply, options and data
+- ReplySize: Number of bytes allocated for ReplyBuffer
+- Timeout: Timeout in milliseconds to wait for a reply
+Returns:
+- the number of replies in uint32 https://docs.microsoft.com/en-us/windows/win32/api/icmpapi/nf-icmpapi-icmpsendecho#return-value
+To retain the reference allocated objects, conversion from pointer to uintptr must happen as arguments to the
+syscall function
 */
-func (ip *icmpProxy) icmpSendEcho(dst netip.Addr, echo *icmp.Echo) (*echoResp, error) {
+func (ip *icmpProxy) icmpSendEcho(dst netip.Addr, echo *icmp.Echo) (*echoV4Resp, error) {
 	dataSize := len(echo.Data)
 	replySize := echoReplySize + uintptr(dataSize)
 	replyBuf := make([]byte, replySize)
@@ -399,7 +419,7 @@ func (ip *icmpProxy) icmpSendEcho(dst netip.Addr, echo *icmp.Echo) (*echoResp, e
 	} else if replyCount > 1 {
 		ip.logger.Warn().Msgf("Received %d ICMP echo replies, only sending 1 back", replyCount)
 	}
-	return newEchoResp(replyBuf)
+	return newEchoV4Resp(replyBuf)
 }
 
 // Third definition of https://docs.microsoft.com/en-us/windows/win32/api/inaddr/ns-inaddr-in_addr#syntax is address in uint32
@@ -411,12 +431,30 @@ func inAddrV4(ip netip.Addr) (uint32, error) {
 	return endian.Uint32(v4[:]), nil
 }
 
-type echoResp struct {
+type echoResp interface {
+	status() ipStatus
+	rtt() uint32
+	payload() []byte
+}
+
+type echoV4Resp struct {
 	reply *echoReply
 	data  []byte
 }
 
-func newEchoResp(replyBuf []byte) (*echoResp, error) {
+func (r *echoV4Resp) status() ipStatus {
+	return r.reply.Status
+}
+
+func (r *echoV4Resp) rtt() uint32 {
+	return r.reply.RoundTripTime
+}
+
+func (r *echoV4Resp) payload() []byte {
+	return r.data
+}
+
+func newEchoV4Resp(replyBuf []byte) (*echoV4Resp, error) {
 	if len(replyBuf) == 0 {
 		return nil, fmt.Errorf("reply buffer is empty")
 	}
@@ -430,7 +468,7 @@ func newEchoResp(replyBuf []byte) (*echoResp, error) {
 	if dataBufStart < int(echoReplySize) {
 		return nil, fmt.Errorf("reply buffer size %d is too small to hold data of size %d", len(replyBuf), int(reply.DataSize))
 	}
-	return &echoResp{
+	return &echoV4Resp{
 		reply: &reply,
 		data:  replyBuf[dataBufStart:],
 	}, nil
@@ -502,6 +540,18 @@ type echoV6Resp struct {
 	data  []byte
 }
 
+func (r *echoV6Resp) status() ipStatus {
+	return r.reply.Status
+}
+
+func (r *echoV6Resp) rtt() uint32 {
+	return r.reply.RoundTripTime
+}
+
+func (r *echoV6Resp) payload() []byte {
+	return r.data
+}
+
 func newEchoV6Resp(replyBuf []byte, dataSize int) (*echoV6Resp, error) {
 	if len(replyBuf) == 0 {
 		return nil, fmt.Errorf("reply buffer is empty")

ingress/icmp_windows_test.go

@@ -59,7 +59,7 @@ func TestParseEchoReply(t *testing.T) {
 	}
 
 	for _, test := range tests {
-		resp, err := newEchoResp(test.replyBuf)
+		resp, err := newEchoV4Resp(test.replyBuf)
 		if test.expectedReply == nil {
 			require.Error(t, err)
 			require.Nil(t, resp)
@@ -125,7 +125,7 @@ func TestParseEchoV6Reply(t *testing.T) {
 	}
 }
 
-//  TestSendEchoErrors makes sure icmpSendEcho handles error cases
+// TestSendEchoErrors makes sure icmpSendEcho handles error cases
 func TestSendEchoErrors(t *testing.T) {
 	testSendEchoErrors(t, netip.IPv4Unspecified())
 	testSendEchoErrors(t, netip.IPv6Unspecified())

ingress/ingress.go

@@ -20,6 +20,7 @@ import (
 
 var (
 	ErrNoIngressRules             = errors.New("The config file doesn't contain any ingress rules")
+	ErrNoIngressRulesCLI          = errors.New("No ingress rules were defined in provided config (if any) nor from the cli, cloudflared will return 503 for all incoming HTTP requests")
 	errLastRuleNotCatchAll        = errors.New("The last ingress rule must match all URLs (i.e. it should not have a hostname or path filter)")
 	errBadWildcard                = errors.New("Hostname patterns can have at most one wildcard character (\"*\") and it can only be used for subdomains, e.g. \"*.example.com\"")
 	errHostnameContainsPort       = errors.New("Hostname cannot contain a port")
@@ -58,7 +59,7 @@ func matchHost(ruleHost, reqHost string) bool {
 
 	// Validate hostnames that use wildcards at the start
 	if strings.HasPrefix(ruleHost, "*.") {
-		toMatch := strings.TrimPrefix(ruleHost, "*.")
+		toMatch := strings.TrimPrefix(ruleHost, "*")
 		return strings.HasSuffix(reqHost, toMatch)
 	}
 	return false
@@ -70,17 +71,52 @@ type Ingress struct {
 	Defaults OriginRequestConfig `json:"originRequest"`
 }
 
-// NewSingleOrigin constructs an Ingress set with only one rule, constructed from
-// legacy CLI parameters like --url or --no-chunked-encoding.
-func NewSingleOrigin(c *cli.Context, allowURLFromArgs bool) (Ingress, error) {
+// ParseIngress parses ingress rules, but does not send HTTP requests to the origins.
+func ParseIngress(conf *config.Configuration) (Ingress, error) {
+	if len(conf.Ingress) == 0 {
+		return Ingress{}, ErrNoIngressRules
+	}
+	return validateIngress(conf.Ingress, originRequestFromConfig(conf.OriginRequest))
+}
 
+// ParseIngressFromConfigAndCLI will parse the configuration rules from config files for ingress
+// rules and then attempt to parse CLI for ingress rules.
+// Will always return at least one valid ingress rule. If none are provided by the user, the default
+// will be to return 502 status code for all incoming requests.
+func ParseIngressFromConfigAndCLI(conf *config.Configuration, c *cli.Context, log *zerolog.Logger) (Ingress, error) {
+	// Attempt to parse ingress rules from configuration
+	ingressRules, err := ParseIngress(conf)
+	if err == nil && !ingressRules.IsEmpty() {
+		return ingressRules, nil
+	}
+	if err != ErrNoIngressRules {
+		return Ingress{}, err
+	}
+	// Attempt to parse ingress rules from CLI:
+	//   --url or --unix-socket flag for a tunnel HTTP ingress
+	//   --hello-world for a basic HTTP ingress self-served
+	//   --bastion for ssh bastion service
+	ingressRules, err = parseCLIIngress(c, false)
+	if errors.Is(err, ErrNoIngressRulesCLI) {
+		log.Warn().Msgf(ErrNoIngressRulesCLI.Error())
+		return newDefaultOrigin(c, log), nil
+	}
+	if err != nil {
+		return Ingress{}, err
+	}
+	return ingressRules, nil
+}
+
+// parseCLIIngress constructs an Ingress set with only one rule constructed from
+// CLI parameters: --url, --hello-world, --bastion, or --unix-socket
+func parseCLIIngress(c *cli.Context, allowURLFromArgs bool) (Ingress, error) {
 	service, err := parseSingleOriginService(c, allowURLFromArgs)
 	if err != nil {
 		return Ingress{}, err
 	}
 
 	// Construct an Ingress with the single rule.
-	defaults := originRequestFromSingeRule(c)
+	defaults := originRequestFromSingleRule(c)
 	ing := Ingress{
 		Rules: []Rule{
 			{
@@ -93,6 +129,22 @@ func NewSingleOrigin(c *cli.Context, allowURLFromArgs bool) (Ingress, error) {
 	return ing, err
 }
 
+// newDefaultOrigin always returns a 503 response code to help indicate that there are no ingress
+// rules setup, but the tunnel is reachable.
+func newDefaultOrigin(c *cli.Context, log *zerolog.Logger) Ingress {
+	noRulesService := newDefaultStatusCode(log)
+	defaults := originRequestFromSingleRule(c)
+	ingress := Ingress{
+		Rules: []Rule{
+			{
+				Service: &noRulesService,
+			},
+		},
+		Defaults: defaults,
+	}
+	return ingress
+}
+
 // WarpRoutingService starts a tcp stream between the origin and requests from
 // warp clients.
 type WarpRoutingService struct {
@@ -113,7 +165,7 @@ func NewWarpRoutingService(config WarpRoutingConfig) *WarpRoutingService {
 
 // Get a single origin service from the CLI/config.
 func parseSingleOriginService(c *cli.Context, allowURLFromArgs bool) (OriginService, error) {
-	if c.IsSet("hello-world") {
+	if c.IsSet(HelloWorldFlag) {
 		return new(helloWorld), nil
 	}
 	if c.IsSet(config.BastionFlag) {
@@ -138,8 +190,7 @@ func parseSingleOriginService(c *cli.Context, allowURLFromArgs bool) (OriginServ
 		}
 		return &unixSocketPath{path: path, scheme: "http"}, nil
 	}
-	u, err := url.Parse("http://localhost:8080")
-	return &httpService{url: u}, err
+	return nil, ErrNoIngressRulesCLI
 }
 
 // IsEmpty checks if there are any ingress rules.
@@ -175,18 +226,10 @@ func validateAccessConfiguration(cfg *config.AccessConfig) error {
 		return nil
 	}
 
-	// It is possible to set `required:true` and not have these two configured yet.
-	// But if one of them is configured, we'd validate for correctness.
-	if len(cfg.AudTag) == 0 && cfg.TeamName == "" {
-		return nil
-	}
-
-	if len(cfg.AudTag) == 0 {
-		return errors.New("access audtag cannot be empty")
-	}
-
-	if cfg.TeamName == "" {
-		return errors.New("access.TeamName cannot be blank")
+	// we allow for an initial setup where user can force Access but not configure the rest of the keys.
+	// however, if the user specified audTags but forgot teamName, we should alert it.
+	if cfg.TeamName == "" && len(cfg.AudTag) > 0 {
+		return errors.New("access.TeamName cannot be blank when access.audTags are present")
 	}
 
 	return nil
@@ -215,7 +258,7 @@ func validateIngress(ingress []config.UnvalidatedIngressRule, defaults OriginReq
 			}
 			srv := newStatusCode(statusCode)
 			service = &srv
-		} else if r.Service == HelloWorldService || r.Service == "hello-world" || r.Service == "helloworld" {
+		} else if r.Service == HelloWorldFlag || r.Service == HelloWorldService {
 			service = new(helloWorld)
 		} else if r.Service == ServiceSocksProxy {
 			rules := make([]ipaccess.Rule, len(r.OriginRequest.IPRules))
@@ -344,14 +387,6 @@ func (e errRuleShouldNotBeCatchAll) Error() string {
 		"will never be triggered.", e.index+1, e.hostname)
 }
 
-// ParseIngress parses ingress rules, but does not send HTTP requests to the origins.
-func ParseIngress(conf *config.Configuration) (Ingress, error) {
-	if len(conf.Ingress) == 0 {
-		return Ingress{}, ErrNoIngressRules
-	}
-	return validateIngress(conf.Ingress, originRequestFromConfig(conf.OriginRequest))
-}
-
 func isHTTPService(url *url.URL) bool {
 	return url.Scheme == "http" || url.Scheme == "https" || url.Scheme == "ws" || url.Scheme == "wss"
 }

ingress/ingress_test.go

@@ -43,7 +43,7 @@ ingress:
 	require.Equal(t, "https", s.scheme)
 }
 
-func Test_parseIngress(t *testing.T) {
+func TestParseIngress(t *testing.T) {
 	localhost8000 := MustParseURL(t, "https://localhost:8000")
 	localhost8001 := MustParseURL(t, "https://localhost:8001")
 	fourOhFour := newStatusCode(404)
@@ -517,7 +517,7 @@ func TestSingleOriginSetsConfig(t *testing.T) {
 
 	allowURLFromArgs := false
 	require.NoError(t, err)
-	ingress, err := NewSingleOrigin(cliCtx, allowURLFromArgs)
+	ingress, err := parseCLIIngress(cliCtx, allowURLFromArgs)
 	require.NoError(t, err)
 
 	assert.Equal(t, config.CustomDuration{Duration: time.Second}, ingress.Rules[0].Config.ConnectTimeout)
@@ -537,6 +537,119 @@ func TestSingleOriginSetsConfig(t *testing.T) {
 	assert.Equal(t, socksProxy, ingress.Rules[0].Config.ProxyType)
 }
 
+func TestSingleOriginServices(t *testing.T) {
+	host := "://localhost:8080"
+	httpURL := urlMustParse("http" + host)
+	tcpURL := urlMustParse("tcp" + host)
+	unix := "unix://service"
+	newCli := func(params ...string) *cli.Context {
+		flagSet := flag.NewFlagSet(t.Name(), flag.PanicOnError)
+		flagSet.Bool("hello-world", false, "")
+		flagSet.Bool("bastion", false, "")
+		flagSet.String("url", "", "")
+		flagSet.String("unix-socket", "", "")
+		cliCtx := cli.NewContext(cli.NewApp(), flagSet, nil)
+		for i := 0; i < len(params); i += 2 {
+			cliCtx.Set(params[i], params[i+1])
+		}
+
+		return cliCtx
+	}
+
+	tests := []struct {
+		name            string
+		cli             *cli.Context
+		expectedService OriginService
+		err             error
+	}{
+		{
+			name:            "Valid hello-world",
+			cli:             newCli("hello-world", "true"),
+			expectedService: &helloWorld{},
+		},
+		{
+			name:            "Valid bastion",
+			cli:             newCli("bastion", "true"),
+			expectedService: newBastionService(),
+		},
+		{
+			name:            "Valid http url",
+			cli:             newCli("url", httpURL.String()),
+			expectedService: &httpService{url: httpURL},
+		},
+		{
+			name:            "Valid tcp url",
+			cli:             newCli("url", tcpURL.String()),
+			expectedService: newTCPOverWSService(tcpURL),
+		},
+		{
+			name:            "Valid unix-socket",
+			cli:             newCli("unix-socket", unix),
+			expectedService: &unixSocketPath{path: unix, scheme: "http"},
+		},
+		{
+			name: "No origins defined",
+			cli:  newCli(),
+			err:  ErrNoIngressRulesCLI,
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			ingress, err := parseCLIIngress(test.cli, false)
+			require.Equal(t, err, test.err)
+			if test.err != nil {
+				return
+			}
+			require.Equal(t, 1, len(ingress.Rules))
+			rule := ingress.Rules[0]
+			require.Equal(t, test.expectedService, rule.Service)
+		})
+	}
+}
+
+func urlMustParse(s string) *url.URL {
+	u, err := url.Parse(s)
+	if err != nil {
+		panic(err)
+	}
+	return u
+}
+
+func TestSingleOriginServices_URL(t *testing.T) {
+	host := "://localhost:8080"
+	newCli := func(param string, value string) *cli.Context {
+		flagSet := flag.NewFlagSet(t.Name(), flag.PanicOnError)
+		flagSet.String("url", "", "")
+		cliCtx := cli.NewContext(cli.NewApp(), flagSet, nil)
+		cliCtx.Set(param, value)
+		return cliCtx
+	}
+
+	httpTests := []string{"http", "https"}
+	for _, test := range httpTests {
+		t.Run(test, func(t *testing.T) {
+			url := urlMustParse(test + host)
+			ingress, err := parseCLIIngress(newCli("url", url.String()), false)
+			require.NoError(t, err)
+			require.Equal(t, 1, len(ingress.Rules))
+			rule := ingress.Rules[0]
+			require.Equal(t, &httpService{url: url}, rule.Service)
+		})
+	}
+
+	tcpTests := []string{"ssh", "rdp", "smb", "tcp"}
+	for _, test := range tcpTests {
+		t.Run(test, func(t *testing.T) {
+			url := urlMustParse(test + host)
+			ingress, err := parseCLIIngress(newCli("url", url.String()), false)
+			require.NoError(t, err)
+			require.Equal(t, 1, len(ingress.Rules))
+			rule := ingress.Rules[0]
+			require.Equal(t, newTCPOverWSService(url), rule.Service)
+		})
+	}
+}
+
 func TestFindMatchingRule(t *testing.T) {
 	ingress := Ingress{
 		Rules: []Rule{
@@ -674,6 +787,46 @@ ingress:
 	}
 }
 
+func TestParseAccessConfig(t *testing.T) {
+	tests := []struct {
+		name        string
+		cfg         config.AccessConfig
+		expectError bool
+	}{
+		{
+			name:        "Config required with teamName only",
+			cfg:         config.AccessConfig{Required: true, TeamName: "team"},
+			expectError: false,
+		},
+		{
+			name:        "required false",
+			cfg:         config.AccessConfig{Required: false},
+			expectError: false,
+		},
+		{
+			name:        "required true but empty config",
+			cfg:         config.AccessConfig{Required: true},
+			expectError: false,
+		},
+		{
+			name:        "complete config",
+			cfg:         config.AccessConfig{Required: true, TeamName: "team", AudTag: []string{"a"}},
+			expectError: false,
+		},
+		{
+			name:        "required true with audTags but no teamName",
+			cfg:         config.AccessConfig{Required: true, AudTag: []string{"a"}},
+			expectError: true,
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			err := validateAccessConfiguration(&test.cfg)
+			require.Equal(t, err != nil, test.expectError)
+		})
+	}
+}
+
 func MustReadIngress(s string) *config.Configuration {
 	var conf config.Configuration
 	err := yaml.Unmarshal([]byte(s), &conf)

ingress/origin_connection.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/cloudflare/cloudflared/ipaccess"
 	"github.com/cloudflare/cloudflared/socks"
+	"github.com/cloudflare/cloudflared/stream"
 	"github.com/cloudflare/cloudflared/websocket"
 )
 
@@ -25,7 +26,7 @@ type streamHandlerFunc func(originConn io.ReadWriter, remoteConn net.Conn, log *
 // DefaultStreamHandler is an implementation of streamHandlerFunc that
 // performs a two way io.Copy between originConn and remoteConn.
 func DefaultStreamHandler(originConn io.ReadWriter, remoteConn net.Conn, log *zerolog.Logger) {
-	websocket.Stream(originConn, remoteConn, log)
+	stream.Pipe(originConn, remoteConn, log)
 }
 
 // tcpConnection is an OriginConnection that directly streams to raw TCP.
@@ -34,7 +35,7 @@ type tcpConnection struct {
 }
 
 func (tc *tcpConnection) Stream(ctx context.Context, tunnelConn io.ReadWriter, log *zerolog.Logger) {
-	websocket.Stream(tunnelConn, tc.conn, log)
+	stream.Pipe(tunnelConn, tc.conn, log)
 }
 
 func (tc *tcpConnection) Close() {

ingress/origin_connection_test.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/socks"
+	"github.com/cloudflare/cloudflared/stream"
 	"github.com/cloudflare/cloudflared/websocket"
 )
 
@@ -50,6 +51,7 @@ func TestStreamTCPConnection(t *testing.T) {
 	errGroup, ctx := errgroup.WithContext(ctx)
 	errGroup.Go(func() error {
 		_, err := eyeballConn.Write(testMessage)
+		require.NoError(t, err)
 
 		readBuffer := make([]byte, len(testResponse))
 		_, err = eyeballConn.Read(readBuffer)
@@ -158,7 +160,7 @@ func TestSocksStreamWSOverTCPConnection(t *testing.T) {
 			require.NoError(t, err)
 			defer wsForwarderInConn.Close()
 
-			websocket.Stream(wsForwarderInConn, &wsEyeball{wsForwarderOutConn}, testLogger)
+			stream.Pipe(wsForwarderInConn, &wsEyeball{wsForwarderOutConn}, testLogger)
 			return nil
 		})
 

ingress/origin_icmp_proxy.go

@@ -37,7 +37,9 @@ func NewICMPRouter(ipv4Addr, ipv6Addr netip.Addr, ipv6Zone string, logger *zerol
 	ipv4Proxy, ipv4Err := newICMPProxy(ipv4Addr, "", logger, funnelIdleTimeout)
 	ipv6Proxy, ipv6Err := newICMPProxy(ipv6Addr, ipv6Zone, logger, funnelIdleTimeout)
 	if ipv4Err != nil && ipv6Err != nil {
-		return nil, fmt.Errorf("cannot create ICMPv4 proxy: %v nor ICMPv6 proxy: %v", ipv4Err, ipv6Err)
+		err := fmt.Errorf("cannot create ICMPv4 proxy: %v nor ICMPv6 proxy: %v", ipv4Err, ipv6Err)
+		logger.Debug().Err(err).Msg("ICMP proxy feature is disabled")
+		return nil, err
 	}
 	if ipv4Err != nil {
 		logger.Debug().Err(ipv4Err).Msg("failed to create ICMPv4 proxy, only ICMPv6 proxy is created")
@@ -73,15 +75,18 @@ func (ir *icmpRouter) Serve(ctx context.Context) error {
 	return fmt.Errorf("ICMPv4 proxy and ICMPv6 proxy are both nil")
 }
 
-func (ir *icmpRouter) Request(pk *packet.ICMP, responder packet.FunnelUniPipe) error {
+func (ir *icmpRouter) Request(ctx context.Context, pk *packet.ICMP, responder *packetResponder) error {
+	if pk == nil {
+		return errPacketNil
+	}
 	if pk.Dst.Is4() {
 		if ir.ipv4Proxy != nil {
-			return ir.ipv4Proxy.Request(pk, responder)
+			return ir.ipv4Proxy.Request(ctx, pk, responder)
 		}
 		return fmt.Errorf("ICMPv4 proxy was not instantiated")
 	}
 	if ir.ipv6Proxy != nil {
-		return ir.ipv6Proxy.Request(pk, responder)
+		return ir.ipv6Proxy.Request(ctx, pk, responder)
 	}
 	return fmt.Errorf("ICMPv6 proxy was not instantiated")
 }

ingress/origin_icmp_proxy_test.go

@@ -1,6 +1,7 @@
 package ingress
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"net"
@@ -17,6 +18,8 @@ import (
 	"golang.org/x/net/ipv6"
 
 	"github.com/cloudflare/cloudflared/packet"
+	quicpogs "github.com/cloudflare/cloudflared/quic"
+	"github.com/cloudflare/cloudflared/tracing"
 )
 
 var (
@@ -52,9 +55,9 @@ func testICMPRouterEcho(t *testing.T, sendIPv4 bool) {
 		close(proxyDone)
 	}()
 
-	responder := echoFlowResponder{
-		decoder:  packet.NewICMPDecoder(),
-		respChan: make(chan []byte, 1),
+	muxer := newMockMuxer(1)
+	responder := packetResponder{
+		datagramMuxer: muxer,
 	}
 
 	protocol := layers.IPProtocolICMPv6
@@ -90,14 +93,113 @@ func testICMPRouterEcho(t *testing.T, sendIPv4 bool) {
 					},
 				},
 			}
-			require.NoError(t, router.Request(&pk, &responder))
-			responder.validate(t, &pk)
+			require.NoError(t, router.Request(ctx, &pk, &responder))
+			validateEchoFlow(t, <-muxer.cfdToEdge, &pk)
 		}
 	}
 	cancel()
 	<-proxyDone
 }
 
+func TestTraceICMPRouterEcho(t *testing.T) {
+	tracingCtx := "ec31ad8a01fde11fdcabe2efdce36873:52726f6cabc144f5:0:1"
+
+	router, err := NewICMPRouter(localhostIP, localhostIPv6, "", &noopLogger)
+	require.NoError(t, err)
+
+	proxyDone := make(chan struct{})
+	ctx, cancel := context.WithCancel(context.Background())
+	go func() {
+		router.Serve(ctx)
+		close(proxyDone)
+	}()
+
+	// Buffer 3 packets, request span, reply span and reply
+	muxer := newMockMuxer(3)
+	tracingIdentity, err := tracing.NewIdentity(tracingCtx)
+	require.NoError(t, err)
+	serializedIdentity, err := tracingIdentity.MarshalBinary()
+	require.NoError(t, err)
+
+	responder := packetResponder{
+		datagramMuxer:      muxer,
+		tracedCtx:          tracing.NewTracedContext(ctx, tracingIdentity.String(), &noopLogger),
+		serializedIdentity: serializedIdentity,
+	}
+
+	echo := &icmp.Echo{
+		ID:   12910,
+		Seq:  182,
+		Data: []byte(t.Name()),
+	}
+	pk := packet.ICMP{
+		IP: &packet.IP{
+			Src:      localhostIP,
+			Dst:      localhostIP,
+			Protocol: layers.IPProtocolICMPv4,
+			TTL:      packet.DefaultTTL,
+		},
+		Message: &icmp.Message{
+			Type: ipv4.ICMPTypeEcho,
+			Code: 0,
+			Body: echo,
+		},
+	}
+
+	require.NoError(t, router.Request(ctx, &pk, &responder))
+	firstPK := <-muxer.cfdToEdge
+	var requestSpan *quicpogs.TracingSpanPacket
+	// The order of receiving reply or request span is not deterministic
+	switch firstPK.Type() {
+	case quicpogs.DatagramTypeIP:
+		// reply packet
+		validateEchoFlow(t, firstPK, &pk)
+	case quicpogs.DatagramTypeTracingSpan:
+		// Request span
+		requestSpan = firstPK.(*quicpogs.TracingSpanPacket)
+		require.NotEmpty(t, requestSpan.Spans)
+		require.True(t, bytes.Equal(serializedIdentity, requestSpan.TracingIdentity))
+	default:
+		panic(fmt.Sprintf("received unexpected packet type %d", firstPK.Type()))
+	}
+
+	secondPK := <-muxer.cfdToEdge
+	if requestSpan != nil {
+		// If first packet is request span, second packet should be the reply
+		validateEchoFlow(t, secondPK, &pk)
+	} else {
+		requestSpan = secondPK.(*quicpogs.TracingSpanPacket)
+		require.NotEmpty(t, requestSpan.Spans)
+		require.True(t, bytes.Equal(serializedIdentity, requestSpan.TracingIdentity))
+	}
+
+	// Reply span
+	thirdPacket := <-muxer.cfdToEdge
+	replySpan, ok := thirdPacket.(*quicpogs.TracingSpanPacket)
+	require.True(t, ok)
+	require.NotEmpty(t, replySpan.Spans)
+	require.True(t, bytes.Equal(serializedIdentity, replySpan.TracingIdentity))
+	require.False(t, bytes.Equal(requestSpan.Spans, replySpan.Spans))
+
+	echo.Seq++
+	pk.Body = echo
+	// Only first request for a flow is traced. The edge will not send tracing context for the second request
+	newResponder := packetResponder{
+		datagramMuxer: muxer,
+	}
+	require.NoError(t, router.Request(ctx, &pk, &newResponder))
+	validateEchoFlow(t, <-muxer.cfdToEdge, &pk)
+
+	select {
+	case receivedPacket := <-muxer.cfdToEdge:
+		panic(fmt.Sprintf("Receive unexpected packet %+v", receivedPacket))
+	default:
+	}
+
+	cancel()
+	<-proxyDone
+}
+
 // TestConcurrentRequests makes sure icmpRouter can send concurrent requests to the same destination with different
 // echo ID. This simulates concurrent ping to the same destination.
 func TestConcurrentRequestsToSameDst(t *testing.T) {
@@ -123,9 +225,10 @@ func TestConcurrentRequestsToSameDst(t *testing.T) {
 		echoID := 38451 + i
 		go func() {
 			defer wg.Done()
-			responder := echoFlowResponder{
-				decoder:  packet.NewICMPDecoder(),
-				respChan: make(chan []byte, 1),
+
+			muxer := newMockMuxer(1)
+			responder := packetResponder{
+				datagramMuxer: muxer,
 			}
 			for seq := 0; seq < endSeq; seq++ {
 				pk := &packet.ICMP{
@@ -145,15 +248,15 @@ func TestConcurrentRequestsToSameDst(t *testing.T) {
 						},
 					},
 				}
-				require.NoError(t, router.Request(pk, &responder))
-				responder.validate(t, pk)
+				require.NoError(t, router.Request(ctx, pk, &responder))
+				validateEchoFlow(t, <-muxer.cfdToEdge, pk)
 			}
 		}()
 		go func() {
 			defer wg.Done()
-			responder := echoFlowResponder{
-				decoder:  packet.NewICMPDecoder(),
-				respChan: make(chan []byte, 1),
+			muxer := newMockMuxer(1)
+			responder := packetResponder{
+				datagramMuxer: muxer,
 			}
 			for seq := 0; seq < endSeq; seq++ {
 				pk := &packet.ICMP{
@@ -173,8 +276,8 @@ func TestConcurrentRequestsToSameDst(t *testing.T) {
 						},
 					},
 				}
-				require.NoError(t, router.Request(pk, &responder))
-				responder.validate(t, pk)
+				require.NoError(t, router.Request(ctx, pk, &responder))
+				validateEchoFlow(t, <-muxer.cfdToEdge, pk)
 			}
 		}()
 	}
@@ -241,9 +344,9 @@ func testICMPRouterRejectNotEcho(t *testing.T, srcDstIP netip.Addr, msgs []icmp.
 	router, err := NewICMPRouter(localhostIP, localhostIPv6, "", &noopLogger)
 	require.NoError(t, err)
 
-	responder := echoFlowResponder{
-		decoder:  packet.NewICMPDecoder(),
-		respChan: make(chan []byte),
+	muxer := newMockMuxer(1)
+	responder := packetResponder{
+		datagramMuxer: muxer,
 	}
 	protocol := layers.IPProtocolICMPv4
 	if srcDstIP.Is6() {
@@ -259,36 +362,14 @@ func testICMPRouterRejectNotEcho(t *testing.T, srcDstIP netip.Addr, msgs []icmp.
 			},
 			Message: &m,
 		}
-		require.Error(t, router.Request(&pk, &responder))
+		require.Error(t, router.Request(context.Background(), &pk, &responder))
 	}
 }
 
-type echoFlowResponder struct {
-	lock     sync.Mutex
-	decoder  *packet.ICMPDecoder
-	respChan chan []byte
-}
-
-func (efr *echoFlowResponder) SendPacket(dst netip.Addr, pk packet.RawPacket) error {
-	efr.lock.Lock()
-	defer efr.lock.Unlock()
-	copiedPacket := make([]byte, len(pk.Data))
-	copy(copiedPacket, pk.Data)
-	efr.respChan <- copiedPacket
-	return nil
-}
-
-func (efr *echoFlowResponder) Close() error {
-	efr.lock.Lock()
-	defer efr.lock.Unlock()
-	close(efr.respChan)
-	return nil
-}
-
-func (efr *echoFlowResponder) validate(t *testing.T, echoReq *packet.ICMP) {
-	pk := <-efr.respChan
-	decoded, err := efr.decoder.Decode(packet.RawPacket{Data: pk})
-	require.NoError(t, err)
+func validateEchoFlow(t *testing.T, pk quicpogs.Packet, echoReq *packet.ICMP) {
+	decoder := packet.NewICMPDecoder()
+	decoded, err := decoder.Decode(packet.RawPacket{Data: pk.Payload()})
+	require.NoError(t, err, pk)
 	require.Equal(t, decoded.Src, echoReq.Dst)
 	require.Equal(t, decoded.Dst, echoReq.Src)
 	require.Equal(t, echoReq.Protocol, decoded.Protocol)

ingress/origin_proxy.go

@@ -44,6 +44,9 @@ func (o *httpService) RoundTrip(req *http.Request) (*http.Response, error) {
 }
 
 func (o *statusCode) RoundTrip(_ *http.Request) (*http.Response, error) {
+	if o.defaultResp {
+		o.log.Warn().Msgf(ErrNoIngressRulesCLI.Error())
+	}
 	resp := &http.Response{
 		StatusCode: o.code,
 		Status:     fmt.Sprintf("%d %s", o.code, http.StatusText(o.code)),

ingress/origin_service.go

@@ -9,6 +9,7 @@ import (
 	"net"
 	"net/http"
 	"net/url"
+	"strconv"
 	"time"
 
 	"github.com/pkg/errors"
@@ -22,6 +23,7 @@ import (
 
 const (
 	HelloWorldService = "hello_world"
+	HelloWorldFlag    = "hello-world"
 	HttpStatusService = "http_status"
 )
 
@@ -155,8 +157,10 @@ func newSocksProxyOverWSService(accessPolicy *ipaccess.Policy) *socksProxyOverWS
 }
 
 func addPortIfMissing(uri *url.URL, port int) {
+	hostname := uri.Hostname()
+
 	if uri.Port() == "" {
-		uri.Host = fmt.Sprintf("%s:%d", uri.Hostname(), port)
+		uri.Host = net.JoinHostPort(hostname, strconv.FormatInt(int64(port), 10))
 	}
 }
 
@@ -243,12 +247,21 @@ func (o helloWorld) MarshalJSON() ([]byte, error) {
 // Typical use-case is "user wants the catch-all rule to just respond 404".
 type statusCode struct {
 	code int
+
+	// Set only when the user has not defined any ingress rules
+	defaultResp bool
+	log         *zerolog.Logger
 }
 
 func newStatusCode(status int) statusCode {
 	return statusCode{code: status}
 }
 
+// default status code (503) that is returned for requests to cloudflared that don't have any ingress rules setup
+func newDefaultStatusCode(log *zerolog.Logger) statusCode {
+	return statusCode{code: 503, defaultResp: true, log: log}
+}
+
 func (o *statusCode) String() string {
 	return fmt.Sprintf("http_status:%d", o.code)
 }

ingress/origin_service_test.go

@@ -0,0 +1,29 @@
+package ingress
+
+import (
+	"net/url"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestAddPortIfMissing(t *testing.T) {
+	testCases := []struct {
+		input    string
+		expected string
+	}{
+		{"ssh://[::1]", "[::1]:22"},
+		{"ssh://[::1]:38", "[::1]:38"},
+		{"ssh://abc:38", "abc:38"},
+		{"ssh://127.0.0.1:38", "127.0.0.1:38"},
+		{"ssh://127.0.0.1", "127.0.0.1:22"},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.input, func(t *testing.T) {
+			url1, _ := url.Parse(tc.input)
+			addPortIfMissing(url1, 22)
+			require.Equal(t, tc.expected, url1.Host)
+		})
+	}
+}

ingress/packet_router.go

@@ -0,0 +1,184 @@
+package ingress
+
+import (
+	"context"
+	"fmt"
+	"net/netip"
+
+	"github.com/rs/zerolog"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
+
+	"github.com/cloudflare/cloudflared/packet"
+	quicpogs "github.com/cloudflare/cloudflared/quic"
+	"github.com/cloudflare/cloudflared/tracing"
+)
+
+// Upstream of raw packets
+type muxer interface {
+	SendPacket(pk quicpogs.Packet) error
+	// ReceivePacket waits for the next raw packet from upstream
+	ReceivePacket(ctx context.Context) (quicpogs.Packet, error)
+}
+
+// PacketRouter routes packets between Upstream and ICMPRouter. Currently it rejects all other type of ICMP packets
+type PacketRouter struct {
+	globalConfig           *GlobalRouterConfig
+	muxer                  muxer
+	logger                 *zerolog.Logger
+	checkRouterEnabledFunc func() bool
+	icmpDecoder            *packet.ICMPDecoder
+	encoder                *packet.Encoder
+}
+
+// GlobalRouterConfig is the configuration shared by all instance of Router.
+type GlobalRouterConfig struct {
+	ICMPRouter *icmpRouter
+	IPv4Src    netip.Addr
+	IPv6Src    netip.Addr
+	Zone       string
+}
+
+// NewPacketRouter creates a PacketRouter that handles ICMP packets. Packets are read from muxer but dropped if globalConfig is nil.
+func NewPacketRouter(globalConfig *GlobalRouterConfig, muxer muxer, logger *zerolog.Logger, checkRouterEnabledFunc func() bool) *PacketRouter {
+	return &PacketRouter{
+		globalConfig:           globalConfig,
+		muxer:                  muxer,
+		logger:                 logger,
+		checkRouterEnabledFunc: checkRouterEnabledFunc,
+		icmpDecoder:            packet.NewICMPDecoder(),
+		encoder:                packet.NewEncoder(),
+	}
+}
+
+func (r *PacketRouter) Serve(ctx context.Context) error {
+	for {
+		rawPacket, responder, err := r.nextPacket(ctx)
+		if err != nil {
+			return err
+		}
+		r.handlePacket(ctx, rawPacket, responder)
+	}
+}
+
+func (r *PacketRouter) nextPacket(ctx context.Context) (packet.RawPacket, *packetResponder, error) {
+	pk, err := r.muxer.ReceivePacket(ctx)
+	if err != nil {
+		return packet.RawPacket{}, nil, err
+	}
+	responder := &packetResponder{
+		datagramMuxer: r.muxer,
+	}
+	switch pk.Type() {
+	case quicpogs.DatagramTypeIP:
+		return packet.RawPacket{Data: pk.Payload()}, responder, nil
+	case quicpogs.DatagramTypeIPWithTrace:
+		var identity tracing.Identity
+		if err := identity.UnmarshalBinary(pk.Metadata()); err != nil {
+			r.logger.Err(err).Bytes("tracingIdentity", pk.Metadata()).Msg("Failed to unmarshal tracing identity")
+		} else {
+			responder.tracedCtx = tracing.NewTracedContext(ctx, identity.String(), r.logger)
+			responder.serializedIdentity = pk.Metadata()
+		}
+		return packet.RawPacket{Data: pk.Payload()}, responder, nil
+	default:
+		return packet.RawPacket{}, nil, fmt.Errorf("unexpected datagram type %d", pk.Type())
+	}
+}
+
+func (r *PacketRouter) handlePacket(ctx context.Context, rawPacket packet.RawPacket, responder *packetResponder) {
+	// ICMP Proxy feature is disabled, drop packets
+	if r.globalConfig == nil {
+		return
+	}
+
+	if enabled := r.checkRouterEnabledFunc(); !enabled {
+		return
+	}
+
+	icmpPacket, err := r.icmpDecoder.Decode(rawPacket)
+	if err != nil {
+		r.logger.Err(err).Msg("Failed to decode ICMP packet from quic datagram")
+		return
+	}
+
+	if icmpPacket.TTL <= 1 {
+		if err := r.sendTTLExceedMsg(ctx, icmpPacket, rawPacket, r.encoder); err != nil {
+			r.logger.Err(err).Msg("Failed to return ICMP TTL exceed error")
+		}
+		return
+	}
+	icmpPacket.TTL--
+
+	if err := r.globalConfig.ICMPRouter.Request(ctx, icmpPacket, responder); err != nil {
+		r.logger.Err(err).
+			Str("src", icmpPacket.Src.String()).
+			Str("dst", icmpPacket.Dst.String()).
+			Interface("type", icmpPacket.Type).
+			Msg("Failed to send ICMP packet")
+	}
+}
+
+func (r *PacketRouter) sendTTLExceedMsg(ctx context.Context, pk *packet.ICMP, rawPacket packet.RawPacket, encoder *packet.Encoder) error {
+	var srcIP netip.Addr
+	if pk.Dst.Is4() {
+		srcIP = r.globalConfig.IPv4Src
+	} else {
+		srcIP = r.globalConfig.IPv6Src
+	}
+	ttlExceedPacket := packet.NewICMPTTLExceedPacket(pk.IP, rawPacket, srcIP)
+
+	encodedTTLExceed, err := encoder.Encode(ttlExceedPacket)
+	if err != nil {
+		return err
+	}
+	return r.muxer.SendPacket(quicpogs.RawPacket(encodedTTLExceed))
+}
+
+// packetResponder should not be used concurrently. This assumption is upheld because reply packets are ready one-by-one
+type packetResponder struct {
+	datagramMuxer      muxer
+	tracedCtx          *tracing.TracedContext
+	serializedIdentity []byte
+	// hadReply tracks if there has been any reply for this flow
+	hadReply bool
+}
+
+func (pr *packetResponder) tracingEnabled() bool {
+	return pr.tracedCtx != nil
+}
+
+func (pr *packetResponder) returnPacket(rawPacket packet.RawPacket) error {
+	pr.hadReply = true
+	return pr.datagramMuxer.SendPacket(quicpogs.RawPacket(rawPacket))
+}
+
+func (pr *packetResponder) requestSpan(ctx context.Context, pk *packet.ICMP) (context.Context, trace.Span) {
+	if !pr.tracingEnabled() {
+		return ctx, tracing.NewNoopSpan()
+	}
+	return pr.tracedCtx.Tracer().Start(pr.tracedCtx, "icmp-echo-request", trace.WithAttributes(
+		attribute.String("src", pk.Src.String()),
+		attribute.String("dst", pk.Dst.String()),
+	))
+}
+
+func (pr *packetResponder) replySpan(ctx context.Context, logger *zerolog.Logger) (context.Context, trace.Span) {
+	if !pr.tracingEnabled() || pr.hadReply {
+		return ctx, tracing.NewNoopSpan()
+	}
+	return pr.tracedCtx.Tracer().Start(pr.tracedCtx, "icmp-echo-reply")
+}
+
+func (pr *packetResponder) exportSpan() {
+	if !pr.tracingEnabled() {
+		return
+	}
+	spans := pr.tracedCtx.GetProtoSpans()
+	if len(spans) > 0 {
+		pr.datagramMuxer.SendPacket(&quicpogs.TracingSpanPacket{
+			Spans:           spans,
+			TracingIdentity: pr.serializedIdentity,
+		})
+	}
+}

ingress/packet_router_test.go

@@ -1,4 +1,4 @@
-package packet
+package ingress
 
 import (
 	"bytes"
@@ -10,32 +10,28 @@ import (
 	"time"
 
 	"github.com/google/gopacket/layers"
-	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/require"
 	"golang.org/x/net/icmp"
 	"golang.org/x/net/ipv4"
 	"golang.org/x/net/ipv6"
+
+	"github.com/cloudflare/cloudflared/packet"
+	quicpogs "github.com/cloudflare/cloudflared/quic"
 )
 
 var (
-	noopLogger   = zerolog.Nop()
 	packetConfig = &GlobalRouterConfig{
-		ICMPRouter: &mockICMPRouter{},
+		ICMPRouter: nil,
 		IPv4Src:    netip.MustParseAddr("172.16.0.1"),
 		IPv6Src:    netip.MustParseAddr("fd51:2391:523:f4ee::1"),
 	}
 )
 
 func TestRouterReturnTTLExceed(t *testing.T) {
-	upstream := &mockUpstream{
-		source: make(chan RawPacket),
-	}
-	returnPipe := &mockFunnelUniPipe{
-		uniPipe: make(chan RawPacket),
-	}
+	muxer := newMockMuxer(0)
 	routerEnabled := &routerEnabledChecker{}
 	routerEnabled.set(true)
-	router := NewRouter(packetConfig, upstream, returnPipe, &noopLogger, routerEnabled.isEnabled)
+	router := NewPacketRouter(packetConfig, muxer, &noopLogger, routerEnabled.isEnabled)
 	ctx, cancel := context.WithCancel(context.Background())
 	routerStopped := make(chan struct{})
 	go func() {
@@ -43,8 +39,8 @@ func TestRouterReturnTTLExceed(t *testing.T) {
 		close(routerStopped)
 	}()
 
-	pk := ICMP{
-		IP: &IP{
+	pk := packet.ICMP{
+		IP: &packet.IP{
 			Src:      netip.MustParseAddr("192.168.1.1"),
 			Dst:      netip.MustParseAddr("10.0.0.1"),
 			Protocol: layers.IPProtocolICMPv4,
@@ -60,9 +56,9 @@ func TestRouterReturnTTLExceed(t *testing.T) {
 			},
 		},
 	}
-	assertTTLExceed(t, &pk, router.globalConfig.IPv4Src, upstream, returnPipe)
-	pk = ICMP{
-		IP: &IP{
+	assertTTLExceed(t, &pk, router.globalConfig.IPv4Src, muxer)
+	pk = packet.ICMP{
+		IP: &packet.IP{
 			Src:      netip.MustParseAddr("fd51:2391:523:f4ee::1"),
 			Dst:      netip.MustParseAddr("fd51:2391:697:f4ee::2"),
 			Protocol: layers.IPProtocolICMPv6,
@@ -78,21 +74,16 @@ func TestRouterReturnTTLExceed(t *testing.T) {
 			},
 		},
 	}
-	assertTTLExceed(t, &pk, router.globalConfig.IPv6Src, upstream, returnPipe)
+	assertTTLExceed(t, &pk, router.globalConfig.IPv6Src, muxer)
 
 	cancel()
 	<-routerStopped
 }
 
 func TestRouterCheckEnabled(t *testing.T) {
-	upstream := &mockUpstream{
-		source: make(chan RawPacket),
-	}
-	returnPipe := &mockFunnelUniPipe{
-		uniPipe: make(chan RawPacket),
-	}
+	muxer := newMockMuxer(0)
 	routerEnabled := &routerEnabledChecker{}
-	router := NewRouter(packetConfig, upstream, returnPipe, &noopLogger, routerEnabled.isEnabled)
+	router := NewPacketRouter(packetConfig, muxer, &noopLogger, routerEnabled.isEnabled)
 	ctx, cancel := context.WithCancel(context.Background())
 	routerStopped := make(chan struct{})
 	go func() {
@@ -100,8 +91,8 @@ func TestRouterCheckEnabled(t *testing.T) {
 		close(routerStopped)
 	}()
 
-	pk := ICMP{
-		IP: &IP{
+	pk := packet.ICMP{
+		IP: &packet.IP{
 			Src:      netip.MustParseAddr("192.168.1.1"),
 			Dst:      netip.MustParseAddr("10.0.0.1"),
 			Protocol: layers.IPProtocolICMPv4,
@@ -119,23 +110,28 @@ func TestRouterCheckEnabled(t *testing.T) {
 	}
 
 	// router is disabled
-	require.NoError(t, upstream.send(&pk))
+	encoder := packet.NewEncoder()
+	encodedPacket, err := encoder.Encode(&pk)
+	require.NoError(t, err)
+	sendPacket := quicpogs.RawPacket(encodedPacket)
+
+	muxer.edgeToCfd <- sendPacket
 	select {
 	case <-time.After(time.Millisecond * 10):
-	case <-returnPipe.uniPipe:
+	case <-muxer.cfdToEdge:
 		t.Error("Unexpected reply when router is disabled")
 	}
 	routerEnabled.set(true)
 	// router is enabled, expects reply
-	require.NoError(t, upstream.send(&pk))
-	<-returnPipe.uniPipe
+	muxer.edgeToCfd <- sendPacket
+	<-muxer.cfdToEdge
 
 	routerEnabled.set(false)
 	// router is disabled
-	require.NoError(t, upstream.send(&pk))
+	muxer.edgeToCfd <- sendPacket
 	select {
 	case <-time.After(time.Millisecond * 10):
-	case <-returnPipe.uniPipe:
+	case <-muxer.cfdToEdge:
 		t.Error("Unexpected reply when router is disabled")
 	}
 
@@ -143,66 +139,88 @@ func TestRouterCheckEnabled(t *testing.T) {
 	<-routerStopped
 }
 
-func assertTTLExceed(t *testing.T, originalPacket *ICMP, expectedSrc netip.Addr, upstream *mockUpstream, returnPipe *mockFunnelUniPipe) {
-	encoder := NewEncoder()
+func assertTTLExceed(t *testing.T, originalPacket *packet.ICMP, expectedSrc netip.Addr, muxer *mockMuxer) {
+	encoder := packet.NewEncoder()
 	rawPacket, err := encoder.Encode(originalPacket)
 	require.NoError(t, err)
-	upstream.source <- rawPacket
+	muxer.edgeToCfd <- quicpogs.RawPacket(rawPacket)
 
-	resp := <-returnPipe.uniPipe
-	decoder := NewICMPDecoder()
-	decoded, err := decoder.Decode(resp)
+	resp := <-muxer.cfdToEdge
+	decoder := packet.NewICMPDecoder()
+	decoded, err := decoder.Decode(packet.RawPacket(resp.(quicpogs.RawPacket)))
 	require.NoError(t, err)
 
 	require.Equal(t, expectedSrc, decoded.Src)
 	require.Equal(t, originalPacket.Src, decoded.Dst)
 	require.Equal(t, originalPacket.Protocol, decoded.Protocol)
-	require.Equal(t, DefaultTTL, decoded.TTL)
+	require.Equal(t, packet.DefaultTTL, decoded.TTL)
 	if originalPacket.Dst.Is4() {
 		require.Equal(t, ipv4.ICMPTypeTimeExceeded, decoded.Type)
 	} else {
 		require.Equal(t, ipv6.ICMPTypeTimeExceeded, decoded.Type)
 	}
 	require.Equal(t, 0, decoded.Code)
-	assertICMPChecksum(t, decoded)
 	timeExceed, ok := decoded.Body.(*icmp.TimeExceeded)
 	require.True(t, ok)
 	require.True(t, bytes.Equal(rawPacket.Data, timeExceed.Data))
 }
 
-type mockUpstream struct {
-	source chan RawPacket
+type mockMuxer struct {
+	cfdToEdge chan quicpogs.Packet
+	edgeToCfd chan quicpogs.Packet
 }
 
-func (ms *mockUpstream) send(pk Packet) error {
-	encoder := NewEncoder()
-	rawPacket, err := encoder.Encode(pk)
-	if err != nil {
-		return err
+func newMockMuxer(capacity int) *mockMuxer {
+	return &mockMuxer{
+		cfdToEdge: make(chan quicpogs.Packet, capacity),
+		edgeToCfd: make(chan quicpogs.Packet, capacity),
 	}
-	ms.source <- rawPacket
+}
+
+// Copy packet, because icmpProxy expects the encoder buffer to be reusable after the packet is sent
+func (mm *mockMuxer) SendPacket(pk quicpogs.Packet) error {
+	payload := pk.Payload()
+	copiedPayload := make([]byte, len(payload))
+	copy(copiedPayload, payload)
+
+	metadata := pk.Metadata()
+	copiedMetadata := make([]byte, len(metadata))
+	copy(copiedMetadata, metadata)
+
+	var copiedPacket quicpogs.Packet
+	switch pk.Type() {
+	case quicpogs.DatagramTypeIP:
+		copiedPacket = quicpogs.RawPacket(packet.RawPacket{
+			Data: copiedPayload,
+		})
+	case quicpogs.DatagramTypeIPWithTrace:
+		copiedPacket = &quicpogs.TracedPacket{
+			Packet: packet.RawPacket{
+				Data: copiedPayload,
+			},
+			TracingIdentity: copiedMetadata,
+		}
+	case quicpogs.DatagramTypeTracingSpan:
+		copiedPacket = &quicpogs.TracingSpanPacket{
+			Spans:           copiedPayload,
+			TracingIdentity: copiedMetadata,
+		}
+	default:
+		return fmt.Errorf("unexpected metadata type %d", pk.Type())
+	}
+	mm.cfdToEdge <- copiedPacket
 	return nil
 }
 
-func (ms *mockUpstream) ReceivePacket(ctx context.Context) (RawPacket, error) {
+func (mm *mockMuxer) ReceivePacket(ctx context.Context) (quicpogs.Packet, error) {
 	select {
 	case <-ctx.Done():
-		return RawPacket{}, ctx.Err()
-	case pk := <-ms.source:
+		return nil, ctx.Err()
+	case pk := <-mm.edgeToCfd:
 		return pk, nil
 	}
 }
 
-type mockICMPRouter struct{}
-
-func (mir mockICMPRouter) Serve(ctx context.Context) error {
-	return fmt.Errorf("Serve not implemented by mockICMPRouter")
-}
-
-func (mir mockICMPRouter) Request(pk *ICMP, responder FunnelUniPipe) error {
-	return fmt.Errorf("Request not implemented by mockICMPRouter")
-}
-
 type routerEnabledChecker struct {
 	enabled uint32
 }

ingress/rule_test.go

@@ -148,6 +148,16 @@ func Test_rule_matches(t *testing.T) {
 			},
 			want: true,
 		},
+		{
+			name: "Hostname with wildcard should not match if no dot present",
+			rule: Rule{
+				Hostname: "*.api.abc.cloud",
+			},
+			args: args{
+				requestURL: MustParseURL(t, "https://testing-api.abc.cloud"),
+			},
+			want: false,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

metrics/metrics.go

@@ -10,7 +10,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/gorilla/mux"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/rs/zerolog"
@@ -18,36 +17,41 @@ import (
 )
 
 const (
-	shutdownTimeout = time.Second * 15
-	startupTime     = time.Millisecond * 500
+	startupTime            = time.Millisecond * 500
+	defaultShutdownTimeout = time.Second * 15
 )
 
+type Config struct {
+	ReadyServer         *ReadyServer
+	QuickTunnelHostname string
+	Orchestrator        orchestrator
+
+	ShutdownTimeout time.Duration
+}
+
 type orchestrator interface {
 	GetVersionedConfigJSON() ([]byte, error)
 }
 
 func newMetricsHandler(
-	readyServer *ReadyServer,
-	quickTunnelHostname string,
-	orchestrator orchestrator,
+	config Config,
 	log *zerolog.Logger,
-) *mux.Router {
-	router := mux.NewRouter()
-	router.PathPrefix("/debug/").Handler(http.DefaultServeMux)
-
+) *http.ServeMux {
+	router := http.NewServeMux()
+	router.Handle("/debug/", http.DefaultServeMux)
 	router.Handle("/metrics", promhttp.Handler())
 	router.HandleFunc("/healthcheck", func(w http.ResponseWriter, r *http.Request) {
 		_, _ = fmt.Fprintf(w, "OK\n")
 	})
-	if readyServer != nil {
-		router.Handle("/ready", readyServer)
+	if config.ReadyServer != nil {
+		router.Handle("/ready", config.ReadyServer)
 	}
 	router.HandleFunc("/quicktunnel", func(w http.ResponseWriter, r *http.Request) {
-		_, _ = fmt.Fprintf(w, `{"hostname":"%s"}`, quickTunnelHostname)
+		_, _ = fmt.Fprintf(w, `{"hostname":"%s"}`, config.QuickTunnelHostname)
 	})
-	if orchestrator != nil {
+	if config.Orchestrator != nil {
 		router.HandleFunc("/config", func(w http.ResponseWriter, r *http.Request) {
-			json, err := orchestrator.GetVersionedConfigJSON()
+			json, err := config.Orchestrator.GetVersionedConfigJSON()
 			if err != nil {
 				w.WriteHeader(500)
 				_, _ = fmt.Fprintf(w, "ERR: %v", err)
@@ -63,10 +67,8 @@ func newMetricsHandler(
 
 func ServeMetrics(
 	l net.Listener,
-	shutdownC <-chan struct{},
-	readyServer *ReadyServer,
-	quickTunnelHostname string,
-	orchestrator orchestrator,
+	ctx context.Context,
+	config Config,
 	log *zerolog.Logger,
 ) (err error) {
 	var wg sync.WaitGroup
@@ -74,7 +76,7 @@ func ServeMetrics(
 	trace.AuthRequest = func(*http.Request) (bool, bool) { return true, true }
 	// TODO: parameterize ReadTimeout and WriteTimeout. The maximum time we can
 	// profile CPU usage depends on WriteTimeout
-	h := newMetricsHandler(readyServer, quickTunnelHostname, orchestrator, log)
+	h := newMetricsHandler(config, log)
 	server := &http.Server{
 		ReadTimeout:  10 * time.Second,
 		WriteTimeout: 10 * time.Second,
@@ -91,7 +93,12 @@ func ServeMetrics(
 	// fully started up. So add artificial delay.
 	time.Sleep(startupTime)
 
-	<-shutdownC
+	<-ctx.Done()
+	shutdownTimeout := config.ShutdownTimeout
+	if shutdownTimeout == 0 {
+		shutdownTimeout = defaultShutdownTimeout
+	}
+
 	ctx, cancel := context.WithTimeout(context.Background(), shutdownTimeout)
 	_ = server.Shutdown(ctx)
 	cancel()

orchestration/orchestrator_test.go

@@ -358,7 +358,7 @@ func proxyHTTP(originProxy connection.OriginProxy, hostname string) (*http.Respo
 		return nil, err
 	}
 
-	err = originProxy.ProxyHTTP(respWriter, tracing.NewTracedHTTPRequest(req, &log), false)
+	err = originProxy.ProxyHTTP(respWriter, tracing.NewTracedHTTPRequest(req, 0, &log), false)
 	if err != nil {
 		return nil, err
 	}
@@ -608,7 +608,7 @@ func TestPersistentConnection(t *testing.T) {
 		respWriter, err := connection.NewHTTP2RespWriter(req, wsRespReadWriter, connection.TypeWebsocket, &log)
 		require.NoError(t, err)
 
-		err = originProxy.ProxyHTTP(respWriter, tracing.NewTracedHTTPRequest(req, &log), true)
+		err = originProxy.ProxyHTTP(respWriter, tracing.NewTracedHTTPRequest(req, 0, &log), true)
 		require.NoError(t, err)
 	}()
 

packet/funnel.go

@@ -16,11 +16,9 @@ var (
 
 // Funnel is an abstraction to pipe from 1 src to 1 or more destinations
 type Funnel interface {
-	// SendToDst sends a raw packet to a destination
-	SendToDst(dst netip.Addr, pk RawPacket) error
-	// ReturnToSrc returns a raw packet to the source
-	ReturnToSrc(pk RawPacket) error
-	// LastActive returns the last time SendToDst or ReturnToSrc is called
+	// Updates the last time traffic went through this funnel
+	UpdateLastActive()
+	// LastActive returns the last time there is traffic through this funnel
 	LastActive() time.Time
 	// Close closes the funnel. Further call to SendToDst or ReturnToSrc should return an error
 	Close() error
@@ -36,73 +34,26 @@ type FunnelUniPipe interface {
 	Close() error
 }
 
-// RawPacketFunnel is an implementation of Funnel that sends raw packets. It can be embedded in other structs to
-// satisfy the Funnel interface.
-type RawPacketFunnel struct {
-	Src netip.Addr
+type ActivityTracker struct {
 	// last active unix time. Unit is seconds
 	lastActive int64
-	sendPipe   FunnelUniPipe
-	returnPipe FunnelUniPipe
 }
 
-func NewRawPacketFunnel(src netip.Addr, sendPipe, returnPipe FunnelUniPipe) *RawPacketFunnel {
-	return &RawPacketFunnel{
-		Src:        src,
+func NewActivityTracker() *ActivityTracker {
+	return &ActivityTracker{
 		lastActive: time.Now().Unix(),
-		sendPipe:   sendPipe,
-		returnPipe: returnPipe,
 	}
 }
 
-func (rpf *RawPacketFunnel) SendToDst(dst netip.Addr, pk RawPacket) error {
-	rpf.updateLastActive()
-	return rpf.sendPipe.SendPacket(dst, pk)
+func (at *ActivityTracker) UpdateLastActive() {
+	atomic.StoreInt64(&at.lastActive, time.Now().Unix())
 }
 
-func (rpf *RawPacketFunnel) ReturnToSrc(pk RawPacket) error {
-	rpf.updateLastActive()
-	return rpf.returnPipe.SendPacket(rpf.Src, pk)
-}
-
-func (rpf *RawPacketFunnel) updateLastActive() {
-	atomic.StoreInt64(&rpf.lastActive, time.Now().Unix())
-}
-
-func (rpf *RawPacketFunnel) LastActive() time.Time {
-	lastActive := atomic.LoadInt64(&rpf.lastActive)
+func (at *ActivityTracker) LastActive() time.Time {
+	lastActive := atomic.LoadInt64(&at.lastActive)
 	return time.Unix(lastActive, 0)
 }
 
-func (rpf *RawPacketFunnel) Close() error {
-	sendPipeErr := rpf.sendPipe.Close()
-	returnPipeErr := rpf.returnPipe.Close()
-	if sendPipeErr != nil {
-		return sendPipeErr
-	}
-	if returnPipeErr != nil {
-		return returnPipeErr
-	}
-	return nil
-}
-
-func (rpf *RawPacketFunnel) Equal(other Funnel) bool {
-	otherRawFunnel, ok := other.(*RawPacketFunnel)
-	if !ok {
-		return false
-	}
-	if rpf.Src != otherRawFunnel.Src {
-		return false
-	}
-	if rpf.sendPipe != otherRawFunnel.sendPipe {
-		return false
-	}
-	if rpf.returnPipe != otherRawFunnel.returnPipe {
-		return false
-	}
-	return true
-}
-
 // FunnelID represents a key type that can be used by FunnelTracker
 type FunnelID interface {
 	// Type returns the name of the type that implements the FunnelID
@@ -157,13 +108,23 @@ func (ft *FunnelTracker) Get(id FunnelID) (Funnel, bool) {
 	return funnel, ok
 }
 
-// Registers a funnel. It replaces the current funnel.
-func (ft *FunnelTracker) GetOrRegister(id FunnelID, newFunnelFunc func() (Funnel, error)) (funnel Funnel, new bool, err error) {
+// Registers a funnel. If the `id` is already registered and `shouldReplaceFunc` returns true, it closes and replaces
+// the current funnel. If `newFunnelFunc` returns an error, the `id` will remain unregistered, even if it was registered
+// when calling this function.
+func (ft *FunnelTracker) GetOrRegister(
+	id FunnelID,
+	shouldReplaceFunc func(Funnel) bool,
+	newFunnelFunc func() (Funnel, error),
+) (funnel Funnel, new bool, err error) {
 	ft.lock.Lock()
 	defer ft.lock.Unlock()
 	currentFunnel, exists := ft.funnels[id]
 	if exists {
-		return currentFunnel, false, nil
+		if !shouldReplaceFunc(currentFunnel) {
+			return currentFunnel, false, nil
+		}
+		currentFunnel.Close()
+		delete(ft.funnels, id)
 	}
 	newFunnel, err := newFunnelFunc()
 	if err != nil {
@@ -173,7 +134,7 @@ func (ft *FunnelTracker) GetOrRegister(id FunnelID, newFunnelFunc func() (Funnel
 	return newFunnel, true, nil
 }
 
-// Unregisters a funnel if the funnel equals to the current funnel
+// Unregisters and closes a funnel if the funnel equals to the current funnel
 func (ft *FunnelTracker) Unregister(id FunnelID, funnel Funnel) (deleted bool) {
 	ft.lock.Lock()
 	defer ft.lock.Unlock()

packet/funnel_test.go

@@ -1,6 +1,13 @@
 package packet
 
-import "net/netip"
+import (
+	"fmt"
+	"net/netip"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
 
 type mockFunnelUniPipe struct {
 	uniPipe chan RawPacket
@@ -14,3 +21,118 @@ func (mfui *mockFunnelUniPipe) SendPacket(dst netip.Addr, pk RawPacket) error {
 func (mfui *mockFunnelUniPipe) Close() error {
 	return nil
 }
+
+func TestFunnelRegistration(t *testing.T) {
+	id := testFunnelID{"id1"}
+	funnelErr := fmt.Errorf("expected error")
+	newFunnelFuncErr := func() (Funnel, error) { return nil, funnelErr }
+	newFunnelFuncUncalled := func() (Funnel, error) {
+		require.FailNow(t, "a new funnel should not be created")
+		panic("unreached")
+	}
+	funnel1, newFunnelFunc1 := newFunnelAndFunc("funnel1")
+	funnel2, newFunnelFunc2 := newFunnelAndFunc("funnel2")
+
+	ft := NewFunnelTracker()
+	// Register funnel1
+	funnel, new, err := ft.GetOrRegister(id, shouldReplaceFalse, newFunnelFunc1)
+	require.NoError(t, err)
+	require.True(t, new)
+	require.Equal(t, funnel1, funnel)
+	// Register funnel, no replace
+	funnel, new, err = ft.GetOrRegister(id, shouldReplaceFalse, newFunnelFuncUncalled)
+	require.NoError(t, err)
+	require.False(t, new)
+	require.Equal(t, funnel1, funnel)
+	// Register funnel2, replace
+	funnel, new, err = ft.GetOrRegister(id, shouldReplaceTrue, newFunnelFunc2)
+	require.NoError(t, err)
+	require.True(t, new)
+	require.Equal(t, funnel2, funnel)
+	require.True(t, funnel1.closed)
+	// Register funnel error, replace
+	funnel, new, err = ft.GetOrRegister(id, shouldReplaceTrue, newFunnelFuncErr)
+	require.ErrorIs(t, err, funnelErr)
+	require.False(t, new)
+	require.Nil(t, funnel)
+	require.True(t, funnel2.closed)
+}
+
+func TestFunnelUnregister(t *testing.T) {
+	id := testFunnelID{"id1"}
+	funnel1, newFunnelFunc1 := newFunnelAndFunc("funnel1")
+	funnel2, newFunnelFunc2 := newFunnelAndFunc("funnel2")
+	funnel3, newFunnelFunc3 := newFunnelAndFunc("funnel3")
+
+	ft := NewFunnelTracker()
+	// Register & unregister
+	_, _, err := ft.GetOrRegister(id, shouldReplaceFalse, newFunnelFunc1)
+	require.NoError(t, err)
+	require.True(t, ft.Unregister(id, funnel1))
+	require.True(t, funnel1.closed)
+	require.True(t, ft.Unregister(id, funnel1))
+	// Register, replace, and unregister
+	_, _, err = ft.GetOrRegister(id, shouldReplaceFalse, newFunnelFunc2)
+	require.NoError(t, err)
+	_, _, err = ft.GetOrRegister(id, shouldReplaceTrue, newFunnelFunc3)
+	require.NoError(t, err)
+	require.True(t, funnel2.closed)
+	require.False(t, ft.Unregister(id, funnel2))
+	require.True(t, ft.Unregister(id, funnel3))
+	require.True(t, funnel3.closed)
+}
+
+func shouldReplaceFalse(_ Funnel) bool {
+	return false
+}
+
+func shouldReplaceTrue(_ Funnel) bool {
+	return true
+}
+
+func newFunnelAndFunc(id string) (*testFunnel, func() (Funnel, error)) {
+	funnel := newTestFunnel(id)
+	funnelFunc := func() (Funnel, error) {
+		return funnel, nil
+	}
+	return funnel, funnelFunc
+}
+
+type testFunnelID struct {
+	id string
+}
+
+func (t testFunnelID) Type() string {
+	return "testFunnelID"
+}
+
+func (t testFunnelID) String() string {
+	return t.id
+}
+
+type testFunnel struct {
+	id     string
+	closed bool
+}
+
+func newTestFunnel(id string) *testFunnel {
+	return &testFunnel{
+		id,
+		false,
+	}
+}
+
+func (tf *testFunnel) Close() error {
+	tf.closed = true
+	return nil
+}
+
+func (tf *testFunnel) Equal(other Funnel) bool {
+	return tf.id == other.(*testFunnel).id
+}
+
+func (tf *testFunnel) LastActive() time.Time {
+	return time.Now()
+}
+
+func (tf *testFunnel) UpdateLastActive() {}

packet/router.go

@@ -1,108 +0,0 @@
-package packet
-
-import (
-	"context"
-	"net/netip"
-
-	"github.com/rs/zerolog"
-)
-
-// ICMPRouter sends ICMP messages and listens for their responses
-type ICMPRouter interface {
-	// Serve starts listening for responses to the requests until context is done
-	Serve(ctx context.Context) error
-	// Request sends an ICMP message. Implementations should not modify pk after the function returns.
-	Request(pk *ICMP, responder FunnelUniPipe) error
-}
-
-// Upstream of raw packets
-type Upstream interface {
-	// ReceivePacket waits for the next raw packet from upstream
-	ReceivePacket(ctx context.Context) (RawPacket, error)
-}
-
-// Router routes packets between Upstream and ICMPRouter. Currently it rejects all other type of ICMP packets
-type Router struct {
-	upstream               Upstream
-	returnPipe             FunnelUniPipe
-	globalConfig           *GlobalRouterConfig
-	logger                 *zerolog.Logger
-	checkRouterEnabledFunc func() bool
-}
-
-// GlobalRouterConfig is the configuration shared by all instance of Router.
-type GlobalRouterConfig struct {
-	ICMPRouter ICMPRouter
-	IPv4Src    netip.Addr
-	IPv6Src    netip.Addr
-	Zone       string
-}
-
-func NewRouter(globalConfig *GlobalRouterConfig, upstream Upstream, returnPipe FunnelUniPipe, logger *zerolog.Logger, checkRouterEnabledFunc func() bool) *Router {
-	return &Router{
-		upstream:               upstream,
-		returnPipe:             returnPipe,
-		globalConfig:           globalConfig,
-		logger:                 logger,
-		checkRouterEnabledFunc: checkRouterEnabledFunc,
-	}
-}
-
-func (r *Router) Serve(ctx context.Context) error {
-	icmpDecoder := NewICMPDecoder()
-	encoder := NewEncoder()
-	for {
-		rawPacket, err := r.upstream.ReceivePacket(ctx)
-		if err != nil {
-			return err
-		}
-
-		// Drop packets if ICMPRouter wasn't created
-		if r.globalConfig == nil {
-			continue
-		}
-
-		if enabled := r.checkRouterEnabledFunc(); !enabled {
-			continue
-		}
-
-		icmpPacket, err := icmpDecoder.Decode(rawPacket)
-		if err != nil {
-			r.logger.Err(err).Msg("Failed to decode ICMP packet from quic datagram")
-			continue
-		}
-
-		if icmpPacket.TTL <= 1 {
-			if err := r.sendTTLExceedMsg(icmpPacket, rawPacket, encoder); err != nil {
-				r.logger.Err(err).Msg("Failed to return ICMP TTL exceed error")
-			}
-			continue
-		}
-		icmpPacket.TTL--
-
-		if err := r.globalConfig.ICMPRouter.Request(icmpPacket, r.returnPipe); err != nil {
-			r.logger.Err(err).
-				Str("src", icmpPacket.Src.String()).
-				Str("dst", icmpPacket.Dst.String()).
-				Interface("type", icmpPacket.Type).
-				Msg("Failed to send ICMP packet")
-			continue
-		}
-	}
-}
-
-func (r *Router) sendTTLExceedMsg(pk *ICMP, rawPacket RawPacket, encoder *Encoder) error {
-	var srcIP netip.Addr
-	if pk.Dst.Is4() {
-		srcIP = r.globalConfig.IPv4Src
-	} else {
-		srcIP = r.globalConfig.IPv6Src
-	}
-	ttlExceedPacket := NewICMPTTLExceedPacket(pk.IP, rawPacket, srcIP)
-
-	encodedTTLExceed, err := encoder.Encode(ttlExceedPacket)
-	if err != nil {
-		return err
-	}
-	return r.returnPipe.SendPacket(pk.Src, encodedTTLExceed)
-}

proxy/proxy.go

@@ -16,9 +16,9 @@ import (
 	"github.com/cloudflare/cloudflared/cfio"
 	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/ingress"
+	"github.com/cloudflare/cloudflared/stream"
 	"github.com/cloudflare/cloudflared/tracing"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
-	"github.com/cloudflare/cloudflared/websocket"
 )
 
 const (
@@ -28,6 +28,7 @@ const (
 	LogFieldRule          = "ingressRule"
 	LogFieldOriginService = "originService"
 	LogFieldFlowID        = "flowID"
+	LogFieldConnIndex     = "connIndex"
 
 	trailerHeaderName = "Trailer"
 )
@@ -94,16 +95,18 @@ func (p *Proxy) ProxyHTTP(
 		trace.WithAttributes(attribute.String("req-host", req.Host)))
 	rule, ruleNum := p.ingressRules.FindMatchingRule(req.Host, req.URL.Path)
 	logFields := logFields{
-		cfRay:   cfRay,
-		lbProbe: lbProbe,
-		rule:    ruleNum,
+		cfRay:     cfRay,
+		lbProbe:   lbProbe,
+		rule:      ruleNum,
+		connIndex: tr.ConnIndex,
 	}
 	p.logRequest(req, logFields)
 	ruleSpan.SetAttributes(attribute.Int("rule-num", ruleNum))
 	ruleSpan.End()
 	if err, applied := p.applyIngressMiddleware(rule, req, w); err != nil {
 		if applied {
-			p.log.Error().Msg(err.Error())
+			rule, srv := ruleField(p.ingressRules, ruleNum)
+			p.logRequestError(err, cfRay, "", rule, srv)
 			return nil
 		}
 		return err
@@ -162,14 +165,14 @@ func (p *Proxy) ProxyTCP(
 
 	tracedCtx := tracing.NewTracedContext(serveCtx, req.CfTraceID, p.log)
 
-	p.log.Debug().Str(LogFieldFlowID, req.FlowID).Msg("tcp proxy stream started")
+	p.log.Debug().Str(LogFieldFlowID, req.FlowID).Uint8(LogFieldConnIndex, req.ConnIndex).Msg("tcp proxy stream started")
 
 	if err := p.proxyStream(tracedCtx, rwa, req.Dest, p.warpRouting.Proxy); err != nil {
 		p.logRequestError(err, req.CFRay, req.FlowID, "", ingress.ServiceWarpRouting)
 		return err
 	}
 
-	p.log.Debug().Str(LogFieldFlowID, req.FlowID).Msg("tcp proxy stream finished successfully")
+	p.log.Debug().Str(LogFieldFlowID, req.FlowID).Uint8(LogFieldConnIndex, req.ConnIndex).Msg("tcp proxy stream finished successfully")
 
 	return nil
 }
@@ -256,11 +259,13 @@ func (p *Proxy) proxyHTTPRequest(
 			reader: tr.Request.Body,
 		}
 
-		websocket.Stream(eyeballStream, rwc, p.log)
+		stream.Pipe(eyeballStream, rwc, p.log)
 		return nil
 	}
 
-	_, _ = cfio.Copy(w, resp.Body)
+	if _, err = cfio.Copy(w, resp.Body); err != nil {
+		return err
+	}
 
 	// copy trailers
 	copyTrailers(w, resp)
@@ -285,6 +290,7 @@ func (p *Proxy) proxyStream(
 		return err
 	}
 	connectSpan.End()
+	defer originConn.Close()
 
 	encodedSpans := tr.GetSpans()
 
@@ -292,15 +298,6 @@ func (p *Proxy) proxyStream(
 		return err
 	}
 
-	streamCtx, cancel := context.WithCancel(ctx)
-	defer cancel()
-
-	go func() {
-		// streamCtx is done if req is cancelled or if Stream returns
-		<-streamCtx.Done()
-		originConn.Close()
-	}()
-
 	originConn.Stream(ctx, rwa, p.log)
 	return nil
 }
@@ -325,10 +322,11 @@ func (p *Proxy) appendTagHeaders(r *http.Request) {
 }
 
 type logFields struct {
-	cfRay   string
-	lbProbe bool
-	rule    interface{}
-	flowID  string
+	cfRay     string
+	lbProbe   bool
+	rule      interface{}
+	flowID    string
+	connIndex uint8
 }
 
 func copyTrailers(w connection.ResponseWriter, response *http.Response) {
@@ -353,6 +351,7 @@ func (p *Proxy) logRequest(r *http.Request, fields logFields) {
 		Str("host", r.Host).
 		Str("path", r.URL.Path).
 		Interface("rule", fields.rule).
+		Uint8(LogFieldConnIndex, fields.connIndex).
 		Msg("Inbound request")
 
 	if contentLen := r.ContentLength; contentLen == -1 {
@@ -365,18 +364,18 @@ func (p *Proxy) logRequest(r *http.Request, fields logFields) {
 func (p *Proxy) logOriginResponse(resp *http.Response, fields logFields) {
 	responseByCode.WithLabelValues(strconv.Itoa(resp.StatusCode)).Inc()
 	if fields.cfRay != "" {
-		p.log.Debug().Msgf("CF-RAY: %s Status: %s served by ingress %d", fields.cfRay, resp.Status, fields.rule)
+		p.log.Debug().Uint8(LogFieldConnIndex, fields.connIndex).Msgf("CF-RAY: %s Status: %s served by ingress %d", fields.cfRay, resp.Status, fields.rule)
 	} else if fields.lbProbe {
-		p.log.Debug().Msgf("Response to Load Balancer health check %s", resp.Status)
+		p.log.Debug().Uint8(LogFieldConnIndex, fields.connIndex).Msgf("Response to Load Balancer health check %s", resp.Status)
 	} else {
-		p.log.Debug().Msgf("Status: %s served by ingress %v", resp.Status, fields.rule)
+		p.log.Debug().Uint8(LogFieldConnIndex, fields.connIndex).Msgf("Status: %s served by ingress %v", resp.Status, fields.rule)
 	}
-	p.log.Debug().Msgf("CF-RAY: %s Response Headers %+v", fields.cfRay, resp.Header)
+	p.log.Debug().Uint8(LogFieldConnIndex, fields.connIndex).Msgf("CF-RAY: %s Response Headers %+v", fields.cfRay, resp.Header)
 
 	if contentLen := resp.ContentLength; contentLen == -1 {
-		p.log.Debug().Msgf("CF-RAY: %s Response content length unknown", fields.cfRay)
+		p.log.Debug().Uint8(LogFieldConnIndex, fields.connIndex).Msgf("CF-RAY: %s Response content length unknown", fields.cfRay)
 	} else {
-		p.log.Debug().Msgf("CF-RAY: %s Response content length %d", fields.cfRay, contentLen)
+		p.log.Debug().Uint8(LogFieldConnIndex, fields.connIndex).Msgf("CF-RAY: %s Response content length %d", fields.cfRay, contentLen)
 	}
 }
 

proxy/proxy_test.go

@@ -23,7 +23,6 @@ import (
 	"golang.org/x/sync/errgroup"
 
 	"github.com/cloudflare/cloudflared/cfio"
-
 	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/hello"
@@ -34,7 +33,7 @@ import (
 )
 
 var (
-	testTags        = []tunnelpogs.Tag{tunnelpogs.Tag{Name: "Name", Value: "value"}}
+	testTags        = []tunnelpogs.Tag{{Name: "Name", Value: "value"}}
 	noWarpRouting   = ingress.WarpRoutingConfig{}
 	testWarpRouting = ingress.WarpRoutingConfig{
 		Enabled:        true,
@@ -150,8 +149,7 @@ func TestProxySingleOrigin(t *testing.T) {
 	err := cliCtx.Set("hello-world", "true")
 	require.NoError(t, err)
 
-	allowURLFromArgs := false
-	ingressRule, err := ingress.NewSingleOrigin(cliCtx, allowURLFromArgs)
+	ingressRule, err := ingress.ParseIngressFromConfigAndCLI(&config.Configuration{}, cliCtx, &log)
 	require.NoError(t, err)
 
 	require.NoError(t, ingressRule.StartOrigins(&log, ctx.Done()))
@@ -170,7 +168,7 @@ func testProxyHTTP(proxy connection.OriginProxy) func(t *testing.T) {
 		require.NoError(t, err)
 
 		log := zerolog.Nop()
-		err = proxy.ProxyHTTP(responseWriter, tracing.NewTracedHTTPRequest(req, &log), false)
+		err = proxy.ProxyHTTP(responseWriter, tracing.NewTracedHTTPRequest(req, 0, &log), false)
 		require.NoError(t, err)
 		for _, tag := range testTags {
 			assert.Equal(t, tag.Value, req.Header.Get(TagHeaderNamePrefix+tag.Name))
@@ -198,7 +196,7 @@ func testProxyWebsocket(proxy connection.OriginProxy) func(t *testing.T) {
 		errGroup, ctx := errgroup.WithContext(ctx)
 		errGroup.Go(func() error {
 			log := zerolog.Nop()
-			err = proxy.ProxyHTTP(responseWriter, tracing.NewTracedHTTPRequest(req, &log), true)
+			err = proxy.ProxyHTTP(responseWriter, tracing.NewTracedHTTPRequest(req, 0, &log), true)
 			require.NoError(t, err)
 
 			require.Equal(t, http.StatusSwitchingProtocols, responseWriter.Code)
@@ -260,8 +258,8 @@ func testProxySSE(proxy connection.OriginProxy) func(t *testing.T) {
 		go func() {
 			defer wg.Done()
 			log := zerolog.Nop()
-			err = proxy.ProxyHTTP(responseWriter, tracing.NewTracedHTTPRequest(req, &log), false)
-			require.NoError(t, err)
+			err = proxy.ProxyHTTP(responseWriter, tracing.NewTracedHTTPRequest(req, 0, &log), false)
+			require.Equal(t, err.Error(), "context canceled")
 
 			require.Equal(t, http.StatusOK, responseWriter.Code)
 		}()
@@ -367,7 +365,7 @@ func runIngressTestScenarios(t *testing.T, unvalidatedIngress []config.Unvalidat
 		req, err := http.NewRequest(http.MethodGet, test.url, nil)
 		require.NoError(t, err)
 
-		err = proxy.ProxyHTTP(responseWriter, tracing.NewTracedHTTPRequest(req, &log), false)
+		err = proxy.ProxyHTTP(responseWriter, tracing.NewTracedHTTPRequest(req, 0, &log), false)
 		require.NoError(t, err)
 
 		assert.Equal(t, test.expectedStatus, responseWriter.Code)
@@ -414,7 +412,7 @@ func TestProxyError(t *testing.T) {
 	req, err := http.NewRequest(http.MethodGet, "http://127.0.0.1", nil)
 	assert.NoError(t, err)
 
-	assert.Error(t, proxy.ProxyHTTP(responseWriter, tracing.NewTracedHTTPRequest(req, &log), false))
+	assert.Error(t, proxy.ProxyHTTP(responseWriter, tracing.NewTracedHTTPRequest(req, 0, &log), false))
 }
 
 type replayer struct {
@@ -695,14 +693,13 @@ func TestConnections(t *testing.T) {
 				err = proxy.ProxyTCP(ctx, rwa, &connection.TCPRequest{Dest: dest})
 			} else {
 				log := zerolog.Nop()
-				err = proxy.ProxyHTTP(respWriter, tracing.NewTracedHTTPRequest(req, &log), test.args.connectionType == connection.TypeWebsocket)
+				err = proxy.ProxyHTTP(respWriter, tracing.NewTracedHTTPRequest(req, 0, &log), test.args.connectionType == connection.TypeWebsocket)
 			}
 
 			cancel()
 			assert.Equal(t, test.want.err, err != nil)
 			assert.Equal(t, test.want.message, replayer.Bytes())
-			respPrinter := respWriter.(responsePrinter)
-			assert.Equal(t, test.want.headers, respPrinter.headers())
+			assert.Equal(t, test.want.headers, respWriter.Header())
 			replayer.rw.Reset()
 		})
 	}
@@ -795,10 +792,6 @@ func (p *pipedRequestBody) Close() error {
 	return nil
 }
 
-type responsePrinter interface {
-	headers() http.Header
-}
-
 type wsRespWriter struct {
 	w               io.Writer
 	responseHeaders http.Header
@@ -837,12 +830,16 @@ func (w *wsRespWriter) AddTrailer(trailerName, trailerValue string) {
 }
 
 // respHeaders is a test function to read respHeaders
-func (w *wsRespWriter) headers() http.Header {
+func (w *wsRespWriter) Header() http.Header {
 	// Removing indeterminstic header because it cannot be asserted.
 	w.responseHeaders.Del("Date")
 	return w.responseHeaders
 }
 
+func (w *wsRespWriter) WriteHeader(status int) {
+	// unused
+}
+
 type mockTCPRespWriter struct {
 	w               io.Writer
 	responseHeaders http.Header
@@ -863,7 +860,7 @@ func (m *mockTCPRespWriter) Write(p []byte) (n int, err error) {
 	return m.w.Write(p)
 }
 
-func (w *mockTCPRespWriter) AddTrailer(trailerName, trailerValue string) {
+func (m *mockTCPRespWriter) AddTrailer(trailerName, trailerValue string) {
 	// do nothing
 }
 
@@ -874,10 +871,14 @@ func (m *mockTCPRespWriter) WriteRespHeaders(status int, header http.Header) err
 }
 
 // respHeaders is a test function to read respHeaders
-func (m *mockTCPRespWriter) headers() http.Header {
+func (m *mockTCPRespWriter) Header() http.Header {
 	return m.responseHeaders
 }
 
+func (m *mockTCPRespWriter) WriteHeader(status int) {
+	// do nothing
+}
+
 func createSingleIngressConfig(t *testing.T, service string) ingress.Ingress {
 	ingressConfig := &config.Configuration{
 		Ingress: []config.UnvalidatedIngressRule{

quic/datagram.go

@@ -113,9 +113,12 @@ func extractSessionID(b []byte) (uuid.UUID, []byte, error) {
 // SuffixSessionID appends the session ID at the end of the payload. Suffix is more performant than prefix because
 // the payload slice might already have enough capacity to append the session ID at the end
 func SuffixSessionID(sessionID uuid.UUID, b []byte) ([]byte, error) {
-	if len(b)+len(sessionID) > MaxDatagramFrameSize {
+	return suffixMetadata(b, sessionID[:])
+}
+
+func suffixMetadata(payload, metadata []byte) ([]byte, error) {
+	if len(payload)+len(metadata) > MaxDatagramFrameSize {
 		return nil, fmt.Errorf("datagram size exceed %d", MaxDatagramFrameSize)
 	}
-	b = append(b, sessionID[:]...)
-	return b, nil
+	return append(payload, metadata...), nil
 }

quic/datagram_test.go

@@ -1,6 +1,7 @@
 package quic
 
 import (
+	"bytes"
 	"context"
 	"crypto/rand"
 	"crypto/rsa"
@@ -23,6 +24,7 @@ import (
 	"golang.org/x/sync/errgroup"
 
 	"github.com/cloudflare/cloudflared/packet"
+	"github.com/cloudflare/cloudflared/tracing"
 )
 
 var (
@@ -121,6 +123,15 @@ func testDatagram(t *testing.T, version uint8, sessionToPayloads []*packet.Sessi
 
 	logger := zerolog.Nop()
 
+	tracingIdentity, err := tracing.NewIdentity("ec31ad8a01fde11fdcabe2efdce36873:52726f6cabc144f5:0:1")
+	require.NoError(t, err)
+	serializedTracingID, err := tracingIdentity.MarshalBinary()
+	require.NoError(t, err)
+	tracingSpan := &TracingSpanPacket{
+		Spans:           []byte("tracing"),
+		TracingIdentity: serializedTracingID,
+	}
+
 	errGroup, ctx := errgroup.WithContext(context.Background())
 	// Run edge side of datagram muxer
 	errGroup.Go(func() error {
@@ -140,18 +151,17 @@ func testDatagram(t *testing.T, version uint8, sessionToPayloads []*packet.Sessi
 			muxer := NewDatagramMuxerV2(quicSession, &logger, sessionDemuxChan)
 			muxer.ServeReceive(ctx)
 
-			icmpDecoder := packet.NewICMPDecoder()
 			for _, pk := range packets {
 				received, err := muxer.ReceivePacket(ctx)
 				require.NoError(t, err)
-
-				receivedICMP, err := icmpDecoder.Decode(received)
+				validateIPPacket(t, received, &pk)
+				received, err = muxer.ReceivePacket(ctx)
 				require.NoError(t, err)
-				require.Equal(t, pk.IP, receivedICMP.IP)
-				require.Equal(t, pk.Type, receivedICMP.Type)
-				require.Equal(t, pk.Code, receivedICMP.Code)
-				require.Equal(t, pk.Body, receivedICMP.Body)
+				validateIPPacketWithTracing(t, received, &pk, serializedTracingID)
 			}
+			received, err := muxer.ReceivePacket(ctx)
+			require.NoError(t, err)
+			validateTracingSpans(t, received, tracingSpan)
 		default:
 			return fmt.Errorf("unknown datagram version %d", version)
 		}
@@ -188,10 +198,15 @@ func testDatagram(t *testing.T, version uint8, sessionToPayloads []*packet.Sessi
 			for _, pk := range packets {
 				encodedPacket, err := encoder.Encode(&pk)
 				require.NoError(t, err)
-				require.NoError(t, muxerV2.SendPacket(encodedPacket))
+				require.NoError(t, muxerV2.SendPacket(RawPacket(encodedPacket)))
+				require.NoError(t, muxerV2.SendPacket(&TracedPacket{
+					Packet:          encodedPacket,
+					TracingIdentity: serializedTracingID,
+				}))
 			}
+			require.NoError(t, muxerV2.SendPacket(tracingSpan))
 			// Payload larger than transport MTU, should not be sent
-			require.Error(t, muxerV2.SendPacket(packet.RawPacket{
+			require.Error(t, muxerV2.SendPacket(RawPacket{
 				Data: largePayload,
 			}))
 			muxer = muxerV2
@@ -217,6 +232,38 @@ func testDatagram(t *testing.T, version uint8, sessionToPayloads []*packet.Sessi
 	require.NoError(t, errGroup.Wait())
 }
 
+func validateIPPacket(t *testing.T, receivedPacket Packet, expectedICMP *packet.ICMP) {
+	require.Equal(t, DatagramTypeIP, receivedPacket.Type())
+	rawPacket := receivedPacket.(RawPacket)
+	decoder := packet.NewICMPDecoder()
+	receivedICMP, err := decoder.Decode(packet.RawPacket(rawPacket))
+	require.NoError(t, err)
+	validateICMP(t, expectedICMP, receivedICMP)
+}
+
+func validateIPPacketWithTracing(t *testing.T, receivedPacket Packet, expectedICMP *packet.ICMP, serializedTracingID []byte) {
+	require.Equal(t, DatagramTypeIPWithTrace, receivedPacket.Type())
+	tracedPacket := receivedPacket.(*TracedPacket)
+	decoder := packet.NewICMPDecoder()
+	receivedICMP, err := decoder.Decode(tracedPacket.Packet)
+	require.NoError(t, err)
+	validateICMP(t, expectedICMP, receivedICMP)
+	require.True(t, bytes.Equal(tracedPacket.TracingIdentity, serializedTracingID))
+}
+
+func validateICMP(t *testing.T, expected, actual *packet.ICMP) {
+	require.Equal(t, expected.IP, actual.IP)
+	require.Equal(t, expected.Type, actual.Type)
+	require.Equal(t, expected.Code, actual.Code)
+	require.Equal(t, expected.Body, actual.Body)
+}
+
+func validateTracingSpans(t *testing.T, receivedPacket Packet, expectedSpan *TracingSpanPacket) {
+	require.Equal(t, DatagramTypeTracingSpan, receivedPacket.Type())
+	tracingSpans := receivedPacket.(*TracingSpanPacket)
+	require.Equal(t, tracingSpans, expectedSpan)
+}
+
 func newQUICListener(t *testing.T, config *quic.Config) quic.Listener {
 	// Create a simple tls config.
 	tlsConfig := generateTLSConfig()

quic/datagramv2.go

@@ -9,15 +9,28 @@ import (
 	"github.com/rs/zerolog"
 
 	"github.com/cloudflare/cloudflared/packet"
+	"github.com/cloudflare/cloudflared/tracing"
 )
 
 type DatagramV2Type byte
 
 const (
+	// UDP payload
 	DatagramTypeUDP DatagramV2Type = iota
+	// Full IP packet
 	DatagramTypeIP
+	// DatagramTypeIP + tracing ID
+	DatagramTypeIPWithTrace
+	// Tracing spans in protobuf format
+	DatagramTypeTracingSpan
 )
 
+type Packet interface {
+	Type() DatagramV2Type
+	Payload() []byte
+	Metadata() []byte
+}
+
 const (
 	typeIDLen = 1
 	// Same as sessionDemuxChan capacity
@@ -41,7 +54,7 @@ type DatagramMuxerV2 struct {
 	session          quic.Connection
 	logger           *zerolog.Logger
 	sessionDemuxChan chan<- *packet.Session
-	packetDemuxChan  chan packet.RawPacket
+	packetDemuxChan  chan Packet
 }
 
 func NewDatagramMuxerV2(
@@ -54,7 +67,7 @@ func NewDatagramMuxerV2(
 		session:          quicSession,
 		logger:           &logger,
 		sessionDemuxChan: sessionDemuxChan,
-		packetDemuxChan:  make(chan packet.RawPacket, packetChanCapacity),
+		packetDemuxChan:  make(chan Packet, packetChanCapacity),
 	}
 }
 
@@ -79,14 +92,19 @@ func (dm *DatagramMuxerV2) SendToSession(session *packet.Session) error {
 	return nil
 }
 
-// SendPacket suffix the datagram type to the packet. The other end of the QUIC connection can demultiplex by parsing
-// the payload as IP and look at the source and destination.
-func (dm *DatagramMuxerV2) SendPacket(pk packet.RawPacket) error {
-	payloadWithVersion, err := SuffixType(pk.Data, DatagramTypeIP)
+// SendPacket sends a packet with datagram version in the suffix. If ctx is a TracedContext, it adds the tracing
+// context between payload and datagram version.
+// The other end of the QUIC connection can demultiplex by parsing the payload as IP and look at the source and destination.
+func (dm *DatagramMuxerV2) SendPacket(pk Packet) error {
+	payloadWithMetadata, err := suffixMetadata(pk.Payload(), pk.Metadata())
+	if err != nil {
+		return err
+	}
+	payloadWithMetadataAndType, err := SuffixType(payloadWithMetadata, pk.Type())
 	if err != nil {
 		return errors.Wrap(err, "Failed to suffix datagram type, it will be dropped")
 	}
-	if err := dm.session.SendMessage(payloadWithVersion); err != nil {
+	if err := dm.session.SendMessage(payloadWithMetadataAndType); err != nil {
 		return errors.Wrap(err, "Failed to send datagram back to edge")
 	}
 	return nil
@@ -108,10 +126,10 @@ func (dm *DatagramMuxerV2) ServeReceive(ctx context.Context) error {
 	}
 }
 
-func (dm *DatagramMuxerV2) ReceivePacket(ctx context.Context) (packet.RawPacket, error) {
+func (dm *DatagramMuxerV2) ReceivePacket(ctx context.Context) (pk Packet, err error) {
 	select {
 	case <-ctx.Done():
-		return packet.RawPacket{}, ctx.Err()
+		return nil, ctx.Err()
 	case pk := <-dm.packetDemuxChan:
 		return pk, nil
 	}
@@ -126,10 +144,8 @@ func (dm *DatagramMuxerV2) demux(ctx context.Context, msgWithType []byte) error
 	switch msgType {
 	case DatagramTypeUDP:
 		return dm.handleSession(ctx, msg)
-	case DatagramTypeIP:
-		return dm.handlePacket(ctx, msg)
 	default:
-		return fmt.Errorf("Unexpected datagram type %d", msgType)
+		return dm.handlePacket(ctx, msg, msgType)
 	}
 }
 
@@ -150,13 +166,93 @@ func (dm *DatagramMuxerV2) handleSession(ctx context.Context, session []byte) er
 	}
 }
 
-func (dm *DatagramMuxerV2) handlePacket(ctx context.Context, pk []byte) error {
+func (dm *DatagramMuxerV2) handlePacket(ctx context.Context, pk []byte, msgType DatagramV2Type) error {
+	var demuxedPacket Packet
+	switch msgType {
+	case DatagramTypeIP:
+		demuxedPacket = RawPacket(packet.RawPacket{Data: pk})
+	case DatagramTypeIPWithTrace:
+		tracingIdentity, payload, err := extractTracingIdentity(pk)
+		if err != nil {
+			return err
+		}
+		demuxedPacket = &TracedPacket{
+			Packet:          packet.RawPacket{Data: payload},
+			TracingIdentity: tracingIdentity,
+		}
+	case DatagramTypeTracingSpan:
+		tracingIdentity, spans, err := extractTracingIdentity(pk)
+		if err != nil {
+			return err
+		}
+		demuxedPacket = &TracingSpanPacket{
+			Spans:           spans,
+			TracingIdentity: tracingIdentity,
+		}
+	default:
+		return fmt.Errorf("Unexpected datagram type %d", msgType)
+	}
 	select {
 	case <-ctx.Done():
 		return ctx.Err()
-	case dm.packetDemuxChan <- packet.RawPacket{
-		Data: pk,
-	}:
+	case dm.packetDemuxChan <- demuxedPacket:
 		return nil
 	}
 }
+
+func extractTracingIdentity(pk []byte) (tracingIdentity []byte, payload []byte, err error) {
+	if len(pk) < tracing.IdentityLength {
+		return nil, nil, fmt.Errorf("packet with tracing context should have at least %d bytes, got %v", tracing.IdentityLength, pk)
+	}
+	tracingIdentity = pk[len(pk)-tracing.IdentityLength:]
+	payload = pk[:len(pk)-tracing.IdentityLength]
+	return tracingIdentity, payload, nil
+}
+
+type RawPacket packet.RawPacket
+
+func (rw RawPacket) Type() DatagramV2Type {
+	return DatagramTypeIP
+}
+
+func (rw RawPacket) Payload() []byte {
+	return rw.Data
+}
+
+func (rw RawPacket) Metadata() []byte {
+	return []byte{}
+}
+
+type TracedPacket struct {
+	Packet          packet.RawPacket
+	TracingIdentity []byte
+}
+
+func (tp *TracedPacket) Type() DatagramV2Type {
+	return DatagramTypeIPWithTrace
+}
+
+func (tp *TracedPacket) Payload() []byte {
+	return tp.Packet.Data
+}
+
+func (tp *TracedPacket) Metadata() []byte {
+	return tp.TracingIdentity
+}
+
+type TracingSpanPacket struct {
+	Spans           []byte
+	TracingIdentity []byte
+}
+
+func (tsp *TracingSpanPacket) Type() DatagramV2Type {
+	return DatagramTypeTracingSpan
+}
+
+func (tsp *TracingSpanPacket) Payload() []byte {
+	return tsp.Spans
+}
+
+func (tsp *TracingSpanPacket) Metadata() []byte {
+	return tsp.TracingIdentity
+}

quic/metrics.go

@@ -15,7 +15,7 @@ var (
 	clientConnLabels = []string{"conn_index"}
 	clientMetrics    = struct {
 		totalConnections  prometheus.Counter
-		closedConnections *prometheus.CounterVec
+		closedConnections prometheus.Counter
 		sentPackets       *prometheus.CounterVec
 		sentBytes         *prometheus.CounterVec
 		receivePackets    *prometheus.CounterVec
@@ -30,9 +30,8 @@ var (
 		totalConnections: prometheus.NewCounter(
 			totalConnectionsOpts(logging.PerspectiveClient),
 		),
-		closedConnections: prometheus.NewCounterVec(
+		closedConnections: prometheus.NewCounter(
 			closedConnectionsOpts(logging.PerspectiveClient),
-			[]string{"error"},
 		),
 		sentPackets: prometheus.NewCounterVec(
 			sentPacketsOpts(logging.PerspectiveClient),
@@ -284,7 +283,7 @@ func (cc *clientCollector) startedConnection() {
 }
 
 func (cc *clientCollector) closedConnection(err error) {
-	clientMetrics.closedConnections.WithLabelValues(err.Error()).Inc()
+	clientMetrics.closedConnections.Inc()
 }
 
 func (cc *clientCollector) sentPackets(size logging.ByteCount) {