Release 2022.11.1

This PR changes protocol initialization of the other N connections to be
the same as the one we know the initial tunnel connected with. This is
so we homogenize connections and not lead to some connections being
QUIC-able and the others not.

There's also an improvement to the connection registered log so we know
what protocol every individual connection connected with from the
cloudflared side.

.docker-images

@@ -2,8 +2,7 @@ images:
   - name: cloudflared
     dockerfile: Dockerfile.$ARCH
     context: .
-    versions: 
-      - latest
+    version_file: versions
     registries:
     - name: docker.io/cloudflare
       user: env:DOCKER_USER

.teamcity/update-homebrew-core.sh

@@ -23,4 +23,4 @@ git config --global user.name "cloudflare-warp-bot"
 git config --global user.email "warp-bot@cloudflare.com"
 
 # bump formula pr
-brew bump-formula-pr cloudflared --version="$VERSION" --no-browse
+brew bump-formula-pr cloudflared --version="$VERSION" --no-browse --no-audit

Dockerfile

@@ -7,8 +7,6 @@ ENV GO111MODULE=on \
     TARGET_GOOS=${TARGET_GOOS} \
     TARGET_GOARCH=${TARGET_GOARCH}
     
-LABEL org.opencontainers.image.source="https://github.com/cloudflare/cloudflared"
-
 WORKDIR /go/src/github.com/cloudflare/cloudflared/
 
 # copy our sources into the builder image
@@ -20,6 +18,8 @@ RUN make cloudflared
 # use a distroless base image with glibc
 FROM gcr.io/distroless/base-debian11:nonroot
 
+LABEL org.opencontainers.image.source="https://github.com/cloudflare/cloudflared"
+
 # copy our compiled binary
 COPY --from=builder --chown=nonroot /go/src/github.com/cloudflare/cloudflared/cloudflared /usr/local/bin/
 

Dockerfile.amd64

@@ -3,8 +3,6 @@ FROM golang:1.19 as builder
 ENV GO111MODULE=on \
     CGO_ENABLED=0 
     
-LABEL org.opencontainers.image.source="https://github.com/cloudflare/cloudflared"
-
 WORKDIR /go/src/github.com/cloudflare/cloudflared/
 
 # copy our sources into the builder image
@@ -16,6 +14,8 @@ RUN GOOS=linux GOARCH=amd64 make cloudflared
 # use a distroless base image with glibc
 FROM gcr.io/distroless/base-debian11:nonroot
 
+LABEL org.opencontainers.image.source="https://github.com/cloudflare/cloudflared"
+
 # copy our compiled binary
 COPY --from=builder --chown=nonroot /go/src/github.com/cloudflare/cloudflared/cloudflared /usr/local/bin/
 

Dockerfile.arm64

@@ -3,8 +3,6 @@ FROM golang:1.19 as builder
 ENV GO111MODULE=on \
     CGO_ENABLED=0 
     
-LABEL org.opencontainers.image.source="https://github.com/cloudflare/cloudflared"
-
 WORKDIR /go/src/github.com/cloudflare/cloudflared/
 
 # copy our sources into the builder image
@@ -16,6 +14,8 @@ RUN GOOS=linux GOARCH=arm64 make cloudflared
 # use a distroless base image with glibc
 FROM gcr.io/distroless/base-debian11:nonroot-arm64
 
+LABEL org.opencontainers.image.source="https://github.com/cloudflare/cloudflared"
+
 # copy our compiled binary
 COPY --from=builder --chown=nonroot /go/src/github.com/cloudflare/cloudflared/cloudflared /usr/local/bin/
 

Makefile

@@ -131,6 +131,10 @@ endif
 container:
 	docker build --build-arg=TARGET_ARCH=$(TARGET_ARCH) --build-arg=TARGET_OS=$(TARGET_OS) -t cloudflare/cloudflared-$(TARGET_OS)-$(TARGET_ARCH):"$(VERSION)" .
 
+.PHONY: generate-docker-version
+generate-docker-version:
+	echo latest $(VERSION) > versions
+
 .PHONY: test
 test: vet
 ifndef CI
@@ -145,24 +149,8 @@ endif
 test-ssh-server:
 	docker-compose -f ssh_server_tests/docker-compose.yml up
 
-define publish_package
-	chmod 664 $(BINARY_NAME)*.$(1); \
-	for HOST in $(CF_PKG_HOSTS); do \
-		ssh-keyscan -t ecdsa $$HOST >> ~/.ssh/known_hosts; \
-		scp -p -4 $(BINARY_NAME)*.$(1) cfsync@$$HOST:/state/cf-pkg/staging/$(2)/$(TARGET_PUBLIC_REPO)/$(BINARY_NAME)/; \
-	done
-endef
-
-.PHONY: publish-deb
-publish-deb: cloudflared-deb
-	$(call publish_package,deb,apt)
-
-.PHONY: publish-rpm
-publish-rpm: cloudflared-rpm
-	$(call publish_package,rpm,yum)
-
 cloudflared.1: cloudflared_man_template
-	cat cloudflared_man_template | sed -e 's/\$${VERSION}/$(VERSION)/; s/\$${DATE}/$(DATE)/' > cloudflared.1
+	sed -e 's/\$${VERSION}/$(VERSION)/; s/\$${DATE}/$(DATE)/' cloudflared_man_template > cloudflared.1
 
 install: cloudflared cloudflared.1
 	mkdir -p $(DESTDIR)$(INSTALL_BINDIR) $(DESTDIR)$(INSTALL_MANDIR)
@@ -309,8 +297,8 @@ quic-deps:
 
 .PHONY: vet
 vet:
-	go vet -v -mod=vendor ./...
+	go vet -v -mod=vendor github.com/cloudflare/cloudflared/...
 
-.PHONY: goimports
-goimports:
-	for d in $$(go list -mod=readonly -f '{{.Dir}}' -a ./... | fgrep -v tunnelrpc) ; do goimports -format-only -local github.com/cloudflare/cloudflared -w $$d ; done
+.PHONY: fmt
+fmt:
+	goimports -l -w -local github.com/cloudflare/cloudflared $$(go list -mod=vendor -f '{{.Dir}}' -a ./... | fgrep -v tunnelrpc)

README.md

@@ -25,12 +25,13 @@ routing), but for legacy reasons this requirement is still necessary:
 
 ## Installing `cloudflared`
 
-Downloads are available as standalone binaries, a Docker image, and Debian, RPM, and Homebrew packages. You can also find releases here on the `cloudflared` GitHub repository.
+Downloads are available as standalone binaries, a Docker image, and Debian, RPM, and Homebrew packages. You can also find releases [here](https://github.com/cloudflare/cloudflared/releases) on the `cloudflared` GitHub repository.
 
 * You can [install on macOS](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/install-and-setup/installation#macos) via Homebrew or by downloading the [latest Darwin amd64 release](https://github.com/cloudflare/cloudflared/releases)
 * Binaries, Debian, and RPM packages for Linux [can be found here](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/install-and-setup/installation#linux)
 * A Docker image of `cloudflared` is [available on DockerHub](https://hub.docker.com/r/cloudflare/cloudflared)
 * You can install on Windows machines with the [steps here](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/install-and-setup/installation#windows)
+* Build from source with the [instructions here](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/install-and-setup/installation#build-from-source)
 
 User documentation for Cloudflare Tunnel can be found at https://developers.cloudflare.com/cloudflare-one/connections/connect-apps
 

RELEASE_NOTES

@@ -1,3 +1,70 @@
+2022.11.1
+- 2022-11-29 TUN-6981: We should close UDP socket if failed to connecto to edge
+- 2022-11-25 CUSTESC-23757: Fix a bug where a wildcard ingress rule would match an host without starting with a dot
+- 2022-11-24 TUN-6970: Print newline when printing tunnel token
+- 2022-11-22 TUN-6963: Refactor Metrics service setup
+2022.11.0
+- 2022-11-16 Revert "TUN-6935: Cloudflared should use APIToken instead of serviceKey"
+- 2022-11-16 TUN-6929: Use same protocol for other connections as first one
+- 2022-11-14 TUN-6941: Reduce log level to debug when failing to proxy ICMP reply
+- 2022-11-14 TUN-6935: Cloudflared should use APIToken instead of serviceKey
+- 2022-11-14 TUN-6935: Cloudflared should use APIToken instead of serviceKey
+- 2022-11-11 TUN-6937: Bump golang.org/x/* packages to new release tags
+- 2022-11-10 ZTC-234: macOS tests
+- 2022-11-09 TUN-6927: Refactor validate access configuration to allow empty audTags only
+- 2022-11-08 ZTC-234: Replace ICMP funnels when ingress connection changes
+- 2022-11-04 TUN-6917: Bump go to 1.19.3
+- 2022-11-02 Issue #574: Better ssh config for short-lived cert (#763)
+- 2022-10-28 TUN-6898: Fix bug handling IPv6 based ingresses with missing port
+- 2022-10-28 TUN-6898: Refactor addPortIfMissing
+2022.10.3
+- 2022-10-24 TUN-6871: Add default feature to cloudflared to support EOF on QUIC connections
+- 2022-10-19 TUN-6876: Fix flaky TestTraceICMPRouterEcho by taking account request span can return before reply
+- 2022-10-18 TUN-6867: Clear spans right after they are serialized to avoid returning duplicate spans
+2022.10.2
+- 2022-10-18 TUN-6869: Fix Makefile complaining about missing GO packages
+- 2022-10-18 TUN-6864: Don't reuse port in quic unit tests
+- 2022-10-18 TUN-6868: Return left padded tracing ID when tracing identity is converted to string
+
+2022.10.1
+- 2022-10-16 TUN-6861: Trace ICMP on Windows
+- 2022-10-15 TUN-6860: Send access configuration keys to the edge
+- 2022-10-14 TUN-6858: Trace ICMP reply
+- 2022-10-13 TUN-6855: Add DatagramV2Type for IP packet with trace and tracing spans
+- 2022-10-13 TUN-6856: Refactor to lay foundation for tracing ICMP
+- 2022-10-13 TUN-6604: Trace icmp echo request on Linux and Darwin
+- 2022-10-12 Fix log message (#591)
+- 2022-10-12 TUN-6853: Reuse source port when connecting to the edge for quic connections
+- 2022-10-11 TUN-6829: Allow user of datagramsession to control logging level of errors
+- 2022-10-10 RTG-2276 Update qtls and go mod tidy
+- 2022-10-05 Add post-quantum flag to quick tunnel
+- 2022-10-05 TUN-6823: Update github release message to pull from KV
+- 2022-10-04 TUN-6825: Fix cloudflared:version images require arch hyphens
+- 2022-10-03 TUN-6806: Add ingress rule number to log when filtering due to middlware handler
+- 2022-08-17 Label correct container
+- 2022-08-16 Fix typo in help text for `cloudflared tunnel route lb`
+- 2022-07-18 drop usage of cat when sed is invoked to generate the manpage
+- 2021-03-15 update-build-readme
+- 2021-03-15 fix link
+
+2022.10.0
+- 2022-09-30 TUN-6755: Remove unused publish functions
+- 2022-09-30 TUN-6813: Only proxy ICMP packets when warp-routing is enabled
+- 2022-09-29 TUN-6811: Ping group range should be parsed as int32
+- 2022-09-29 TUN-6812: Drop IP packets if ICMP proxy is not initialized
+- 2022-09-28 TUN-6716: Document limitation of Windows ICMP proxy
+- 2022-09-28 TUN-6810: Add component test for post-quantum
+- 2022-09-27 TUN-6715: Provide suggestion to add cloudflared to ping_group_range if it failed to open ICMP socket
+- 2022-09-22 TUN-6792: Fix brew core release by not auditing the formula
+- 2022-09-22 TUN-6774: Validate OriginRequest.Access to add Ingress.Middleware
+- 2022-09-22 TUN-6775: Add middleware.Handler verification to ProxyHTTP
+- 2022-09-22 TUN-6791: Calculate ICMPv6 checksum
+- 2022-09-22 TUN-6801: Add punycode alternatives for ingress rules
+- 2022-09-21 TUN-6772: Add a JWT Validator as an ingress verifier
+- 2022-09-21 TUN-6772: Add a JWT Validator as an ingress verifier
+- 2022-09-21 TUN-6774: Validate OriginRequest.Access to add Ingress.Middleware
+- 2022-09-21 TUN-6772: Add a JWT Validator as an ingress verifier
+- 2022-09-20 TUN-6741: ICMP proxy tries to listen on specific IPv4 & IPv6 when possible
 2022.9.1
 - 2022-09-20 TUN-6777: Fix race condition in TestFunnelIdleTimeout
 - 2022-09-20 TUN-6595: Enable datagramv2 and icmp proxy by default

carrier/carrier.go

@@ -1,6 +1,6 @@
-//Package carrier provides a WebSocket proxy to carry or proxy a connection
-//from the local client to the edge. See it as a wrapper around any protocol
-//that it packages up in a WebSocket connection to the edge.
+// Package carrier provides a WebSocket proxy to carry or proxy a connection
+// from the local client to the edge. See it as a wrapper around any protocol
+// that it packages up in a WebSocket connection to the edge.
 package carrier
 
 import (

certutil/certutil.go

@@ -1,25 +1,21 @@
 package certutil
 
 import (
-	"crypto/x509"
 	"encoding/json"
 	"encoding/pem"
 	"fmt"
-	"strings"
 )
 
 type namedTunnelToken struct {
-	ZoneID     string `json:"zoneID"`
-	AccountID  string `json:"accountID"`
-	ServiceKey string `json:"serviceKey"`
+	ZoneID    string `json:"zoneID"`
+	AccountID string `json:"accountID"`
+	APIToken  string `json:"apiToken"`
 }
 
 type OriginCert struct {
-	PrivateKey interface{}
-	Cert       *x509.Certificate
-	ZoneID     string
-	ServiceKey string
-	AccountID  string
+	ZoneID    string
+	APIToken  string
+	AccountID string
 }
 
 func DecodeOriginCert(blocks []byte) (*OriginCert, error) {
@@ -33,29 +29,11 @@ func DecodeOriginCert(blocks []byte) (*OriginCert, error) {
 			break
 		}
 		switch block.Type {
-		case "PRIVATE KEY":
-			if originCert.PrivateKey != nil {
-				return nil, fmt.Errorf("Found multiple private key in the certificate")
-			}
-			// RSA private key
-			privateKey, err := x509.ParsePKCS8PrivateKey(block.Bytes)
-			if err != nil {
-				return nil, fmt.Errorf("Cannot parse private key")
-			}
-			originCert.PrivateKey = privateKey
-		case "CERTIFICATE":
-			if originCert.Cert != nil {
-				return nil, fmt.Errorf("Found multiple certificates in the certificate")
-			}
-			cert, err := x509.ParseCertificates(block.Bytes)
-			if err != nil {
-				return nil, fmt.Errorf("Cannot parse certificate")
-			} else if len(cert) > 1 {
-				return nil, fmt.Errorf("Found multiple certificates in the certificate")
-			}
-			originCert.Cert = cert[0]
-		case "WARP TOKEN", "ARGO TUNNEL TOKEN":
-			if originCert.ZoneID != "" || originCert.ServiceKey != "" {
+		case "PRIVATE KEY", "CERTIFICATE":
+			// this is for legacy purposes.
+			break
+		case "ARGO TUNNEL TOKEN":
+			if originCert.ZoneID != "" || originCert.APIToken != "" {
 				return nil, fmt.Errorf("Found multiple tokens in the certificate")
 			}
 			// The token is a string,
@@ -63,18 +41,8 @@ func DecodeOriginCert(blocks []byte) (*OriginCert, error) {
 			ntt := namedTunnelToken{}
 			if err := json.Unmarshal(block.Bytes, &ntt); err == nil {
 				originCert.ZoneID = ntt.ZoneID
-				originCert.ServiceKey = ntt.ServiceKey
+				originCert.APIToken = ntt.APIToken
 				originCert.AccountID = ntt.AccountID
-			} else {
-				// Try the older format, where the zoneID and service key are separated by
-				// a new line character
-				token := string(block.Bytes)
-				s := strings.Split(token, "\n")
-				if len(s) != 2 {
-					return nil, fmt.Errorf("Cannot parse token")
-				}
-				originCert.ZoneID = s[0]
-				originCert.ServiceKey = s[1]
 			}
 		default:
 			return nil, fmt.Errorf("Unknown block %s in the certificate", block.Type)
@@ -82,11 +50,7 @@ func DecodeOriginCert(blocks []byte) (*OriginCert, error) {
 		block, rest = pem.Decode(rest)
 	}
 
-	if originCert.PrivateKey == nil {
-		return nil, fmt.Errorf("Missing private key in the certificate")
-	} else if originCert.Cert == nil {
-		return nil, fmt.Errorf("Missing certificate in the certificate")
-	} else if originCert.ZoneID == "" || originCert.ServiceKey == "" {
+	if originCert.ZoneID == "" || originCert.APIToken == "" {
 		return nil, fmt.Errorf("Missing token in the certificate")
 	}
 

certutil/certutil_test.go

@@ -13,49 +13,33 @@ func TestLoadOriginCert(t *testing.T) {
 	assert.Equal(t, fmt.Errorf("Cannot decode empty certificate"), err)
 	assert.Nil(t, cert)
 
-	blocks, err := ioutil.ReadFile("test-cert-no-key.pem")
-	assert.Nil(t, err)
-	cert, err = DecodeOriginCert(blocks)
-	assert.Equal(t, fmt.Errorf("Missing private key in the certificate"), err)
-	assert.Nil(t, cert)
-
-	blocks, err = ioutil.ReadFile("test-cert-two-certificates.pem")
-	assert.Nil(t, err)
-	cert, err = DecodeOriginCert(blocks)
-	assert.Equal(t, fmt.Errorf("Found multiple certificates in the certificate"), err)
-	assert.Nil(t, cert)
-
-	blocks, err = ioutil.ReadFile("test-cert-unknown-block.pem")
+	blocks, err := ioutil.ReadFile("test-cert-unknown-block.pem")
 	assert.Nil(t, err)
 	cert, err = DecodeOriginCert(blocks)
 	assert.Equal(t, fmt.Errorf("Unknown block RSA PRIVATE KEY in the certificate"), err)
 	assert.Nil(t, cert)
-
-	blocks, err = ioutil.ReadFile("test-cert.pem")
-	assert.Nil(t, err)
-	cert, err = DecodeOriginCert(blocks)
-	assert.Nil(t, err)
-	assert.NotNil(t, cert)
-	assert.Equal(t, "7b0a4d77dfb881c1a3b7d61ea9443e19", cert.ZoneID)
-	key := "v1.0-58bd4f9e28f7b3c28e05a35ff3e80ab4fd9644ef3fece537eb0d12e2e9258217-183442fbb0bbdb3e571558fec9b5589ebd77aafc87498ee3f09f64a4ad79ffe8791edbae08b36c1d8f1d70a8670de56922dff92b15d214a524f4ebfa1958859e-7ce80f79921312a6022c5d25e2d380f82ceaefe3fbdc43dd13b080e3ef1e26f7"
-	assert.Equal(t, key, cert.ServiceKey)
 }
 
-func TestNewlineArgoTunnelToken(t *testing.T) {
-	ArgoTunnelTokenTest(t, "test-argo-tunnel-cert.pem")
+func TestJSONArgoTunnelTokenEmpty(t *testing.T) {
+	cert, err := DecodeOriginCert([]byte{})
+	blocks, err := ioutil.ReadFile("test-cert-no-token.pem")
+	assert.Nil(t, err)
+	cert, err = DecodeOriginCert(blocks)
+	assert.Equal(t, fmt.Errorf("Missing token in the certificate"), err)
+	assert.Nil(t, cert)
 }
 
 func TestJSONArgoTunnelToken(t *testing.T) {
 	// The given cert's Argo Tunnel Token was generated by base64 encoding this JSON:
 	// {
 	// "zoneID": "7b0a4d77dfb881c1a3b7d61ea9443e19",
-	// "serviceKey": "test-service-key",
+	// "apiToken": "test-service-key",
 	// "accountID": "abcdabcdabcdabcd1234567890abcdef"
 	// }
-	ArgoTunnelTokenTest(t, "test-argo-tunnel-cert-json.pem")
+	CloudflareTunnelTokenTest(t, "test-cloudflare-tunnel-cert-json.pem")
 }
 
-func ArgoTunnelTokenTest(t *testing.T, path string) {
+func CloudflareTunnelTokenTest(t *testing.T, path string) {
 	blocks, err := ioutil.ReadFile(path)
 	assert.Nil(t, err)
 	cert, err := DecodeOriginCert(blocks)
@@ -63,5 +47,5 @@ func ArgoTunnelTokenTest(t *testing.T, path string) {
 	assert.NotNil(t, cert)
 	assert.Equal(t, "7b0a4d77dfb881c1a3b7d61ea9443e19", cert.ZoneID)
 	key := "test-service-key"
-	assert.Equal(t, key, cert.ServiceKey)
+	assert.Equal(t, key, cert.APIToken)
 }

certutil/test-cert-no-key.pem

@@ -1,33 +0,0 @@
------BEGIN CERTIFICATE-----
-MIID+jCCA6CgAwIBAgIUJhFxUKEGvTRc3CjCok6dbPGH/P4wCgYIKoZIzj0EAwIw
-gagxCzAJBgNVBAYTAlVTMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMTgwNgYD
-VQQLEy9DbG91ZEZsYXJlIE9yaWdpbiBTU0wgRUNDIENlcnRpZmljYXRlIEF1dGhv
-cml0eTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzETMBEGA1UECBMKQ2FsaWZvcm5p
-YTEXMBUGA1UEAxMOKGRldiB1c2Ugb25seSkwHhcNMTcxMDEzMTM1OTAwWhcNMzIx
-MDA5MTM1OTAwWjBiMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMR0wGwYDVQQL
-ExRDbG91ZEZsYXJlIE9yaWdpbiBDQTEmMCQGA1UEAxMdQ2xvdWRGbGFyZSBPcmln
-aW4gQ2VydGlmaWNhdGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCf
-GswL16Fz9Ei3sAg5AmBizoN2nZdyXHP8T57UxUMcrlJXEEXCVS5RR4m9l+EmK0ng
-6yHR1H5oX1Lg1WKyXgWwr0whwmdTD+qWFJW2M8HyefyBKLrsGPuxw4CVYT0h72bx
-tG0uyrXYh7Mtz0lHjGV90qrFpq5o0jx0sLbDlDvpFPbIO58uYzKG4Sn2VTC4rOyX
-PE6SuDvMHIeX6Ekw4wSVQ9eTbksLQqTyxSqM3zp2ygc56SjGjy1nGQT8ZBGFzSbZ
-AzNOxVKrUsySx7LzZVl+zCGCPlQwaYLKObKXadZJmrqSFmErC5jcbVgBz7oJQOgl
-HJ2n0sMcZ+Ja1Y649mPVAgMBAAGjggEgMIIBHDAOBgNVHQ8BAf8EBAMCBaAwEwYD
-VR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUzA6f2Ajq
-zhX67c6piY2a1uTiUkwwHwYDVR0jBBgwFoAU2qfBlqxKMZnf0QeTeYiMelfqJfgw
-RAYIKwYBBQUHAQEEODA2MDQGCCsGAQUFBzABhihodHRwOi8vb2NzcC5jbG91ZGZs
-YXJlLmNvbS9vcmlnaW5fZWNjX2NhMCMGA1UdEQQcMBqCDCouYXJub2xkLmNvbYIK
-YXJub2xkLmNvbTA8BgNVHR8ENTAzMDGgL6AthitodHRwOi8vY3JsLmNsb3VkZmxh
-cmUuY29tL29yaWdpbl9lY2NfY2EuY3JsMAoGCCqGSM49BAMCA0gAMEUCIDV7HoMj
-K5rShE/l+90YAOzHC89OH/wUz3I5KYOFuehoAiEA8e92aIf9XBkr0K6EvFCiSsD+
-x+Yo/cL8fGfVpPt4UM8=
------END CERTIFICATE-----
------BEGIN WARP TOKEN-----
-N2IwYTRkNzdkZmI4ODFjMWEzYjdkNjFlYTk0NDNlMTkKdjEuMC01OGJkNGY5ZTI4
-ZjdiM2MyOGUwNWEzNWZmM2U4MGFiNGZkOTY0NGVmM2ZlY2U1MzdlYjBkMTJlMmU5
-MjU4MjE3LTE4MzQ0MmZiYjBiYmRiM2U1NzE1NThmZWM5YjU1ODllYmQ3N2FhZmM4
-NzQ5OGVlM2YwOWY2NGE0YWQ3OWZmZTg3OTFlZGJhZTA4YjM2YzFkOGYxZDcwYTg2
-NzBkZTU2OTIyZGZmOTJiMTVkMjE0YTUyNGY0ZWJmYTE5NTg4NTllLTdjZTgwZjc5
-OTIxMzEyYTYwMjJjNWQyNWUyZDM4MGY4MmNlYWVmZTNmYmRjNDNkZDEzYjA4MGUz
-ZWYxZTI2Zjc=
------END WARP TOKEN-----

certutil/test-cert-no-token.pem

@@ -51,7 +51,6 @@ K5rShE/l+90YAOzHC89OH/wUz3I5KYOFuehoAiEA8e92aIf9XBkr0K6EvFCiSsD+
 x+Yo/cL8fGfVpPt4UM8=
 -----END CERTIFICATE-----
 -----BEGIN ARGO TUNNEL TOKEN-----
-eyJ6b25lSUQiOiAiN2IwYTRkNzdkZmI4ODFjMWEzYjdkNjFlYTk0NDNlMTkiLCAi
-c2VydmljZUtleSI6ICJ0ZXN0LXNlcnZpY2Uta2V5IiwgImFjY291bnRJRCI6ICJh
-YmNkYWJjZGFiY2RhYmNkMTIzNDU2Nzg5MGFiY2RlZiJ9
+eyJ6b25lSUQiOiAiN2IwYTRkNzdkZmI4ODFjMWEzYjdkNjFlYTk0NDNlMTkiLCAiYWNjb3VudElE
+IjogImFiY2RhYmNkYWJjZGFiY2QxMjM0NTY3ODkwYWJjZGVmIn0=
 -----END ARGO TUNNEL TOKEN-----

certutil/test-cert-two-certificates.pem

@@ -1,85 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQCfGswL16Fz9Ei3
-sAg5AmBizoN2nZdyXHP8T57UxUMcrlJXEEXCVS5RR4m9l+EmK0ng6yHR1H5oX1Lg
-1WKyXgWwr0whwmdTD+qWFJW2M8HyefyBKLrsGPuxw4CVYT0h72bxtG0uyrXYh7Mt
-z0lHjGV90qrFpq5o0jx0sLbDlDvpFPbIO58uYzKG4Sn2VTC4rOyXPE6SuDvMHIeX
-6Ekw4wSVQ9eTbksLQqTyxSqM3zp2ygc56SjGjy1nGQT8ZBGFzSbZAzNOxVKrUsyS
-x7LzZVl+zCGCPlQwaYLKObKXadZJmrqSFmErC5jcbVgBz7oJQOglHJ2n0sMcZ+Ja
-1Y649mPVAgMBAAECggEAEbPF0ah9fH0IzTU/CPbIeh3flyY8GDuMpR1HvwUurSWB
-IFI9bLyVAXKb8vYP1TMaTnXi5qmFof+/JShgyZc3+1tZtWTfoaiC8Y1bRfE2yk+D
-xmwddhDmijYGG7i8uEaeddSdFEh2GKAqkbV/QgBvN2Nl4EVmIOAJXXNe9l5LFyjy
-sR10aNVJRYV1FahrCTwZ3SovHP4d4AUvHh/3FFZDukHc37CFA0+CcR4uehp5yedi
-2UdqaszXqunFo/3h+Tn9dW2C7gTTZx4+mfyaws3p3YOmdYArXvpejxHIc0FGwLBm
-sb9K7wGVUiF0Bt0ch+C1mdYrCaFNHnPuDswjmm3FwQKBgQDYtxOwwSLA6ZyppozX
-Doyx9a7PhiMHCFKSdVB4l8rpK545a+AmpG6LRScTtBsMTHBhT3IQ3QPWlVm1AhjF
-AvXMa1rOeaGbCbDn1xqEoEVPtj4tys8eTfyWmtU73jWTFauOt4/xpf/urEpg91xj
-m+Gl/8qgBrpm5rQxV5Y4MysRlQKBgQC78jzzlhocXGNvw0wT/K2NsknyeoZXqpIE
-QYL60FMl4geZn6w9hwxaL1r+g/tUjTnpBPQtS1r2Ed2gXby5zspN1g/PW8U3t3to
-P7zHIJ/sLBXrCh5RJko3hUgGhDNOOCIQj4IaKUfvHYvEIbIxlyI0vdsXsgXgMuQ8
-pb9Yifn5QQKBgQCmGu0EtYQlyOlDP10EGSrN3Dm45l9CrKZdi326cN4eCkikSoLs
-G2x/YumouItiydP5QiNzuXOPrbmse4bwumwb2s0nJSMw6iSmDsFMlmuJxW2zO5e0
-6qGH7fUyhgcaTanJIfk6hrm7/mKkH/S4hGpYCc8NCRsmc/35M+D4AoAoYQKBgQC0
-LWpZaxDlF30MbAHHN3l6We2iU+vup0sMYXGb2ZOcwa/fir+ozIr++l8VmJmdWTan
-OWSM96zgMghx8Os4hhJTxF+rvqK242OfcVsc2x31X94zUaP2z+peh5uhA6Pb3Nxr
-W+iyA9k+Vujiwhr+h5D3VvtvH++aG6/KpGtoCf5nAQKBgQDXX2+d7bd5CLNLLFNd
-M2i4QoOFcSKIG+v4SuvgEJHgG8vGvxh2qlSxnMWuPV+7/1P5ATLqDj1PlKms+BNR
-y7sc5AT9PclkL3Y9MNzOu0LXyBkGYcl8M0EQfLv9VPbWT+NXiMg/O2CHiT02pAAz
-uQicoQq3yzeQh20wtrtaXzTNmA==
------END PRIVATE KEY-----
------BEGIN CERTIFICATE-----
-MIID+jCCA6CgAwIBAgIUJhFxUKEGvTRc3CjCok6dbPGH/P4wCgYIKoZIzj0EAwIw
-gagxCzAJBgNVBAYTAlVTMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMTgwNgYD
-VQQLEy9DbG91ZEZsYXJlIE9yaWdpbiBTU0wgRUNDIENlcnRpZmljYXRlIEF1dGhv
-cml0eTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzETMBEGA1UECBMKQ2FsaWZvcm5p
-YTEXMBUGA1UEAxMOKGRldiB1c2Ugb25seSkwHhcNMTcxMDEzMTM1OTAwWhcNMzIx
-MDA5MTM1OTAwWjBiMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMR0wGwYDVQQL
-ExRDbG91ZEZsYXJlIE9yaWdpbiBDQTEmMCQGA1UEAxMdQ2xvdWRGbGFyZSBPcmln
-aW4gQ2VydGlmaWNhdGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCf
-GswL16Fz9Ei3sAg5AmBizoN2nZdyXHP8T57UxUMcrlJXEEXCVS5RR4m9l+EmK0ng
-6yHR1H5oX1Lg1WKyXgWwr0whwmdTD+qWFJW2M8HyefyBKLrsGPuxw4CVYT0h72bx
-tG0uyrXYh7Mtz0lHjGV90qrFpq5o0jx0sLbDlDvpFPbIO58uYzKG4Sn2VTC4rOyX
-PE6SuDvMHIeX6Ekw4wSVQ9eTbksLQqTyxSqM3zp2ygc56SjGjy1nGQT8ZBGFzSbZ
-AzNOxVKrUsySx7LzZVl+zCGCPlQwaYLKObKXadZJmrqSFmErC5jcbVgBz7oJQOgl
-HJ2n0sMcZ+Ja1Y649mPVAgMBAAGjggEgMIIBHDAOBgNVHQ8BAf8EBAMCBaAwEwYD
-VR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUzA6f2Ajq
-zhX67c6piY2a1uTiUkwwHwYDVR0jBBgwFoAU2qfBlqxKMZnf0QeTeYiMelfqJfgw
-RAYIKwYBBQUHAQEEODA2MDQGCCsGAQUFBzABhihodHRwOi8vb2NzcC5jbG91ZGZs
-YXJlLmNvbS9vcmlnaW5fZWNjX2NhMCMGA1UdEQQcMBqCDCouYXJub2xkLmNvbYIK
-YXJub2xkLmNvbTA8BgNVHR8ENTAzMDGgL6AthitodHRwOi8vY3JsLmNsb3VkZmxh
-cmUuY29tL29yaWdpbl9lY2NfY2EuY3JsMAoGCCqGSM49BAMCA0gAMEUCIDV7HoMj
-K5rShE/l+90YAOzHC89OH/wUz3I5KYOFuehoAiEA8e92aIf9XBkr0K6EvFCiSsD+
-x+Yo/cL8fGfVpPt4UM8=
------END CERTIFICATE-----
------BEGIN CERTIFICATE-----
-MIID+jCCA6CgAwIBAgIUJhFxUKEGvTRc3CjCok6dbPGH/P4wCgYIKoZIzj0EAwIw
-gagxCzAJBgNVBAYTAlVTMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMTgwNgYD
-VQQLEy9DbG91ZEZsYXJlIE9yaWdpbiBTU0wgRUNDIENlcnRpZmljYXRlIEF1dGhv
-cml0eTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzETMBEGA1UECBMKQ2FsaWZvcm5p
-YTEXMBUGA1UEAxMOKGRldiB1c2Ugb25seSkwHhcNMTcxMDEzMTM1OTAwWhcNMzIx
-MDA5MTM1OTAwWjBiMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMR0wGwYDVQQL
-ExRDbG91ZEZsYXJlIE9yaWdpbiBDQTEmMCQGA1UEAxMdQ2xvdWRGbGFyZSBPcmln
-aW4gQ2VydGlmaWNhdGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCf
-GswL16Fz9Ei3sAg5AmBizoN2nZdyXHP8T57UxUMcrlJXEEXCVS5RR4m9l+EmK0ng
-6yHR1H5oX1Lg1WKyXgWwr0whwmdTD+qWFJW2M8HyefyBKLrsGPuxw4CVYT0h72bx
-tG0uyrXYh7Mtz0lHjGV90qrFpq5o0jx0sLbDlDvpFPbIO58uYzKG4Sn2VTC4rOyX
-PE6SuDvMHIeX6Ekw4wSVQ9eTbksLQqTyxSqM3zp2ygc56SjGjy1nGQT8ZBGFzSbZ
-AzNOxVKrUsySx7LzZVl+zCGCPlQwaYLKObKXadZJmrqSFmErC5jcbVgBz7oJQOgl
-HJ2n0sMcZ+Ja1Y649mPVAgMBAAGjggEgMIIBHDAOBgNVHQ8BAf8EBAMCBaAwEwYD
-VR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUzA6f2Ajq
-zhX67c6piY2a1uTiUkwwHwYDVR0jBBgwFoAU2qfBlqxKMZnf0QeTeYiMelfqJfgw
-RAYIKwYBBQUHAQEEODA2MDQGCCsGAQUFBzABhihodHRwOi8vb2NzcC5jbG91ZGZs
-YXJlLmNvbS9vcmlnaW5fZWNjX2NhMCMGA1UdEQQcMBqCDCouYXJub2xkLmNvbYIK
-YXJub2xkLmNvbTA8BgNVHR8ENTAzMDGgL6AthitodHRwOi8vY3JsLmNsb3VkZmxh
-cmUuY29tL29yaWdpbl9lY2NfY2EuY3JsMAoGCCqGSM49BAMCA0gAMEUCIDV7HoMj
-K5rShE/l+90YAOzHC89OH/wUz3I5KYOFuehoAiEA8e92aIf9XBkr0K6EvFCiSsD+
-x+Yo/cL8fGfVpPt4UM8=
------END CERTIFICATE-----
------BEGIN WARP TOKEN-----
-N2IwYTRkNzdkZmI4ODFjMWEzYjdkNjFlYTk0NDNlMTkKdjEuMC01OGJkNGY5ZTI4
-ZjdiM2MyOGUwNWEzNWZmM2U4MGFiNGZkOTY0NGVmM2ZlY2U1MzdlYjBkMTJlMmU5
-MjU4MjE3LTE4MzQ0MmZiYjBiYmRiM2U1NzE1NThmZWM5YjU1ODllYmQ3N2FhZmM4
-NzQ5OGVlM2YwOWY2NGE0YWQ3OWZmZTg3OTFlZGJhZTA4YjM2YzFkOGYxZDcwYTg2
-NzBkZTU2OTIyZGZmOTJiMTVkMjE0YTUyNGY0ZWJmYTE5NTg4NTllLTdjZTgwZjc5
-OTIxMzEyYTYwMjJjNWQyNWUyZDM4MGY4MmNlYWVmZTNmYmRjNDNkZDEzYjA4MGUz
-ZWYxZTI2Zjc=
------END WARP TOKEN-----

certutil/test-cert-unknown-block.pem

@@ -50,7 +50,7 @@ cmUuY29tL29yaWdpbl9lY2NfY2EuY3JsMAoGCCqGSM49BAMCA0gAMEUCIDV7HoMj
 K5rShE/l+90YAOzHC89OH/wUz3I5KYOFuehoAiEA8e92aIf9XBkr0K6EvFCiSsD+
 x+Yo/cL8fGfVpPt4UM8=
 -----END CERTIFICATE-----
------BEGIN WARP TOKEN-----
+-----BEGIN ARGO TUNNEL TOKEN-----
 N2IwYTRkNzdkZmI4ODFjMWEzYjdkNjFlYTk0NDNlMTkKdjEuMC01OGJkNGY5ZTI4
 ZjdiM2MyOGUwNWEzNWZmM2U4MGFiNGZkOTY0NGVmM2ZlY2U1MzdlYjBkMTJlMmU5
 MjU4MjE3LTE4MzQ0MmZiYjBiYmRiM2U1NzE1NThmZWM5YjU1ODllYmQ3N2FhZmM4
@@ -58,7 +58,7 @@ NzQ5OGVlM2YwOWY2NGE0YWQ3OWZmZTg3OTFlZGJhZTA4YjM2YzFkOGYxZDcwYTg2
 NzBkZTU2OTIyZGZmOTJiMTVkMjE0YTUyNGY0ZWJmYTE5NTg4NTllLTdjZTgwZjc5
 OTIxMzEyYTYwMjJjNWQyNWUyZDM4MGY4MmNlYWVmZTNmYmRjNDNkZDEzYjA4MGUz
 ZWYxZTI2Zjc=
------END WARP TOKEN-----
+-----END ARGO TUNNEL TOKEN-----
 -----BEGIN RSA PRIVATE KEY-----
 MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQCfGswL16Fz9Ei3
 sAg5AmBizoN2nZdyXHP8T57UxUMcrlJXEEXCVS5RR4m9l+EmK0ng6yHR1H5oX1Lg

certutil/test-cert.pem

@@ -1,61 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQCfGswL16Fz9Ei3
-sAg5AmBizoN2nZdyXHP8T57UxUMcrlJXEEXCVS5RR4m9l+EmK0ng6yHR1H5oX1Lg
-1WKyXgWwr0whwmdTD+qWFJW2M8HyefyBKLrsGPuxw4CVYT0h72bxtG0uyrXYh7Mt
-z0lHjGV90qrFpq5o0jx0sLbDlDvpFPbIO58uYzKG4Sn2VTC4rOyXPE6SuDvMHIeX
-6Ekw4wSVQ9eTbksLQqTyxSqM3zp2ygc56SjGjy1nGQT8ZBGFzSbZAzNOxVKrUsyS
-x7LzZVl+zCGCPlQwaYLKObKXadZJmrqSFmErC5jcbVgBz7oJQOglHJ2n0sMcZ+Ja
-1Y649mPVAgMBAAECggEAEbPF0ah9fH0IzTU/CPbIeh3flyY8GDuMpR1HvwUurSWB
-IFI9bLyVAXKb8vYP1TMaTnXi5qmFof+/JShgyZc3+1tZtWTfoaiC8Y1bRfE2yk+D
-xmwddhDmijYGG7i8uEaeddSdFEh2GKAqkbV/QgBvN2Nl4EVmIOAJXXNe9l5LFyjy
-sR10aNVJRYV1FahrCTwZ3SovHP4d4AUvHh/3FFZDukHc37CFA0+CcR4uehp5yedi
-2UdqaszXqunFo/3h+Tn9dW2C7gTTZx4+mfyaws3p3YOmdYArXvpejxHIc0FGwLBm
-sb9K7wGVUiF0Bt0ch+C1mdYrCaFNHnPuDswjmm3FwQKBgQDYtxOwwSLA6ZyppozX
-Doyx9a7PhiMHCFKSdVB4l8rpK545a+AmpG6LRScTtBsMTHBhT3IQ3QPWlVm1AhjF
-AvXMa1rOeaGbCbDn1xqEoEVPtj4tys8eTfyWmtU73jWTFauOt4/xpf/urEpg91xj
-m+Gl/8qgBrpm5rQxV5Y4MysRlQKBgQC78jzzlhocXGNvw0wT/K2NsknyeoZXqpIE
-QYL60FMl4geZn6w9hwxaL1r+g/tUjTnpBPQtS1r2Ed2gXby5zspN1g/PW8U3t3to
-P7zHIJ/sLBXrCh5RJko3hUgGhDNOOCIQj4IaKUfvHYvEIbIxlyI0vdsXsgXgMuQ8
-pb9Yifn5QQKBgQCmGu0EtYQlyOlDP10EGSrN3Dm45l9CrKZdi326cN4eCkikSoLs
-G2x/YumouItiydP5QiNzuXOPrbmse4bwumwb2s0nJSMw6iSmDsFMlmuJxW2zO5e0
-6qGH7fUyhgcaTanJIfk6hrm7/mKkH/S4hGpYCc8NCRsmc/35M+D4AoAoYQKBgQC0
-LWpZaxDlF30MbAHHN3l6We2iU+vup0sMYXGb2ZOcwa/fir+ozIr++l8VmJmdWTan
-OWSM96zgMghx8Os4hhJTxF+rvqK242OfcVsc2x31X94zUaP2z+peh5uhA6Pb3Nxr
-W+iyA9k+Vujiwhr+h5D3VvtvH++aG6/KpGtoCf5nAQKBgQDXX2+d7bd5CLNLLFNd
-M2i4QoOFcSKIG+v4SuvgEJHgG8vGvxh2qlSxnMWuPV+7/1P5ATLqDj1PlKms+BNR
-y7sc5AT9PclkL3Y9MNzOu0LXyBkGYcl8M0EQfLv9VPbWT+NXiMg/O2CHiT02pAAz
-uQicoQq3yzeQh20wtrtaXzTNmA==
------END PRIVATE KEY-----
------BEGIN CERTIFICATE-----
-MIID+jCCA6CgAwIBAgIUJhFxUKEGvTRc3CjCok6dbPGH/P4wCgYIKoZIzj0EAwIw
-gagxCzAJBgNVBAYTAlVTMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMTgwNgYD
-VQQLEy9DbG91ZEZsYXJlIE9yaWdpbiBTU0wgRUNDIENlcnRpZmljYXRlIEF1dGhv
-cml0eTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzETMBEGA1UECBMKQ2FsaWZvcm5p
-YTEXMBUGA1UEAxMOKGRldiB1c2Ugb25seSkwHhcNMTcxMDEzMTM1OTAwWhcNMzIx
-MDA5MTM1OTAwWjBiMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMR0wGwYDVQQL
-ExRDbG91ZEZsYXJlIE9yaWdpbiBDQTEmMCQGA1UEAxMdQ2xvdWRGbGFyZSBPcmln
-aW4gQ2VydGlmaWNhdGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCf
-GswL16Fz9Ei3sAg5AmBizoN2nZdyXHP8T57UxUMcrlJXEEXCVS5RR4m9l+EmK0ng
-6yHR1H5oX1Lg1WKyXgWwr0whwmdTD+qWFJW2M8HyefyBKLrsGPuxw4CVYT0h72bx
-tG0uyrXYh7Mtz0lHjGV90qrFpq5o0jx0sLbDlDvpFPbIO58uYzKG4Sn2VTC4rOyX
-PE6SuDvMHIeX6Ekw4wSVQ9eTbksLQqTyxSqM3zp2ygc56SjGjy1nGQT8ZBGFzSbZ
-AzNOxVKrUsySx7LzZVl+zCGCPlQwaYLKObKXadZJmrqSFmErC5jcbVgBz7oJQOgl
-HJ2n0sMcZ+Ja1Y649mPVAgMBAAGjggEgMIIBHDAOBgNVHQ8BAf8EBAMCBaAwEwYD
-VR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUzA6f2Ajq
-zhX67c6piY2a1uTiUkwwHwYDVR0jBBgwFoAU2qfBlqxKMZnf0QeTeYiMelfqJfgw
-RAYIKwYBBQUHAQEEODA2MDQGCCsGAQUFBzABhihodHRwOi8vb2NzcC5jbG91ZGZs
-YXJlLmNvbS9vcmlnaW5fZWNjX2NhMCMGA1UdEQQcMBqCDCouYXJub2xkLmNvbYIK
-YXJub2xkLmNvbTA8BgNVHR8ENTAzMDGgL6AthitodHRwOi8vY3JsLmNsb3VkZmxh
-cmUuY29tL29yaWdpbl9lY2NfY2EuY3JsMAoGCCqGSM49BAMCA0gAMEUCIDV7HoMj
-K5rShE/l+90YAOzHC89OH/wUz3I5KYOFuehoAiEA8e92aIf9XBkr0K6EvFCiSsD+
-x+Yo/cL8fGfVpPt4UM8=
------END CERTIFICATE-----
------BEGIN WARP TOKEN-----
-N2IwYTRkNzdkZmI4ODFjMWEzYjdkNjFlYTk0NDNlMTkKdjEuMC01OGJkNGY5ZTI4
-ZjdiM2MyOGUwNWEzNWZmM2U4MGFiNGZkOTY0NGVmM2ZlY2U1MzdlYjBkMTJlMmU5
-MjU4MjE3LTE4MzQ0MmZiYjBiYmRiM2U1NzE1NThmZWM5YjU1ODllYmQ3N2FhZmM4
-NzQ5OGVlM2YwOWY2NGE0YWQ3OWZmZTg3OTFlZGJhZTA4YjM2YzFkOGYxZDcwYTg2
-NzBkZTU2OTIyZGZmOTJiMTVkMjE0YTUyNGY0ZWJmYTE5NTg4NTllLTdjZTgwZjc5
-OTIxMzEyYTYwMjJjNWQyNWUyZDM4MGY4MmNlYWVmZTNmYmRjNDNkZDEzYjA4MGUz
-ZWYxZTI2Zjc=
------END WARP TOKEN-----

certutil/test-cloudflare-tunnel-cert-json.pem

@@ -51,6 +51,7 @@ K5rShE/l+90YAOzHC89OH/wUz3I5KYOFuehoAiEA8e92aIf9XBkr0K6EvFCiSsD+
 x+Yo/cL8fGfVpPt4UM8=
 -----END CERTIFICATE-----
 -----BEGIN ARGO TUNNEL TOKEN-----
-N2IwYTRkNzdkZmI4ODFjMWEzYjdkNjFlYTk0NDNlMTkKdGVzdC1zZXJ2aWNlLWtl
-eQ==
+eyJ6b25lSUQiOiAiN2IwYTRkNzdkZmI4ODFjMWEzYjdkNjFlYTk0NDNlMTkiLCAiYXBpVG9rZW4i
+OiAidGVzdC1zZXJ2aWNlLWtleSIsICJhY2NvdW50SUQiOiAiYWJjZGFiY2RhYmNkYWJjZDEyMzQ1
+Njc4OTBhYmNkZWYifQ==
 -----END ARGO TUNNEL TOKEN-----

cfapi/base_client.go

@@ -104,7 +104,7 @@ func (r *RESTClient) sendRequest(method string, url url.URL, body interface{}) (
 	if bodyReader != nil {
 		req.Header.Set("Content-Type", jsonContentType)
 	}
-	req.Header.Add("X-Auth-User-Service-Key", r.authToken)
+	req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", r.authToken))
 	req.Header.Add("Accept", "application/json;version=1")
 	return r.client.Do(req)
 }

cfsetup.yaml

@@ -1,23 +1,29 @@
-pinned_go: &pinned_go go=1.18.6-1
-pinned_go_fips: &pinned_go_fips go-boring=1.18.6-1
+pinned_go: &pinned_go go=1.19.3-1
+pinned_go_fips: &pinned_go_fips go-boring=1.19.3-1
 
 build_dir: &build_dir /cfsetup_build
 default-flavor: bullseye
 stretch: &stretch
   build:
     build_dir: *build_dir
-    builddeps:
+    builddeps: &build_deps
       - *pinned_go
       - build-essential
+      - gotest-to-teamcity
+    pre-cache: &build_pre_cache
+      - export GOCACHE=/cfsetup_build/.cache/go-build
+      - go install golang.org/x/tools/cmd/goimports@latest
     post-cache:
       - export GOOS=linux
       - export GOARCH=amd64
       - make cloudflared
   build-fips:
     build_dir: *build_dir
-    builddeps:
+    builddeps: &build_deps_fips
       - *pinned_go_fips
       - build-essential
+      - gotest-to-teamcity
+    pre-cache: *build_pre_cache
     post-cache:
       - export GOOS=linux
       - export GOARCH=amd64
@@ -78,6 +84,13 @@ stretch: &stretch
       # same logic as above, but for FIPS packages only
       - ./build-packages-fips.sh
       - make github-release-built-pkgs
+  generate-versions-file:
+    build_dir: *build_dir
+    builddeps: 
+      - *pinned_go
+      - build-essential
+    post-cache:
+      - make generate-docker-version
   build-deb:
     build_dir: *build_dir
     builddeps: &build_deb_deps
@@ -119,18 +132,6 @@ stretch: &stretch
       - export GOOS=linux
       - export GOARCH=arm64
       - make cloudflared-deb
-  publish-deb:
-    build_dir: *build_dir
-    builddeps:
-      - *pinned_go
-      - build-essential
-      - fakeroot
-      - rubygem-fpm
-      - openssh-client
-    post-cache:
-      - export GOOS=linux
-      - export GOARCH=amd64
-      - make publish-deb
   github-release-macos-amd64:
     build_dir: *build_dir
     builddeps: &build_pygithub
@@ -168,12 +169,8 @@ stretch: &stretch
       - make github-windows-upload
   test:
     build_dir: *build_dir
-    builddeps:
-      - *pinned_go
-      - build-essential
-      - gotest-to-teamcity
-    pre-cache: &test_pre_cache
-      - go install golang.org/x/tools/cmd/goimports@latest
+    builddeps: *build_deps
+    pre-cache: *build_pre_cache
     post-cache:
       - export GOOS=linux
       - export GOARCH=amd64
@@ -182,11 +179,8 @@ stretch: &stretch
       - make test | gotest-to-teamcity
   test-fips:
     build_dir: *build_dir
-    builddeps:
-      - *pinned_go_fips
-      - build-essential
-      - gotest-to-teamcity
-    pre-cache: *test_pre_cache
+    builddeps: *build_deps_fips
+    pre-cache: *build_pre_cache
     post-cache:
       - export GOOS=linux
       - export GOARCH=amd64
@@ -195,6 +189,26 @@ stretch: &stretch
       - ./fmt-check.sh
       - make test | gotest-to-teamcity
   component-test:
+    build_dir: *build_dir
+    builddeps:
+      - *pinned_go
+      - python3.7
+      - python3-pip
+      - python3-setuptools
+      # procps installs the ps command which is needed in test_sysv_service because the init script
+      # uses ps pid to determine if the agent is running
+      - procps
+    pre-cache-copy-paths:
+      - component-tests/requirements.txt
+    pre-cache: &component_test_pre_cache
+      - sudo pip3 install --upgrade -r component-tests/requirements.txt
+    post-cache: &component_test_post_cache
+      # Creates and routes a Named Tunnel for this build. Also constructs config file from env vars.
+      - python3 component-tests/setup.py --type create
+      - pytest component-tests -o log_cli=true --log-cli-level=INFO
+      # The Named Tunnel is deleted and its route unprovisioned here.
+      - python3 component-tests/setup.py --type cleanup
+  component-test-fips:
     build_dir: *build_dir
     builddeps:
       - *pinned_go_fips
@@ -206,20 +220,15 @@ stretch: &stretch
       - procps
     pre-cache-copy-paths:
       - component-tests/requirements.txt
-    pre-cache:
-      - sudo pip3 install --upgrade -r component-tests/requirements.txt
-    post-cache:
-      # Creates and routes a Named Tunnel for this build. Also constructs config file from env vars.
-      - python3 component-tests/setup.py --type create
-      - pytest component-tests -o log_cli=true --log-cli-level=INFO
-      # The Named Tunnel is deleted and its route unprovisioned here.
-      - python3 component-tests/setup.py --type cleanup
+    pre-cache: *component_test_pre_cache
+    post-cache: *component_test_post_cache
   update-homebrew:
     builddeps:
       - openssh-client
       - s3cmd
       - jq
       - build-essential
+      - procps
     post-cache:
       - .teamcity/update-homebrew.sh
       - .teamcity/update-homebrew-core.sh
@@ -262,18 +271,3 @@ stretch: &stretch
 buster: *stretch
 bullseye: *stretch
 bookworm: *stretch
-centos-7:
-  publish-rpm:
-    build_dir: *build_dir
-    builddeps: &el7_builddeps
-      - https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
-    pre-cache:
-      - yum install -y fakeroot
-      - yum upgrade -y binutils-2.27-44.base.el7.x86_64
-      - wget https://go.dev/dl/go1.18.6.linux-amd64.tar.gz -P /tmp/
-      - tar -C /usr/local -xzf /tmp/go1.18.6.linux-amd64.tar.gz
-    post-cache:
-      - export PATH=$PATH:/usr/local/go/bin
-      - export GOOS=linux
-      - export GOARCH=amd64
-      - make publish-rpm

cmd/cloudflared/access/cmd.go

@@ -37,16 +37,13 @@ const (
 	sshConfigTemplate  = `
 Add to your {{.Home}}/.ssh/config:
 
-Host {{.Hostname}}
 {{- if .ShortLivedCerts}}
-  ProxyCommand bash -c '{{.Cloudflared}} access ssh-gen --hostname %h; ssh -tt %r@cfpipe-{{.Hostname}} >&2 <&1'
-
-Host cfpipe-{{.Hostname}}
-  HostName {{.Hostname}}
+Match host {{.Hostname}} exec "{{.Cloudflared}} access ssh-gen --hostname %h"
   ProxyCommand {{.Cloudflared}} access ssh --hostname %h
-  IdentityFile ~/.cloudflared/{{.Hostname}}-cf_key
-  CertificateFile ~/.cloudflared/{{.Hostname}}-cf_key-cert.pub
+  IdentityFile ~/.cloudflared/%h-cf_key
+  CertificateFile ~/.cloudflared/%h-cf_key-cert.pub
 {{- else}}
+Host {{.Hostname}}
   ProxyCommand {{.Cloudflared}} access ssh --hostname %h
 {{end}}
 `

cmd/cloudflared/proxydns/cmd.go

@@ -1,6 +1,7 @@
 package proxydns
 
 import (
+	"context"
 	"net"
 	"os"
 	"os/signal"
@@ -73,7 +74,7 @@ func Run(c *cli.Context) error {
 		log.Fatal().Err(err).Msg("Failed to open the metrics listener")
 	}
 
-	go metrics.ServeMetrics(metricsListener, nil, nil, "", nil, log)
+	go metrics.ServeMetrics(metricsListener, context.Background(), metrics.Config{}, log)
 
 	listener, err := tunneldns.CreateListener(
 		c.String("address"),

cmd/cloudflared/tunnel/cmd.go

@@ -372,7 +372,12 @@ func StartServer(
 		defer wg.Done()
 		readinessServer := metrics.NewReadyServer(log, clientID)
 		observer.RegisterSink(readinessServer)
-		errC <- metrics.ServeMetrics(metricsListener, ctx.Done(), readinessServer, quickTunnelURL, orchestrator, log)
+		metricsConfig := metrics.Config{
+			ReadyServer:         readinessServer,
+			QuickTunnelHostname: quickTunnelURL,
+			Orchestrator:        orchestrator,
+		}
+		errC <- metrics.ServeMetrics(metricsListener, ctx, metricsConfig, log)
 	}()
 
 	reconnectCh := make(chan supervisor.ReconnectSignal, c.Int("ha-connections"))
@@ -665,6 +670,13 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			EnvVars: []string{"TUNNEL_MAX_FETCH_SIZE"},
 			Hidden:  true,
 		}),
+		altsrc.NewBoolFlag(&cli.BoolFlag{
+			Name:    "post-quantum",
+			Usage:   "When given creates an experimental post-quantum secure tunnel",
+			Aliases: []string{"pq"},
+			EnvVars: []string{"TUNNEL_POST_QUANTUM"},
+			Hidden:  FipsEnabled,
+		}),
 		selectProtocolFlag,
 		overwriteDNSFlag,
 	}...)

cmd/cloudflared/tunnel/configuration.go

@@ -5,6 +5,8 @@ import (
 	"fmt"
 	"io/ioutil"
 	mathRand "math/rand"
+	"net"
+	"net/netip"
 	"os"
 	"path/filepath"
 	"strings"
@@ -44,7 +46,7 @@ var (
 	LogFieldHostname = "hostname"
 
 	secretFlags     = [2]*altsrc.StringFlag{credentialsContentsFlag, tunnelTokenFlag}
-	defaultFeatures = []string{supervisor.FeatureAllowRemoteConfig, supervisor.FeatureSerializedHeaders, supervisor.FeatureDatagramV2}
+	defaultFeatures = []string{supervisor.FeatureAllowRemoteConfig, supervisor.FeatureSerializedHeaders, supervisor.FeatureDatagramV2, supervisor.FeatureQUICSupportEOF}
 
 	configFlags = []string{"autoupdate-freq", "no-autoupdate", "retries", "protocol", "loglevel", "transport-loglevel", "origincert", "metrics", "metrics-update-freq", "edge-ip-version"}
 )
@@ -384,6 +386,12 @@ func prepareTunnelConfig(
 		NeedPQ:           needPQ,
 		PQKexIdx:         pqKexIdx,
 	}
+	packetConfig, err := newPacketConfig(c, log)
+	if err != nil {
+		log.Warn().Err(err).Msg("ICMP proxy feature is disabled")
+	} else {
+		tunnelConfig.PacketConfig = packetConfig
+	}
 	orchestratorConfig := &orchestration.Config{
 		Ingress:            &ingressRules,
 		WarpRouting:        ingress.NewWarpRoutingConfig(&cfg.WarpRouting),
@@ -453,3 +461,147 @@ func parseConfigIPVersion(version string) (v allregions.ConfigIPVersion, err err
 	}
 	return
 }
+
+func newPacketConfig(c *cli.Context, logger *zerolog.Logger) (*ingress.GlobalRouterConfig, error) {
+	ipv4Src, err := determineICMPv4Src(c.String("icmpv4-src"), logger)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to determine IPv4 source address for ICMP proxy")
+	}
+	logger.Info().Msgf("ICMP proxy will use %s as source for IPv4", ipv4Src)
+
+	ipv6Src, zone, err := determineICMPv6Src(c.String("icmpv6-src"), logger, ipv4Src)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to determine IPv6 source address for ICMP proxy")
+	}
+	if zone != "" {
+		logger.Info().Msgf("ICMP proxy will use %s in zone %s as source for IPv6", ipv6Src, zone)
+	} else {
+		logger.Info().Msgf("ICMP proxy will use %s as source for IPv6", ipv6Src)
+	}
+
+	icmpRouter, err := ingress.NewICMPRouter(ipv4Src, ipv6Src, zone, logger)
+	if err != nil {
+		return nil, err
+	}
+	return &ingress.GlobalRouterConfig{
+		ICMPRouter: icmpRouter,
+		IPv4Src:    ipv4Src,
+		IPv6Src:    ipv6Src,
+		Zone:       zone,
+	}, nil
+}
+
+func determineICMPv4Src(userDefinedSrc string, logger *zerolog.Logger) (netip.Addr, error) {
+	if userDefinedSrc != "" {
+		addr, err := netip.ParseAddr(userDefinedSrc)
+		if err != nil {
+			return netip.Addr{}, err
+		}
+		if addr.Is4() {
+			return addr, nil
+		}
+		return netip.Addr{}, fmt.Errorf("expect IPv4, but %s is IPv6", userDefinedSrc)
+	}
+
+	addr, err := findLocalAddr(net.ParseIP("192.168.0.1"), 53)
+	if err != nil {
+		addr = netip.IPv4Unspecified()
+		logger.Debug().Err(err).Msgf("Failed to determine the IPv4 for this machine. It will use %s to send/listen for ICMPv4 echo", addr)
+	}
+	return addr, nil
+}
+
+type interfaceIP struct {
+	name string
+	ip   net.IP
+}
+
+func determineICMPv6Src(userDefinedSrc string, logger *zerolog.Logger, ipv4Src netip.Addr) (addr netip.Addr, zone string, err error) {
+	if userDefinedSrc != "" {
+		userDefinedIP, zone, _ := strings.Cut(userDefinedSrc, "%")
+		addr, err := netip.ParseAddr(userDefinedIP)
+		if err != nil {
+			return netip.Addr{}, "", err
+		}
+		if addr.Is6() {
+			return addr, zone, nil
+		}
+		return netip.Addr{}, "", fmt.Errorf("expect IPv6, but %s is IPv4", userDefinedSrc)
+	}
+
+	// Loop through all the interfaces, the preference is
+	// 1. The interface where ipv4Src is in
+	// 2. Interface with IPv6 address
+	// 3. Unspecified interface
+
+	interfaces, err := net.Interfaces()
+	if err != nil {
+		return netip.IPv6Unspecified(), "", nil
+	}
+
+	interfacesWithIPv6 := make([]interfaceIP, 0)
+	for _, interf := range interfaces {
+		interfaceAddrs, err := interf.Addrs()
+		if err != nil {
+			continue
+		}
+
+		foundIPv4SrcInterface := false
+		for _, interfaceAddr := range interfaceAddrs {
+			if ipnet, ok := interfaceAddr.(*net.IPNet); ok {
+				ip := ipnet.IP
+				if ip.Equal(ipv4Src.AsSlice()) {
+					foundIPv4SrcInterface = true
+				}
+				if ip.To4() == nil {
+					interfacesWithIPv6 = append(interfacesWithIPv6, interfaceIP{
+						name: interf.Name,
+						ip:   ip,
+					})
+				}
+			}
+		}
+		// Found the interface of ipv4Src. Loop through the addresses to see if there is an IPv6
+		if foundIPv4SrcInterface {
+			for _, interfaceAddr := range interfaceAddrs {
+				if ipnet, ok := interfaceAddr.(*net.IPNet); ok {
+					ip := ipnet.IP
+					if ip.To4() == nil {
+						addr, err := netip.ParseAddr(ip.String())
+						if err == nil {
+							return addr, interf.Name, nil
+						}
+					}
+				}
+			}
+		}
+	}
+
+	for _, interf := range interfacesWithIPv6 {
+		addr, err := netip.ParseAddr(interf.ip.String())
+		if err == nil {
+			return addr, interf.name, nil
+		}
+	}
+	logger.Debug().Err(err).Msgf("Failed to determine the IPv6 for this machine. It will use %s to send/listen for ICMPv6 echo", netip.IPv6Unspecified())
+
+	return netip.IPv6Unspecified(), "", nil
+}
+
+// FindLocalAddr tries to dial UDP and returns the local address picked by the OS
+func findLocalAddr(dst net.IP, port int) (netip.Addr, error) {
+	udpConn, err := net.DialUDP("udp", nil, &net.UDPAddr{
+		IP:   dst,
+		Port: port,
+	})
+	if err != nil {
+		return netip.Addr{}, err
+	}
+	defer udpConn.Close()
+	localAddrPort, err := netip.ParseAddrPort(udpConn.LocalAddr().String())
+	if err != nil {
+		return netip.Addr{}, err
+	}
+	localAddr := localAddrPort.Addr()
+	return localAddr, nil
+}

cmd/cloudflared/tunnel/subcommand_context.go

@@ -74,7 +74,7 @@ func (sc *subcommandContext) client() (cfapi.Client, error) {
 		sc.c.String("api-url"),
 		credential.cert.AccountID,
 		credential.cert.ZoneID,
-		credential.cert.ServiceKey,
+		credential.cert.APIToken,
 		userAgent,
 		sc.log,
 	)

cmd/cloudflared/tunnel/subcommands.go

@@ -176,6 +176,16 @@ var (
 		Usage:   "Base64 encoded secret to set for the tunnel. The decoded secret must be at least 32 bytes long. If not specified, a random 32-byte secret will be generated.",
 		EnvVars: []string{"TUNNEL_CREATE_SECRET"},
 	}
+	icmpv4SrcFlag = &cli.StringFlag{
+		Name:    "icmpv4-src",
+		Usage:   "Source address to send/receive ICMPv4 messages. If not provided cloudflared will dial a local address to determine the source IP or fallback to 0.0.0.0.",
+		EnvVars: []string{"TUNNEL_ICMPV4_SRC"},
+	}
+	icmpv6SrcFlag = &cli.StringFlag{
+		Name:    "icmpv6-src",
+		Usage:   "Source address and the interface name to send/receive ICMPv6 messages. If not provided cloudflared will dial a local address to determine the source IP or fallback to ::.",
+		EnvVars: []string{"TUNNEL_ICMPV6_SRC"},
+	}
 )
 
 func buildCreateCommand() *cli.Command {
@@ -613,6 +623,8 @@ func buildRunCommand() *cli.Command {
 		selectProtocolFlag,
 		featuresFlag,
 		tunnelTokenFlag,
+		icmpv4SrcFlag,
+		icmpv6SrcFlag,
 	}
 	flags = append(flags, configureProxyFlags(false)...)
 	return &cli.Command{
@@ -771,7 +783,7 @@ func tokenCommand(c *cli.Context) error {
 		return err
 	}
 
-	fmt.Printf("%s", encodedToken)
+	fmt.Println(encodedToken)
 	return nil
 }
 
@@ -809,7 +821,7 @@ Further information about managing Cloudflare WARP traffic to your tunnel is ava
 				Name:        "lb",
 				Action:      cliutil.ConfiguredAction(routeLbCommand),
 				Usage:       "Use this tunnel as a load balancer origin, creating pool and load balancer if necessary",
-				UsageText:   "cloudflared tunnel route dns [TUNNEL] [HOSTNAME] [LB-POOL]",
+				UsageText:   "cloudflared tunnel route lb [TUNNEL] [HOSTNAME] [LB-POOL]",
 				Description: `Creates Load Balancer with an origin pool that points to the tunnel.`,
 			},
 			buildRouteIPSubcommand(),

component-tests/constants.py

@@ -1,6 +1,7 @@
 METRICS_PORT = 51000
 MAX_RETRIES = 5
 BACKOFF_SECS = 7
+MAX_LOG_LINES = 50
 
 PROXY_DNS_PORT = 9053
 

component-tests/test_logging.py

@@ -2,6 +2,7 @@
 import json
 import os
 
+from constants import MAX_LOG_LINES
 from util import start_cloudflared, wait_tunnel_ready, send_requests
 
 # Rolling logger rotate log files after 1 MB
@@ -11,14 +12,24 @@ expect_message = "Starting Hello"
 
 
 def assert_log_to_terminal(cloudflared):
-    stderr = cloudflared.stderr.read(1500)
-    assert expect_message.encode() in stderr, f"{stderr} doesn't contain {expect_message}"
+    for _ in range(0, MAX_LOG_LINES):
+        line = cloudflared.stderr.readline()
+        if not line:
+            break
+        if expect_message.encode() in line:
+            return
+    raise Exception(f"terminal log doesn't contain {expect_message}")
 
 
 def assert_log_in_file(file):
     with open(file, "r") as f:
-        log = f.read(2000)
-        assert expect_message in log, f"{log} doesn't contain {expect_message}"
+        for _ in range(0, MAX_LOG_LINES):
+            line = f.readline()
+            if not line:
+                break
+            if expect_message in line:
+                return
+    raise Exception(f"log file doesn't contain {expect_message}")
 
 
 def assert_json_log(file):

component-tests/test_pq.py

@@ -0,0 +1,17 @@
+from util import LOGGER, nofips, start_cloudflared, wait_tunnel_ready
+
+
+@nofips
+class TestPostQuantum:
+    def _extra_config(self):
+        config = {
+            "protocol": "quic",
+        }
+        return config
+
+    def test_post_quantum(self, tmp_path, component_tests_config):
+        config = component_tests_config(self._extra_config())
+        LOGGER.debug(config)
+        with start_cloudflared(tmp_path, config, cfd_args=["run", "--post-quantum"], new_process=True):
+            wait_tunnel_ready(tunnel_url=config.get_url(),
+                              require_min_connections=4)

component-tests/util.py

@@ -20,6 +20,12 @@ def select_platform(plat):
     return pytest.mark.skipif(
         platform.system() != plat, reason=f"Only runs on {plat}")
 
+def fips_enabled():
+    env_fips = os.getenv("COMPONENT_TESTS_FIPS")
+    return env_fips is not None and env_fips != "0"
+
+nofips = pytest.mark.skipif(
+        fips_enabled(), reason=f"Only runs without FIPS (COMPONENT_TESTS_FIPS=0)")
 
 def write_config(directory, config):
     config_path = directory / "config.yml"

config/configuration.go

@@ -234,14 +234,14 @@ type OriginRequestConfig struct {
 }
 
 type AccessConfig struct {
-	// Enabled when set to true will fail every request that does not arrive through an access authenticated endpoint.
-	Enabled bool
+	// Required when set to true will fail every request that does not arrive through an access authenticated endpoint.
+	Required bool `yaml:"required" json:"required,omitempty"`
 
 	// TeamName is the organization team name to get the public key certificates for.
-	TeamName string `yaml:"teamName" json:"teamName,omitempty"`
+	TeamName string `yaml:"teamName" json:"teamName"`
 
 	// AudTag is the AudTag to verify access JWT against.
-	AudTag []string `yaml:"audTag" json:"audTag,omitempty"`
+	AudTag []string `yaml:"audTag" json:"audTag"`
 }
 
 type IngressIPRule struct {

connection/connection.go

@@ -39,6 +39,7 @@ type Orchestrator interface {
 	UpdateConfig(version int32, config []byte) *pogs.UpdateConfigurationResponse
 	GetConfigJSON() ([]byte, error)
 	GetOriginProxy() (OriginProxy, error)
+	WarpRoutingEnabled() (enabled bool)
 }
 
 type NamedTunnelProperties struct {

connection/connection_test.go

@@ -56,6 +56,10 @@ func (mcr *mockOrchestrator) GetOriginProxy() (OriginProxy, error) {
 	return mcr.originProxy, nil
 }
 
+func (mcr *mockOrchestrator) WarpRoutingEnabled() (enabled bool) {
+	return true
+}
+
 type mockOriginProxy struct{}
 
 func (moc *mockOriginProxy) ProxyHTTP(

connection/control.go

@@ -85,7 +85,7 @@ func (c *controlStream) ServeControlStream(
 		return err
 	}
 
-	c.observer.logServerInfo(c.connIndex, registrationDetails.Location, c.edgeAddress, fmt.Sprintf("Connection %s registered", registrationDetails.UUID))
+	c.observer.logServerInfo(c.connIndex, registrationDetails.Location, c.edgeAddress, fmt.Sprintf("Connection %s registered with protocol: %s", registrationDetails.UUID, c.protocol))
 	c.observer.sendConnectedEvent(c.connIndex, c.protocol, registrationDetails.Location)
 	c.connectedFuse.Connected()
 

connection/quic.go

@@ -10,6 +10,7 @@ import (
 	"net/netip"
 	"strconv"
 	"strings"
+	"sync"
 	"sync/atomic"
 	"time"
 
@@ -42,6 +43,11 @@ const (
 	demuxChanCapacity = 16
 )
 
+var (
+	portForConnIndex = make(map[uint8]int, 0)
+	portMapMutex     sync.Mutex
+)
+
 // QUICConnection represents the type that facilitates Proxying via QUIC streams.
 type QUICConnection struct {
 	session      quic.Connection
@@ -50,8 +56,8 @@ type QUICConnection struct {
 	// sessionManager tracks active sessions. It receives datagrams from quic connection via datagramMuxer
 	sessionManager datagramsession.Manager
 	// datagramMuxer mux/demux datagrams from quic connection
-	datagramMuxer        quicpogs.BaseDatagramMuxer
-	packetRouter         *packet.Router
+	datagramMuxer        *quicpogs.DatagramMuxerV2
+	packetRouter         *ingress.PacketRouter
 	controlStreamHandler ControlStreamHandler
 	connOptions          *tunnelpogs.ConnectionOptions
 }
@@ -60,31 +66,36 @@ type QUICConnection struct {
 func NewQUICConnection(
 	quicConfig *quic.Config,
 	edgeAddr net.Addr,
+	connIndex uint8,
 	tlsConfig *tls.Config,
 	orchestrator Orchestrator,
 	connOptions *tunnelpogs.ConnectionOptions,
 	controlStreamHandler ControlStreamHandler,
 	logger *zerolog.Logger,
-	icmpRouter packet.ICMPRouter,
+	packetRouterConfig *ingress.GlobalRouterConfig,
 ) (*QUICConnection, error) {
-	session, err := quic.DialAddr(edgeAddr.String(), tlsConfig, quicConfig)
+	udpConn, err := createUDPConnForConnIndex(connIndex, logger)
 	if err != nil {
+		return nil, err
+	}
+
+	session, err := quic.Dial(udpConn, edgeAddr, edgeAddr.String(), tlsConfig, quicConfig)
+	if err != nil {
+		// close the udp server socket in case of error connecting to the edge
+		udpConn.Close()
 		return nil, &EdgeQuicDialError{Cause: err}
 	}
 
-	sessionDemuxChan := make(chan *packet.Session, demuxChanCapacity)
-	var (
-		datagramMuxer quicpogs.BaseDatagramMuxer
-		pr            *packet.Router
-	)
-	if icmpRouter != nil {
-		datagramMuxerV2 := quicpogs.NewDatagramMuxerV2(session, logger, sessionDemuxChan)
-		pr = packet.NewRouter(datagramMuxerV2, &returnPipe{muxer: datagramMuxerV2}, icmpRouter, logger)
-		datagramMuxer = datagramMuxerV2
-	} else {
-		datagramMuxer = quicpogs.NewDatagramMuxer(session, logger, sessionDemuxChan)
+	// wrap the session, so that the UDPConn is closed after session is closed.
+	session = &wrapCloseableConnQuicConnection{
+		session,
+		udpConn,
 	}
+
+	sessionDemuxChan := make(chan *packet.Session, demuxChanCapacity)
+	datagramMuxer := quicpogs.NewDatagramMuxerV2(session, logger, sessionDemuxChan)
 	sessionManager := datagramsession.NewManager(logger, datagramMuxer.SendToSession, sessionDemuxChan)
+	packetRouter := ingress.NewPacketRouter(packetRouterConfig, datagramMuxer, logger, orchestrator.WarpRoutingEnabled)
 
 	return &QUICConnection{
 		session:              session,
@@ -92,7 +103,7 @@ func NewQUICConnection(
 		logger:               logger,
 		sessionManager:       sessionManager,
 		datagramMuxer:        datagramMuxer,
-		packetRouter:         pr,
+		packetRouter:         packetRouter,
 		controlStreamHandler: controlStreamHandler,
 		connOptions:          connOptions,
 	}, nil
@@ -128,17 +139,14 @@ func (q *QUICConnection) Serve(ctx context.Context) error {
 		defer cancel()
 		return q.sessionManager.Serve(ctx)
 	})
-
 	errGroup.Go(func() error {
 		defer cancel()
 		return q.datagramMuxer.ServeReceive(ctx)
 	})
-	if q.packetRouter != nil {
-		errGroup.Go(func() error {
-			defer cancel()
-			return q.packetRouter.Serve(ctx)
-		})
-	}
+	errGroup.Go(func() error {
+		defer cancel()
+		return q.packetRouter.Serve(ctx)
+	})
 
 	return errGroup.Wait()
 }
@@ -492,15 +500,69 @@ func (np *nopCloserReadWriter) Close() error {
 	return nil
 }
 
-// returnPipe wraps DatagramMuxerV2 to satisfy the packet.FunnelUniPipe interface
-type returnPipe struct {
+// muxerWrapper wraps DatagramMuxerV2 to satisfy the packet.FunnelUniPipe interface
+type muxerWrapper struct {
 	muxer *quicpogs.DatagramMuxerV2
 }
 
-func (rp *returnPipe) SendPacket(dst netip.Addr, pk packet.RawPacket) error {
-	return rp.muxer.SendPacket(pk)
+func (rp *muxerWrapper) SendPacket(dst netip.Addr, pk packet.RawPacket) error {
+	return rp.muxer.SendPacket(quicpogs.RawPacket(pk))
 }
 
-func (rp *returnPipe) Close() error {
+func (rp *muxerWrapper) ReceivePacket(ctx context.Context) (packet.RawPacket, error) {
+	pk, err := rp.muxer.ReceivePacket(ctx)
+	if err != nil {
+		return packet.RawPacket{}, err
+	}
+	rawPacket, ok := pk.(quicpogs.RawPacket)
+	if ok {
+		return packet.RawPacket(rawPacket), nil
+	}
+	return packet.RawPacket{}, fmt.Errorf("unexpected packet type %+v", pk)
+}
+
+func (rp *muxerWrapper) Close() error {
 	return nil
 }
+
+func createUDPConnForConnIndex(connIndex uint8, logger *zerolog.Logger) (*net.UDPConn, error) {
+	portMapMutex.Lock()
+	defer portMapMutex.Unlock()
+
+	// if port was not set yet, it will be zero, so bind will randomly allocate one.
+	if port, ok := portForConnIndex[connIndex]; ok {
+		udpConn, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.IPv4zero, Port: port})
+		// if there wasn't an error, or if port was 0 (independently of error or not, just return)
+		if err == nil {
+			return udpConn, nil
+		} else {
+			logger.Debug().Err(err).Msgf("Unable to reuse port %d for connIndex %d. Falling back to random allocation.", port, connIndex)
+		}
+	}
+
+	// if we reached here, then there was an error or port as not been allocated it.
+	udpConn, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.IPv4zero, Port: 0})
+	if err == nil {
+		udpAddr, ok := (udpConn.LocalAddr()).(*net.UDPAddr)
+		if !ok {
+			return nil, fmt.Errorf("unable to cast to udpConn")
+		}
+		portForConnIndex[connIndex] = udpAddr.Port
+	} else {
+		delete(portForConnIndex, connIndex)
+	}
+
+	return udpConn, err
+}
+
+type wrapCloseableConnQuicConnection struct {
+	quic.Connection
+	udpConn *net.UDPConn
+}
+
+func (w *wrapCloseableConnQuicConnection) CloseWithError(errorCode quic.ApplicationErrorCode, reason string) error {
+	err := w.Connection.CloseWithError(errorCode, reason)
+	w.udpConn.Close()
+
+	return err
+}

connection/quic_test.go

@@ -11,7 +11,6 @@ import (
 	"net/url"
 	"os"
 	"strings"
-	"sync"
 	"testing"
 	"time"
 
@@ -142,24 +141,33 @@ func TestQUICServer(t *testing.T) {
 		},
 	}
 
-	for _, test := range tests {
+	for i, test := range tests {
+		test := test // capture range variable
 		t.Run(test.desc, func(t *testing.T) {
 			ctx, cancel := context.WithCancel(context.Background())
-			var wg sync.WaitGroup
-			wg.Add(1)
+
+			quicListener, err := quic.Listen(udpListener, testTLSServerConfig, testQUICConfig)
+			require.NoError(t, err)
+
+			serverDone := make(chan struct{})
 			go func() {
-				defer wg.Done()
 				quicServer(
-					t, udpListener, testTLSServerConfig, testQUICConfig,
-					test.dest, test.connectionType, test.metadata, test.message, test.expectedResponse,
+					ctx, t, quicListener, test.dest, test.connectionType, test.metadata, test.message, test.expectedResponse,
 				)
+				close(serverDone)
 			}()
 
-			qc := testQUICConnection(udpListener.LocalAddr(), t)
-			go qc.Serve(ctx)
+			qc := testQUICConnection(udpListener.LocalAddr(), t, uint8(i))
 
-			wg.Wait()
+			connDone := make(chan struct{})
+			go func() {
+				qc.Serve(ctx)
+				close(connDone)
+			}()
+
+			<-serverDone
 			cancel()
+			<-connDone
 		})
 	}
 }
@@ -177,23 +185,16 @@ func (fakeControlStream) IsStopped() bool {
 }
 
 func quicServer(
+	ctx context.Context,
 	t *testing.T,
-	conn net.PacketConn,
-	tlsConf *tls.Config,
-	config *quic.Config,
+	listener quic.Listener,
 	dest string,
 	connectionType quicpogs.ConnectionType,
 	metadata []quicpogs.Metadata,
 	message []byte,
 	expectedResponse []byte,
 ) {
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	earlyListener, err := quic.Listen(conn, tlsConf, config)
-	require.NoError(t, err)
-
-	session, err := earlyListener.Accept(ctx)
+	session, err := listener.Accept(ctx)
 	require.NoError(t, err)
 
 	quicStream, err := session.OpenStreamSync(context.Background())
@@ -482,6 +483,7 @@ func TestBuildHTTPRequest(t *testing.T) {
 
 	log := zerolog.Nop()
 	for _, test := range tests {
+		test := test // capture range variable
 		t.Run(test.name, func(t *testing.T) {
 			req, err := buildHTTPRequest(context.Background(), test.connectRequest, test.body, &log)
 			assert.NoError(t, err)
@@ -519,7 +521,8 @@ func TestServeUDPSession(t *testing.T) {
 		edgeQUICSessionChan <- edgeQUICSession
 	}()
 
-	qc := testQUICConnection(val, t)
+	// Random index to avoid reusing port
+	qc := testQUICConnection(val, t, 28)
 	go qc.Serve(ctx)
 
 	edgeQUICSession := <-edgeQUICSessionChan
@@ -567,6 +570,37 @@ func TestNopCloserReadWriterCloseAfterEOF(t *testing.T) {
 	require.Equal(t, err, io.EOF)
 }
 
+func TestCreateUDPConnReuseSourcePort(t *testing.T) {
+	logger := zerolog.Nop()
+	conn, err := createUDPConnForConnIndex(0, &logger)
+	require.NoError(t, err)
+
+	getPortFunc := func(conn *net.UDPConn) int {
+		addr := conn.LocalAddr().(*net.UDPAddr)
+		return addr.Port
+	}
+
+	initialPort := getPortFunc(conn)
+
+	// close conn
+	conn.Close()
+
+	// should get the same port as before.
+	conn, err = createUDPConnForConnIndex(0, &logger)
+	require.NoError(t, err)
+	require.Equal(t, initialPort, getPortFunc(conn))
+
+	// new index, should get a different port
+	conn1, err := createUDPConnForConnIndex(1, &logger)
+	require.NoError(t, err)
+	require.NotEqual(t, initialPort, getPortFunc(conn1))
+
+	// not closing the conn and trying to obtain a new conn for same index should give a different random port
+	conn, err = createUDPConnForConnIndex(0, &logger)
+	require.NoError(t, err)
+	require.NotEqual(t, initialPort, getPortFunc(conn))
+}
+
 func serveSession(ctx context.Context, qc *QUICConnection, edgeQUICSession quic.Connection, closeType closeReason, expectedReason string, t *testing.T) {
 	var (
 		payload = []byte(t.Name())
@@ -583,8 +617,12 @@ func serveSession(ctx context.Context, qc *QUICConnection, edgeQUICSession quic.
 		close(sessionDone)
 	}()
 
-	// Send a message to the quic session on edge side, it should be deumx to this datagram session
-	muxedPayload := append(payload, sessionID[:]...)
+	// Send a message to the quic session on edge side, it should be deumx to this datagram v2 session
+	muxedPayload, err := quicpogs.SuffixSessionID(sessionID, payload)
+	require.NoError(t, err)
+	muxedPayload, err = quicpogs.SuffixType(muxedPayload, quicpogs.DatagramTypeUDP)
+	require.NoError(t, err)
+
 	err = edgeQUICSession.SendMessage(muxedPayload)
 	require.NoError(t, err)
 
@@ -668,7 +706,7 @@ func (s mockSessionRPCServer) UnregisterUdpSession(ctx context.Context, sessionI
 	return nil
 }
 
-func testQUICConnection(udpListenerAddr net.Addr, t *testing.T) *QUICConnection {
+func testQUICConnection(udpListenerAddr net.Addr, t *testing.T, index uint8) *QUICConnection {
 	tlsClientConfig := &tls.Config{
 		InsecureSkipVerify: true,
 		NextProtos:         []string{"argotunnel"},
@@ -678,6 +716,7 @@ func testQUICConnection(udpListenerAddr net.Addr, t *testing.T) *QUICConnection
 	qc, err := NewQUICConnection(
 		testQUICConfig,
 		udpListenerAddr,
+		index,
 		tlsClientConfig,
 		&mockOrchestrator{originProxy: &mockOriginProxyWithRequest{}},
 		&tunnelpogs.ConnectionOptions{},

datagramsession/session.go

@@ -24,6 +24,13 @@ func SessionIdleErr(timeout time.Duration) error {
 
 type transportSender func(session *packet.Session) error
 
+// ErrVithVariableSeverity are errors that have variable severity
+type ErrVithVariableSeverity interface {
+	error
+	// LogLevel return the severity of this error
+	LogLevel() zerolog.Level
+}
+
 // Session is a bidirectional pipe of datagrams between transport and dstConn
 // Destination can be a connection with origin or with eyeball
 // When the destination is origin:
@@ -53,7 +60,11 @@ func (s *Session) Serve(ctx context.Context, closeAfterIdle time.Duration) (clos
 				if errors.Is(err, net.ErrClosed) {
 					s.log.Debug().Msg("Destination connection closed")
 				} else {
-					s.log.Error().Err(err).Msg("Failed to send session payload from destination to transport")
+					level := zerolog.ErrorLevel
+					if variableErr, ok := err.(ErrVithVariableSeverity); ok {
+						level = variableErr.LogLevel()
+					}
+					s.log.WithLevel(level).Err(err).Msg("Failed to send session payload from destination to transport")
 				}
 				if closeSession {
 					s.closeChan <- err

dev.Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.18 as builder
+FROM golang:1.19 as builder
 ENV GO111MODULE=on \
     CGO_ENABLED=0
 WORKDIR /go/src/github.com/cloudflare/cloudflared/

edgediscovery/allregions/discovery.go

@@ -71,11 +71,14 @@ type EdgeAddr struct {
 // If the call to net.LookupSRV fails, try to fall back to DoT from Cloudflare directly.
 //
 // Note: Instead of DoT, we could also have used DoH. Either of these:
-//     - directly via the JSON API (https://1.1.1.1/dns-query?ct=application/dns-json&name=_origintunneld._tcp.argotunnel.com&type=srv)
-//     - indirectly via `tunneldns.NewUpstreamHTTPS()`
+//   - directly via the JSON API (https://1.1.1.1/dns-query?ct=application/dns-json&name=_origintunneld._tcp.argotunnel.com&type=srv)
+//   - indirectly via `tunneldns.NewUpstreamHTTPS()`
+//
 // But both of these cases miss out on a key feature from the stdlib:
-//     "The returned records are sorted by priority and randomized by weight within a priority."
-//     (https://golang.org/pkg/net/#Resolver.LookupSRV)
+//
+//	"The returned records are sorted by priority and randomized by weight within a priority."
+//	(https://golang.org/pkg/net/#Resolver.LookupSRV)
+//
 // Does this matter? I don't know. It may someday. Let's use DoT so we don't need to worry about it.
 // See also: Go feature request for stdlib-supported DoH: https://github.com/golang/go/issues/27552
 var fallbackLookupSRV = lookupSRVWithDOT

fmt-check.sh

@@ -2,12 +2,12 @@
 
 set -e -o pipefail
 
-OUTPUT=$(for d in $(go list -mod=vendor -f '{{.Dir}}' -a ./... | fgrep -v tunnelrpc) ; do goimports -format-only -local github.com/cloudflare/cloudflared -d $d ; done)
+OUTPUT=$(goimports -l -d -local github.com/cloudflare/cloudflared $(go list -mod=vendor -f '{{.Dir}}' -a ./... | fgrep -v tunnelrpc))
 
 if [ -n "$OUTPUT" ] ; then
   PAGER=$(which colordiff || echo cat)
   echo
-  echo "Code formatting issues found, use 'goimports -format-only -local github.com/cloudflare/cloudflared' to correct them"
+  echo "Code formatting issues found, use 'make fmt' to correct them"
   echo
   echo "$OUTPUT" | $PAGER
   exit 1

github_message.py

@@ -1,29 +1,56 @@
 #!/usr/bin/python3
 """
-Creates Github Releases Notes with content hashes
+Create Github Releases Notes with binary checksums from Workers KV
 """
 
 import argparse
 import logging
 import os
-import hashlib
-import glob
+import requests
 
-from github import Github, GithubException, UnknownObjectException
+from github import Github, UnknownObjectException
 
 FORMAT = "%(levelname)s - %(asctime)s: %(message)s"
-logging.basicConfig(format=FORMAT)
+logging.basicConfig(format=FORMAT, level=logging.INFO)
 
 CLOUDFLARED_REPO = os.environ.get("GITHUB_REPO", "cloudflare/cloudflared")
 GITHUB_CONFLICT_CODE = "already_exists"
+BASE_KV_URL = 'https://api.cloudflare.com/client/v4/accounts/'
 
-def get_sha256(filename):
-    """ get the sha256 of a file """
-    sha256_hash = hashlib.sha256()
-    with open(filename,"rb") as f:
-        for byte_block in iter(lambda: f.read(4096),b""):
-            sha256_hash.update(byte_block)
-        return sha256_hash.hexdigest()
+
+def kv_get_keys(prefix, account, namespace, api_token):
+    """ get the KV keys for a given prefix """
+    response = requests.get(
+        BASE_KV_URL + account + "/storage/kv/namespaces/" +
+        namespace + "/keys" + "?prefix=" + prefix,
+        headers={
+            "Content-Type": "application/json",
+            "Authorization": "Bearer " + api_token,
+        },
+    )
+    if response.status_code != 200:
+        jsonResponse = response.json()
+        errors = jsonResponse["errors"]
+        if len(errors) > 0:
+            raise Exception("failed to get checksums: {0}", errors[0])
+    return response.json()["result"]
+
+
+def kv_get_value(key, account, namespace, api_token):
+    """ get the KV value for a provided key """
+    response = requests.get(
+        BASE_KV_URL + account + "/storage/kv/namespaces/" + namespace + "/values/" + key,
+        headers={
+            "Content-Type": "application/json",
+            "Authorization": "Bearer " + api_token,
+        },
+    )
+    if response.status_code != 200:
+        jsonResponse = response.json()
+        errors = jsonResponse["errors"]
+        if len(errors) > 0:
+            raise Exception("failed to get checksums: {0}", errors[0])
+    return response.text
 
 
 def update_or_add_message(msg, name, sha):
@@ -42,6 +69,7 @@ def update_or_add_message(msg, name, sha):
         return '{0}{1}```'.format(msg[:back], new_text)
     return '{0} \n### SHA256 Checksums:\n```\n{1}```'.format(msg, new_text)
 
+
 def get_release(repo, version):
     """ Get a Github Release matching the version tag. """
     try:
@@ -55,11 +83,20 @@ def get_release(repo, version):
 def parse_args():
     """ Parse and validate args """
     parser = argparse.ArgumentParser(
-        description="Creates Github Releases and uploads assets."
+        description="Updates a Github Release with checksums from KV"
     )
     parser.add_argument(
         "--api-key", default=os.environ.get("API_KEY"), help="Github API key"
     )
+    parser.add_argument(
+        "--kv-namespace-id", default=os.environ.get("KV_NAMESPACE"), help="workers KV namespace id"
+    )
+    parser.add_argument(
+        "--kv-account-id", default=os.environ.get("KV_ACCOUNT"), help="workers KV account id"
+    )
+    parser.add_argument(
+        "--kv-api-token", default=os.environ.get("KV_API_TOKEN"), help="workers KV API Token"
+    )
     parser.add_argument(
         "--release-version",
         metavar="version",
@@ -80,6 +117,18 @@ def parse_args():
         logging.error("Missing API key")
         is_valid = False
 
+    if not args.kv_namespace_id:
+        logging.error("Missing KV namespace id")
+        is_valid = False
+
+    if not args.kv_account_id:
+        logging.error("Missing KV account id")
+        is_valid = False
+
+    if not args.kv_api_token:
+        logging.error("Missing KV API token")
+        is_valid = False
+
     if is_valid:
         return args
 
@@ -95,16 +144,20 @@ def main():
         repo = client.get_repo(CLOUDFLARED_REPO)
         release = get_release(repo, args.release_version)
 
-        msg = release.body
+        msg = ""
 
-        for filepath in glob.glob("artifacts/*"):
-            pkg_hash = get_sha256(filepath)
-            # add the sha256 of the new artifact to the release message body
-            name = os.path.basename(filepath)
-            msg = update_or_add_message(msg, name, pkg_hash)
+        prefix = f"update_{args.release_version}_"
+        keys = kv_get_keys(prefix, args.kv_account_id,
+                           args.kv_namespace_id, args.kv_api_token)
+        for key in [k["name"] for k in keys]:
+            checksum = kv_get_value(
+                key, args.kv_account_id, args.kv_namespace_id, args.kv_api_token)
+            binary_name = key[len(prefix):]
+            msg = update_or_add_message(msg, binary_name, checksum)
 
         if args.dry_run:
-            logging.info("Skipping asset upload because of dry-run")
+            logging.info("Skipping release message update because of dry-run")
+            logging.info(f"Github message:\n{msg}")
             return
 
         # update the release body text
@@ -115,4 +168,4 @@ def main():
         exit(1)
 
 
-main()
+main()

go.mod

@@ -1,11 +1,12 @@
 module github.com/cloudflare/cloudflared
 
-go 1.18
+go 1.19
 
 require (
 	github.com/cloudflare/brotli-go v0.0.0-20191101163834-d34379f7ff93
 	github.com/cloudflare/golibs v0.0.0-20170913112048-333127dbecfc
 	github.com/coredns/coredns v1.8.7
+	github.com/coreos/go-oidc/v3 v3.4.0
 	github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf
 	github.com/facebookgo/grace v0.0.0-20180706040059-75cf19382434
 	github.com/fsnotify/fsnotify v1.4.9
@@ -34,11 +35,11 @@ require (
 	go.opentelemetry.io/otel/trace v1.6.3
 	go.opentelemetry.io/proto/otlp v0.15.0
 	go.uber.org/automaxprocs v1.4.0
-	golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa
-	golang.org/x/net v0.0.0-20220909164309-bea034e7d591
-	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
-	golang.org/x/sys v0.0.0-20220808155132-1c4a2a72c664
-	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211
+	golang.org/x/crypto v0.2.0
+	golang.org/x/net v0.2.0
+	golang.org/x/sync v0.1.0
+	golang.org/x/sys v0.2.0
+	golang.org/x/term v0.2.0
 	google.golang.org/protobuf v1.28.0
 	gopkg.in/coreos/go-oidc.v2 v2.2.1
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
@@ -88,14 +89,13 @@ require (
 	github.com/prometheus/common v0.32.1 // indirect
 	github.com/prometheus/procfs v0.7.3 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
-	golang.org/x/mod v0.4.2 // indirect
-	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
-	golang.org/x/text v0.3.7 // indirect
-	golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2 // indirect
-	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
+	golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
+	golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094 // indirect
+	golang.org/x/text v0.4.0 // indirect
+	golang.org/x/tools v0.1.12 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20211223182754-3ac035c7e7cb // indirect
-	google.golang.org/grpc v1.45.0 // indirect
+	google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90 // indirect
+	google.golang.org/grpc v1.47.0 // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 )
@@ -112,8 +112,8 @@ replace gopkg.in/yaml.v3 => gopkg.in/yaml.v3 v3.0.1
 // Post-quantum tunnel RTG-1339
 replace (
 	// branch go1.18
-	github.com/marten-seemann/qtls-go1-18 => github.com/cloudflare/qtls-pq v0.0.0-20220824105406-fb955667e0af
+	github.com/marten-seemann/qtls-go1-18 => github.com/cloudflare/qtls-pq v0.0.0-20221010110824-0053225e48b2
 
 	// branch go1.19
-	github.com/marten-seemann/qtls-go1-19 => github.com/cloudflare/qtls-pq v0.0.0-20220824104809-96561a41e0af
+	github.com/marten-seemann/qtls-go1-19 => github.com/cloudflare/qtls-pq v0.0.0-20221010110800-4f3769902fe0
 )

go.sum

@@ -28,14 +28,23 @@ cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+Y
 cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4=
 cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc=
 cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA=
+cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A=
+cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+nc=
 cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
 cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
 cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
 cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
 cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
 cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
+cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow=
+cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM=
+cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M=
+cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s=
+cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU=
+cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U=
 cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
 cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
+cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY=
 cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
 cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
 cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
@@ -45,6 +54,7 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo
 cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
 cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
+cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y=
 dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBrvjyP0v+ecvNYvCpyZgu5/xkfAUhi6wJj28eUfSU=
@@ -122,10 +132,10 @@ github.com/cloudflare/circl v1.2.1-0.20220809205628-0a9554f37a47 h1:YzpECHxZ9TzO
 github.com/cloudflare/circl v1.2.1-0.20220809205628-0a9554f37a47/go.mod h1:qhx8gBILsYlbam7h09SvHDSkjpe3TfLA7b/z4rxJvkE=
 github.com/cloudflare/golibs v0.0.0-20170913112048-333127dbecfc h1:Dvk3ySBsOm5EviLx6VCyILnafPcQinXGP5jbTdHUJgE=
 github.com/cloudflare/golibs v0.0.0-20170913112048-333127dbecfc/go.mod h1:HlgKKR8V5a1wroIDDIz3/A+T+9Janfq+7n1P5sEFdi0=
-github.com/cloudflare/qtls-pq v0.0.0-20220824104809-96561a41e0af h1:JMpOQAaXjRRBkUX73fTNe9mConJLFl6FsIp9fHdLm7Y=
-github.com/cloudflare/qtls-pq v0.0.0-20220824104809-96561a41e0af/go.mod h1:aIsWqC0WXyUiUxBl/RfxAjDyWE9CCLqvSMnCMTd/+bc=
-github.com/cloudflare/qtls-pq v0.0.0-20220824105406-fb955667e0af h1:bhCmedjwrOSyzLtHVeQ+KhimcNTSfs0P5T7kbRQS+gA=
-github.com/cloudflare/qtls-pq v0.0.0-20220824105406-fb955667e0af/go.mod h1:mW0BgKFFDAiSmOdUwoORtjo0V2vqw5QzVYRtKQqw/Jg=
+github.com/cloudflare/qtls-pq v0.0.0-20221010110800-4f3769902fe0 h1:LEsjEHfKnIJEJU9QEIPVRuslxpBu+2kG2DXhxpkGT+o=
+github.com/cloudflare/qtls-pq v0.0.0-20221010110800-4f3769902fe0/go.mod h1:aIsWqC0WXyUiUxBl/RfxAjDyWE9CCLqvSMnCMTd/+bc=
+github.com/cloudflare/qtls-pq v0.0.0-20221010110824-0053225e48b2 h1:ErNoeVNqFXV+emlf4gY7Ms7/0DbQ8PT2UFxNyWBc51Q=
+github.com/cloudflare/qtls-pq v0.0.0-20221010110824-0053225e48b2/go.mod h1:mW0BgKFFDAiSmOdUwoORtjo0V2vqw5QzVYRtKQqw/Jg=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
@@ -133,11 +143,14 @@ github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XP
 github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
+github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/coredns/caddy v1.1.1 h1:2eYKZT7i6yxIfGP3qLJoJ7HAsDJqYB+X68g4NYjSrE0=
 github.com/coredns/caddy v1.1.1/go.mod h1:A6ntJQlAWuQfFlsd9hvigKbo2WS0VUs2l1e2F+BawD4=
 github.com/coredns/coredns v1.8.7 h1:wVMjAnyFnY7Mc18AFO+9qbGD6ODPtdVUIlzoWrHr3hk=
 github.com/coredns/coredns v1.8.7/go.mod h1:bFmbgEfeRz5aizL2VsQ5LRlsvJuXWkgG/MWG9zxqjVM=
+github.com/coreos/go-oidc/v3 v3.4.0 h1:xz7elHb/LDwm/ERpwHd+5nb7wFHL32rsr6bBOgaeu6g=
+github.com/coreos/go-oidc/v3 v3.4.0/go.mod h1:eHUXhZtXPQLgEaDrOVTgwbgmz1xGOkJNye6h3zkD2Pw=
 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
@@ -171,6 +184,7 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m
 github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
 github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ=
 github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
+github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 h1:0JZ+dUmQeA8IIVUMzysrX4/AKuQwWhV2dYQuPZdvdSQ=
@@ -292,8 +306,9 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
 github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
+github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
+github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -325,14 +340,19 @@ github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8=
 github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY=
 github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0=
 github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM=
+github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM=
+github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM=
+github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c=
 github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU=
 github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA=
+github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
 github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
@@ -624,8 +644,9 @@ golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20210920023735-84f357641f63/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa h1:zuSxTR4o9y82ebqCUJYNGJbGPo6sKVl54f/TVDObg1c=
 golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.2.0 h1:BRXPfhNivWL5Yq0BGQ39a2sW6t44aODpfxkWjYdzewE=
+golang.org/x/crypto v0.2.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -661,8 +682,9 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -714,9 +736,16 @@ golang.org/x/net v0.0.0-20210917221730-978cfadd31cf/go.mod h1:9nx3DQGgdP8bBQD5qx
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.0.0-20220909164309-bea034e7d591 h1:D0B/7al0LLrVC8aWF4+oxpv/m8bc7ViFfVS8/gXGdqI=
-golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
+golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
+golang.org/x/net v0.2.0 h1:sZfSu1wtKLGlWI4ZZayP0ck9Y73K1ynO6gqzTdBVdPU=
+golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -734,8 +763,13 @@ golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ
 golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 h1:RerP+noqYHUQ8CMRcPlC2nvTa4dcBIjegkuWdcUDuqg=
 golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
+golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
+golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
+golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE=
+golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094 h1:2o1E+E8TpNLklK9nHiPiK1uzIYrIHt+cQx3ynCwq9V8=
+golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg=
 golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -747,8 +781,10 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -819,15 +855,27 @@ golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220808155132-1c4a2a72c664 h1:v1W7bwXHsnLLloWYTVEdvGvA7BHMeBYsPcF0GLDxIRs=
+golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220808155132-1c4a2a72c664/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A=
+golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.2.0 h1:z85xZCsEl7bi/KwbNADeBYoOP0++7W1ipu+aGnpwzRM=
+golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -836,8 +884,9 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg=
+golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -902,13 +951,16 @@ golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
-golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2 h1:BonxutuHCTL0rBDnZlKjpGIQFTjyUVTexFOdWkB6Fg0=
 golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
+golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
 google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
 google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
 google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y=
@@ -942,7 +994,16 @@ google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqiv
 google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE=
 google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI=
 google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I=
+google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo=
 google.golang.org/api v0.64.0/go.mod h1:931CdxA8Rm4t6zqTFGSsgwbAEZ2+GMYurbndwSimebM=
+google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g=
+google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA=
+google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8=
+google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs=
+google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA=
+google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw=
+google.golang.org/api v0.80.0/go.mod h1:xY3nI94gbvBrE0J6NHXhxOmW97HG7Khjkku6AFB3Hyg=
+google.golang.org/api v0.84.0/go.mod h1:NTsGnUFJMYROtiquksZHBWtHfeMC7iYthki7Eq3pa8o=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@@ -997,6 +1058,7 @@ google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6D
 google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20210329143202-679c6ae281ee/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A=
 google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A=
 google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A=
 google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
@@ -1016,8 +1078,27 @@ google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEc
 google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
 google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
 google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20211223182754-3ac035c7e7cb h1:ZrsicilzPCS/Xr8qtBZZLpy4P9TYXAfl49ctG1/5tgw=
+google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
+google.golang.org/genproto v0.0.0-20211221195035-429b39de9b1c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
 google.golang.org/genproto v0.0.0-20211223182754-3ac035c7e7cb/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
+google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
+google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
+google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI=
+google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI=
+google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI=
+google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI=
+google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E=
+google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
+google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
+google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
+google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
+google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
+google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4=
+google.golang.org/genproto v0.0.0-20220518221133-4f43b3371335/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4=
+google.golang.org/genproto v0.0.0-20220523171625-347a074981d8/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4=
+google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA=
+google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90 h1:4SPz2GL2CXJt28MTF8V6Ap/9ZiVbQlJeGSd9qtA7DLs=
+google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA=
 google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
 google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio=
 google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
@@ -1052,8 +1133,12 @@ google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9K
 google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k=
 google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
 google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
-google.golang.org/grpc v1.45.0 h1:NEpgUqV3Z+ZjkqMsxMg11IaDrXY4RY6CQukSGK0uI1M=
+google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
 google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ=
+google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
+google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
+google.golang.org/grpc v1.47.0 h1:9n77onPX5F3qfFCqjy9dhn8PbNQsIKeVU04J9G7umt8=
+google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=

ingress/config.go

@@ -427,12 +427,19 @@ func (defaults *OriginRequestConfig) setHttp2Origin(overrides config.OriginReque
 	}
 }
 
+func (defaults *OriginRequestConfig) setAccess(overrides config.OriginRequestConfig) {
+	if val := overrides.Access; val != nil {
+		defaults.Access = *val
+	}
+}
+
 // SetConfig gets config for the requests that cloudflared sends to origins.
 // Each field has a setter method which sets a value for the field by trying to find:
-//   1. The user config for this rule
-//   2. The user config for the overall ingress config
-//   3. Defaults chosen by the cloudflared team
-//   4. Golang zero values for that type
+//  1. The user config for this rule
+//  2. The user config for the overall ingress config
+//  3. Defaults chosen by the cloudflared team
+//  4. Golang zero values for that type
+//
 // If an earlier option isn't set, it will try the next option down.
 func setConfig(defaults OriginRequestConfig, overrides config.OriginRequestConfig) OriginRequestConfig {
 	cfg := defaults
@@ -453,6 +460,8 @@ func setConfig(defaults OriginRequestConfig, overrides config.OriginRequestConfi
 	cfg.setProxyType(overrides)
 	cfg.setIPRules(overrides)
 	cfg.setHttp2Origin(overrides)
+	cfg.setAccess(overrides)
+
 	return cfg
 }
 
@@ -463,6 +472,7 @@ func ConvertToRawOriginConfig(c OriginRequestConfig) config.OriginRequestConfig
 	var keepAliveConnections *int
 	var keepAliveTimeout *config.CustomDuration
 	var proxyAddress *string
+	var access *config.AccessConfig
 
 	if c.ConnectTimeout != defaultHTTPConnectTimeout {
 		connectTimeout = &c.ConnectTimeout
@@ -482,6 +492,9 @@ func ConvertToRawOriginConfig(c OriginRequestConfig) config.OriginRequestConfig
 	if c.ProxyAddress != defaultProxyAddress {
 		proxyAddress = &c.ProxyAddress
 	}
+	if c.Access.Required {
+		access = &c.Access
+	}
 
 	return config.OriginRequestConfig{
 		ConnectTimeout:         connectTimeout,
@@ -501,6 +514,7 @@ func ConvertToRawOriginConfig(c OriginRequestConfig) config.OriginRequestConfig
 		ProxyType:              emptyStringToNil(c.ProxyType),
 		IPRules:                convertToRawIPRules(c.IPRules),
 		Http2Origin:            defaultBoolToNil(c.Http2Origin),
+		Access:                 access,
 	}
 }
 

ingress/icmp_darwin.go

@@ -11,16 +11,17 @@ import (
 	"context"
 	"fmt"
 	"math"
-	"net"
 	"net/netip"
 	"strconv"
 	"sync"
 	"time"
 
 	"github.com/rs/zerolog"
+	"go.opentelemetry.io/otel/attribute"
 	"golang.org/x/net/icmp"
 
 	"github.com/cloudflare/cloudflared/packet"
+	"github.com/cloudflare/cloudflared/tracing"
 )
 
 type icmpProxy struct {
@@ -113,11 +114,12 @@ func (snf echoFunnelID) String() string {
 	return strconv.FormatUint(uint64(snf), 10)
 }
 
-func newICMPProxy(listenIP netip.Addr, logger *zerolog.Logger, idleTimeout time.Duration) (*icmpProxy, error) {
-	conn, err := newICMPConn(listenIP)
+func newICMPProxy(listenIP netip.Addr, zone string, logger *zerolog.Logger, idleTimeout time.Duration) (*icmpProxy, error) {
+	conn, err := newICMPConn(listenIP, zone)
 	if err != nil {
 		return nil, err
 	}
+	logger.Info().Msgf("Created ICMP proxy listening on %s", conn.LocalAddr())
 	return &icmpProxy{
 		srcFunnelTracker: packet.NewFunnelTracker(),
 		echoIDTracker:    newEchoIDTracker(),
@@ -128,44 +130,53 @@ func newICMPProxy(listenIP netip.Addr, logger *zerolog.Logger, idleTimeout time.
 	}, nil
 }
 
-func (ip *icmpProxy) Request(pk *packet.ICMP, responder packet.FunnelUniPipe) error {
-	if pk == nil {
-		return errPacketNil
-	}
+func (ip *icmpProxy) Request(ctx context.Context, pk *packet.ICMP, responder *packetResponder) error {
+	ctx, span := responder.requestSpan(ctx, pk)
+	defer responder.exportSpan()
+
 	originalEcho, err := getICMPEcho(pk.Message)
 	if err != nil {
+		tracing.EndWithErrorStatus(span, err)
 		return err
 	}
+	span.SetAttributes(
+		attribute.Int("originalEchoID", originalEcho.ID),
+		attribute.Int("seq", originalEcho.Seq),
+	)
 	echoIDTrackerKey := flow3Tuple{
 		srcIP:          pk.Src,
 		dstIP:          pk.Dst,
 		originalEchoID: originalEcho.ID,
 	}
-	// TODO: TUN-6744 assign unique flow per (src, echo ID)
 	assignedEchoID, success := ip.echoIDTracker.getOrAssign(echoIDTrackerKey)
 	if !success {
-		return fmt.Errorf("failed to assign unique echo ID")
+		err := fmt.Errorf("failed to assign unique echo ID")
+		tracing.EndWithErrorStatus(span, err)
+		return err
 	}
+	span.SetAttributes(attribute.Int("assignedEchoID", int(assignedEchoID)))
+
+	shouldReplaceFunnelFunc := createShouldReplaceFunnelFunc(ip.logger, responder.datagramMuxer, pk, originalEcho.ID)
 	newFunnelFunc := func() (packet.Funnel, error) {
 		originalEcho, err := getICMPEcho(pk.Message)
 		if err != nil {
 			return nil, err
 		}
-		originSender := originSender{
-			conn:             ip.conn,
-			echoIDTracker:    ip.echoIDTracker,
-			echoIDTrackerKey: echoIDTrackerKey,
-			assignedEchoID:   assignedEchoID,
+		closeCallback := func() error {
+			ip.echoIDTracker.release(echoIDTrackerKey, assignedEchoID)
+			return nil
 		}
-		icmpFlow := newICMPEchoFlow(pk.Src, &originSender, responder, int(assignedEchoID), originalEcho.ID, ip.encoder)
+		icmpFlow := newICMPEchoFlow(pk.Src, closeCallback, ip.conn, responder, int(assignedEchoID), originalEcho.ID, ip.encoder)
 		return icmpFlow, nil
 	}
 	funnelID := echoFunnelID(assignedEchoID)
-	funnel, isNew, err := ip.srcFunnelTracker.GetOrRegister(funnelID, newFunnelFunc)
+	funnel, isNew, err := ip.srcFunnelTracker.GetOrRegister(funnelID, shouldReplaceFunnelFunc, newFunnelFunc)
 	if err != nil {
+		tracing.EndWithErrorStatus(span, err)
 		return err
 	}
 	if isNew {
+		span.SetAttributes(attribute.Bool("newFlow", true))
 		ip.logger.Debug().
 			Str("src", pk.Src.String()).
 			Str("dst", pk.Dst.String()).
@@ -175,9 +186,16 @@ func (ip *icmpProxy) Request(pk *packet.ICMP, responder packet.FunnelUniPipe) er
 	}
 	icmpFlow, err := toICMPEchoFlow(funnel)
 	if err != nil {
+		tracing.EndWithErrorStatus(span, err)
 		return err
 	}
-	return icmpFlow.sendToDst(pk.Dst, pk.Message)
+	err = icmpFlow.sendToDst(pk.Dst, pk.Message)
+	if err != nil {
+		tracing.EndWithErrorStatus(span, err)
+		return err
+	}
+	tracing.End(span)
+	return nil
 }
 
 // Serve listens for responses to the requests until context is done
@@ -201,7 +219,7 @@ func (ip *icmpProxy) Serve(ctx context.Context) error {
 			ip.logger.Debug().Err(err).Str("dst", from.String()).Msg("Failed to parse ICMP reply, continue to parse as full packet")
 			// In unit test, we found out when the listener listens on 0.0.0.0, the socket reads the full packet after
 			// the second reply
-			if err := ip.handleFullPacket(icmpDecoder, buf[:n]); err != nil {
+			if err := ip.handleFullPacket(ctx, icmpDecoder, buf[:n]); err != nil {
 				ip.logger.Debug().Err(err).Str("dst", from.String()).Msg("Failed to parse ICMP reply as full packet")
 			}
 			continue
@@ -210,14 +228,14 @@ func (ip *icmpProxy) Serve(ctx context.Context) error {
 			ip.logger.Debug().Str("dst", from.String()).Msgf("Drop ICMP %s from reply", reply.msg.Type)
 			continue
 		}
-		if err := ip.sendReply(reply); err != nil {
-			ip.logger.Error().Err(err).Str("dst", from.String()).Msg("Failed to send ICMP reply")
+		if err := ip.sendReply(ctx, reply); err != nil {
+			ip.logger.Debug().Err(err).Str("dst", from.String()).Msg("Failed to send ICMP reply")
 			continue
 		}
 	}
 }
 
-func (ip *icmpProxy) handleFullPacket(decoder *packet.ICMPDecoder, rawPacket []byte) error {
+func (ip *icmpProxy) handleFullPacket(ctx context.Context, decoder *packet.ICMPDecoder, rawPacket []byte) error {
 	icmpPacket, err := decoder.Decode(packet.RawPacket{Data: rawPacket})
 	if err != nil {
 		return err
@@ -231,13 +249,13 @@ func (ip *icmpProxy) handleFullPacket(decoder *packet.ICMPDecoder, rawPacket []b
 		msg:  icmpPacket.Message,
 		echo: echo,
 	}
-	if ip.sendReply(&reply); err != nil {
+	if ip.sendReply(ctx, &reply); err != nil {
 		return err
 	}
 	return nil
 }
 
-func (ip *icmpProxy) sendReply(reply *echoReply) error {
+func (ip *icmpProxy) sendReply(ctx context.Context, reply *echoReply) error {
 	funnelID := echoFunnelID(reply.echo.ID)
 	funnel, ok := ip.srcFunnelTracker.Get(funnelID)
 	if !ok {
@@ -247,25 +265,19 @@ func (ip *icmpProxy) sendReply(reply *echoReply) error {
 	if err != nil {
 		return err
 	}
-	return icmpFlow.returnToSrc(reply)
-}
 
-// originSender wraps icmp.PacketConn to implement packet.FunnelUniPipe interface
-type originSender struct {
-	conn             *icmp.PacketConn
-	echoIDTracker    *echoIDTracker
-	echoIDTrackerKey flow3Tuple
-	assignedEchoID   uint16
-}
+	_, span := icmpFlow.responder.replySpan(ctx, ip.logger)
+	defer icmpFlow.responder.exportSpan()
 
-func (os *originSender) SendPacket(dst netip.Addr, pk packet.RawPacket) error {
-	_, err := os.conn.WriteTo(pk.Data, &net.UDPAddr{
-		IP: dst.AsSlice(),
-	})
-	return err
-}
-
-func (os *originSender) Close() error {
-	os.echoIDTracker.release(os.echoIDTrackerKey, os.assignedEchoID)
+	span.SetAttributes(
+		attribute.String("dst", reply.from.String()),
+		attribute.Int("echoID", reply.echo.ID),
+		attribute.Int("seq", reply.echo.Seq),
+		attribute.Int("originalEchoID", icmpFlow.originalEchoID),
+	)
+	if err := icmpFlow.returnToSrc(reply); err != nil {
+		tracing.EndWithErrorStatus(span, err)
+	}
+	tracing.End(span)
 	return nil
 }

ingress/icmp_darwin_test.go

@@ -8,6 +8,8 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/require"
+
+	"github.com/cloudflare/cloudflared/packet"
 )
 
 func TestSingleEchoIDTracker(t *testing.T) {
@@ -119,3 +121,9 @@ func (eit *echoIDTracker) get(key flow3Tuple) (id uint16, exist bool) {
 	id, exists := eit.mapping[key]
 	return id, exists
 }
+
+func getFunnel(t *testing.T, proxy *icmpProxy, tuple flow3Tuple) (packet.Funnel, bool) {
+	assignedEchoID, success := proxy.echoIDTracker.getOrAssign(tuple)
+	require.True(t, success)
+	return proxy.srcFunnelTracker.Get(echoFunnelID(assignedEchoID))
+}

ingress/icmp_generic.go

@@ -18,7 +18,7 @@ var errICMPProxyNotImplemented = fmt.Errorf("ICMP proxy is not implemented on %s
 
 type icmpProxy struct{}
 
-func (ip icmpProxy) Request(pk *packet.ICMP, responder packet.FunnelUniPipe) error {
+func (ip icmpProxy) Request(ctx context.Context, pk *packet.ICMP, responder *packetResponder) error {
 	return errICMPProxyNotImplemented
 }
 
@@ -26,6 +26,6 @@ func (ip *icmpProxy) Serve(ctx context.Context) error {
 	return errICMPProxyNotImplemented
 }
 
-func newICMPProxy(listenIP netip.Addr, logger *zerolog.Logger, idleTimeout time.Duration) (*icmpProxy, error) {
+func newICMPProxy(listenIP netip.Addr, zone string, logger *zerolog.Logger, idleTimeout time.Duration) (*icmpProxy, error) {
 	return nil, errICMPProxyNotImplemented
 }

ingress/icmp_linux.go

@@ -12,40 +12,60 @@ import (
 	"fmt"
 	"net"
 	"net/netip"
+	"os"
+	"regexp"
+	"strconv"
 	"time"
 
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
-	"golang.org/x/net/icmp"
+	"go.opentelemetry.io/otel/attribute"
 
 	"github.com/cloudflare/cloudflared/packet"
+	"github.com/cloudflare/cloudflared/tracing"
+)
+
+const (
+	// https://lwn.net/Articles/550551/ IPv4 and IPv6 share the same path
+	pingGroupPath = "/proc/sys/net/ipv4/ping_group_range"
+)
+
+var (
+	findGroupIDRegex = regexp.MustCompile(`\d+`)
 )
 
 type icmpProxy struct {
 	srcFunnelTracker *packet.FunnelTracker
 	listenIP         netip.Addr
+	ipv6Zone         string
 	logger           *zerolog.Logger
 	idleTimeout      time.Duration
 }
 
-func newICMPProxy(listenIP netip.Addr, logger *zerolog.Logger, idleTimeout time.Duration) (*icmpProxy, error) {
-	if err := testPermission(listenIP); err != nil {
+func newICMPProxy(listenIP netip.Addr, zone string, logger *zerolog.Logger, idleTimeout time.Duration) (*icmpProxy, error) {
+	if err := testPermission(listenIP, zone, logger); err != nil {
 		return nil, err
 	}
 	return &icmpProxy{
 		srcFunnelTracker: packet.NewFunnelTracker(),
 		listenIP:         listenIP,
+		ipv6Zone:         zone,
 		logger:           logger,
 		idleTimeout:      idleTimeout,
 	}, nil
 }
 
-func testPermission(listenIP netip.Addr) error {
+func testPermission(listenIP netip.Addr, zone string, logger *zerolog.Logger) error {
 	// Opens a non-privileged ICMP socket. On Linux the group ID of the process needs to be in ping_group_range
-	conn, err := newICMPConn(listenIP)
+	// Only check ping_group_range once for IPv4
+	if listenIP.Is4() {
+		if err := checkInPingGroup(); err != nil {
+			logger.Warn().Err(err).Msgf("The user running cloudflared process has a GID (group ID) that is not within ping_group_range. You might need to add that user to a group within that range, or instead update the range to encompass a group the user is already in by modifying %s. Otherwise cloudflared will not be able to ping this network", pingGroupPath)
+			return err
+		}
+	}
+	conn, err := newICMPConn(listenIP, zone)
 	if err != nil {
-		// TODO: TUN-6715 check if cloudflared is in ping_group_range if the check failed. If not log instruction to
-		// change the group ID
 		return err
 	}
 	// This conn is only to test if cloudflared has permission to open this type of socket
@@ -53,28 +73,64 @@ func testPermission(listenIP netip.Addr) error {
 	return nil
 }
 
-func (ip *icmpProxy) Request(pk *packet.ICMP, responder packet.FunnelUniPipe) error {
-	if pk == nil {
-		return errPacketNil
-	}
-	originalEcho, err := getICMPEcho(pk.Message)
+func checkInPingGroup() error {
+	file, err := os.ReadFile(pingGroupPath)
 	if err != nil {
 		return err
 	}
-	newConnChan := make(chan *icmp.PacketConn, 1)
-	newFunnelFunc := func() (packet.Funnel, error) {
-		conn, err := newICMPConn(ip.listenIP)
+	groupID := os.Getgid()
+	// Example content: 999	   59999
+	found := findGroupIDRegex.FindAll(file, 2)
+	if len(found) == 2 {
+		groupMin, err := strconv.ParseInt(string(found[0]), 10, 32)
 		if err != nil {
+			return errors.Wrapf(err, "failed to determine minimum ping group ID")
+		}
+		groupMax, err := strconv.ParseInt(string(found[1]), 10, 32)
+		if err != nil {
+			return errors.Wrapf(err, "failed to determine minimum ping group ID")
+		}
+		if groupID < int(groupMin) || groupID > int(groupMax) {
+			return fmt.Errorf("Group ID %d is not between ping group %d to %d", groupID, groupMin, groupMax)
+		}
+		return nil
+	}
+	return fmt.Errorf("did not find group range in %s", pingGroupPath)
+}
+
+func (ip *icmpProxy) Request(ctx context.Context, pk *packet.ICMP, responder *packetResponder) error {
+	ctx, span := responder.requestSpan(ctx, pk)
+	defer responder.exportSpan()
+
+	originalEcho, err := getICMPEcho(pk.Message)
+	if err != nil {
+		tracing.EndWithErrorStatus(span, err)
+		return err
+	}
+	span.SetAttributes(
+		attribute.Int("originalEchoID", originalEcho.ID),
+		attribute.Int("seq", originalEcho.Seq),
+	)
+
+	shouldReplaceFunnelFunc := createShouldReplaceFunnelFunc(ip.logger, responder.datagramMuxer, pk, originalEcho.ID)
+	newFunnelFunc := func() (packet.Funnel, error) {
+		conn, err := newICMPConn(ip.listenIP, ip.ipv6Zone)
+		if err != nil {
+			tracing.EndWithErrorStatus(span, err)
 			return nil, errors.Wrap(err, "failed to open ICMP socket")
 		}
-		newConnChan <- conn
+		ip.logger.Debug().Msgf("Opened ICMP socket listen on %s", conn.LocalAddr())
+		closeCallback := func() error {
+			return conn.Close()
+		}
 		localUDPAddr, ok := conn.LocalAddr().(*net.UDPAddr)
 		if !ok {
 			return nil, fmt.Errorf("ICMP listener address %s is not net.UDPAddr", conn.LocalAddr())
 		}
-		originSender := originSender{conn: conn}
+		span.SetAttributes(attribute.Int("port", localUDPAddr.Port))
+
 		echoID := localUDPAddr.Port
-		icmpFlow := newICMPEchoFlow(pk.Src, &originSender, responder, echoID, originalEcho.ID, packet.NewEncoder())
+		icmpFlow := newICMPEchoFlow(pk.Src, closeCallback, conn, responder, echoID, originalEcho.ID, packet.NewEncoder())
 		return icmpFlow, nil
 	}
 	funnelID := flow3Tuple{
@@ -82,24 +138,26 @@ func (ip *icmpProxy) Request(pk *packet.ICMP, responder packet.FunnelUniPipe) er
 		dstIP:          pk.Dst,
 		originalEchoID: originalEcho.ID,
 	}
-	funnel, isNew, err := ip.srcFunnelTracker.GetOrRegister(funnelID, newFunnelFunc)
+	funnel, isNew, err := ip.srcFunnelTracker.GetOrRegister(funnelID, shouldReplaceFunnelFunc, newFunnelFunc)
 	if err != nil {
+		tracing.EndWithErrorStatus(span, err)
 		return err
 	}
 	icmpFlow, err := toICMPEchoFlow(funnel)
 	if err != nil {
+		tracing.EndWithErrorStatus(span, err)
 		return err
 	}
 	if isNew {
+		span.SetAttributes(attribute.Bool("newFlow", true))
 		ip.logger.Debug().
 			Str("src", pk.Src.String()).
 			Str("dst", pk.Dst.String()).
 			Int("originalEchoID", originalEcho.ID).
 			Msg("New flow")
-		conn := <-newConnChan
 		go func() {
 			defer ip.srcFunnelTracker.Unregister(funnelID, icmpFlow)
-			if err := ip.listenResponse(icmpFlow, conn); err != nil {
+			if err := ip.listenResponse(ctx, icmpFlow); err != nil {
 				ip.logger.Debug().Err(err).
 					Str("src", pk.Src.String()).
 					Str("dst", pk.Dst.String()).
@@ -109,8 +167,10 @@ func (ip *icmpProxy) Request(pk *packet.ICMP, responder packet.FunnelUniPipe) er
 		}()
 	}
 	if err := icmpFlow.sendToDst(pk.Dst, pk.Message); err != nil {
+		tracing.EndWithErrorStatus(span, err)
 		return errors.Wrap(err, "failed to send ICMP echo request")
 	}
+	tracing.End(span)
 	return nil
 }
 
@@ -119,43 +179,53 @@ func (ip *icmpProxy) Serve(ctx context.Context) error {
 	return ctx.Err()
 }
 
-func (ip *icmpProxy) listenResponse(flow *icmpEchoFlow, conn *icmp.PacketConn) error {
+func (ip *icmpProxy) listenResponse(ctx context.Context, flow *icmpEchoFlow) error {
 	buf := make([]byte, mtu)
 	for {
-		n, from, err := conn.ReadFrom(buf)
-		if err != nil {
+		retryable, err := ip.handleResponse(ctx, flow, buf)
+		if err != nil && !retryable {
 			return err
 		}
-		reply, err := parseReply(from, buf[:n])
-		if err != nil {
-			ip.logger.Error().Err(err).Str("dst", from.String()).Msg("Failed to parse ICMP reply")
-			continue
-		}
-		if !isEchoReply(reply.msg) {
-			ip.logger.Debug().Str("dst", from.String()).Msgf("Drop ICMP %s from reply", reply.msg.Type)
-			continue
-		}
-		if err := flow.returnToSrc(reply); err != nil {
-			ip.logger.Err(err).Str("dst", from.String()).Msg("Failed to send ICMP reply")
-			continue
-		}
 	}
 }
 
-// originSender wraps icmp.PacketConn to implement packet.FunnelUniPipe interface
-type originSender struct {
-	conn *icmp.PacketConn
-}
+func (ip *icmpProxy) handleResponse(ctx context.Context, flow *icmpEchoFlow, buf []byte) (retryableErr bool, err error) {
+	_, span := flow.responder.replySpan(ctx, ip.logger)
+	defer flow.responder.exportSpan()
 
-func (os *originSender) SendPacket(dst netip.Addr, pk packet.RawPacket) error {
-	_, err := os.conn.WriteTo(pk.Data, &net.UDPAddr{
-		IP: dst.AsSlice(),
-	})
-	return err
-}
+	span.SetAttributes(
+		attribute.Int("originalEchoID", flow.originalEchoID),
+	)
 
-func (os *originSender) Close() error {
-	return os.conn.Close()
+	n, from, err := flow.originConn.ReadFrom(buf)
+	if err != nil {
+		tracing.EndWithErrorStatus(span, err)
+		return false, err
+	}
+	reply, err := parseReply(from, buf[:n])
+	if err != nil {
+		ip.logger.Error().Err(err).Str("dst", from.String()).Msg("Failed to parse ICMP reply")
+		tracing.EndWithErrorStatus(span, err)
+		return true, err
+	}
+	if !isEchoReply(reply.msg) {
+		err := fmt.Errorf("Expect ICMP echo reply, got %s", reply.msg.Type)
+		ip.logger.Debug().Str("dst", from.String()).Msgf("Drop ICMP %s from reply", reply.msg.Type)
+		tracing.EndWithErrorStatus(span, err)
+		return true, err
+	}
+	span.SetAttributes(
+		attribute.String("dst", reply.from.String()),
+		attribute.Int("echoID", reply.echo.ID),
+		attribute.Int("seq", reply.echo.Seq),
+	)
+	if err := flow.returnToSrc(reply); err != nil {
+		ip.logger.Debug().Err(err).Str("dst", from.String()).Msg("Failed to send ICMP reply")
+		tracing.EndWithErrorStatus(span, err)
+		return true, err
+	}
+	tracing.End(span)
+	return true, nil
 }
 
 // Only linux uses flow3Tuple as FunnelID

ingress/icmp_linux_test.go

@@ -0,0 +1,13 @@
+//go:build linux
+
+package ingress
+
+import (
+	"testing"
+
+	"github.com/cloudflare/cloudflared/packet"
+)
+
+func getFunnel(t *testing.T, proxy *icmpProxy, tuple flow3Tuple) (packet.Funnel, bool) {
+	return proxy.srcFunnelTracker.Get(tuple)
+}

ingress/icmp_posix.go

@@ -10,18 +10,22 @@ import (
 	"net/netip"
 
 	"github.com/google/gopacket/layers"
+	"github.com/rs/zerolog"
 	"golang.org/x/net/icmp"
 
 	"github.com/cloudflare/cloudflared/packet"
 )
 
 // Opens a non-privileged ICMP socket on Linux and Darwin
-func newICMPConn(listenIP netip.Addr) (*icmp.PacketConn, error) {
-	network := "udp6"
+func newICMPConn(listenIP netip.Addr, zone string) (*icmp.PacketConn, error) {
 	if listenIP.Is4() {
-		network = "udp4"
+		return icmp.ListenPacket("udp4", listenIP.String())
 	}
-	return icmp.ListenPacket(network, listenIP.String())
+	listenAddr := listenIP.String()
+	if zone != "" {
+		listenAddr = listenAddr + "%" + zone
+	}
+	return icmp.ListenPacket("udp6", listenAddr)
 }
 
 func netipAddr(addr net.Addr) (netip.Addr, bool) {
@@ -40,24 +44,54 @@ type flow3Tuple struct {
 
 // icmpEchoFlow implements the packet.Funnel interface.
 type icmpEchoFlow struct {
-	*packet.RawPacketFunnel
+	*packet.ActivityTracker
+	closeCallback  func() error
+	src            netip.Addr
+	originConn     *icmp.PacketConn
+	responder      *packetResponder
 	assignedEchoID int
 	originalEchoID int
 	// it's up to the user to ensure respEncoder is not used concurrently
 	respEncoder *packet.Encoder
 }
 
-func newICMPEchoFlow(src netip.Addr, sendPipe, returnPipe packet.FunnelUniPipe, assignedEchoID, originalEchoID int, respEncoder *packet.Encoder) *icmpEchoFlow {
+func newICMPEchoFlow(src netip.Addr, closeCallback func() error, originConn *icmp.PacketConn, responder *packetResponder, assignedEchoID, originalEchoID int, respEncoder *packet.Encoder) *icmpEchoFlow {
 	return &icmpEchoFlow{
-		RawPacketFunnel: packet.NewRawPacketFunnel(src, sendPipe, returnPipe),
+		ActivityTracker: packet.NewActivityTracker(),
+		closeCallback:   closeCallback,
+		src:             src,
+		originConn:      originConn,
+		responder:       responder,
 		assignedEchoID:  assignedEchoID,
 		originalEchoID:  originalEchoID,
 		respEncoder:     respEncoder,
 	}
 }
 
+func (ief *icmpEchoFlow) Equal(other packet.Funnel) bool {
+	otherICMPFlow, ok := other.(*icmpEchoFlow)
+	if !ok {
+		return false
+	}
+	if otherICMPFlow.src != ief.src {
+		return false
+	}
+	if otherICMPFlow.originalEchoID != ief.originalEchoID {
+		return false
+	}
+	if otherICMPFlow.assignedEchoID != ief.assignedEchoID {
+		return false
+	}
+	return true
+}
+
+func (ief *icmpEchoFlow) Close() error {
+	return ief.closeCallback()
+}
+
 // sendToDst rewrites the echo ID to the one assigned to this flow
 func (ief *icmpEchoFlow) sendToDst(dst netip.Addr, msg *icmp.Message) error {
+	ief.UpdateLastActive()
 	originalEcho, err := getICMPEcho(msg)
 	if err != nil {
 		return err
@@ -77,17 +111,21 @@ func (ief *icmpEchoFlow) sendToDst(dst netip.Addr, msg *icmp.Message) error {
 	if err != nil {
 		return err
 	}
-	return ief.SendToDst(dst, packet.RawPacket{Data: serializedPacket})
+	_, err = ief.originConn.WriteTo(serializedPacket, &net.UDPAddr{
+		IP: dst.AsSlice(),
+	})
+	return err
 }
 
 // returnToSrc rewrites the echo ID to the original echo ID from the eyeball
 func (ief *icmpEchoFlow) returnToSrc(reply *echoReply) error {
+	ief.UpdateLastActive()
 	reply.echo.ID = ief.originalEchoID
 	reply.msg.Body = reply.echo
 	pk := packet.ICMP{
 		IP: &packet.IP{
 			Src:      reply.from,
-			Dst:      ief.Src,
+			Dst:      ief.src,
 			Protocol: layers.IPProtocol(reply.msg.Type.Protocol()),
 			TTL:      packet.DefaultTTL,
 		},
@@ -97,7 +135,7 @@ func (ief *icmpEchoFlow) returnToSrc(reply *echoReply) error {
 	if err != nil {
 		return err
 	}
-	return ief.ReturnToSrc(serializedPacket)
+	return ief.responder.returnPacket(serializedPacket)
 }
 
 type echoReply struct {
@@ -137,3 +175,30 @@ func toICMPEchoFlow(funnel packet.Funnel) (*icmpEchoFlow, error) {
 	}
 	return icmpFlow, nil
 }
+
+func createShouldReplaceFunnelFunc(logger *zerolog.Logger, muxer muxer, pk *packet.ICMP, originalEchoID int) func(packet.Funnel) bool {
+	return func(existing packet.Funnel) bool {
+		existingFlow, err := toICMPEchoFlow(existing)
+		if err != nil {
+			logger.Err(err).
+				Str("src", pk.Src.String()).
+				Str("dst", pk.Dst.String()).
+				Int("originalEchoID", originalEchoID).
+				Msg("Funnel of wrong type found")
+			return true
+		}
+		// Each quic connection should have a unique muxer.
+		// If the existing flow has a different muxer, there's a new quic connection where return packets should be
+		// routed. Otherwise, return packets will be send to the first observed incoming connection, rather than the
+		// most recently observed connection.
+		if existingFlow.responder.datagramMuxer != muxer {
+			logger.Debug().
+				Str("src", pk.Src.String()).
+				Str("dst", pk.Dst.String()).
+				Int("originalEchoID", originalEchoID).
+				Msg("Replacing funnel with new responder")
+			return true
+		}
+		return false
+	}
+}

ingress/icmp_posix_test.go

@@ -24,7 +24,7 @@ func TestFunnelIdleTimeout(t *testing.T) {
 		startSeq    = 8129
 	)
 	logger := zerolog.New(os.Stderr)
-	proxy, err := newICMPProxy(localhostIP, &logger, idleTimeout)
+	proxy, err := newICMPProxy(localhostIP, "", &logger, idleTimeout)
 	require.NoError(t, err)
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -52,24 +52,88 @@ func TestFunnelIdleTimeout(t *testing.T) {
 			},
 		},
 	}
-	responder := echoFlowResponder{
-		decoder:  packet.NewICMPDecoder(),
-		respChan: make(chan []byte),
+	muxer := newMockMuxer(0)
+	responder := packetResponder{
+		datagramMuxer: muxer,
 	}
-	require.NoError(t, proxy.Request(&pk, &responder))
-	responder.validate(t, &pk)
+	require.NoError(t, proxy.Request(ctx, &pk, &responder))
+	validateEchoFlow(t, <-muxer.cfdToEdge, &pk)
 
 	// Send second request, should reuse the funnel
-	require.NoError(t, proxy.Request(&pk, nil))
-	responder.validate(t, &pk)
+	require.NoError(t, proxy.Request(ctx, &pk, &packetResponder{
+		datagramMuxer: muxer,
+	}))
+	validateEchoFlow(t, <-muxer.cfdToEdge, &pk)
 
 	time.Sleep(idleTimeout * 2)
-	newResponder := echoFlowResponder{
-		decoder:  packet.NewICMPDecoder(),
-		respChan: make(chan []byte),
+	newMuxer := newMockMuxer(0)
+	newResponder := packetResponder{
+		datagramMuxer: newMuxer,
 	}
-	require.NoError(t, proxy.Request(&pk, &newResponder))
-	newResponder.validate(t, &pk)
+	require.NoError(t, proxy.Request(ctx, &pk, &newResponder))
+	validateEchoFlow(t, <-newMuxer.cfdToEdge, &pk)
+
+	cancel()
+	<-proxyDone
+}
+
+func TestReuseFunnel(t *testing.T) {
+	const (
+		idleTimeout = time.Second
+		echoID      = 42573
+		startSeq    = 8129
+	)
+	logger := zerolog.New(os.Stderr)
+	proxy, err := newICMPProxy(localhostIP, "", &logger, idleTimeout)
+	require.NoError(t, err)
+
+	ctx, cancel := context.WithCancel(context.Background())
+
+	proxyDone := make(chan struct{})
+	go func() {
+		proxy.Serve(ctx)
+		close(proxyDone)
+	}()
+
+	// Send a packet to register the flow
+	pk := packet.ICMP{
+		IP: &packet.IP{
+			Src:      localhostIP,
+			Dst:      localhostIP,
+			Protocol: layers.IPProtocolICMPv4,
+		},
+		Message: &icmp.Message{
+			Type: ipv4.ICMPTypeEcho,
+			Code: 0,
+			Body: &icmp.Echo{
+				ID:   echoID,
+				Seq:  startSeq,
+				Data: []byte(t.Name()),
+			},
+		},
+	}
+	tuple := flow3Tuple{
+		srcIP:          pk.Src,
+		dstIP:          pk.Dst,
+		originalEchoID: echoID,
+	}
+	muxer := newMockMuxer(0)
+	responder := packetResponder{
+		datagramMuxer: muxer,
+	}
+	require.NoError(t, proxy.Request(ctx, &pk, &responder))
+	validateEchoFlow(t, <-muxer.cfdToEdge, &pk)
+	funnel1, found := getFunnel(t, proxy, tuple)
+	require.True(t, found)
+
+	// Send second request, should reuse the funnel
+	require.NoError(t, proxy.Request(ctx, &pk, &packetResponder{
+		datagramMuxer: muxer,
+	}))
+	validateEchoFlow(t, <-muxer.cfdToEdge, &pk)
+	funnel2, found := getFunnel(t, proxy, tuple)
+	require.True(t, found)
+	require.Equal(t, funnel1, funnel2)
 
 	cancel()
 	<-proxyDone

ingress/icmp_windows.go

@@ -21,11 +21,13 @@ import (
 	"github.com/google/gopacket/layers"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
+	"go.opentelemetry.io/otel/attribute"
 	"golang.org/x/net/icmp"
 	"golang.org/x/net/ipv4"
 	"golang.org/x/net/ipv6"
 
 	"github.com/cloudflare/cloudflared/packet"
+	"github.com/cloudflare/cloudflared/tracing"
 )
 
 const (
@@ -224,7 +226,7 @@ type icmpProxy struct {
 	encoderPool sync.Pool
 }
 
-func newICMPProxy(listenIP netip.Addr, logger *zerolog.Logger, idleTimeout time.Duration) (*icmpProxy, error) {
+func newICMPProxy(listenIP netip.Addr, zone string, logger *zerolog.Logger, idleTimeout time.Duration) (*icmpProxy, error) {
 	var (
 		srcSocketAddr *sockAddrIn6
 		handle        uintptr
@@ -262,34 +264,54 @@ func (ip *icmpProxy) Serve(ctx context.Context) error {
 	return ctx.Err()
 }
 
-func (ip *icmpProxy) Request(pk *packet.ICMP, responder packet.FunnelUniPipe) error {
-	if pk == nil {
-		return errPacketNil
-	}
+// Request sends an ICMP echo request and wait for a reply or timeout.
+// The async version of Win32 APIs take a callback whose memory is not garbage collected, so we use the synchronous version.
+// It's possible that a slow request will block other requests, so we set the timeout to only 1s.
+func (ip *icmpProxy) Request(ctx context.Context, pk *packet.ICMP, responder *packetResponder) error {
 	defer func() {
 		if r := recover(); r != nil {
 			ip.logger.Error().Interface("error", r).Msgf("Recover panic from sending icmp request/response, error %s", debug.Stack())
 		}
 	}()
 
+	_, requestSpan := responder.requestSpan(ctx, pk)
+	defer responder.exportSpan()
+
 	echo, err := getICMPEcho(pk.Message)
 	if err != nil {
 		return err
 	}
-	respData, err := ip.icmpEchoRoundtrip(pk.Dst, echo)
+	requestSpan.SetAttributes(
+		attribute.Int("originalEchoID", echo.ID),
+		attribute.Int("seq", echo.Seq),
+	)
+
+	resp, err := ip.icmpEchoRoundtrip(pk.Dst, echo)
 	if err != nil {
 		ip.logger.Err(err).Msg("ICMP echo roundtrip failed")
+		tracing.EndWithErrorStatus(requestSpan, err)
 		return err
 	}
+	tracing.End(requestSpan)
+	responder.exportSpan()
 
-	err = ip.handleEchoReply(pk, echo, respData, responder)
+	_, replySpan := responder.replySpan(ctx, ip.logger)
+	replySpan.SetAttributes(
+		attribute.Int("originalEchoID", echo.ID),
+		attribute.Int("seq", echo.Seq),
+		attribute.Int64("rtt", int64(resp.rtt())),
+		attribute.String("status", resp.status().String()),
+	)
+	err = ip.handleEchoReply(pk, echo, resp, responder)
 	if err != nil {
+		tracing.EndWithErrorStatus(replySpan, err)
 		return errors.Wrap(err, "failed to handle ICMP echo reply")
 	}
+	tracing.End(replySpan)
 	return nil
 }
 
-func (ip *icmpProxy) handleEchoReply(request *packet.ICMP, echoReq *icmp.Echo, data []byte, responder packet.FunnelUniPipe) error {
+func (ip *icmpProxy) handleEchoReply(request *packet.ICMP, echoReq *icmp.Echo, resp echoResp, responder *packetResponder) error {
 	var replyType icmp.Type
 	if request.Dst.Is4() {
 		replyType = ipv4.ICMPTypeEchoReply
@@ -310,7 +332,7 @@ func (ip *icmpProxy) handleEchoReply(request *packet.ICMP, echoReq *icmp.Echo, d
 			Body: &icmp.Echo{
 				ID:   echoReq.ID,
 				Seq:  echoReq.Seq,
-				Data: data,
+				Data: resp.payload(),
 			},
 		},
 	}
@@ -328,19 +350,20 @@ func (ip *icmpProxy) handleEchoReply(request *packet.ICMP, echoReq *icmp.Echo, d
 	if err != nil {
 		return err
 	}
-	return responder.SendPacket(request.Src, serializedPacket)
+	return responder.returnPacket(serializedPacket)
 }
 
-func (ip *icmpProxy) icmpEchoRoundtrip(dst netip.Addr, echo *icmp.Echo) ([]byte, error) {
+func (ip *icmpProxy) icmpEchoRoundtrip(dst netip.Addr, echo *icmp.Echo) (echoResp, error) {
 	if dst.Is6() {
 		if ip.srcSocketAddr == nil {
 			return nil, fmt.Errorf("cannot send ICMPv6 using ICMPv4 proxy")
 		}
 		resp, err := ip.icmp6SendEcho(dst, echo)
 		if err != nil {
+
 			return nil, errors.Wrap(err, "failed to send/receive ICMPv6 echo")
 		}
-		return resp.data, nil
+		return resp, nil
 	}
 	if ip.srcSocketAddr != nil {
 		return nil, fmt.Errorf("cannot send ICMPv4 using ICMPv6 proxy")
@@ -349,26 +372,26 @@ func (ip *icmpProxy) icmpEchoRoundtrip(dst netip.Addr, echo *icmp.Echo) ([]byte,
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to send/receive ICMPv4 echo")
 	}
-	return resp.data, nil
+	return resp, nil
 }
 
 /*
-	Wrapper to call https://docs.microsoft.com/en-us/windows/win32/api/icmpapi/nf-icmpapi-icmpsendecho
-	Parameters:
-	- IcmpHandle: Handle created by IcmpCreateFile
-	- DestinationAddress: IPv4 in the form of https://docs.microsoft.com/en-us/windows/win32/api/inaddr/ns-inaddr-in_addr#syntax
-	- RequestData: A pointer to echo data
-	- RequestSize: Number of bytes in buffer pointed by echo data
-	- RequestOptions: IP header options
-	- ReplyBuffer: A pointer to the buffer for echoReply, options and data
-	- ReplySize: Number of bytes allocated for ReplyBuffer
-	- Timeout: Timeout in milliseconds to wait for a reply
-	Returns:
-	- the number of replies in uint32 https://docs.microsoft.com/en-us/windows/win32/api/icmpapi/nf-icmpapi-icmpsendecho#return-value
-	To retain the reference allocated objects, conversion from pointer to uintptr must happen as arguments to the
-	syscall function
+Wrapper to call https://docs.microsoft.com/en-us/windows/win32/api/icmpapi/nf-icmpapi-icmpsendecho
+Parameters:
+- IcmpHandle: Handle created by IcmpCreateFile
+- DestinationAddress: IPv4 in the form of https://docs.microsoft.com/en-us/windows/win32/api/inaddr/ns-inaddr-in_addr#syntax
+- RequestData: A pointer to echo data
+- RequestSize: Number of bytes in buffer pointed by echo data
+- RequestOptions: IP header options
+- ReplyBuffer: A pointer to the buffer for echoReply, options and data
+- ReplySize: Number of bytes allocated for ReplyBuffer
+- Timeout: Timeout in milliseconds to wait for a reply
+Returns:
+- the number of replies in uint32 https://docs.microsoft.com/en-us/windows/win32/api/icmpapi/nf-icmpapi-icmpsendecho#return-value
+To retain the reference allocated objects, conversion from pointer to uintptr must happen as arguments to the
+syscall function
 */
-func (ip *icmpProxy) icmpSendEcho(dst netip.Addr, echo *icmp.Echo) (*echoResp, error) {
+func (ip *icmpProxy) icmpSendEcho(dst netip.Addr, echo *icmp.Echo) (*echoV4Resp, error) {
 	dataSize := len(echo.Data)
 	replySize := echoReplySize + uintptr(dataSize)
 	replyBuf := make([]byte, replySize)
@@ -396,7 +419,7 @@ func (ip *icmpProxy) icmpSendEcho(dst netip.Addr, echo *icmp.Echo) (*echoResp, e
 	} else if replyCount > 1 {
 		ip.logger.Warn().Msgf("Received %d ICMP echo replies, only sending 1 back", replyCount)
 	}
-	return newEchoResp(replyBuf)
+	return newEchoV4Resp(replyBuf)
 }
 
 // Third definition of https://docs.microsoft.com/en-us/windows/win32/api/inaddr/ns-inaddr-in_addr#syntax is address in uint32
@@ -408,12 +431,30 @@ func inAddrV4(ip netip.Addr) (uint32, error) {
 	return endian.Uint32(v4[:]), nil
 }
 
-type echoResp struct {
+type echoResp interface {
+	status() ipStatus
+	rtt() uint32
+	payload() []byte
+}
+
+type echoV4Resp struct {
 	reply *echoReply
 	data  []byte
 }
 
-func newEchoResp(replyBuf []byte) (*echoResp, error) {
+func (r *echoV4Resp) status() ipStatus {
+	return r.reply.Status
+}
+
+func (r *echoV4Resp) rtt() uint32 {
+	return r.reply.RoundTripTime
+}
+
+func (r *echoV4Resp) payload() []byte {
+	return r.data
+}
+
+func newEchoV4Resp(replyBuf []byte) (*echoV4Resp, error) {
 	if len(replyBuf) == 0 {
 		return nil, fmt.Errorf("reply buffer is empty")
 	}
@@ -427,7 +468,7 @@ func newEchoResp(replyBuf []byte) (*echoResp, error) {
 	if dataBufStart < int(echoReplySize) {
 		return nil, fmt.Errorf("reply buffer size %d is too small to hold data of size %d", len(replyBuf), int(reply.DataSize))
 	}
-	return &echoResp{
+	return &echoV4Resp{
 		reply: &reply,
 		data:  replyBuf[dataBufStart:],
 	}, nil
@@ -499,6 +540,18 @@ type echoV6Resp struct {
 	data  []byte
 }
 
+func (r *echoV6Resp) status() ipStatus {
+	return r.reply.Status
+}
+
+func (r *echoV6Resp) rtt() uint32 {
+	return r.reply.RoundTripTime
+}
+
+func (r *echoV6Resp) payload() []byte {
+	return r.data
+}
+
 func newEchoV6Resp(replyBuf []byte, dataSize int) (*echoV6Resp, error) {
 	if len(replyBuf) == 0 {
 		return nil, fmt.Errorf("reply buffer is empty")

ingress/icmp_windows_test.go

@@ -59,7 +59,7 @@ func TestParseEchoReply(t *testing.T) {
 	}
 
 	for _, test := range tests {
-		resp, err := newEchoResp(test.replyBuf)
+		resp, err := newEchoV4Resp(test.replyBuf)
 		if test.expectedReply == nil {
 			require.Error(t, err)
 			require.Nil(t, resp)
@@ -125,14 +125,14 @@ func TestParseEchoV6Reply(t *testing.T) {
 	}
 }
 
-//  TestSendEchoErrors makes sure icmpSendEcho handles error cases
+// TestSendEchoErrors makes sure icmpSendEcho handles error cases
 func TestSendEchoErrors(t *testing.T) {
 	testSendEchoErrors(t, netip.IPv4Unspecified())
 	testSendEchoErrors(t, netip.IPv6Unspecified())
 }
 
 func testSendEchoErrors(t *testing.T, listenIP netip.Addr) {
-	proxy, err := newICMPProxy(listenIP, &noopLogger, time.Second)
+	proxy, err := newICMPProxy(listenIP, "", &noopLogger, time.Second)
 	require.NoError(t, err)
 
 	echo := icmp.Echo{

ingress/ingress.go

@@ -11,8 +11,10 @@ import (
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
 	"github.com/urfave/cli/v2"
+	"golang.org/x/net/idna"
 
 	"github.com/cloudflare/cloudflared/config"
+	"github.com/cloudflare/cloudflared/ingress/middleware"
 	"github.com/cloudflare/cloudflared/ipaccess"
 )
 
@@ -56,7 +58,7 @@ func matchHost(ruleHost, reqHost string) bool {
 
 	// Validate hostnames that use wildcards at the start
 	if strings.HasPrefix(ruleHost, "*.") {
-		toMatch := strings.TrimPrefix(ruleHost, "*.")
+		toMatch := strings.TrimPrefix(ruleHost, "*")
 		return strings.HasSuffix(reqHost, toMatch)
 	}
 	return false
@@ -168,6 +170,20 @@ func (ing Ingress) CatchAll() *Rule {
 	return &ing.Rules[len(ing.Rules)-1]
 }
 
+func validateAccessConfiguration(cfg *config.AccessConfig) error {
+	if !cfg.Required {
+		return nil
+	}
+
+	// we allow for an initial setup where user can force Access but not configure the rest of the keys.
+	// however, if the user specified audTags but forgot teamName, we should alert it.
+	if cfg.TeamName == "" && len(cfg.AudTag) > 0 {
+		return errors.New("access.TeamName cannot be blank when access.audTags are present")
+	}
+
+	return nil
+}
+
 func validateIngress(ingress []config.UnvalidatedIngressRule, defaults OriginRequestConfig) (Ingress, error) {
 	rules := make([]Rule, len(ingress))
 	for i, r := range ingress {
@@ -237,10 +253,31 @@ func validateIngress(ingress []config.UnvalidatedIngressRule, defaults OriginReq
 			}
 		}
 
+		var handlers []middleware.Handler
+		if access := r.OriginRequest.Access; access != nil {
+			if err := validateAccessConfiguration(access); err != nil {
+				return Ingress{}, err
+			}
+			if access.Required {
+				verifier := middleware.NewJWTValidator(access.TeamName, "", access.AudTag)
+				handlers = append(handlers, verifier)
+			}
+		}
+
 		if err := validateHostname(r, i, len(ingress)); err != nil {
 			return Ingress{}, err
 		}
 
+		isCatchAllRule := (r.Hostname == "" || r.Hostname == "*") && r.Path == ""
+		punycodeHostname := ""
+		if !isCatchAllRule {
+			punycode, err := idna.Lookup.ToASCII(r.Hostname)
+			// Don't provide the punycode hostname if it is the same as the original hostname
+			if err == nil && punycode != r.Hostname {
+				punycodeHostname = punycode
+			}
+		}
+
 		var pathRegexp *Regexp
 		if r.Path != "" {
 			var err error
@@ -252,10 +289,12 @@ func validateIngress(ingress []config.UnvalidatedIngressRule, defaults OriginReq
 		}
 
 		rules[i] = Rule{
-			Hostname: r.Hostname,
-			Service:  service,
-			Path:     pathRegexp,
-			Config:   cfg,
+			Hostname:         r.Hostname,
+			punycodeHostname: punycodeHostname,
+			Service:          service,
+			Path:             pathRegexp,
+			Handlers:         handlers,
+			Config:           cfg,
 		}
 	}
 	return Ingress{Rules: rules, Defaults: defaults}, nil

ingress/ingress_test.go

@@ -130,6 +130,36 @@ ingress:
 				},
 			},
 		},
+		{
+			name: "Unicode domain",
+			args: args{rawYAML: `
+ingress:
+ - hostname: môô.cloudflare.com
+   service: https://localhost:8000
+ - service: https://localhost:8001
+`},
+			want: []Rule{
+				{
+					Hostname:         "môô.cloudflare.com",
+					punycodeHostname: "xn--m-xgaa.cloudflare.com",
+					Service:          &httpService{url: localhost8000},
+					Config:           defaultConfig,
+				},
+				{
+					Service: &httpService{url: localhost8001},
+					Config:  defaultConfig,
+				},
+			},
+		},
+		{
+			name: "Invalid unicode domain",
+			args: args{rawYAML: fmt.Sprintf(`
+ingress:
+ - hostname: %s
+   service: https://localhost:8000
+`, string(rune(0xd8f3))+".cloudflare.com")},
+			wantErr: true,
+		},
 		{
 			name: "Invalid service",
 			args: args{rawYAML: `
@@ -644,6 +674,46 @@ ingress:
 	}
 }
 
+func TestParseAccessConfig(t *testing.T) {
+	tests := []struct {
+		name        string
+		cfg         config.AccessConfig
+		expectError bool
+	}{
+		{
+			name:        "Config required with teamName only",
+			cfg:         config.AccessConfig{Required: true, TeamName: "team"},
+			expectError: false,
+		},
+		{
+			name:        "required false",
+			cfg:         config.AccessConfig{Required: false},
+			expectError: false,
+		},
+		{
+			name:        "required true but empty config",
+			cfg:         config.AccessConfig{Required: true},
+			expectError: false,
+		},
+		{
+			name:        "complete config",
+			cfg:         config.AccessConfig{Required: true, TeamName: "team", AudTag: []string{"a"}},
+			expectError: false,
+		},
+		{
+			name:        "required true with audTags but no teamName",
+			cfg:         config.AccessConfig{Required: true, AudTag: []string{"a"}},
+			expectError: true,
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			err := validateAccessConfiguration(&test.cfg)
+			require.Equal(t, err != nil, test.expectError)
+		})
+	}
+}
+
 func MustReadIngress(s string) *config.Configuration {
 	var conf config.Configuration
 	err := yaml.Unmarshal([]byte(s), &conf)

ingress/middleware/jwtvalidator.go

@@ -0,0 +1,79 @@
+package middleware
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+
+	"github.com/coreos/go-oidc/v3/oidc"
+)
+
+const (
+	headerKeyAccessJWTAssertion = "Cf-Access-Jwt-Assertion"
+)
+
+var (
+	cloudflareAccessCertsURL = "https://%s.cloudflareaccess.com"
+)
+
+// JWTValidator is an implementation of Verifier that validates access based JWT tokens.
+type JWTValidator struct {
+	*oidc.IDTokenVerifier
+	audTags []string
+}
+
+func NewJWTValidator(teamName string, certsURL string, audTags []string) *JWTValidator {
+	if certsURL == "" {
+		certsURL = fmt.Sprintf(cloudflareAccessCertsURL, teamName)
+	}
+	certsEndpoint := fmt.Sprintf("%s/cdn-cgi/access/certs", certsURL)
+
+	config := &oidc.Config{
+		SkipClientIDCheck: true,
+	}
+
+	ctx := context.Background()
+	keySet := oidc.NewRemoteKeySet(ctx, certsEndpoint)
+	verifier := oidc.NewVerifier(certsURL, keySet, config)
+	return &JWTValidator{
+		IDTokenVerifier: verifier,
+		audTags:         audTags,
+	}
+}
+
+func (v *JWTValidator) Name() string {
+	return "AccessJWTValidator"
+}
+
+func (v *JWTValidator) Handle(ctx context.Context, r *http.Request) (*HandleResult, error) {
+	accessJWT := r.Header.Get(headerKeyAccessJWTAssertion)
+	if accessJWT == "" {
+		// log the exact error message here. the message is specific to the handler implementation logic, we don't gain anything
+		// in passing it upstream. and each handler impl know what logging level to use for each.
+		return &HandleResult{
+			ShouldFilterRequest: true,
+			StatusCode:          http.StatusForbidden,
+			Reason:              "no access token in request",
+		}, nil
+	}
+
+	token, err := v.IDTokenVerifier.Verify(ctx, accessJWT)
+	if err != nil {
+		return nil, err
+	}
+
+	// We want at least one audTag to match
+	for _, jwtAudTag := range token.Audience {
+		for _, acceptedAudTag := range v.audTags {
+			if acceptedAudTag == jwtAudTag {
+				return &HandleResult{ShouldFilterRequest: false}, nil
+			}
+		}
+	}
+
+	return &HandleResult{
+		ShouldFilterRequest: true,
+		StatusCode:          http.StatusForbidden,
+		Reason:              fmt.Sprintf("Invalid token in jwt: %v", token.Audience),
+	}, nil
+}

ingress/middleware/middleware.go

@@ -0,0 +1,19 @@
+package middleware
+
+import (
+	"context"
+	"net/http"
+)
+
+type HandleResult struct {
+	// Tells that the request didn't passed the handler and should be filtered
+	ShouldFilterRequest bool
+	// The status code to return in case ShouldFilterRequest is true.
+	StatusCode int
+	Reason     string
+}
+
+type Handler interface {
+	Name() string
+	Handle(ctx context.Context, r *http.Request) (result *HandleResult, err error)
+}

ingress/origin_icmp_proxy.go

@@ -33,12 +33,13 @@ type icmpRouter struct {
 
 // NewICMPRouter doesn't return an error if either ipv4 proxy or ipv6 proxy can be created. The machine might only
 // support one of them
-func NewICMPRouter(logger *zerolog.Logger) (*icmpRouter, error) {
-	// TODO: TUN-6741: don't bind to all interface
-	ipv4Proxy, ipv4Err := newICMPProxy(netip.IPv4Unspecified(), logger, funnelIdleTimeout)
-	ipv6Proxy, ipv6Err := newICMPProxy(netip.IPv6Unspecified(), logger, funnelIdleTimeout)
+func NewICMPRouter(ipv4Addr, ipv6Addr netip.Addr, ipv6Zone string, logger *zerolog.Logger) (*icmpRouter, error) {
+	ipv4Proxy, ipv4Err := newICMPProxy(ipv4Addr, "", logger, funnelIdleTimeout)
+	ipv6Proxy, ipv6Err := newICMPProxy(ipv6Addr, ipv6Zone, logger, funnelIdleTimeout)
 	if ipv4Err != nil && ipv6Err != nil {
-		return nil, fmt.Errorf("cannot create ICMPv4 proxy: %v nor ICMPv6 proxy: %v", ipv4Err, ipv6Err)
+		err := fmt.Errorf("cannot create ICMPv4 proxy: %v nor ICMPv6 proxy: %v", ipv4Err, ipv6Err)
+		logger.Debug().Err(err).Msg("ICMP proxy feature is disabled")
+		return nil, err
 	}
 	if ipv4Err != nil {
 		logger.Debug().Err(ipv4Err).Msg("failed to create ICMPv4 proxy, only ICMPv6 proxy is created")
@@ -74,15 +75,18 @@ func (ir *icmpRouter) Serve(ctx context.Context) error {
 	return fmt.Errorf("ICMPv4 proxy and ICMPv6 proxy are both nil")
 }
 
-func (ir *icmpRouter) Request(pk *packet.ICMP, responder packet.FunnelUniPipe) error {
+func (ir *icmpRouter) Request(ctx context.Context, pk *packet.ICMP, responder *packetResponder) error {
+	if pk == nil {
+		return errPacketNil
+	}
 	if pk.Dst.Is4() {
 		if ir.ipv4Proxy != nil {
-			return ir.ipv4Proxy.Request(pk, responder)
+			return ir.ipv4Proxy.Request(ctx, pk, responder)
 		}
 		return fmt.Errorf("ICMPv4 proxy was not instantiated")
 	}
 	if ir.ipv6Proxy != nil {
-		return ir.ipv6Proxy.Request(pk, responder)
+		return ir.ipv6Proxy.Request(ctx, pk, responder)
 	}
 	return fmt.Errorf("ICMPv6 proxy was not instantiated")
 }

ingress/origin_icmp_proxy_test.go

@@ -1,6 +1,7 @@
 package ingress
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"net"
@@ -17,6 +18,8 @@ import (
 	"golang.org/x/net/ipv6"
 
 	"github.com/cloudflare/cloudflared/packet"
+	quicpogs "github.com/cloudflare/cloudflared/quic"
+	"github.com/cloudflare/cloudflared/tracing"
 )
 
 var (
@@ -42,7 +45,7 @@ func testICMPRouterEcho(t *testing.T, sendIPv4 bool) {
 		endSeq = 20
 	)
 
-	router, err := NewICMPRouter(&noopLogger)
+	router, err := NewICMPRouter(localhostIP, localhostIPv6, "", &noopLogger)
 	require.NoError(t, err)
 
 	proxyDone := make(chan struct{})
@@ -52,9 +55,9 @@ func testICMPRouterEcho(t *testing.T, sendIPv4 bool) {
 		close(proxyDone)
 	}()
 
-	responder := echoFlowResponder{
-		decoder:  packet.NewICMPDecoder(),
-		respChan: make(chan []byte, 1),
+	muxer := newMockMuxer(1)
+	responder := packetResponder{
+		datagramMuxer: muxer,
 	}
 
 	protocol := layers.IPProtocolICMPv6
@@ -90,14 +93,113 @@ func testICMPRouterEcho(t *testing.T, sendIPv4 bool) {
 					},
 				},
 			}
-			require.NoError(t, router.Request(&pk, &responder))
-			responder.validate(t, &pk)
+			require.NoError(t, router.Request(ctx, &pk, &responder))
+			validateEchoFlow(t, <-muxer.cfdToEdge, &pk)
 		}
 	}
 	cancel()
 	<-proxyDone
 }
 
+func TestTraceICMPRouterEcho(t *testing.T) {
+	tracingCtx := "ec31ad8a01fde11fdcabe2efdce36873:52726f6cabc144f5:0:1"
+
+	router, err := NewICMPRouter(localhostIP, localhostIPv6, "", &noopLogger)
+	require.NoError(t, err)
+
+	proxyDone := make(chan struct{})
+	ctx, cancel := context.WithCancel(context.Background())
+	go func() {
+		router.Serve(ctx)
+		close(proxyDone)
+	}()
+
+	// Buffer 3 packets, request span, reply span and reply
+	muxer := newMockMuxer(3)
+	tracingIdentity, err := tracing.NewIdentity(tracingCtx)
+	require.NoError(t, err)
+	serializedIdentity, err := tracingIdentity.MarshalBinary()
+	require.NoError(t, err)
+
+	responder := packetResponder{
+		datagramMuxer:      muxer,
+		tracedCtx:          tracing.NewTracedContext(ctx, tracingIdentity.String(), &noopLogger),
+		serializedIdentity: serializedIdentity,
+	}
+
+	echo := &icmp.Echo{
+		ID:   12910,
+		Seq:  182,
+		Data: []byte(t.Name()),
+	}
+	pk := packet.ICMP{
+		IP: &packet.IP{
+			Src:      localhostIP,
+			Dst:      localhostIP,
+			Protocol: layers.IPProtocolICMPv4,
+			TTL:      packet.DefaultTTL,
+		},
+		Message: &icmp.Message{
+			Type: ipv4.ICMPTypeEcho,
+			Code: 0,
+			Body: echo,
+		},
+	}
+
+	require.NoError(t, router.Request(ctx, &pk, &responder))
+	firstPK := <-muxer.cfdToEdge
+	var requestSpan *quicpogs.TracingSpanPacket
+	// The order of receiving reply or request span is not deterministic
+	switch firstPK.Type() {
+	case quicpogs.DatagramTypeIP:
+		// reply packet
+		validateEchoFlow(t, firstPK, &pk)
+	case quicpogs.DatagramTypeTracingSpan:
+		// Request span
+		requestSpan = firstPK.(*quicpogs.TracingSpanPacket)
+		require.NotEmpty(t, requestSpan.Spans)
+		require.True(t, bytes.Equal(serializedIdentity, requestSpan.TracingIdentity))
+	default:
+		panic(fmt.Sprintf("received unexpected packet type %d", firstPK.Type()))
+	}
+
+	secondPK := <-muxer.cfdToEdge
+	if requestSpan != nil {
+		// If first packet is request span, second packet should be the reply
+		validateEchoFlow(t, secondPK, &pk)
+	} else {
+		requestSpan = secondPK.(*quicpogs.TracingSpanPacket)
+		require.NotEmpty(t, requestSpan.Spans)
+		require.True(t, bytes.Equal(serializedIdentity, requestSpan.TracingIdentity))
+	}
+
+	// Reply span
+	thirdPacket := <-muxer.cfdToEdge
+	replySpan, ok := thirdPacket.(*quicpogs.TracingSpanPacket)
+	require.True(t, ok)
+	require.NotEmpty(t, replySpan.Spans)
+	require.True(t, bytes.Equal(serializedIdentity, replySpan.TracingIdentity))
+	require.False(t, bytes.Equal(requestSpan.Spans, replySpan.Spans))
+
+	echo.Seq++
+	pk.Body = echo
+	// Only first request for a flow is traced. The edge will not send tracing context for the second request
+	newResponder := packetResponder{
+		datagramMuxer: muxer,
+	}
+	require.NoError(t, router.Request(ctx, &pk, &newResponder))
+	validateEchoFlow(t, <-muxer.cfdToEdge, &pk)
+
+	select {
+	case receivedPacket := <-muxer.cfdToEdge:
+		panic(fmt.Sprintf("Receive unexpected packet %+v", receivedPacket))
+	default:
+	}
+
+	cancel()
+	<-proxyDone
+}
+
 // TestConcurrentRequests makes sure icmpRouter can send concurrent requests to the same destination with different
 // echo ID. This simulates concurrent ping to the same destination.
 func TestConcurrentRequestsToSameDst(t *testing.T) {
@@ -106,7 +208,7 @@ func TestConcurrentRequestsToSameDst(t *testing.T) {
 		endSeq          = 5
 	)
 
-	router, err := NewICMPRouter(&noopLogger)
+	router, err := NewICMPRouter(localhostIP, localhostIPv6, "", &noopLogger)
 	require.NoError(t, err)
 
 	proxyDone := make(chan struct{})
@@ -123,9 +225,10 @@ func TestConcurrentRequestsToSameDst(t *testing.T) {
 		echoID := 38451 + i
 		go func() {
 			defer wg.Done()
-			responder := echoFlowResponder{
-				decoder:  packet.NewICMPDecoder(),
-				respChan: make(chan []byte, 1),
+
+			muxer := newMockMuxer(1)
+			responder := packetResponder{
+				datagramMuxer: muxer,
 			}
 			for seq := 0; seq < endSeq; seq++ {
 				pk := &packet.ICMP{
@@ -145,15 +248,15 @@ func TestConcurrentRequestsToSameDst(t *testing.T) {
 						},
 					},
 				}
-				require.NoError(t, router.Request(pk, &responder))
-				responder.validate(t, pk)
+				require.NoError(t, router.Request(ctx, pk, &responder))
+				validateEchoFlow(t, <-muxer.cfdToEdge, pk)
 			}
 		}()
 		go func() {
 			defer wg.Done()
-			responder := echoFlowResponder{
-				decoder:  packet.NewICMPDecoder(),
-				respChan: make(chan []byte, 1),
+			muxer := newMockMuxer(1)
+			responder := packetResponder{
+				datagramMuxer: muxer,
 			}
 			for seq := 0; seq < endSeq; seq++ {
 				pk := &packet.ICMP{
@@ -173,8 +276,8 @@ func TestConcurrentRequestsToSameDst(t *testing.T) {
 						},
 					},
 				}
-				require.NoError(t, router.Request(pk, &responder))
-				responder.validate(t, pk)
+				require.NoError(t, router.Request(ctx, pk, &responder))
+				validateEchoFlow(t, <-muxer.cfdToEdge, pk)
 			}
 		}()
 	}
@@ -238,12 +341,12 @@ func TestICMPRouterRejectNotEcho(t *testing.T) {
 }
 
 func testICMPRouterRejectNotEcho(t *testing.T, srcDstIP netip.Addr, msgs []icmp.Message) {
-	router, err := NewICMPRouter(&noopLogger)
+	router, err := NewICMPRouter(localhostIP, localhostIPv6, "", &noopLogger)
 	require.NoError(t, err)
 
-	responder := echoFlowResponder{
-		decoder:  packet.NewICMPDecoder(),
-		respChan: make(chan []byte),
+	muxer := newMockMuxer(1)
+	responder := packetResponder{
+		datagramMuxer: muxer,
 	}
 	protocol := layers.IPProtocolICMPv4
 	if srcDstIP.Is6() {
@@ -259,36 +362,14 @@ func testICMPRouterRejectNotEcho(t *testing.T, srcDstIP netip.Addr, msgs []icmp.
 			},
 			Message: &m,
 		}
-		require.Error(t, router.Request(&pk, &responder))
+		require.Error(t, router.Request(context.Background(), &pk, &responder))
 	}
 }
 
-type echoFlowResponder struct {
-	lock     sync.Mutex
-	decoder  *packet.ICMPDecoder
-	respChan chan []byte
-}
-
-func (efr *echoFlowResponder) SendPacket(dst netip.Addr, pk packet.RawPacket) error {
-	efr.lock.Lock()
-	defer efr.lock.Unlock()
-	copiedPacket := make([]byte, len(pk.Data))
-	copy(copiedPacket, pk.Data)
-	efr.respChan <- copiedPacket
-	return nil
-}
-
-func (efr *echoFlowResponder) Close() error {
-	efr.lock.Lock()
-	defer efr.lock.Unlock()
-	close(efr.respChan)
-	return nil
-}
-
-func (efr *echoFlowResponder) validate(t *testing.T, echoReq *packet.ICMP) {
-	pk := <-efr.respChan
-	decoded, err := efr.decoder.Decode(packet.RawPacket{Data: pk})
-	require.NoError(t, err)
+func validateEchoFlow(t *testing.T, pk quicpogs.Packet, echoReq *packet.ICMP) {
+	decoder := packet.NewICMPDecoder()
+	decoded, err := decoder.Decode(packet.RawPacket{Data: pk.Payload()})
+	require.NoError(t, err, pk)
 	require.Equal(t, decoded.Src, echoReq.Dst)
 	require.Equal(t, decoded.Dst, echoReq.Src)
 	require.Equal(t, echoReq.Protocol, decoded.Protocol)
@@ -299,12 +380,7 @@ func (efr *echoFlowResponder) validate(t *testing.T, echoReq *packet.ICMP) {
 		require.Equal(t, ipv6.ICMPTypeEchoReply, decoded.Type)
 	}
 	require.Equal(t, 0, decoded.Code)
-	if echoReq.Type == ipv4.ICMPTypeEcho {
-		require.NotZero(t, decoded.Checksum)
-	} else {
-		// For ICMPv6, the kernel will compute the checksum during transmission unless pseudo header is not nil
-		require.Zero(t, decoded.Checksum)
-	}
+	require.NotZero(t, decoded.Checksum)
 
 	require.Equal(t, echoReq.Body, decoded.Body)
 }

ingress/origin_service.go

@@ -9,6 +9,7 @@ import (
 	"net"
 	"net/http"
 	"net/url"
+	"strconv"
 	"time"
 
 	"github.com/pkg/errors"
@@ -155,8 +156,10 @@ func newSocksProxyOverWSService(accessPolicy *ipaccess.Policy) *socksProxyOverWS
 }
 
 func addPortIfMissing(uri *url.URL, port int) {
+	hostname := uri.Hostname()
+
 	if uri.Port() == "" {
-		uri.Host = fmt.Sprintf("%s:%d", uri.Hostname(), port)
+		uri.Host = net.JoinHostPort(hostname, strconv.FormatInt(int64(port), 10))
 	}
 }
 

ingress/origin_service_test.go

@@ -0,0 +1,29 @@
+package ingress
+
+import (
+	"net/url"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestAddPortIfMissing(t *testing.T) {
+	testCases := []struct {
+		input    string
+		expected string
+	}{
+		{"ssh://[::1]", "[::1]:22"},
+		{"ssh://[::1]:38", "[::1]:38"},
+		{"ssh://abc:38", "abc:38"},
+		{"ssh://127.0.0.1:38", "127.0.0.1:38"},
+		{"ssh://127.0.0.1", "127.0.0.1:22"},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.input, func(t *testing.T) {
+			url1, _ := url.Parse(tc.input)
+			addPortIfMissing(url1, 22)
+			require.Equal(t, tc.expected, url1.Host)
+		})
+	}
+}

ingress/packet_router.go

@@ -0,0 +1,184 @@
+package ingress
+
+import (
+	"context"
+	"fmt"
+	"net/netip"
+
+	"github.com/rs/zerolog"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
+
+	"github.com/cloudflare/cloudflared/packet"
+	quicpogs "github.com/cloudflare/cloudflared/quic"
+	"github.com/cloudflare/cloudflared/tracing"
+)
+
+// Upstream of raw packets
+type muxer interface {
+	SendPacket(pk quicpogs.Packet) error
+	// ReceivePacket waits for the next raw packet from upstream
+	ReceivePacket(ctx context.Context) (quicpogs.Packet, error)
+}
+
+// PacketRouter routes packets between Upstream and ICMPRouter. Currently it rejects all other type of ICMP packets
+type PacketRouter struct {
+	globalConfig           *GlobalRouterConfig
+	muxer                  muxer
+	logger                 *zerolog.Logger
+	checkRouterEnabledFunc func() bool
+	icmpDecoder            *packet.ICMPDecoder
+	encoder                *packet.Encoder
+}
+
+// GlobalRouterConfig is the configuration shared by all instance of Router.
+type GlobalRouterConfig struct {
+	ICMPRouter *icmpRouter
+	IPv4Src    netip.Addr
+	IPv6Src    netip.Addr
+	Zone       string
+}
+
+// NewPacketRouter creates a PacketRouter that handles ICMP packets. Packets are read from muxer but dropped if globalConfig is nil.
+func NewPacketRouter(globalConfig *GlobalRouterConfig, muxer muxer, logger *zerolog.Logger, checkRouterEnabledFunc func() bool) *PacketRouter {
+	return &PacketRouter{
+		globalConfig:           globalConfig,
+		muxer:                  muxer,
+		logger:                 logger,
+		checkRouterEnabledFunc: checkRouterEnabledFunc,
+		icmpDecoder:            packet.NewICMPDecoder(),
+		encoder:                packet.NewEncoder(),
+	}
+}
+
+func (r *PacketRouter) Serve(ctx context.Context) error {
+	for {
+		rawPacket, responder, err := r.nextPacket(ctx)
+		if err != nil {
+			return err
+		}
+		r.handlePacket(ctx, rawPacket, responder)
+	}
+}
+
+func (r *PacketRouter) nextPacket(ctx context.Context) (packet.RawPacket, *packetResponder, error) {
+	pk, err := r.muxer.ReceivePacket(ctx)
+	if err != nil {
+		return packet.RawPacket{}, nil, err
+	}
+	responder := &packetResponder{
+		datagramMuxer: r.muxer,
+	}
+	switch pk.Type() {
+	case quicpogs.DatagramTypeIP:
+		return packet.RawPacket{Data: pk.Payload()}, responder, nil
+	case quicpogs.DatagramTypeIPWithTrace:
+		var identity tracing.Identity
+		if err := identity.UnmarshalBinary(pk.Metadata()); err != nil {
+			r.logger.Err(err).Bytes("tracingIdentity", pk.Metadata()).Msg("Failed to unmarshal tracing identity")
+		} else {
+			responder.tracedCtx = tracing.NewTracedContext(ctx, identity.String(), r.logger)
+			responder.serializedIdentity = pk.Metadata()
+		}
+		return packet.RawPacket{Data: pk.Payload()}, responder, nil
+	default:
+		return packet.RawPacket{}, nil, fmt.Errorf("unexpected datagram type %d", pk.Type())
+	}
+}
+
+func (r *PacketRouter) handlePacket(ctx context.Context, rawPacket packet.RawPacket, responder *packetResponder) {
+	// ICMP Proxy feature is disabled, drop packets
+	if r.globalConfig == nil {
+		return
+	}
+
+	if enabled := r.checkRouterEnabledFunc(); !enabled {
+		return
+	}
+
+	icmpPacket, err := r.icmpDecoder.Decode(rawPacket)
+	if err != nil {
+		r.logger.Err(err).Msg("Failed to decode ICMP packet from quic datagram")
+		return
+	}
+
+	if icmpPacket.TTL <= 1 {
+		if err := r.sendTTLExceedMsg(ctx, icmpPacket, rawPacket, r.encoder); err != nil {
+			r.logger.Err(err).Msg("Failed to return ICMP TTL exceed error")
+		}
+		return
+	}
+	icmpPacket.TTL--
+
+	if err := r.globalConfig.ICMPRouter.Request(ctx, icmpPacket, responder); err != nil {
+		r.logger.Err(err).
+			Str("src", icmpPacket.Src.String()).
+			Str("dst", icmpPacket.Dst.String()).
+			Interface("type", icmpPacket.Type).
+			Msg("Failed to send ICMP packet")
+	}
+}
+
+func (r *PacketRouter) sendTTLExceedMsg(ctx context.Context, pk *packet.ICMP, rawPacket packet.RawPacket, encoder *packet.Encoder) error {
+	var srcIP netip.Addr
+	if pk.Dst.Is4() {
+		srcIP = r.globalConfig.IPv4Src
+	} else {
+		srcIP = r.globalConfig.IPv6Src
+	}
+	ttlExceedPacket := packet.NewICMPTTLExceedPacket(pk.IP, rawPacket, srcIP)
+
+	encodedTTLExceed, err := encoder.Encode(ttlExceedPacket)
+	if err != nil {
+		return err
+	}
+	return r.muxer.SendPacket(quicpogs.RawPacket(encodedTTLExceed))
+}
+
+// packetResponder should not be used concurrently. This assumption is upheld because reply packets are ready one-by-one
+type packetResponder struct {
+	datagramMuxer      muxer
+	tracedCtx          *tracing.TracedContext
+	serializedIdentity []byte
+	// hadReply tracks if there has been any reply for this flow
+	hadReply bool
+}
+
+func (pr *packetResponder) tracingEnabled() bool {
+	return pr.tracedCtx != nil
+}
+
+func (pr *packetResponder) returnPacket(rawPacket packet.RawPacket) error {
+	pr.hadReply = true
+	return pr.datagramMuxer.SendPacket(quicpogs.RawPacket(rawPacket))
+}
+
+func (pr *packetResponder) requestSpan(ctx context.Context, pk *packet.ICMP) (context.Context, trace.Span) {
+	if !pr.tracingEnabled() {
+		return ctx, tracing.NewNoopSpan()
+	}
+	return pr.tracedCtx.Tracer().Start(pr.tracedCtx, "icmp-echo-request", trace.WithAttributes(
+		attribute.String("src", pk.Src.String()),
+		attribute.String("dst", pk.Dst.String()),
+	))
+}
+
+func (pr *packetResponder) replySpan(ctx context.Context, logger *zerolog.Logger) (context.Context, trace.Span) {
+	if !pr.tracingEnabled() || pr.hadReply {
+		return ctx, tracing.NewNoopSpan()
+	}
+	return pr.tracedCtx.Tracer().Start(pr.tracedCtx, "icmp-echo-reply")
+}
+
+func (pr *packetResponder) exportSpan() {
+	if !pr.tracingEnabled() {
+		return
+	}
+	spans := pr.tracedCtx.GetProtoSpans()
+	if len(spans) > 0 {
+		pr.datagramMuxer.SendPacket(&quicpogs.TracingSpanPacket{
+			Spans:           spans,
+			TracingIdentity: pr.serializedIdentity,
+		})
+	}
+}

ingress/packet_router_test.go

@@ -0,0 +1,241 @@
+package ingress
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"net/netip"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/google/gopacket/layers"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/net/icmp"
+	"golang.org/x/net/ipv4"
+	"golang.org/x/net/ipv6"
+
+	"github.com/cloudflare/cloudflared/packet"
+	quicpogs "github.com/cloudflare/cloudflared/quic"
+)
+
+var (
+	packetConfig = &GlobalRouterConfig{
+		ICMPRouter: nil,
+		IPv4Src:    netip.MustParseAddr("172.16.0.1"),
+		IPv6Src:    netip.MustParseAddr("fd51:2391:523:f4ee::1"),
+	}
+)
+
+func TestRouterReturnTTLExceed(t *testing.T) {
+	muxer := newMockMuxer(0)
+	routerEnabled := &routerEnabledChecker{}
+	routerEnabled.set(true)
+	router := NewPacketRouter(packetConfig, muxer, &noopLogger, routerEnabled.isEnabled)
+	ctx, cancel := context.WithCancel(context.Background())
+	routerStopped := make(chan struct{})
+	go func() {
+		router.Serve(ctx)
+		close(routerStopped)
+	}()
+
+	pk := packet.ICMP{
+		IP: &packet.IP{
+			Src:      netip.MustParseAddr("192.168.1.1"),
+			Dst:      netip.MustParseAddr("10.0.0.1"),
+			Protocol: layers.IPProtocolICMPv4,
+			TTL:      1,
+		},
+		Message: &icmp.Message{
+			Type: ipv4.ICMPTypeEcho,
+			Code: 0,
+			Body: &icmp.Echo{
+				ID:   12481,
+				Seq:  8036,
+				Data: []byte("TTL exceed"),
+			},
+		},
+	}
+	assertTTLExceed(t, &pk, router.globalConfig.IPv4Src, muxer)
+	pk = packet.ICMP{
+		IP: &packet.IP{
+			Src:      netip.MustParseAddr("fd51:2391:523:f4ee::1"),
+			Dst:      netip.MustParseAddr("fd51:2391:697:f4ee::2"),
+			Protocol: layers.IPProtocolICMPv6,
+			TTL:      1,
+		},
+		Message: &icmp.Message{
+			Type: ipv6.ICMPTypeEchoRequest,
+			Code: 0,
+			Body: &icmp.Echo{
+				ID:   42583,
+				Seq:  7039,
+				Data: []byte("TTL exceed"),
+			},
+		},
+	}
+	assertTTLExceed(t, &pk, router.globalConfig.IPv6Src, muxer)
+
+	cancel()
+	<-routerStopped
+}
+
+func TestRouterCheckEnabled(t *testing.T) {
+	muxer := newMockMuxer(0)
+	routerEnabled := &routerEnabledChecker{}
+	router := NewPacketRouter(packetConfig, muxer, &noopLogger, routerEnabled.isEnabled)
+	ctx, cancel := context.WithCancel(context.Background())
+	routerStopped := make(chan struct{})
+	go func() {
+		router.Serve(ctx)
+		close(routerStopped)
+	}()
+
+	pk := packet.ICMP{
+		IP: &packet.IP{
+			Src:      netip.MustParseAddr("192.168.1.1"),
+			Dst:      netip.MustParseAddr("10.0.0.1"),
+			Protocol: layers.IPProtocolICMPv4,
+			TTL:      1,
+		},
+		Message: &icmp.Message{
+			Type: ipv4.ICMPTypeEcho,
+			Code: 0,
+			Body: &icmp.Echo{
+				ID:   12481,
+				Seq:  8036,
+				Data: []byte(t.Name()),
+			},
+		},
+	}
+
+	// router is disabled
+	encoder := packet.NewEncoder()
+	encodedPacket, err := encoder.Encode(&pk)
+	require.NoError(t, err)
+	sendPacket := quicpogs.RawPacket(encodedPacket)
+
+	muxer.edgeToCfd <- sendPacket
+	select {
+	case <-time.After(time.Millisecond * 10):
+	case <-muxer.cfdToEdge:
+		t.Error("Unexpected reply when router is disabled")
+	}
+	routerEnabled.set(true)
+	// router is enabled, expects reply
+	muxer.edgeToCfd <- sendPacket
+	<-muxer.cfdToEdge
+
+	routerEnabled.set(false)
+	// router is disabled
+	muxer.edgeToCfd <- sendPacket
+	select {
+	case <-time.After(time.Millisecond * 10):
+	case <-muxer.cfdToEdge:
+		t.Error("Unexpected reply when router is disabled")
+	}
+
+	cancel()
+	<-routerStopped
+}
+
+func assertTTLExceed(t *testing.T, originalPacket *packet.ICMP, expectedSrc netip.Addr, muxer *mockMuxer) {
+	encoder := packet.NewEncoder()
+	rawPacket, err := encoder.Encode(originalPacket)
+	require.NoError(t, err)
+	muxer.edgeToCfd <- quicpogs.RawPacket(rawPacket)
+
+	resp := <-muxer.cfdToEdge
+	decoder := packet.NewICMPDecoder()
+	decoded, err := decoder.Decode(packet.RawPacket(resp.(quicpogs.RawPacket)))
+	require.NoError(t, err)
+
+	require.Equal(t, expectedSrc, decoded.Src)
+	require.Equal(t, originalPacket.Src, decoded.Dst)
+	require.Equal(t, originalPacket.Protocol, decoded.Protocol)
+	require.Equal(t, packet.DefaultTTL, decoded.TTL)
+	if originalPacket.Dst.Is4() {
+		require.Equal(t, ipv4.ICMPTypeTimeExceeded, decoded.Type)
+	} else {
+		require.Equal(t, ipv6.ICMPTypeTimeExceeded, decoded.Type)
+	}
+	require.Equal(t, 0, decoded.Code)
+	timeExceed, ok := decoded.Body.(*icmp.TimeExceeded)
+	require.True(t, ok)
+	require.True(t, bytes.Equal(rawPacket.Data, timeExceed.Data))
+}
+
+type mockMuxer struct {
+	cfdToEdge chan quicpogs.Packet
+	edgeToCfd chan quicpogs.Packet
+}
+
+func newMockMuxer(capacity int) *mockMuxer {
+	return &mockMuxer{
+		cfdToEdge: make(chan quicpogs.Packet, capacity),
+		edgeToCfd: make(chan quicpogs.Packet, capacity),
+	}
+}
+
+// Copy packet, because icmpProxy expects the encoder buffer to be reusable after the packet is sent
+func (mm *mockMuxer) SendPacket(pk quicpogs.Packet) error {
+	payload := pk.Payload()
+	copiedPayload := make([]byte, len(payload))
+	copy(copiedPayload, payload)
+
+	metadata := pk.Metadata()
+	copiedMetadata := make([]byte, len(metadata))
+	copy(copiedMetadata, metadata)
+
+	var copiedPacket quicpogs.Packet
+	switch pk.Type() {
+	case quicpogs.DatagramTypeIP:
+		copiedPacket = quicpogs.RawPacket(packet.RawPacket{
+			Data: copiedPayload,
+		})
+	case quicpogs.DatagramTypeIPWithTrace:
+		copiedPacket = &quicpogs.TracedPacket{
+			Packet: packet.RawPacket{
+				Data: copiedPayload,
+			},
+			TracingIdentity: copiedMetadata,
+		}
+	case quicpogs.DatagramTypeTracingSpan:
+		copiedPacket = &quicpogs.TracingSpanPacket{
+			Spans:           copiedPayload,
+			TracingIdentity: copiedMetadata,
+		}
+	default:
+		return fmt.Errorf("unexpected metadata type %d", pk.Type())
+	}
+	mm.cfdToEdge <- copiedPacket
+	return nil
+}
+
+func (mm *mockMuxer) ReceivePacket(ctx context.Context) (quicpogs.Packet, error) {
+	select {
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	case pk := <-mm.edgeToCfd:
+		return pk, nil
+	}
+}
+
+type routerEnabledChecker struct {
+	enabled uint32
+}
+
+func (rec *routerEnabledChecker) isEnabled() bool {
+	if atomic.LoadUint32(&rec.enabled) == 0 {
+		return false
+	}
+	return true
+}
+
+func (rec *routerEnabledChecker) set(enabled bool) {
+	if enabled {
+		atomic.StoreUint32(&rec.enabled, 1)
+	} else {
+		atomic.StoreUint32(&rec.enabled, 0)
+	}
+}

ingress/rule.go

@@ -4,6 +4,8 @@ import (
 	"encoding/json"
 	"regexp"
 	"strings"
+
+	"github.com/cloudflare/cloudflared/ingress/middleware"
 )
 
 // Rule routes traffic from a hostname/path on the public internet to the
@@ -12,6 +14,9 @@ type Rule struct {
 	// Requests for this hostname will be proxied to this rule's service.
 	Hostname string `json:"hostname"`
 
+	// punycodeHostname is an additional optional hostname converted to punycode.
+	punycodeHostname string
+
 	// Path is an optional regex that can specify path-driven ingress rules.
 	Path *Regexp `json:"path"`
 
@@ -20,6 +25,9 @@ type Rule struct {
 	// address.
 	Service OriginService `json:"service"`
 
+	// Handlers is a list of functions that acts as a middleware during ProxyHTTP
+	Handlers []middleware.Handler
+
 	// Configure the request cloudflared sends to this specific origin.
 	Config OriginRequestConfig `json:"originRequest"`
 }
@@ -45,9 +53,18 @@ func (r Rule) MultiLineString() string {
 
 // Matches checks if the rule matches a given hostname/path combination.
 func (r *Rule) Matches(hostname, path string) bool {
-	hostMatch := r.Hostname == "" || r.Hostname == "*" || matchHost(r.Hostname, hostname)
+	hostMatch := false
+	if r.Hostname == "" || r.Hostname == "*" {
+		hostMatch = true
+	} else {
+		hostMatch = matchHost(r.Hostname, hostname)
+	}
+	punycodeHostMatch := false
+	if r.punycodeHostname != "" {
+		punycodeHostMatch = matchHost(r.punycodeHostname, hostname)
+	}
 	pathMatch := r.Path == nil || r.Path.Regexp == nil || r.Path.Regexp.MatchString(path)
-	return hostMatch && pathMatch
+	return (hostMatch || punycodeHostMatch) && pathMatch
 }
 
 // Regexp adds unmarshalling from json for regexp.Regexp

ingress/rule_test.go

@@ -14,23 +14,18 @@ import (
 )
 
 func Test_rule_matches(t *testing.T) {
-	type fields struct {
-		Hostname string
-		Path     *Regexp
-		Service  OriginService
-	}
 	type args struct {
 		requestURL *url.URL
 	}
 	tests := []struct {
-		name   string
-		fields fields
-		args   args
-		want   bool
+		name string
+		rule Rule
+		args args
+		want bool
 	}{
 		{
 			name: "Just hostname, pass",
-			fields: fields{
+			rule: Rule{
 				Hostname: "example.com",
 			},
 			args: args{
@@ -38,9 +33,31 @@ func Test_rule_matches(t *testing.T) {
 			},
 			want: true,
 		},
+		{
+			name: "Unicode hostname with unicode request, pass",
+			rule: Rule{
+				Hostname:         "môô.cloudflare.com",
+				punycodeHostname: "xn--m-xgaa.cloudflare.com",
+			},
+			args: args{
+				requestURL: MustParseURL(t, "https://môô.cloudflare.com"),
+			},
+			want: true,
+		},
+		{
+			name: "Unicode hostname with punycode request, pass",
+			rule: Rule{
+				Hostname:         "môô.cloudflare.com",
+				punycodeHostname: "xn--m-xgaa.cloudflare.com",
+			},
+			args: args{
+				requestURL: MustParseURL(t, "https://xn--m-xgaa.cloudflare.com"),
+			},
+			want: true,
+		},
 		{
 			name: "Entire hostname is wildcard, should match everything",
-			fields: fields{
+			rule: Rule{
 				Hostname: "*",
 			},
 			args: args{
@@ -50,7 +67,7 @@ func Test_rule_matches(t *testing.T) {
 		},
 		{
 			name: "Just hostname, fail",
-			fields: fields{
+			rule: Rule{
 				Hostname: "example.com",
 			},
 			args: args{
@@ -60,7 +77,7 @@ func Test_rule_matches(t *testing.T) {
 		},
 		{
 			name: "Just wildcard hostname, pass",
-			fields: fields{
+			rule: Rule{
 				Hostname: "*.example.com",
 			},
 			args: args{
@@ -70,7 +87,7 @@ func Test_rule_matches(t *testing.T) {
 		},
 		{
 			name: "Just wildcard hostname, fail",
-			fields: fields{
+			rule: Rule{
 				Hostname: "*.example.com",
 			},
 			args: args{
@@ -80,7 +97,7 @@ func Test_rule_matches(t *testing.T) {
 		},
 		{
 			name: "Just wildcard outside of subdomain in hostname, fail",
-			fields: fields{
+			rule: Rule{
 				Hostname: "*example.com",
 			},
 			args: args{
@@ -90,7 +107,7 @@ func Test_rule_matches(t *testing.T) {
 		},
 		{
 			name: "Wildcard over multiple subdomains",
-			fields: fields{
+			rule: Rule{
 				Hostname: "*.example.com",
 			},
 			args: args{
@@ -100,7 +117,7 @@ func Test_rule_matches(t *testing.T) {
 		},
 		{
 			name: "Hostname and path",
-			fields: fields{
+			rule: Rule{
 				Hostname: "*.example.com",
 				Path:     &Regexp{Regexp: regexp.MustCompile("/static/.*\\.html")},
 			},
@@ -111,7 +128,7 @@ func Test_rule_matches(t *testing.T) {
 		},
 		{
 			name: "Hostname and empty Regex",
-			fields: fields{
+			rule: Rule{
 				Hostname: "example.com",
 				Path:     &Regexp{},
 			},
@@ -122,7 +139,7 @@ func Test_rule_matches(t *testing.T) {
 		},
 		{
 			name: "Hostname and nil path",
-			fields: fields{
+			rule: Rule{
 				Hostname: "example.com",
 				Path:     nil,
 			},
@@ -131,16 +148,21 @@ func Test_rule_matches(t *testing.T) {
 			},
 			want: true,
 		},
+		{
+			name: "Hostname with wildcard should not match if no dot present",
+			rule: Rule{
+				Hostname: "*.api.abc.cloud",
+			},
+			args: args{
+				requestURL: MustParseURL(t, "https://testing-api.abc.cloud"),
+			},
+			want: false,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			r := Rule{
-				Hostname: tt.fields.Hostname,
-				Path:     tt.fields.Path,
-				Service:  tt.fields.Service,
-			}
 			u := tt.args.requestURL
-			if got := r.Matches(u.Hostname(), u.Path); got != tt.want {
+			if got := tt.rule.Matches(u.Hostname(), u.Path); got != tt.want {
 				t.Errorf("rule.matches() = %v, want %v", got, tt.want)
 			}
 		})
@@ -182,25 +204,25 @@ func TestMarshalJSON(t *testing.T) {
 		{
 			name:     "Nil",
 			path:     nil,
-			expected: `{"hostname":"example.com","path":null,"service":"https://localhost:8000","originRequest":{"connectTimeout":30,"tlsTimeout":10,"tcpKeepAlive":30,"noHappyEyeballs":false,"keepAliveTimeout":90,"keepAliveConnections":100,"httpHostHeader":"","originServerName":"","caPool":"","noTLSVerify":false,"disableChunkedEncoding":false,"bastionMode":false,"proxyAddress":"127.0.0.1","proxyPort":0,"proxyType":"","ipRules":null,"http2Origin":false,"access":{"Enabled":false}}}`,
+			expected: `{"hostname":"example.com","path":null,"service":"https://localhost:8000","Handlers":null,"originRequest":{"connectTimeout":30,"tlsTimeout":10,"tcpKeepAlive":30,"noHappyEyeballs":false,"keepAliveTimeout":90,"keepAliveConnections":100,"httpHostHeader":"","originServerName":"","caPool":"","noTLSVerify":false,"disableChunkedEncoding":false,"bastionMode":false,"proxyAddress":"127.0.0.1","proxyPort":0,"proxyType":"","ipRules":null,"http2Origin":false,"access":{"teamName":"","audTag":null}}}`,
 			want:     true,
 		},
 		{
 			name:     "Nil regex",
 			path:     &Regexp{Regexp: nil},
-			expected: `{"hostname":"example.com","path":null,"service":"https://localhost:8000","originRequest":{"connectTimeout":30,"tlsTimeout":10,"tcpKeepAlive":30,"noHappyEyeballs":false,"keepAliveTimeout":90,"keepAliveConnections":100,"httpHostHeader":"","originServerName":"","caPool":"","noTLSVerify":false,"disableChunkedEncoding":false,"bastionMode":false,"proxyAddress":"127.0.0.1","proxyPort":0,"proxyType":"","ipRules":null,"http2Origin":false,"access":{"Enabled":false}}}`,
+			expected: `{"hostname":"example.com","path":null,"service":"https://localhost:8000","Handlers":null,"originRequest":{"connectTimeout":30,"tlsTimeout":10,"tcpKeepAlive":30,"noHappyEyeballs":false,"keepAliveTimeout":90,"keepAliveConnections":100,"httpHostHeader":"","originServerName":"","caPool":"","noTLSVerify":false,"disableChunkedEncoding":false,"bastionMode":false,"proxyAddress":"127.0.0.1","proxyPort":0,"proxyType":"","ipRules":null,"http2Origin":false,"access":{"teamName":"","audTag":null}}}`,
 			want:     true,
 		},
 		{
 			name:     "Empty",
 			path:     &Regexp{Regexp: regexp.MustCompile("")},
-			expected: `{"hostname":"example.com","path":"","service":"https://localhost:8000","originRequest":{"connectTimeout":30,"tlsTimeout":10,"tcpKeepAlive":30,"noHappyEyeballs":false,"keepAliveTimeout":90,"keepAliveConnections":100,"httpHostHeader":"","originServerName":"","caPool":"","noTLSVerify":false,"disableChunkedEncoding":false,"bastionMode":false,"proxyAddress":"127.0.0.1","proxyPort":0,"proxyType":"","ipRules":null,"http2Origin":false,"access":{"Enabled":false}}}`,
+			expected: `{"hostname":"example.com","path":"","service":"https://localhost:8000","Handlers":null,"originRequest":{"connectTimeout":30,"tlsTimeout":10,"tcpKeepAlive":30,"noHappyEyeballs":false,"keepAliveTimeout":90,"keepAliveConnections":100,"httpHostHeader":"","originServerName":"","caPool":"","noTLSVerify":false,"disableChunkedEncoding":false,"bastionMode":false,"proxyAddress":"127.0.0.1","proxyPort":0,"proxyType":"","ipRules":null,"http2Origin":false,"access":{"teamName":"","audTag":null}}}`,
 			want:     true,
 		},
 		{
 			name:     "Basic",
 			path:     &Regexp{Regexp: regexp.MustCompile("/echo")},
-			expected: `{"hostname":"example.com","path":"/echo","service":"https://localhost:8000","originRequest":{"connectTimeout":30,"tlsTimeout":10,"tcpKeepAlive":30,"noHappyEyeballs":false,"keepAliveTimeout":90,"keepAliveConnections":100,"httpHostHeader":"","originServerName":"","caPool":"","noTLSVerify":false,"disableChunkedEncoding":false,"bastionMode":false,"proxyAddress":"127.0.0.1","proxyPort":0,"proxyType":"","ipRules":null,"http2Origin":false,"access":{"Enabled":false}}}`,
+			expected: `{"hostname":"example.com","path":"/echo","service":"https://localhost:8000","Handlers":null,"originRequest":{"connectTimeout":30,"tlsTimeout":10,"tcpKeepAlive":30,"noHappyEyeballs":false,"keepAliveTimeout":90,"keepAliveConnections":100,"httpHostHeader":"","originServerName":"","caPool":"","noTLSVerify":false,"disableChunkedEncoding":false,"bastionMode":false,"proxyAddress":"127.0.0.1","proxyPort":0,"proxyType":"","ipRules":null,"http2Origin":false,"access":{"teamName":"","audTag":null}}}`,
 			want:     true,
 		},
 	}

metrics/metrics.go

@@ -18,18 +18,24 @@ import (
 )
 
 const (
-	shutdownTimeout = time.Second * 15
-	startupTime     = time.Millisecond * 500
+	startupTime            = time.Millisecond * 500
+	defaultShutdownTimeout = time.Second * 15
 )
 
+type Config struct {
+	ReadyServer         *ReadyServer
+	QuickTunnelHostname string
+	Orchestrator        orchestrator
+
+	ShutdownTimeout time.Duration
+}
+
 type orchestrator interface {
 	GetVersionedConfigJSON() ([]byte, error)
 }
 
 func newMetricsHandler(
-	readyServer *ReadyServer,
-	quickTunnelHostname string,
-	orchestrator orchestrator,
+	config Config,
 	log *zerolog.Logger,
 ) *mux.Router {
 	router := mux.NewRouter()
@@ -39,15 +45,15 @@ func newMetricsHandler(
 	router.HandleFunc("/healthcheck", func(w http.ResponseWriter, r *http.Request) {
 		_, _ = fmt.Fprintf(w, "OK\n")
 	})
-	if readyServer != nil {
-		router.Handle("/ready", readyServer)
+	if config.ReadyServer != nil {
+		router.Handle("/ready", config.ReadyServer)
 	}
 	router.HandleFunc("/quicktunnel", func(w http.ResponseWriter, r *http.Request) {
-		_, _ = fmt.Fprintf(w, `{"hostname":"%s"}`, quickTunnelHostname)
+		_, _ = fmt.Fprintf(w, `{"hostname":"%s"}`, config.QuickTunnelHostname)
 	})
-	if orchestrator != nil {
+	if config.Orchestrator != nil {
 		router.HandleFunc("/config", func(w http.ResponseWriter, r *http.Request) {
-			json, err := orchestrator.GetVersionedConfigJSON()
+			json, err := config.Orchestrator.GetVersionedConfigJSON()
 			if err != nil {
 				w.WriteHeader(500)
 				_, _ = fmt.Fprintf(w, "ERR: %v", err)
@@ -63,10 +69,8 @@ func newMetricsHandler(
 
 func ServeMetrics(
 	l net.Listener,
-	shutdownC <-chan struct{},
-	readyServer *ReadyServer,
-	quickTunnelHostname string,
-	orchestrator orchestrator,
+	ctx context.Context,
+	config Config,
 	log *zerolog.Logger,
 ) (err error) {
 	var wg sync.WaitGroup
@@ -74,7 +78,7 @@ func ServeMetrics(
 	trace.AuthRequest = func(*http.Request) (bool, bool) { return true, true }
 	// TODO: parameterize ReadTimeout and WriteTimeout. The maximum time we can
 	// profile CPU usage depends on WriteTimeout
-	h := newMetricsHandler(readyServer, quickTunnelHostname, orchestrator, log)
+	h := newMetricsHandler(config, log)
 	server := &http.Server{
 		ReadTimeout:  10 * time.Second,
 		WriteTimeout: 10 * time.Second,
@@ -91,7 +95,12 @@ func ServeMetrics(
 	// fully started up. So add artificial delay.
 	time.Sleep(startupTime)
 
-	<-shutdownC
+	<-ctx.Done()
+	shutdownTimeout := config.ShutdownTimeout
+	if shutdownTimeout == 0 {
+		shutdownTimeout = defaultShutdownTimeout
+	}
+
 	ctx, cancel := context.WithTimeout(context.Background(), shutdownTimeout)
 	_ = server.Shutdown(ctx)
 	cancel()

orchestration/orchestrator.go

@@ -27,10 +27,12 @@ type Orchestrator struct {
 	// Used by UpdateConfig to make sure one update at a time
 	lock sync.RWMutex
 	// Underlying value is proxy.Proxy, can be read without the lock, but still needs the lock to update
-	proxy  atomic.Value
-	config *Config
-	tags   []tunnelpogs.Tag
-	log    *zerolog.Logger
+	proxy atomic.Value
+	// TODO: TUN-6815 Use atomic.Bool once we upgrade to go 1.19. 1 Means enabled and 0 means disabled
+	warpRoutingEnabled uint32
+	config             *Config
+	tags               []tunnelpogs.Tag
+	log                *zerolog.Logger
 
 	// orchestrator must not handle any more updates after shutdownC is closed
 	shutdownC <-chan struct{}
@@ -122,6 +124,11 @@ func (o *Orchestrator) updateIngress(ingressRules ingress.Ingress, warpRouting i
 	o.proxy.Store(newProxy)
 	o.config.Ingress = &ingressRules
 	o.config.WarpRouting = warpRouting
+	if warpRouting.Enabled {
+		atomic.StoreUint32(&o.warpRoutingEnabled, 1)
+	} else {
+		atomic.StoreUint32(&o.warpRoutingEnabled, 0)
+	}
 
 	// If proxyShutdownC is nil, there is no previous running proxy
 	if o.proxyShutdownC != nil {
@@ -190,6 +197,14 @@ func (o *Orchestrator) GetOriginProxy() (connection.OriginProxy, error) {
 	return proxy, nil
 }
 
+// TODO: TUN-6815 consider storing WarpRouting.Enabled as atomic.Bool once we upgrade to go 1.19
+func (o *Orchestrator) WarpRoutingEnabled() (enabled bool) {
+	if atomic.LoadUint32(&o.warpRoutingEnabled) == 0 {
+		return false
+	}
+	return true
+}
+
 func (o *Orchestrator) waitToCloseLastProxy() {
 	<-o.shutdownC
 	o.lock.Lock()

orchestration/orchestrator_test.go

@@ -55,6 +55,7 @@ func TestUpdateConfiguration(t *testing.T) {
 	initOriginProxy, err := orchestrator.GetOriginProxy()
 	require.NoError(t, err)
 	require.IsType(t, &proxy.Proxy{}, initOriginProxy)
+	require.False(t, orchestrator.WarpRoutingEnabled())
 
 	configJSONV2 := []byte(`
 {
@@ -122,6 +123,7 @@ func TestUpdateConfiguration(t *testing.T) {
 	require.Equal(t, false, configV2.Ingress.Rules[2].Config.NoTLSVerify)
 	require.Equal(t, true, configV2.Ingress.Rules[2].Config.NoHappyEyeballs)
 	require.True(t, configV2.WarpRouting.Enabled)
+	require.Equal(t, configV2.WarpRouting.Enabled, orchestrator.WarpRoutingEnabled())
 	require.Equal(t, configV2.WarpRouting.ConnectTimeout.Duration, 10*time.Second)
 
 	originProxyV2, err := orchestrator.GetOriginProxy()
@@ -166,6 +168,7 @@ func TestUpdateConfiguration(t *testing.T) {
 	require.True(t, configV10.Ingress.Rules[0].Matches("blogs.tunnel.io", "/2022/02/10"))
 	require.Equal(t, ingress.HelloWorldService, configV10.Ingress.Rules[0].Service.String())
 	require.False(t, configV10.WarpRouting.Enabled)
+	require.Equal(t, configV10.WarpRouting.Enabled, orchestrator.WarpRoutingEnabled())
 
 	originProxyV10, err := orchestrator.GetOriginProxy()
 	require.NoError(t, err)

packet/decoder_test.go

@@ -152,6 +152,7 @@ func TestDecodeICMP(t *testing.T) {
 
 		require.Equal(t, test.packet.Type, icmpPacket.Type)
 		require.Equal(t, test.packet.Code, icmpPacket.Code)
+		assertICMPChecksum(t, icmpPacket)
 		require.Equal(t, test.packet.Body, icmpPacket.Body)
 		expectedBody, err := test.packet.Body.Marshal(test.packet.Type.Protocol())
 		require.NoError(t, err)

packet/funnel.go

@@ -16,11 +16,9 @@ var (
 
 // Funnel is an abstraction to pipe from 1 src to 1 or more destinations
 type Funnel interface {
-	// SendToDst sends a raw packet to a destination
-	SendToDst(dst netip.Addr, pk RawPacket) error
-	// ReturnToSrc returns a raw packet to the source
-	ReturnToSrc(pk RawPacket) error
-	// LastActive returns the last time SendToDst or ReturnToSrc is called
+	// Updates the last time traffic went through this funnel
+	UpdateLastActive()
+	// LastActive returns the last time there is traffic through this funnel
 	LastActive() time.Time
 	// Close closes the funnel. Further call to SendToDst or ReturnToSrc should return an error
 	Close() error
@@ -36,73 +34,26 @@ type FunnelUniPipe interface {
 	Close() error
 }
 
-// RawPacketFunnel is an implementation of Funnel that sends raw packets. It can be embedded in other structs to
-// satisfy the Funnel interface.
-type RawPacketFunnel struct {
-	Src netip.Addr
+type ActivityTracker struct {
 	// last active unix time. Unit is seconds
 	lastActive int64
-	sendPipe   FunnelUniPipe
-	returnPipe FunnelUniPipe
 }
 
-func NewRawPacketFunnel(src netip.Addr, sendPipe, returnPipe FunnelUniPipe) *RawPacketFunnel {
-	return &RawPacketFunnel{
-		Src:        src,
+func NewActivityTracker() *ActivityTracker {
+	return &ActivityTracker{
 		lastActive: time.Now().Unix(),
-		sendPipe:   sendPipe,
-		returnPipe: returnPipe,
 	}
 }
 
-func (rpf *RawPacketFunnel) SendToDst(dst netip.Addr, pk RawPacket) error {
-	rpf.updateLastActive()
-	return rpf.sendPipe.SendPacket(dst, pk)
+func (at *ActivityTracker) UpdateLastActive() {
+	atomic.StoreInt64(&at.lastActive, time.Now().Unix())
 }
 
-func (rpf *RawPacketFunnel) ReturnToSrc(pk RawPacket) error {
-	rpf.updateLastActive()
-	return rpf.returnPipe.SendPacket(rpf.Src, pk)
-}
-
-func (rpf *RawPacketFunnel) updateLastActive() {
-	atomic.StoreInt64(&rpf.lastActive, time.Now().Unix())
-}
-
-func (rpf *RawPacketFunnel) LastActive() time.Time {
-	lastActive := atomic.LoadInt64(&rpf.lastActive)
+func (at *ActivityTracker) LastActive() time.Time {
+	lastActive := atomic.LoadInt64(&at.lastActive)
 	return time.Unix(lastActive, 0)
 }
 
-func (rpf *RawPacketFunnel) Close() error {
-	sendPipeErr := rpf.sendPipe.Close()
-	returnPipeErr := rpf.returnPipe.Close()
-	if sendPipeErr != nil {
-		return sendPipeErr
-	}
-	if returnPipeErr != nil {
-		return returnPipeErr
-	}
-	return nil
-}
-
-func (rpf *RawPacketFunnel) Equal(other Funnel) bool {
-	otherRawFunnel, ok := other.(*RawPacketFunnel)
-	if !ok {
-		return false
-	}
-	if rpf.Src != otherRawFunnel.Src {
-		return false
-	}
-	if rpf.sendPipe != otherRawFunnel.sendPipe {
-		return false
-	}
-	if rpf.returnPipe != otherRawFunnel.returnPipe {
-		return false
-	}
-	return true
-}
-
 // FunnelID represents a key type that can be used by FunnelTracker
 type FunnelID interface {
 	// Type returns the name of the type that implements the FunnelID
@@ -157,13 +108,23 @@ func (ft *FunnelTracker) Get(id FunnelID) (Funnel, bool) {
 	return funnel, ok
 }
 
-// Registers a funnel. It replaces the current funnel.
-func (ft *FunnelTracker) GetOrRegister(id FunnelID, newFunnelFunc func() (Funnel, error)) (funnel Funnel, new bool, err error) {
+// Registers a funnel. If the `id` is already registered and `shouldReplaceFunc` returns true, it closes and replaces
+// the current funnel. If `newFunnelFunc` returns an error, the `id` will remain unregistered, even if it was registered
+// when calling this function.
+func (ft *FunnelTracker) GetOrRegister(
+	id FunnelID,
+	shouldReplaceFunc func(Funnel) bool,
+	newFunnelFunc func() (Funnel, error),
+) (funnel Funnel, new bool, err error) {
 	ft.lock.Lock()
 	defer ft.lock.Unlock()
 	currentFunnel, exists := ft.funnels[id]
 	if exists {
-		return currentFunnel, false, nil
+		if !shouldReplaceFunc(currentFunnel) {
+			return currentFunnel, false, nil
+		}
+		currentFunnel.Close()
+		delete(ft.funnels, id)
 	}
 	newFunnel, err := newFunnelFunc()
 	if err != nil {
@@ -173,7 +134,7 @@ func (ft *FunnelTracker) GetOrRegister(id FunnelID, newFunnelFunc func() (Funnel
 	return newFunnel, true, nil
 }
 
-// Unregisters a funnel if the funnel equals to the current funnel
+// Unregisters and closes a funnel if the funnel equals to the current funnel
 func (ft *FunnelTracker) Unregister(id FunnelID, funnel Funnel) (deleted bool) {
 	ft.lock.Lock()
 	defer ft.lock.Unlock()

packet/funnel_test.go

@@ -1,6 +1,13 @@
 package packet
 
-import "net/netip"
+import (
+	"fmt"
+	"net/netip"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
 
 type mockFunnelUniPipe struct {
 	uniPipe chan RawPacket
@@ -14,3 +21,118 @@ func (mfui *mockFunnelUniPipe) SendPacket(dst netip.Addr, pk RawPacket) error {
 func (mfui *mockFunnelUniPipe) Close() error {
 	return nil
 }
+
+func TestFunnelRegistration(t *testing.T) {
+	id := testFunnelID{"id1"}
+	funnelErr := fmt.Errorf("expected error")
+	newFunnelFuncErr := func() (Funnel, error) { return nil, funnelErr }
+	newFunnelFuncUncalled := func() (Funnel, error) {
+		require.FailNow(t, "a new funnel should not be created")
+		panic("unreached")
+	}
+	funnel1, newFunnelFunc1 := newFunnelAndFunc("funnel1")
+	funnel2, newFunnelFunc2 := newFunnelAndFunc("funnel2")
+
+	ft := NewFunnelTracker()
+	// Register funnel1
+	funnel, new, err := ft.GetOrRegister(id, shouldReplaceFalse, newFunnelFunc1)
+	require.NoError(t, err)
+	require.True(t, new)
+	require.Equal(t, funnel1, funnel)
+	// Register funnel, no replace
+	funnel, new, err = ft.GetOrRegister(id, shouldReplaceFalse, newFunnelFuncUncalled)
+	require.NoError(t, err)
+	require.False(t, new)
+	require.Equal(t, funnel1, funnel)
+	// Register funnel2, replace
+	funnel, new, err = ft.GetOrRegister(id, shouldReplaceTrue, newFunnelFunc2)
+	require.NoError(t, err)
+	require.True(t, new)
+	require.Equal(t, funnel2, funnel)
+	require.True(t, funnel1.closed)
+	// Register funnel error, replace
+	funnel, new, err = ft.GetOrRegister(id, shouldReplaceTrue, newFunnelFuncErr)
+	require.ErrorIs(t, err, funnelErr)
+	require.False(t, new)
+	require.Nil(t, funnel)
+	require.True(t, funnel2.closed)
+}
+
+func TestFunnelUnregister(t *testing.T) {
+	id := testFunnelID{"id1"}
+	funnel1, newFunnelFunc1 := newFunnelAndFunc("funnel1")
+	funnel2, newFunnelFunc2 := newFunnelAndFunc("funnel2")
+	funnel3, newFunnelFunc3 := newFunnelAndFunc("funnel3")
+
+	ft := NewFunnelTracker()
+	// Register & unregister
+	_, _, err := ft.GetOrRegister(id, shouldReplaceFalse, newFunnelFunc1)
+	require.NoError(t, err)
+	require.True(t, ft.Unregister(id, funnel1))
+	require.True(t, funnel1.closed)
+	require.True(t, ft.Unregister(id, funnel1))
+	// Register, replace, and unregister
+	_, _, err = ft.GetOrRegister(id, shouldReplaceFalse, newFunnelFunc2)
+	require.NoError(t, err)
+	_, _, err = ft.GetOrRegister(id, shouldReplaceTrue, newFunnelFunc3)
+	require.NoError(t, err)
+	require.True(t, funnel2.closed)
+	require.False(t, ft.Unregister(id, funnel2))
+	require.True(t, ft.Unregister(id, funnel3))
+	require.True(t, funnel3.closed)
+}
+
+func shouldReplaceFalse(_ Funnel) bool {
+	return false
+}
+
+func shouldReplaceTrue(_ Funnel) bool {
+	return true
+}
+
+func newFunnelAndFunc(id string) (*testFunnel, func() (Funnel, error)) {
+	funnel := newTestFunnel(id)
+	funnelFunc := func() (Funnel, error) {
+		return funnel, nil
+	}
+	return funnel, funnelFunc
+}
+
+type testFunnelID struct {
+	id string
+}
+
+func (t testFunnelID) Type() string {
+	return "testFunnelID"
+}
+
+func (t testFunnelID) String() string {
+	return t.id
+}
+
+type testFunnel struct {
+	id     string
+	closed bool
+}
+
+func newTestFunnel(id string) *testFunnel {
+	return &testFunnel{
+		id,
+		false,
+	}
+}
+
+func (tf *testFunnel) Close() error {
+	tf.closed = true
+	return nil
+}
+
+func (tf *testFunnel) Equal(other Funnel) bool {
+	return tf.id == other.(*testFunnel).id
+}
+
+func (tf *testFunnel) LastActive() time.Time {
+	return time.Now()
+}
+
+func (tf *testFunnel) UpdateLastActive() {}

packet/packet.go

@@ -1,6 +1,7 @@
 package packet
 
 import (
+	"encoding/binary"
 	"fmt"
 	"net/netip"
 
@@ -21,6 +22,7 @@ const (
 	// 0 = ttl exceed in transit, 1 = fragment reassembly time exceeded
 	icmpTTLExceedInTransitCode       = 0
 	DefaultTTL                 uint8 = 255
+	pseudoHeaderLen                  = 40
 )
 
 // Packet represents an IP packet or a packet that is encapsulated by IP
@@ -117,7 +119,18 @@ func (i *ICMP) EncodeLayers() ([]gopacket.SerializableLayer, error) {
 		return nil, err
 	}
 
-	msg, err := i.Marshal(nil)
+	var serializedPsh []byte = nil
+	if i.Protocol == layers.IPProtocolICMPv6 {
+		psh := &PseudoHeader{
+			SrcIP: i.Src.As16(),
+			DstIP: i.Dst.As16(),
+			// i.Marshal re-calculates the UpperLayerPacketLength
+			UpperLayerPacketLength: 0,
+			NextHeader:             uint8(i.Protocol),
+		}
+		serializedPsh = psh.Marshal()
+	}
+	msg, err := i.Marshal(serializedPsh)
 	if err != nil {
 		return nil, err
 	}
@@ -125,6 +138,29 @@ func (i *ICMP) EncodeLayers() ([]gopacket.SerializableLayer, error) {
 	return append(ipLayers, icmpLayer), nil
 }
 
+// https://www.rfc-editor.org/rfc/rfc2460#section-8.1
+type PseudoHeader struct {
+	SrcIP                  [16]byte
+	DstIP                  [16]byte
+	UpperLayerPacketLength uint32
+	zero                   [3]byte
+	NextHeader             uint8
+}
+
+func (ph *PseudoHeader) Marshal() []byte {
+	buf := make([]byte, pseudoHeaderLen)
+	index := 0
+	copy(buf, ph.SrcIP[:])
+	index += 16
+	copy(buf[index:], ph.DstIP[:])
+	index += 16
+	binary.BigEndian.PutUint32(buf[index:], ph.UpperLayerPacketLength)
+	index += 4
+	copy(buf[index:], ph.zero[:])
+	buf[pseudoHeaderLen-1] = ph.NextHeader
+	return buf
+}
+
 func NewICMPTTLExceedPacket(originalIP *IP, originalPacket RawPacket, routerIP netip.Addr) *ICMP {
 	var (
 		protocol layers.IPProtocol
@@ -137,6 +173,7 @@ func NewICMPTTLExceedPacket(originalIP *IP, originalPacket RawPacket, routerIP n
 		protocol = layers.IPProtocolICMPv6
 		icmpType = ipv6.ICMPTypeTimeExceeded
 	}
+
 	return &ICMP{
 		IP: &IP{
 			Src:      routerIP,

packet/packet_test.go

@@ -5,6 +5,7 @@ import (
 	"net/netip"
 	"testing"
 
+	"github.com/google/gopacket"
 	"github.com/google/gopacket/layers"
 	"github.com/stretchr/testify/require"
 	"golang.org/x/net/icmp"
@@ -101,4 +102,96 @@ func assertTTLExceedPacket(t *testing.T, pk *ICMP) {
 		require.Len(t, rawTTLExceedPacket.Data, headerLen+icmpHeaderLen+len(rawPacket.Data))
 		require.True(t, bytes.Equal(rawPacket.Data, rawTTLExceedPacket.Data[headerLen+icmpHeaderLen:]))
 	}
+
+	decoder := NewICMPDecoder()
+	decodedPacket, err := decoder.Decode(rawTTLExceedPacket)
+	require.NoError(t, err)
+	assertICMPChecksum(t, decodedPacket)
+}
+
+func assertICMPChecksum(t *testing.T, icmpPacket *ICMP) {
+	buf := gopacket.NewSerializeBuffer()
+	if icmpPacket.Protocol == layers.IPProtocolICMPv4 {
+		icmpv4 := layers.ICMPv4{
+			TypeCode: layers.CreateICMPv4TypeCode(uint8(icmpPacket.Type.(ipv4.ICMPType)), uint8(icmpPacket.Code)),
+		}
+		switch body := icmpPacket.Body.(type) {
+		case *icmp.Echo:
+			icmpv4.Id = uint16(body.ID)
+			icmpv4.Seq = uint16(body.Seq)
+			payload := gopacket.Payload(body.Data)
+			require.NoError(t, payload.SerializeTo(buf, serializeOpts))
+		default:
+			require.NoError(t, serializeICMPAsPayload(icmpPacket.Message, buf))
+		}
+		// SerializeTo sets the checksum in icmpv4
+		require.NoError(t, icmpv4.SerializeTo(buf, serializeOpts))
+		require.Equal(t, icmpv4.Checksum, uint16(icmpPacket.Checksum))
+	} else {
+		switch body := icmpPacket.Body.(type) {
+		case *icmp.Echo:
+			payload := gopacket.Payload(body.Data)
+			require.NoError(t, payload.SerializeTo(buf, serializeOpts))
+			echo := layers.ICMPv6Echo{
+				Identifier: uint16(body.ID),
+				SeqNumber:  uint16(body.Seq),
+			}
+			require.NoError(t, echo.SerializeTo(buf, serializeOpts))
+		default:
+			require.NoError(t, serializeICMPAsPayload(icmpPacket.Message, buf))
+		}
+
+		icmpv6 := layers.ICMPv6{
+			TypeCode: layers.CreateICMPv6TypeCode(uint8(icmpPacket.Type.(ipv6.ICMPType)), uint8(icmpPacket.Code)),
+		}
+		ipLayer := layers.IPv6{
+			Version:    6,
+			SrcIP:      icmpPacket.Src.AsSlice(),
+			DstIP:      icmpPacket.Dst.AsSlice(),
+			NextHeader: icmpPacket.Protocol,
+			HopLimit:   icmpPacket.TTL,
+		}
+		require.NoError(t, icmpv6.SetNetworkLayerForChecksum(&ipLayer))
+
+		// SerializeTo sets the checksum in icmpv4
+		require.NoError(t, icmpv6.SerializeTo(buf, serializeOpts))
+		require.Equal(t, icmpv6.Checksum, uint16(icmpPacket.Checksum))
+	}
+}
+
+func serializeICMPAsPayload(message *icmp.Message, buf gopacket.SerializeBuffer) error {
+	serializedBody, err := message.Body.Marshal(message.Type.Protocol())
+	if err != nil {
+		return err
+	}
+	payload := gopacket.Payload(serializedBody)
+	return payload.SerializeTo(buf, serializeOpts)
+}
+
+func TestChecksum(t *testing.T) {
+	data := []byte{0x63, 0x2c, 0x49, 0xd6, 0x00, 0x0d, 0xc1, 0xda}
+	pk := ICMP{
+		IP: &IP{
+			Src:      netip.MustParseAddr("2606:4700:110:89c1:c63a:861:e08c:b049"),
+			Dst:      netip.MustParseAddr("fde8:b693:d420:109b::2"),
+			Protocol: layers.IPProtocolICMPv6,
+			TTL:      3,
+		},
+		Message: &icmp.Message{
+			Type: ipv6.ICMPTypeEchoRequest,
+			Code: 0,
+			Body: &icmp.Echo{
+				ID:   0x20a7,
+				Seq:  8,
+				Data: data,
+			},
+		},
+	}
+	encoder := NewEncoder()
+	encoded, err := encoder.Encode(&pk)
+	require.NoError(t, err)
+
+	decoder := NewICMPDecoder()
+	decoded, err := decoder.Decode(encoded)
+	require.Equal(t, 0xff96, decoded.Checksum)
 }

packet/router.go

@@ -1,126 +0,0 @@
-package packet
-
-import (
-	"context"
-	"net"
-	"net/netip"
-
-	"github.com/rs/zerolog"
-)
-
-var (
-	// Source IP in documentation range to return ICMP error messages if we can't determine the IP of this machine
-	icmpv4ErrFallbackSrc = netip.MustParseAddr("192.0.2.30")
-	icmpv6ErrFallbackSrc = netip.MustParseAddr("2001:db8::")
-)
-
-// ICMPRouter sends ICMP messages and listens for their responses
-type ICMPRouter interface {
-	// Serve starts listening for responses to the requests until context is done
-	Serve(ctx context.Context) error
-	// Request sends an ICMP message
-	Request(pk *ICMP, responder FunnelUniPipe) error
-}
-
-// Upstream of raw packets
-type Upstream interface {
-	// ReceivePacket waits for the next raw packet from upstream
-	ReceivePacket(ctx context.Context) (RawPacket, error)
-}
-
-type Router struct {
-	upstream   Upstream
-	returnPipe FunnelUniPipe
-	icmpProxy  ICMPRouter
-	ipv4Src    netip.Addr
-	ipv6Src    netip.Addr
-	logger     *zerolog.Logger
-}
-
-func NewRouter(upstream Upstream, returnPipe FunnelUniPipe, icmpProxy ICMPRouter, logger *zerolog.Logger) *Router {
-	ipv4Src, err := findLocalAddr(net.ParseIP("1.1.1.1"), 53)
-	if err != nil {
-		logger.Warn().Err(err).Msgf("Failed to determine the IPv4 for this machine. It will use %s as source IP for error messages such as ICMP TTL exceed", icmpv4ErrFallbackSrc)
-		ipv4Src = icmpv4ErrFallbackSrc
-	}
-	ipv6Src, err := findLocalAddr(net.ParseIP("2606:4700:4700::1111"), 53)
-	if err != nil {
-		logger.Warn().Err(err).Msgf("Failed to determine the IPv6 for this machine. It will use %s as source IP for error messages such as ICMP TTL exceed", icmpv6ErrFallbackSrc)
-		ipv6Src = icmpv6ErrFallbackSrc
-	}
-	return &Router{
-		upstream:   upstream,
-		returnPipe: returnPipe,
-		icmpProxy:  icmpProxy,
-		ipv4Src:    ipv4Src,
-		ipv6Src:    ipv6Src,
-		logger:     logger,
-	}
-}
-
-func (r *Router) Serve(ctx context.Context) error {
-	icmpDecoder := NewICMPDecoder()
-	encoder := NewEncoder()
-	for {
-		rawPacket, err := r.upstream.ReceivePacket(ctx)
-		if err != nil {
-			return err
-		}
-		icmpPacket, err := icmpDecoder.Decode(rawPacket)
-		if err != nil {
-			r.logger.Err(err).Msg("Failed to decode ICMP packet from quic datagram")
-			continue
-		}
-
-		if icmpPacket.TTL <= 1 {
-			if err := r.sendTTLExceedMsg(icmpPacket, rawPacket, encoder); err != nil {
-				r.logger.Err(err).Msg("Failed to return ICMP TTL exceed error")
-			}
-			continue
-		}
-		icmpPacket.TTL--
-
-		if err := r.icmpProxy.Request(icmpPacket, r.returnPipe); err != nil {
-			r.logger.Err(err).
-				Str("src", icmpPacket.Src.String()).
-				Str("dst", icmpPacket.Dst.String()).
-				Interface("type", icmpPacket.Type).
-				Msg("Failed to send ICMP packet")
-			continue
-		}
-	}
-}
-
-func (r *Router) sendTTLExceedMsg(pk *ICMP, rawPacket RawPacket, encoder *Encoder) error {
-	var srcIP netip.Addr
-	if pk.Dst.Is4() {
-		srcIP = r.ipv4Src
-	} else {
-		srcIP = r.ipv6Src
-	}
-	ttlExceedPacket := NewICMPTTLExceedPacket(pk.IP, rawPacket, srcIP)
-
-	encodedTTLExceed, err := encoder.Encode(ttlExceedPacket)
-	if err != nil {
-		return err
-	}
-	return r.returnPipe.SendPacket(pk.Src, encodedTTLExceed)
-}
-
-// findLocalAddr tries to dial UDP and returns the local address picked by the OS
-func findLocalAddr(dst net.IP, port int) (netip.Addr, error) {
-	udpConn, err := net.DialUDP("udp", nil, &net.UDPAddr{
-		IP:   dst,
-		Port: port,
-	})
-	if err != nil {
-		return netip.Addr{}, err
-	}
-	defer udpConn.Close()
-	localAddrPort, err := netip.ParseAddrPort(udpConn.LocalAddr().String())
-	if err != nil {
-		return netip.Addr{}, err
-	}
-	localAddr := localAddrPort.Addr()
-	return localAddr, nil
-}

packet/router_test.go

@@ -1,125 +0,0 @@
-package packet
-
-import (
-	"bytes"
-	"context"
-	"fmt"
-	"net/netip"
-	"testing"
-
-	"github.com/google/gopacket/layers"
-	"github.com/rs/zerolog"
-	"github.com/stretchr/testify/require"
-	"golang.org/x/net/icmp"
-	"golang.org/x/net/ipv4"
-	"golang.org/x/net/ipv6"
-)
-
-var (
-	noopLogger = zerolog.Nop()
-)
-
-func TestRouterReturnTTLExceed(t *testing.T) {
-	upstream := &mockUpstream{
-		source: make(chan RawPacket),
-	}
-	returnPipe := &mockFunnelUniPipe{
-		uniPipe: make(chan RawPacket),
-	}
-	router := NewRouter(upstream, returnPipe, &mockICMPRouter{}, &noopLogger)
-	ctx, cancel := context.WithCancel(context.Background())
-	routerStopped := make(chan struct{})
-	go func() {
-		router.Serve(ctx)
-		close(routerStopped)
-	}()
-
-	pk := ICMP{
-		IP: &IP{
-			Src:      netip.MustParseAddr("192.168.1.1"),
-			Dst:      netip.MustParseAddr("10.0.0.1"),
-			Protocol: layers.IPProtocolICMPv4,
-			TTL:      1,
-		},
-		Message: &icmp.Message{
-			Type: ipv4.ICMPTypeEcho,
-			Code: 0,
-			Body: &icmp.Echo{
-				ID:   12481,
-				Seq:  8036,
-				Data: []byte("TTL exceed"),
-			},
-		},
-	}
-	assertTTLExceed(t, &pk, router.ipv4Src, upstream, returnPipe)
-	pk = ICMP{
-		IP: &IP{
-			Src:      netip.MustParseAddr("fd51:2391:523:f4ee::1"),
-			Dst:      netip.MustParseAddr("fd51:2391:697:f4ee::2"),
-			Protocol: layers.IPProtocolICMPv6,
-			TTL:      1,
-		},
-		Message: &icmp.Message{
-			Type: ipv6.ICMPTypeEchoRequest,
-			Code: 0,
-			Body: &icmp.Echo{
-				ID:   42583,
-				Seq:  7039,
-				Data: []byte("TTL exceed"),
-			},
-		},
-	}
-	assertTTLExceed(t, &pk, router.ipv6Src, upstream, returnPipe)
-
-	cancel()
-	<-routerStopped
-}
-
-func assertTTLExceed(t *testing.T, originalPacket *ICMP, expectedSrc netip.Addr, upstream *mockUpstream, returnPipe *mockFunnelUniPipe) {
-	encoder := NewEncoder()
-	rawPacket, err := encoder.Encode(originalPacket)
-	require.NoError(t, err)
-
-	upstream.source <- rawPacket
-	resp := <-returnPipe.uniPipe
-	decoder := NewICMPDecoder()
-	decoded, err := decoder.Decode(resp)
-	require.NoError(t, err)
-
-	require.Equal(t, expectedSrc, decoded.Src)
-	require.Equal(t, originalPacket.Src, decoded.Dst)
-	require.Equal(t, originalPacket.Protocol, decoded.Protocol)
-	require.Equal(t, DefaultTTL, decoded.TTL)
-	if originalPacket.Dst.Is4() {
-		require.Equal(t, ipv4.ICMPTypeTimeExceeded, decoded.Type)
-	} else {
-		require.Equal(t, ipv6.ICMPTypeTimeExceeded, decoded.Type)
-	}
-	require.Equal(t, 0, decoded.Code)
-	timeExceed, ok := decoded.Body.(*icmp.TimeExceeded)
-	require.True(t, ok)
-	require.True(t, bytes.Equal(rawPacket.Data, timeExceed.Data))
-}
-
-type mockUpstream struct {
-	source chan RawPacket
-}
-
-func (ms *mockUpstream) ReceivePacket(ctx context.Context) (RawPacket, error) {
-	select {
-	case <-ctx.Done():
-		return RawPacket{}, ctx.Err()
-	case pk := <-ms.source:
-		return pk, nil
-	}
-}
-
-type mockICMPRouter struct{}
-
-func (mir mockICMPRouter) Serve(ctx context.Context) error {
-	return fmt.Errorf("Serve not implemented by mockICMPRouter")
-}
-
-func (mir mockICMPRouter) Request(pk *ICMP, responder FunnelUniPipe) error {
-	return fmt.Errorf("Request not implemented by mockICMPRouter")
-}

proxy/proxy.go

@@ -60,6 +60,21 @@ func NewOriginProxy(
 	return proxy
 }
 
+func (p *Proxy) applyIngressMiddleware(rule *ingress.Rule, r *http.Request, w connection.ResponseWriter) (error, bool) {
+	for _, handler := range rule.Handlers {
+		result, err := handler.Handle(r.Context(), r)
+		if err != nil {
+			return errors.Wrap(err, fmt.Sprintf("error while processing middleware handler %s", handler.Name())), false
+		}
+
+		if result.ShouldFilterRequest {
+			w.WriteRespHeaders(result.StatusCode, nil)
+			return fmt.Errorf("request filtered by middleware handler (%s) due to: %s", handler.Name(), result.Reason), true
+		}
+	}
+	return nil, true
+}
+
 // ProxyHTTP further depends on ingress rules to establish a connection with the origin service. This may be
 // a simple roundtrip or a tcp/websocket dial depending on ingres rule setup.
 func (p *Proxy) ProxyHTTP(
@@ -86,6 +101,14 @@ func (p *Proxy) ProxyHTTP(
 	p.logRequest(req, logFields)
 	ruleSpan.SetAttributes(attribute.Int("rule-num", ruleNum))
 	ruleSpan.End()
+	if err, applied := p.applyIngressMiddleware(rule, req, w); err != nil {
+		if applied {
+			rule, srv := ruleField(p.ingressRules, ruleNum)
+			p.logRequestError(err, cfRay, "", rule, srv)
+			return nil
+		}
+		return err
+	}
 
 	switch originProxy := rule.Service.(type) {
 	case ingress.HTTPOriginProxy:

quic/datagram.go

@@ -49,7 +49,7 @@ func (dm *DatagramMuxer) SendToSession(session *packet.Session) error {
 		packetTooBigDropped.Inc()
 		return fmt.Errorf("origin UDP payload has %d bytes, which exceeds transport MTU %d", len(session.Payload), dm.mtu())
 	}
-	payloadWithMetadata, err := suffixSessionID(session.ID, session.Payload)
+	payloadWithMetadata, err := SuffixSessionID(session.ID, session.Payload)
 	if err != nil {
 		return errors.Wrap(err, "Failed to suffix session ID to datagram, it will be dropped")
 	}
@@ -112,10 +112,13 @@ func extractSessionID(b []byte) (uuid.UUID, []byte, error) {
 
 // SuffixSessionID appends the session ID at the end of the payload. Suffix is more performant than prefix because
 // the payload slice might already have enough capacity to append the session ID at the end
-func suffixSessionID(sessionID uuid.UUID, b []byte) ([]byte, error) {
-	if len(b)+len(sessionID) > MaxDatagramFrameSize {
+func SuffixSessionID(sessionID uuid.UUID, b []byte) ([]byte, error) {
+	return suffixMetadata(b, sessionID[:])
+}
+
+func suffixMetadata(payload, metadata []byte) ([]byte, error) {
+	if len(payload)+len(metadata) > MaxDatagramFrameSize {
 		return nil, fmt.Errorf("datagram size exceed %d", MaxDatagramFrameSize)
 	}
-	b = append(b, sessionID[:]...)
-	return b, nil
+	return append(payload, metadata...), nil
 }

quic/datagram_test.go

@@ -1,6 +1,7 @@
 package quic
 
 import (
+	"bytes"
 	"context"
 	"crypto/rand"
 	"crypto/rsa"
@@ -23,6 +24,7 @@ import (
 	"golang.org/x/sync/errgroup"
 
 	"github.com/cloudflare/cloudflared/packet"
+	"github.com/cloudflare/cloudflared/tracing"
 )
 
 var (
@@ -31,7 +33,7 @@ var (
 
 func TestSuffixThenRemoveSessionID(t *testing.T) {
 	msg := []byte(t.Name())
-	msgWithID, err := suffixSessionID(testSessionID, msg)
+	msgWithID, err := SuffixSessionID(testSessionID, msg)
 	require.NoError(t, err)
 	require.Len(t, msgWithID, len(msg)+sessionIDLen)
 
@@ -50,11 +52,11 @@ func TestRemoveSessionIDError(t *testing.T) {
 
 func TestSuffixSessionIDError(t *testing.T) {
 	msg := make([]byte, MaxDatagramFrameSize-sessionIDLen)
-	_, err := suffixSessionID(testSessionID, msg)
+	_, err := SuffixSessionID(testSessionID, msg)
 	require.NoError(t, err)
 
 	msg = make([]byte, MaxDatagramFrameSize-sessionIDLen+1)
-	_, err = suffixSessionID(testSessionID, msg)
+	_, err = SuffixSessionID(testSessionID, msg)
 	require.Error(t, err)
 }
 
@@ -121,6 +123,15 @@ func testDatagram(t *testing.T, version uint8, sessionToPayloads []*packet.Sessi
 
 	logger := zerolog.Nop()
 
+	tracingIdentity, err := tracing.NewIdentity("ec31ad8a01fde11fdcabe2efdce36873:52726f6cabc144f5:0:1")
+	require.NoError(t, err)
+	serializedTracingID, err := tracingIdentity.MarshalBinary()
+	require.NoError(t, err)
+	tracingSpan := &TracingSpanPacket{
+		Spans:           []byte("tracing"),
+		TracingIdentity: serializedTracingID,
+	}
+
 	errGroup, ctx := errgroup.WithContext(context.Background())
 	// Run edge side of datagram muxer
 	errGroup.Go(func() error {
@@ -140,18 +151,17 @@ func testDatagram(t *testing.T, version uint8, sessionToPayloads []*packet.Sessi
 			muxer := NewDatagramMuxerV2(quicSession, &logger, sessionDemuxChan)
 			muxer.ServeReceive(ctx)
 
-			icmpDecoder := packet.NewICMPDecoder()
 			for _, pk := range packets {
 				received, err := muxer.ReceivePacket(ctx)
 				require.NoError(t, err)
-
-				receivedICMP, err := icmpDecoder.Decode(received)
+				validateIPPacket(t, received, &pk)
+				received, err = muxer.ReceivePacket(ctx)
 				require.NoError(t, err)
-				require.Equal(t, pk.IP, receivedICMP.IP)
-				require.Equal(t, pk.Type, receivedICMP.Type)
-				require.Equal(t, pk.Code, receivedICMP.Code)
-				require.Equal(t, pk.Body, receivedICMP.Body)
+				validateIPPacketWithTracing(t, received, &pk, serializedTracingID)
 			}
+			received, err := muxer.ReceivePacket(ctx)
+			require.NoError(t, err)
+			validateTracingSpans(t, received, tracingSpan)
 		default:
 			return fmt.Errorf("unknown datagram version %d", version)
 		}
@@ -188,10 +198,15 @@ func testDatagram(t *testing.T, version uint8, sessionToPayloads []*packet.Sessi
 			for _, pk := range packets {
 				encodedPacket, err := encoder.Encode(&pk)
 				require.NoError(t, err)
-				require.NoError(t, muxerV2.SendPacket(encodedPacket))
+				require.NoError(t, muxerV2.SendPacket(RawPacket(encodedPacket)))
+				require.NoError(t, muxerV2.SendPacket(&TracedPacket{
+					Packet:          encodedPacket,
+					TracingIdentity: serializedTracingID,
+				}))
 			}
+			require.NoError(t, muxerV2.SendPacket(tracingSpan))
 			// Payload larger than transport MTU, should not be sent
-			require.Error(t, muxerV2.SendPacket(packet.RawPacket{
+			require.Error(t, muxerV2.SendPacket(RawPacket{
 				Data: largePayload,
 			}))
 			muxer = muxerV2
@@ -217,6 +232,38 @@ func testDatagram(t *testing.T, version uint8, sessionToPayloads []*packet.Sessi
 	require.NoError(t, errGroup.Wait())
 }
 
+func validateIPPacket(t *testing.T, receivedPacket Packet, expectedICMP *packet.ICMP) {
+	require.Equal(t, DatagramTypeIP, receivedPacket.Type())
+	rawPacket := receivedPacket.(RawPacket)
+	decoder := packet.NewICMPDecoder()
+	receivedICMP, err := decoder.Decode(packet.RawPacket(rawPacket))
+	require.NoError(t, err)
+	validateICMP(t, expectedICMP, receivedICMP)
+}
+
+func validateIPPacketWithTracing(t *testing.T, receivedPacket Packet, expectedICMP *packet.ICMP, serializedTracingID []byte) {
+	require.Equal(t, DatagramTypeIPWithTrace, receivedPacket.Type())
+	tracedPacket := receivedPacket.(*TracedPacket)
+	decoder := packet.NewICMPDecoder()
+	receivedICMP, err := decoder.Decode(tracedPacket.Packet)
+	require.NoError(t, err)
+	validateICMP(t, expectedICMP, receivedICMP)
+	require.True(t, bytes.Equal(tracedPacket.TracingIdentity, serializedTracingID))
+}
+
+func validateICMP(t *testing.T, expected, actual *packet.ICMP) {
+	require.Equal(t, expected.IP, actual.IP)
+	require.Equal(t, expected.Type, actual.Type)
+	require.Equal(t, expected.Code, actual.Code)
+	require.Equal(t, expected.Body, actual.Body)
+}
+
+func validateTracingSpans(t *testing.T, receivedPacket Packet, expectedSpan *TracingSpanPacket) {
+	require.Equal(t, DatagramTypeTracingSpan, receivedPacket.Type())
+	tracingSpans := receivedPacket.(*TracingSpanPacket)
+	require.Equal(t, tracingSpans, expectedSpan)
+}
+
 func newQUICListener(t *testing.T, config *quic.Config) quic.Listener {
 	// Create a simple tls config.
 	tlsConfig := generateTLSConfig()

quic/datagramv2.go

@@ -9,22 +9,35 @@ import (
 	"github.com/rs/zerolog"
 
 	"github.com/cloudflare/cloudflared/packet"
+	"github.com/cloudflare/cloudflared/tracing"
 )
 
-type datagramV2Type byte
+type DatagramV2Type byte
 
 const (
-	udp datagramV2Type = iota
-	ip
+	// UDP payload
+	DatagramTypeUDP DatagramV2Type = iota
+	// Full IP packet
+	DatagramTypeIP
+	// DatagramTypeIP + tracing ID
+	DatagramTypeIPWithTrace
+	// Tracing spans in protobuf format
+	DatagramTypeTracingSpan
 )
 
+type Packet interface {
+	Type() DatagramV2Type
+	Payload() []byte
+	Metadata() []byte
+}
+
 const (
 	typeIDLen = 1
 	// Same as sessionDemuxChan capacity
 	packetChanCapacity = 128
 )
 
-func suffixType(b []byte, datagramType datagramV2Type) ([]byte, error) {
+func SuffixType(b []byte, datagramType DatagramV2Type) ([]byte, error) {
 	if len(b)+typeIDLen > MaxDatagramFrameSize {
 		return nil, fmt.Errorf("datagram size %d exceeds max frame size %d", len(b), MaxDatagramFrameSize)
 	}
@@ -41,7 +54,7 @@ type DatagramMuxerV2 struct {
 	session          quic.Connection
 	logger           *zerolog.Logger
 	sessionDemuxChan chan<- *packet.Session
-	packetDemuxChan  chan packet.RawPacket
+	packetDemuxChan  chan Packet
 }
 
 func NewDatagramMuxerV2(
@@ -54,7 +67,7 @@ func NewDatagramMuxerV2(
 		session:          quicSession,
 		logger:           &logger,
 		sessionDemuxChan: sessionDemuxChan,
-		packetDemuxChan:  make(chan packet.RawPacket, packetChanCapacity),
+		packetDemuxChan:  make(chan Packet, packetChanCapacity),
 	}
 }
 
@@ -65,11 +78,11 @@ func (dm *DatagramMuxerV2) SendToSession(session *packet.Session) error {
 		packetTooBigDropped.Inc()
 		return fmt.Errorf("origin UDP payload has %d bytes, which exceeds transport MTU %d", len(session.Payload), dm.mtu())
 	}
-	msgWithID, err := suffixSessionID(session.ID, session.Payload)
+	msgWithID, err := SuffixSessionID(session.ID, session.Payload)
 	if err != nil {
 		return errors.Wrap(err, "Failed to suffix session ID to datagram, it will be dropped")
 	}
-	msgWithIDAndType, err := suffixType(msgWithID, udp)
+	msgWithIDAndType, err := SuffixType(msgWithID, DatagramTypeUDP)
 	if err != nil {
 		return errors.Wrap(err, "Failed to suffix datagram type, it will be dropped")
 	}
@@ -79,14 +92,19 @@ func (dm *DatagramMuxerV2) SendToSession(session *packet.Session) error {
 	return nil
 }
 
-// SendPacket suffix the datagram type to the packet. The other end of the QUIC connection can demultiplex by parsing
-// the payload as IP and look at the source and destination.
-func (dm *DatagramMuxerV2) SendPacket(pk packet.RawPacket) error {
-	payloadWithVersion, err := suffixType(pk.Data, ip)
+// SendPacket sends a packet with datagram version in the suffix. If ctx is a TracedContext, it adds the tracing
+// context between payload and datagram version.
+// The other end of the QUIC connection can demultiplex by parsing the payload as IP and look at the source and destination.
+func (dm *DatagramMuxerV2) SendPacket(pk Packet) error {
+	payloadWithMetadata, err := suffixMetadata(pk.Payload(), pk.Metadata())
+	if err != nil {
+		return err
+	}
+	payloadWithMetadataAndType, err := SuffixType(payloadWithMetadata, pk.Type())
 	if err != nil {
 		return errors.Wrap(err, "Failed to suffix datagram type, it will be dropped")
 	}
-	if err := dm.session.SendMessage(payloadWithVersion); err != nil {
+	if err := dm.session.SendMessage(payloadWithMetadataAndType); err != nil {
 		return errors.Wrap(err, "Failed to send datagram back to edge")
 	}
 	return nil
@@ -108,10 +126,10 @@ func (dm *DatagramMuxerV2) ServeReceive(ctx context.Context) error {
 	}
 }
 
-func (dm *DatagramMuxerV2) ReceivePacket(ctx context.Context) (packet.RawPacket, error) {
+func (dm *DatagramMuxerV2) ReceivePacket(ctx context.Context) (pk Packet, err error) {
 	select {
 	case <-ctx.Done():
-		return packet.RawPacket{}, ctx.Err()
+		return nil, ctx.Err()
 	case pk := <-dm.packetDemuxChan:
 		return pk, nil
 	}
@@ -121,15 +139,13 @@ func (dm *DatagramMuxerV2) demux(ctx context.Context, msgWithType []byte) error
 	if len(msgWithType) < typeIDLen {
 		return fmt.Errorf("QUIC datagram should have at least %d byte", typeIDLen)
 	}
-	msgType := datagramV2Type(msgWithType[len(msgWithType)-typeIDLen])
+	msgType := DatagramV2Type(msgWithType[len(msgWithType)-typeIDLen])
 	msg := msgWithType[0 : len(msgWithType)-typeIDLen]
 	switch msgType {
-	case udp:
+	case DatagramTypeUDP:
 		return dm.handleSession(ctx, msg)
-	case ip:
-		return dm.handlePacket(ctx, msg)
 	default:
-		return fmt.Errorf("Unexpected datagram type %d", msgType)
+		return dm.handlePacket(ctx, msg, msgType)
 	}
 }
 
@@ -150,13 +166,93 @@ func (dm *DatagramMuxerV2) handleSession(ctx context.Context, session []byte) er
 	}
 }
 
-func (dm *DatagramMuxerV2) handlePacket(ctx context.Context, pk []byte) error {
+func (dm *DatagramMuxerV2) handlePacket(ctx context.Context, pk []byte, msgType DatagramV2Type) error {
+	var demuxedPacket Packet
+	switch msgType {
+	case DatagramTypeIP:
+		demuxedPacket = RawPacket(packet.RawPacket{Data: pk})
+	case DatagramTypeIPWithTrace:
+		tracingIdentity, payload, err := extractTracingIdentity(pk)
+		if err != nil {
+			return err
+		}
+		demuxedPacket = &TracedPacket{
+			Packet:          packet.RawPacket{Data: payload},
+			TracingIdentity: tracingIdentity,
+		}
+	case DatagramTypeTracingSpan:
+		tracingIdentity, spans, err := extractTracingIdentity(pk)
+		if err != nil {
+			return err
+		}
+		demuxedPacket = &TracingSpanPacket{
+			Spans:           spans,
+			TracingIdentity: tracingIdentity,
+		}
+	default:
+		return fmt.Errorf("Unexpected datagram type %d", msgType)
+	}
 	select {
 	case <-ctx.Done():
 		return ctx.Err()
-	case dm.packetDemuxChan <- packet.RawPacket{
-		Data: pk,
-	}:
+	case dm.packetDemuxChan <- demuxedPacket:
 		return nil
 	}
 }
+
+func extractTracingIdentity(pk []byte) (tracingIdentity []byte, payload []byte, err error) {
+	if len(pk) < tracing.IdentityLength {
+		return nil, nil, fmt.Errorf("packet with tracing context should have at least %d bytes, got %v", tracing.IdentityLength, pk)
+	}
+	tracingIdentity = pk[len(pk)-tracing.IdentityLength:]
+	payload = pk[:len(pk)-tracing.IdentityLength]
+	return tracingIdentity, payload, nil
+}
+
+type RawPacket packet.RawPacket
+
+func (rw RawPacket) Type() DatagramV2Type {
+	return DatagramTypeIP
+}
+
+func (rw RawPacket) Payload() []byte {
+	return rw.Data
+}
+
+func (rw RawPacket) Metadata() []byte {
+	return []byte{}
+}
+
+type TracedPacket struct {
+	Packet          packet.RawPacket
+	TracingIdentity []byte
+}
+
+func (tp *TracedPacket) Type() DatagramV2Type {
+	return DatagramTypeIPWithTrace
+}
+
+func (tp *TracedPacket) Payload() []byte {
+	return tp.Packet.Data
+}
+
+func (tp *TracedPacket) Metadata() []byte {
+	return tp.TracingIdentity
+}
+
+type TracingSpanPacket struct {
+	Spans           []byte
+	TracingIdentity []byte
+}
+
+func (tsp *TracingSpanPacket) Type() DatagramV2Type {
+	return DatagramTypeTracingSpan
+}
+
+func (tsp *TracingSpanPacket) Payload() []byte {
+	return tsp.Spans
+}
+
+func (tsp *TracingSpanPacket) Metadata() []byte {
+	return tsp.TracingIdentity
+}

supervisor/supervisor.go

@@ -16,7 +16,6 @@ import (
 	"github.com/cloudflare/cloudflared/edgediscovery"
 	"github.com/cloudflare/cloudflared/edgediscovery/allregions"
 	"github.com/cloudflare/cloudflared/h2mux"
-	"github.com/cloudflare/cloudflared/ingress"
 	"github.com/cloudflare/cloudflared/orchestration"
 	"github.com/cloudflare/cloudflared/retry"
 	"github.com/cloudflare/cloudflared/signal"
@@ -46,7 +45,7 @@ type Supervisor struct {
 	config                  *TunnelConfig
 	orchestrator            *orchestration.Orchestrator
 	edgeIPs                 *edgediscovery.Edge
-	edgeTunnelServer        *EdgeTunnelServer
+	edgeTunnelServer        TunnelServer
 	tunnelErrors            chan tunnelError
 	tunnelsConnecting       map[int]chan struct{}
 	tunnelsProtocolFallback map[int]*protocolFallback
@@ -117,13 +116,6 @@ func NewSupervisor(config *TunnelConfig, orchestrator *orchestration.Orchestrato
 		connAwareLogger:   log,
 	}
 
-	icmpRouter, err := ingress.NewICMPRouter(config.Log)
-	if err != nil {
-		log.Logger().Warn().Err(err).Msg("Failed to create icmp router, ICMP proxy feature is disabled")
-	} else {
-		edgeTunnelServer.icmpRouter = icmpRouter
-	}
-
 	useReconnectToken := false
 	if config.ClassicTunnel != nil {
 		useReconnectToken = config.ClassicTunnel.UseReconnectToken
@@ -151,9 +143,9 @@ func (s *Supervisor) Run(
 	ctx context.Context,
 	connectedSignal *signal.Signal,
 ) error {
-	if s.edgeTunnelServer.icmpRouter != nil {
+	if s.config.PacketConfig != nil {
 		go func() {
-			if err := s.edgeTunnelServer.icmpRouter.Serve(ctx); err != nil {
+			if err := s.config.PacketConfig.ICMPRouter.Serve(ctx); err != nil {
 				if errors.Is(err, net.ErrClosed) {
 					s.log.Logger().Info().Err(err).Msg("icmp router terminated")
 				} else {
@@ -293,7 +285,8 @@ func (s *Supervisor) initialize(
 	for i := 1; i < s.config.HAConnections; i++ {
 		s.tunnelsProtocolFallback[i] = &protocolFallback{
 			retry.BackoffHandler{MaxRetries: s.config.Retries, RetryForever: true},
-			s.config.ProtocolSelector.Current(),
+			// Set the protocol we know the first tunnel connected with.
+			s.tunnelsProtocolFallback[0].protocol,
 			false,
 		}
 		go s.startTunnel(ctx, i, s.newConnectedTunnelSignal(i))

supervisor/supervisor_test.go

@@ -0,0 +1,74 @@
+package supervisor
+
+import (
+	"context"
+	"testing"
+
+	"github.com/rs/zerolog"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/cloudflare/cloudflared/connection"
+	"github.com/cloudflare/cloudflared/edgediscovery"
+	"github.com/cloudflare/cloudflared/edgediscovery/allregions"
+	"github.com/cloudflare/cloudflared/signal"
+)
+
+type mockProtocolSelector struct {
+	protocols []connection.Protocol
+	index     int
+}
+
+func (m *mockProtocolSelector) Current() connection.Protocol {
+	return m.protocols[m.index]
+}
+
+func (m *mockProtocolSelector) Fallback() (connection.Protocol, bool) {
+	m.index++
+	if m.index == len(m.protocols) {
+		return m.protocols[len(m.protocols)-1], false
+	}
+
+	return m.protocols[m.index], true
+}
+
+type mockEdgeTunnelServer struct {
+	config *TunnelConfig
+}
+
+func (m *mockEdgeTunnelServer) Serve(ctx context.Context, connIndex uint8, protocolFallback *protocolFallback, connectedSignal *signal.Signal) error {
+	// This is to mock the first connection falling back because of connectivity issues.
+	protocolFallback.protocol, _ = m.config.ProtocolSelector.Fallback()
+	connectedSignal.Notify()
+	return nil
+}
+
+// Test to check if initialize sets all the different connections to the same protocol should the first
+// tunnel fall back.
+func Test_Initialize_Same_Protocol(t *testing.T) {
+	edgeIPs, err := edgediscovery.ResolveEdge(&zerolog.Logger{}, "us", allregions.Auto)
+	assert.Nil(t, err)
+	s := Supervisor{
+		edgeIPs: edgeIPs,
+		config: &TunnelConfig{
+			ProtocolSelector: &mockProtocolSelector{protocols: []connection.Protocol{connection.QUIC, connection.HTTP2, connection.H2mux}},
+		},
+		tunnelsProtocolFallback: make(map[int]*protocolFallback),
+		edgeTunnelServer: &mockEdgeTunnelServer{
+			config: &TunnelConfig{
+				ProtocolSelector: &mockProtocolSelector{protocols: []connection.Protocol{connection.QUIC, connection.HTTP2, connection.H2mux}},
+			},
+		},
+	}
+
+	ctx := context.Background()
+	connectedSignal := signal.New(make(chan struct{}))
+	s.initialize(ctx, connectedSignal)
+
+	// Make sure we fell back to http2 as the mock Serve is wont to do.
+	assert.Equal(t, s.tunnelsProtocolFallback[0].protocol, connection.HTTP2)
+
+	// Ensure all the protocols we set to try are the same as what the first tunnel has fallen back to.
+	for _, protocolFallback := range s.tunnelsProtocolFallback {
+		assert.Equal(t, protocolFallback.protocol, s.tunnelsProtocolFallback[0].protocol)
+	}
+}

supervisor/tunnel.go

@@ -20,8 +20,8 @@ import (
 	"github.com/cloudflare/cloudflared/edgediscovery"
 	"github.com/cloudflare/cloudflared/edgediscovery/allregions"
 	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/ingress"
 	"github.com/cloudflare/cloudflared/orchestration"
-	"github.com/cloudflare/cloudflared/packet"
 	quicpogs "github.com/cloudflare/cloudflared/quic"
 	"github.com/cloudflare/cloudflared/retry"
 	"github.com/cloudflare/cloudflared/signal"
@@ -37,6 +37,7 @@ const (
 	FeatureAllowRemoteConfig = "allow_remote_config"
 	FeatureDatagramV2        = "support_datagram_v2"
 	FeaturePostQuantum       = "postquantum"
+	FeatureQUICSupportEOF    = "support_quic_eof"
 )
 
 type TunnelConfig struct {
@@ -70,6 +71,7 @@ type TunnelConfig struct {
 	MuxerConfig      *connection.MuxerConfig
 	ProtocolSelector connection.ProtocolSelector
 	EdgeTLSConfigs   map[connection.Protocol]*tls.Config
+	PacketConfig     *ingress.GlobalRouterConfig
 }
 
 func (c *TunnelConfig) registrationOptions(connectionID uint8, OriginLocalIP string, uuid uuid.UUID) *tunnelpogs.RegistrationOptions {
@@ -200,11 +202,14 @@ type EdgeTunnelServer struct {
 	reconnectCh       chan ReconnectSignal
 	gracefulShutdownC <-chan struct{}
 	tracker           *tunnelstate.ConnTracker
-	icmpRouter        packet.ICMPRouter
 
 	connAwareLogger *ConnAwareLogger
 }
 
+type TunnelServer interface {
+	Serve(ctx context.Context, connIndex uint8, protocolFallback *protocolFallback, connectedSignal *signal.Signal) error
+}
+
 func (e *EdgeTunnelServer) Serve(ctx context.Context, connIndex uint8, protocolFallback *protocolFallback, connectedSignal *signal.Signal) error {
 	haConnections.Inc()
 	defer haConnections.Dec()
@@ -254,8 +259,9 @@ func (e *EdgeTunnelServer) Serve(ctx context.Context, connIndex uint8, protocolF
 		if !ok {
 			return err
 		}
+
 		e.config.Observer.SendReconnect(connIndex)
-		connLog.Logger().Info().Msgf("Retrying connection in up to %s seconds", duration)
+		connLog.Logger().Info().Msgf("Retrying connection in up to %s", duration)
 	}
 
 	// Check if the connection error was from an IP issue with the host or
@@ -656,12 +662,13 @@ func (e *EdgeTunnelServer) serveQUIC(
 	quicConn, err := connection.NewQUICConnection(
 		quicConfig,
 		edgeAddr,
+		connIndex,
 		tlsConfig,
 		e.orchestrator,
 		connOptions,
 		controlStreamHandler,
 		connLogger.Logger(),
-		e.icmpRouter)
+		e.config.PacketConfig)
 	if err != nil {
 		if e.config.NeedPQ {
 			handlePQTunnelError(err, e.config)

token/encrypt.go

@@ -3,26 +3,26 @@
 // tldr is it uses Elliptic Curves (Curve25519) for the keys, XSalsa20 and Poly1305 for encryption.
 // You can read more here https://godoc.org/golang.org/x/crypto/nacl/box.
 //
-//		msg := []byte("super safe message.")
-//		alice, err := NewEncrypter("alice_priv_key.pem", "alice_pub_key.pem")
-//		if err != nil {
-//			log.Fatal(err)
-//		}
+//	msg := []byte("super safe message.")
+//	alice, err := NewEncrypter("alice_priv_key.pem", "alice_pub_key.pem")
+//	if err != nil {
+//		log.Fatal(err)
+//	}
 //
-//		bob, err := NewEncrypter("bob_priv_key.pem", "bob_pub_key.pem")
-//		if err != nil {
-//			log.Fatal(err)
-//		}
-//		encrypted, err := alice.Encrypt(msg, bob.PublicKey())
-//		if err != nil {
-//			log.Fatal(err)
-//		}
+//	bob, err := NewEncrypter("bob_priv_key.pem", "bob_pub_key.pem")
+//	if err != nil {
+//		log.Fatal(err)
+//	}
+//	encrypted, err := alice.Encrypt(msg, bob.PublicKey())
+//	if err != nil {
+//		log.Fatal(err)
+//	}
 //
-//		data, err := bob.Decrypt(encrypted, alice.PublicKey())
-//		if err != nil {
-//			log.Fatal(err)
-//		}
-//		fmt.Println(string(data))
+//	data, err := bob.Decrypt(encrypted, alice.PublicKey())
+//	if err != nil {
+//		log.Fatal(err)
+//	}
+//	fmt.Println(string(data))
 package token
 
 import (

tracing/client.go

@@ -24,9 +24,9 @@ type InMemoryClient interface {
 	// Spans returns a copy of the list of in-memory stored spans as a base64
 	// encoded otlp protobuf string.
 	Spans() (string, error)
-	// ProtoSpans returns a copy of the list of in-memory stored spans as otlp
-	// protobuf byte array.
-	ProtoSpans() ([]byte, error)
+	// ExportProtoSpans returns a copy of the list of in-memory stored spans as otlp
+	// protobuf byte array and clears the in-memory spans.
+	ExportProtoSpans() ([]byte, error)
 }
 
 // InMemoryOtlpClient is a client implementation for otlptrace.Client
@@ -58,7 +58,7 @@ func (mc *InMemoryOtlpClient) UploadTraces(_ context.Context, protoSpans []*trac
 
 // Spans returns the list of in-memory stored spans as a base64 encoded otlp protobuf string.
 func (mc *InMemoryOtlpClient) Spans() (string, error) {
-	data, err := mc.ProtoSpans()
+	data, err := mc.ExportProtoSpans()
 	if err != nil {
 		return "", err
 	}
@@ -66,7 +66,7 @@ func (mc *InMemoryOtlpClient) Spans() (string, error) {
 }
 
 // ProtoSpans returns the list of in-memory stored spans as the protobuf byte array.
-func (mc *InMemoryOtlpClient) ProtoSpans() ([]byte, error) {
+func (mc *InMemoryOtlpClient) ExportProtoSpans() ([]byte, error) {
 	mc.mu.Lock()
 	defer mc.mu.Unlock()
 	if len(mc.spans) <= 0 {
@@ -75,7 +75,12 @@ func (mc *InMemoryOtlpClient) ProtoSpans() ([]byte, error) {
 	pbRequest := &coltracepb.ExportTraceServiceRequest{
 		ResourceSpans: mc.spans,
 	}
-	return proto.Marshal(pbRequest)
+	serializedSpans, err := proto.Marshal(pbRequest)
+	if err != nil {
+		return nil, err
+	}
+	mc.spans = make([]*tracepb.ResourceSpans, 0)
+	return serializedSpans, nil
 }
 
 // NoopOtlpClient is a client implementation for otlptrace.Client that does nothing
@@ -99,6 +104,8 @@ func (mc *NoopOtlpClient) Spans() (string, error) {
 }
 
 // Spans always returns no traces error
-func (mc *NoopOtlpClient) ProtoSpans() ([]byte, error) {
+func (mc *NoopOtlpClient) ExportProtoSpans() ([]byte, error) {
 	return nil, errNoopTracer
 }
+
+func (mc *NoopOtlpClient) ClearSpans() {}

tracing/identity.go

@@ -0,0 +1,110 @@
+package tracing
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"strconv"
+	"strings"
+)
+
+const (
+	// 16 bytes for tracing ID, 8 bytes for span ID and 1 byte for flags
+	IdentityLength = 16 + 8 + 1
+)
+
+type Identity struct {
+	// Based on https://www.jaegertracing.io/docs/1.36/client-libraries/#value
+	// parent span ID is always 0 for our case
+	traceIDUpper uint64
+	traceIDLower uint64
+	spanID       uint64
+	flags        uint8
+}
+
+// TODO: TUN-6604 Remove this. To reconstruct into Jaeger propagation format, convert tracingContext to tracing.Identity
+func (tc *Identity) String() string {
+	return fmt.Sprintf("%016x%016x:%x:0:%x", tc.traceIDUpper, tc.traceIDLower, tc.spanID, tc.flags)
+}
+
+func (tc *Identity) MarshalBinary() ([]byte, error) {
+	buf := bytes.NewBuffer(make([]byte, 0, IdentityLength))
+	for _, field := range []interface{}{
+		tc.traceIDUpper,
+		tc.traceIDLower,
+		tc.spanID,
+		tc.flags,
+	} {
+		if err := binary.Write(buf, binary.BigEndian, field); err != nil {
+			return nil, err
+		}
+	}
+	return buf.Bytes(), nil
+}
+
+func (tc *Identity) UnmarshalBinary(data []byte) error {
+	if len(data) < IdentityLength {
+		return fmt.Errorf("expect tracingContext to have at least %d bytes, got %d", IdentityLength, len(data))
+	}
+
+	buf := bytes.NewBuffer(data)
+	for _, field := range []interface{}{
+		&tc.traceIDUpper,
+		&tc.traceIDLower,
+		&tc.spanID,
+		&tc.flags,
+	} {
+		if err := binary.Read(buf, binary.BigEndian, field); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func NewIdentity(trace string) (*Identity, error) {
+	parts := strings.Split(trace, separator)
+	if len(parts) != 4 {
+		return nil, fmt.Errorf("trace '%s' doesn't have exactly 4 parts separated by %s", trace, separator)
+	}
+	const base = 16
+	tracingID, err := padTracingID(parts[0])
+	if err != nil {
+		return nil, err
+	}
+	traceIDUpper, err := strconv.ParseUint(tracingID[:16], base, 64)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse first 16 bytes of tracing ID as uint64, err: %w", err)
+	}
+	traceIDLower, err := strconv.ParseUint(tracingID[16:], base, 64)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse last 16 bytes of tracing ID as uint64, err: %w", err)
+	}
+	spanID, err := strconv.ParseUint(parts[1], base, 64)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse span ID as uint64, err: %w", err)
+	}
+	flags, err := strconv.ParseUint(parts[3], base, 8)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse flag as uint8, err: %w", err)
+	}
+	return &Identity{
+		traceIDUpper: traceIDUpper,
+		traceIDLower: traceIDLower,
+		spanID:       spanID,
+		flags:        uint8(flags),
+	}, nil
+}
+
+func padTracingID(tracingID string) (string, error) {
+	if len(tracingID) == 0 {
+		return "", fmt.Errorf("missing tracing ID")
+	}
+	if len(tracingID) == traceID128bitsWidth {
+		return tracingID, nil
+	}
+	// Correctly left pad the trace to a length of 32
+	left := traceID128bitsWidth - len(tracingID)
+	paddedTracingID := strings.Repeat("0", left) + tracingID
+	return paddedTracingID, nil
+}

tracing/identity_test.go

@@ -0,0 +1,67 @@
+package tracing
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestNewIdentity(t *testing.T) {
+	testCases := []struct {
+		testCase string
+		trace    string
+		expected string
+	}{
+		{
+			testCase: "full length trace",
+			trace:    "ec31ad8a01fde11fdcabe2efdce36873:52726f6cabc144f5:0:1",
+			expected: "ec31ad8a01fde11fdcabe2efdce36873:52726f6cabc144f5:0:1",
+		},
+		{
+			testCase: "short trace ID",
+			trace:    "ad8a01fde11fdcabe2efdce36873:52726f6cabc144f5:0:1",
+			expected: "0000ad8a01fde11fdcabe2efdce36873:52726f6cabc144f5:0:1",
+		},
+		{
+			testCase: "short trace ID with 0s in the middle",
+			trace:    "ad8a01fde11f000002efdce36873:52726f6cabc144f5:0:1",
+			expected: "0000ad8a01fde11f000002efdce36873:52726f6cabc144f5:0:1",
+		},
+		{
+			testCase: "short trace ID with 0s in the beginning and middle",
+			trace:    "001ad8a01fde11fdcabe2efdce36873:52726f6cabc144f5:0:1",
+			expected: "0001ad8a01fde11fdcabe2efdce36873:52726f6cabc144f5:0:1",
+		},
+		{
+			testCase: "no trace",
+			trace:    "",
+		},
+		{
+			testCase: "missing flags",
+			trace:    "ec31ad8a01fde11fdcabe2efdce36873:52726f6cabc144f5:0",
+		},
+		{
+			testCase: "missing separator",
+			trace:    "ec31ad8a01fde11fdcabe2efdce3687352726f6cabc144f501",
+		},
+	}
+
+	for _, testCase := range testCases {
+		identity, err := NewIdentity(testCase.trace)
+		if testCase.expected != "" {
+			require.NoError(t, err)
+			require.Equal(t, testCase.expected, identity.String())
+
+			serializedIdentity, err := identity.MarshalBinary()
+			require.NoError(t, err)
+			deserializedIdentity := new(Identity)
+			err = deserializedIdentity.UnmarshalBinary(serializedIdentity)
+			require.NoError(t, err)
+			require.Equal(t, identity, deserializedIdentity)
+
+		} else {
+			require.Error(t, err)
+			require.Nil(t, identity)
+		}
+	}
+}

tracing/tracing.go

@@ -157,7 +157,7 @@ func (cft *cfdTracer) GetSpans() (enc string) {
 
 // GetProtoSpans returns the spans as the otlp traces in protobuf byte array.
 func (cft *cfdTracer) GetProtoSpans() (proto []byte) {
-	proto, err := cft.exporter.ProtoSpans()
+	proto, err := cft.exporter.ExportProtoSpans()
 	switch err {
 	case nil:
 		break
@@ -246,7 +246,6 @@ func extractTraceFromString(ctx context.Context, trace string) (context.Context,
 		parts[0] = strings.Repeat("0", left) + parts[0]
 		trace = strings.Join(parts, separator)
 	}
-
 	// Override the 'cf-trace-id' as 'uber-trace-id' so the jaeger propagator can extract it.
 	traceHeader := map[string]string{TracerContextNameOverride: trace}
 	remoteCtx := otel.GetTextMapPropagator().Extract(ctx, propagation.MapCarrier(traceHeader))
@@ -277,6 +276,11 @@ func extractTrace(req *http.Request) (context.Context, bool) {
 	if traceHeader[TracerContextNameOverride] == "" {
 		return nil, false
 	}
+
 	remoteCtx := otel.GetTextMapPropagator().Extract(req.Context(), propagation.MapCarrier(traceHeader))
 	return remoteCtx, true
 }
+
+func NewNoopSpan() trace.Span {
+	return trace.SpanFromContext(nil)
+}

tunnelrpc/pogs/auth_outcome.go

@@ -40,7 +40,6 @@ func (ar AuthenticateResponse) Outcome() AuthOutcome {
 }
 
 // AuthOutcome is a programmer-friendly sum type denoting the possible outcomes of Authenticate.
-//go-sumtype:decl AuthOutcome
 type AuthOutcome interface {
 	isAuthOutcome()
 	// Serialize into an AuthenticateResponse which can be sent via Capnp

validation/validation.go

@@ -61,9 +61,13 @@ func ValidateHostname(hostname string) (string, error) {
 
 // ValidateUrl returns a validated version of `originUrl` with a scheme prepended (by default http://).
 // Note: when originUrl contains a scheme, the path is removed:
-//   ValidateUrl("https://localhost:8080/api/") => "https://localhost:8080"
+//
+//	ValidateUrl("https://localhost:8080/api/") => "https://localhost:8080"
+//
 // but when it does not, the path is preserved:
-//   ValidateUrl("localhost:8080/api/") => "http://localhost:8080/api/"
+//
+//	ValidateUrl("localhost:8080/api/") => "http://localhost:8080/api/"
+//
 // This is arguably a bug, but changing it might break some cloudflared users.
 func ValidateUrl(originUrl string) (*url.URL, error) {
 	urlStr, err := validateUrlString(originUrl)

vendor/github.com/coreos/go-oidc/v3/LICENSE

@@ -0,0 +1,202 @@
+Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "{}"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright {yyyy} {name of copyright owner}
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+

vendor/github.com/coreos/go-oidc/v3/NOTICE

@@ -0,0 +1,5 @@
+CoreOS Project
+Copyright 2014 CoreOS, Inc
+
+This product includes software developed at CoreOS, Inc.
+(http://www.coreos.com/).

vendor/github.com/coreos/go-oidc/v3/oidc/jose.go

@@ -0,0 +1,16 @@
+package oidc
+
+// JOSE asymmetric signing algorithm values as defined by RFC 7518
+//
+// see: https://tools.ietf.org/html/rfc7518#section-3.1
+const (
+	RS256 = "RS256" // RSASSA-PKCS-v1.5 using SHA-256
+	RS384 = "RS384" // RSASSA-PKCS-v1.5 using SHA-384
+	RS512 = "RS512" // RSASSA-PKCS-v1.5 using SHA-512
+	ES256 = "ES256" // ECDSA using P-256 and SHA-256
+	ES384 = "ES384" // ECDSA using P-384 and SHA-384
+	ES512 = "ES512" // ECDSA using P-521 and SHA-512
+	PS256 = "PS256" // RSASSA-PSS using SHA256 and MGF1-SHA256
+	PS384 = "PS384" // RSASSA-PSS using SHA384 and MGF1-SHA384
+	PS512 = "PS512" // RSASSA-PSS using SHA512 and MGF1-SHA512
+)

vendor/github.com/coreos/go-oidc/v3/oidc/jwks.go

@@ -0,0 +1,248 @@
+package oidc
+
+import (
+	"context"
+	"crypto"
+	"crypto/ecdsa"
+	"crypto/rsa"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"sync"
+	"time"
+
+	jose "gopkg.in/square/go-jose.v2"
+)
+
+// StaticKeySet is a verifier that validates JWT against a static set of public keys.
+type StaticKeySet struct {
+	// PublicKeys used to verify the JWT. Supported types are *rsa.PublicKey and
+	// *ecdsa.PublicKey.
+	PublicKeys []crypto.PublicKey
+}
+
+// VerifySignature compares the signature against a static set of public keys.
+func (s *StaticKeySet) VerifySignature(ctx context.Context, jwt string) ([]byte, error) {
+	jws, err := jose.ParseSigned(jwt)
+	if err != nil {
+		return nil, fmt.Errorf("parsing jwt: %v", err)
+	}
+	for _, pub := range s.PublicKeys {
+		switch pub.(type) {
+		case *rsa.PublicKey:
+		case *ecdsa.PublicKey:
+		default:
+			return nil, fmt.Errorf("invalid public key type provided: %T", pub)
+		}
+		payload, err := jws.Verify(pub)
+		if err != nil {
+			continue
+		}
+		return payload, nil
+	}
+	return nil, fmt.Errorf("no public keys able to verify jwt")
+}
+
+// NewRemoteKeySet returns a KeySet that can validate JSON web tokens by using HTTP
+// GETs to fetch JSON web token sets hosted at a remote URL. This is automatically
+// used by NewProvider using the URLs returned by OpenID Connect discovery, but is
+// exposed for providers that don't support discovery or to prevent round trips to the
+// discovery URL.
+//
+// The returned KeySet is a long lived verifier that caches keys based on any
+// keys change. Reuse a common remote key set instead of creating new ones as needed.
+func NewRemoteKeySet(ctx context.Context, jwksURL string) *RemoteKeySet {
+	return newRemoteKeySet(ctx, jwksURL, time.Now)
+}
+
+func newRemoteKeySet(ctx context.Context, jwksURL string, now func() time.Time) *RemoteKeySet {
+	if now == nil {
+		now = time.Now
+	}
+	return &RemoteKeySet{jwksURL: jwksURL, ctx: cloneContext(ctx), now: now}
+}
+
+// RemoteKeySet is a KeySet implementation that validates JSON web tokens against
+// a jwks_uri endpoint.
+type RemoteKeySet struct {
+	jwksURL string
+	ctx     context.Context
+	now     func() time.Time
+
+	// guard all other fields
+	mu sync.RWMutex
+
+	// inflight suppresses parallel execution of updateKeys and allows
+	// multiple goroutines to wait for its result.
+	inflight *inflight
+
+	// A set of cached keys.
+	cachedKeys []jose.JSONWebKey
+}
+
+// inflight is used to wait on some in-flight request from multiple goroutines.
+type inflight struct {
+	doneCh chan struct{}
+
+	keys []jose.JSONWebKey
+	err  error
+}
+
+func newInflight() *inflight {
+	return &inflight{doneCh: make(chan struct{})}
+}
+
+// wait returns a channel that multiple goroutines can receive on. Once it returns
+// a value, the inflight request is done and result() can be inspected.
+func (i *inflight) wait() <-chan struct{} {
+	return i.doneCh
+}
+
+// done can only be called by a single goroutine. It records the result of the
+// inflight request and signals other goroutines that the result is safe to
+// inspect.
+func (i *inflight) done(keys []jose.JSONWebKey, err error) {
+	i.keys = keys
+	i.err = err
+	close(i.doneCh)
+}
+
+// result cannot be called until the wait() channel has returned a value.
+func (i *inflight) result() ([]jose.JSONWebKey, error) {
+	return i.keys, i.err
+}
+
+// paresdJWTKey is a context key that allows common setups to avoid parsing the
+// JWT twice. It holds a *jose.JSONWebSignature value.
+var parsedJWTKey contextKey
+
+// VerifySignature validates a payload against a signature from the jwks_uri.
+//
+// Users MUST NOT call this method directly and should use an IDTokenVerifier
+// instead. This method skips critical validations such as 'alg' values and is
+// only exported to implement the KeySet interface.
+func (r *RemoteKeySet) VerifySignature(ctx context.Context, jwt string) ([]byte, error) {
+	jws, ok := ctx.Value(parsedJWTKey).(*jose.JSONWebSignature)
+	if !ok {
+		var err error
+		jws, err = jose.ParseSigned(jwt)
+		if err != nil {
+			return nil, fmt.Errorf("oidc: malformed jwt: %v", err)
+		}
+	}
+	return r.verify(ctx, jws)
+}
+
+func (r *RemoteKeySet) verify(ctx context.Context, jws *jose.JSONWebSignature) ([]byte, error) {
+	// We don't support JWTs signed with multiple signatures.
+	keyID := ""
+	for _, sig := range jws.Signatures {
+		keyID = sig.Header.KeyID
+		break
+	}
+
+	keys := r.keysFromCache()
+	for _, key := range keys {
+		if keyID == "" || key.KeyID == keyID {
+			if payload, err := jws.Verify(&key); err == nil {
+				return payload, nil
+			}
+		}
+	}
+
+	// If the kid doesn't match, check for new keys from the remote. This is the
+	// strategy recommended by the spec.
+	//
+	// https://openid.net/specs/openid-connect-core-1_0.html#RotateSigKeys
+	keys, err := r.keysFromRemote(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("fetching keys %v", err)
+	}
+
+	for _, key := range keys {
+		if keyID == "" || key.KeyID == keyID {
+			if payload, err := jws.Verify(&key); err == nil {
+				return payload, nil
+			}
+		}
+	}
+	return nil, errors.New("failed to verify id token signature")
+}
+
+func (r *RemoteKeySet) keysFromCache() (keys []jose.JSONWebKey) {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+	return r.cachedKeys
+}
+
+// keysFromRemote syncs the key set from the remote set, records the values in the
+// cache, and returns the key set.
+func (r *RemoteKeySet) keysFromRemote(ctx context.Context) ([]jose.JSONWebKey, error) {
+	// Need to lock to inspect the inflight request field.
+	r.mu.Lock()
+	// If there's not a current inflight request, create one.
+	if r.inflight == nil {
+		r.inflight = newInflight()
+
+		// This goroutine has exclusive ownership over the current inflight
+		// request. It releases the resource by nil'ing the inflight field
+		// once the goroutine is done.
+		go func() {
+			// Sync keys and finish inflight when that's done.
+			keys, err := r.updateKeys()
+
+			r.inflight.done(keys, err)
+
+			// Lock to update the keys and indicate that there is no longer an
+			// inflight request.
+			r.mu.Lock()
+			defer r.mu.Unlock()
+
+			if err == nil {
+				r.cachedKeys = keys
+			}
+
+			// Free inflight so a different request can run.
+			r.inflight = nil
+		}()
+	}
+	inflight := r.inflight
+	r.mu.Unlock()
+
+	select {
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	case <-inflight.wait():
+		return inflight.result()
+	}
+}
+
+func (r *RemoteKeySet) updateKeys() ([]jose.JSONWebKey, error) {
+	req, err := http.NewRequest("GET", r.jwksURL, nil)
+	if err != nil {
+		return nil, fmt.Errorf("oidc: can't create request: %v", err)
+	}
+
+	resp, err := doRequest(r.ctx, req)
+	if err != nil {
+		return nil, fmt.Errorf("oidc: get keys failed %v", err)
+	}
+	defer resp.Body.Close()
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("unable to read response body: %v", err)
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("oidc: get keys failed: %s %s", resp.Status, body)
+	}
+
+	var keySet jose.JSONWebKeySet
+	err = unmarshalResp(resp, body, &keySet)
+	if err != nil {
+		return nil, fmt.Errorf("oidc: failed to decode keys: %v %s", err, body)
+	}
+	return keySet.Keys, nil
+}

vendor/github.com/coreos/go-oidc/v3/oidc/oidc.go

@@ -0,0 +1,522 @@
+// Package oidc implements OpenID Connect client logic for the golang.org/x/oauth2 package.
+package oidc
+
+import (
+	"context"
+	"crypto/sha256"
+	"crypto/sha512"
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"hash"
+	"io/ioutil"
+	"mime"
+	"net/http"
+	"strings"
+	"time"
+
+	"golang.org/x/oauth2"
+)
+
+const (
+	// ScopeOpenID is the mandatory scope for all OpenID Connect OAuth2 requests.
+	ScopeOpenID = "openid"
+
+	// ScopeOfflineAccess is an optional scope defined by OpenID Connect for requesting
+	// OAuth2 refresh tokens.
+	//
+	// Support for this scope differs between OpenID Connect providers. For instance
+	// Google rejects it, favoring appending "access_type=offline" as part of the
+	// authorization request instead.
+	//
+	// See: https://openid.net/specs/openid-connect-core-1_0.html#OfflineAccess
+	ScopeOfflineAccess = "offline_access"
+)
+
+var (
+	errNoAtHash      = errors.New("id token did not have an access token hash")
+	errInvalidAtHash = errors.New("access token hash does not match value in ID token")
+)
+
+type contextKey int
+
+var issuerURLKey contextKey
+
+// ClientContext returns a new Context that carries the provided HTTP client.
+//
+// This method sets the same context key used by the golang.org/x/oauth2 package,
+// so the returned context works for that package too.
+//
+//    myClient := &http.Client{}
+//    ctx := oidc.ClientContext(parentContext, myClient)
+//
+//    // This will use the custom client
+//    provider, err := oidc.NewProvider(ctx, "https://accounts.example.com")
+//
+func ClientContext(ctx context.Context, client *http.Client) context.Context {
+	return context.WithValue(ctx, oauth2.HTTPClient, client)
+}
+
+// cloneContext copies a context's bag-of-values into a new context that isn't
+// associated with its cancellation. This is used to initialize remote keys sets
+// which run in the background and aren't associated with the initial context.
+func cloneContext(ctx context.Context) context.Context {
+	cp := context.Background()
+	if c, ok := ctx.Value(oauth2.HTTPClient).(*http.Client); ok {
+		cp = ClientContext(cp, c)
+	}
+	return cp
+}
+
+// InsecureIssuerURLContext allows discovery to work when the issuer_url reported
+// by upstream is mismatched with the discovery URL. This is meant for integration
+// with off-spec providers such as Azure.
+//
+//    discoveryBaseURL := "https://login.microsoftonline.com/organizations/v2.0"
+//    issuerURL := "https://login.microsoftonline.com/my-tenantid/v2.0"
+//
+//    ctx := oidc.InsecureIssuerURLContext(parentContext, issuerURL)
+//
+//    // Provider will be discovered with the discoveryBaseURL, but use issuerURL
+//    // for future issuer validation.
+//    provider, err := oidc.NewProvider(ctx, discoveryBaseURL)
+//
+// This is insecure because validating the correct issuer is critical for multi-tenant
+// proivders. Any overrides here MUST be carefully reviewed.
+func InsecureIssuerURLContext(ctx context.Context, issuerURL string) context.Context {
+	return context.WithValue(ctx, issuerURLKey, issuerURL)
+}
+
+func doRequest(ctx context.Context, req *http.Request) (*http.Response, error) {
+	client := http.DefaultClient
+	if c, ok := ctx.Value(oauth2.HTTPClient).(*http.Client); ok {
+		client = c
+	}
+	return client.Do(req.WithContext(ctx))
+}
+
+// Provider represents an OpenID Connect server's configuration.
+type Provider struct {
+	issuer      string
+	authURL     string
+	tokenURL    string
+	userInfoURL string
+	algorithms  []string
+
+	// Raw claims returned by the server.
+	rawClaims []byte
+
+	remoteKeySet KeySet
+}
+
+type providerJSON struct {
+	Issuer      string   `json:"issuer"`
+	AuthURL     string   `json:"authorization_endpoint"`
+	TokenURL    string   `json:"token_endpoint"`
+	JWKSURL     string   `json:"jwks_uri"`
+	UserInfoURL string   `json:"userinfo_endpoint"`
+	Algorithms  []string `json:"id_token_signing_alg_values_supported"`
+}
+
+// supportedAlgorithms is a list of algorithms explicitly supported by this
+// package. If a provider supports other algorithms, such as HS256 or none,
+// those values won't be passed to the IDTokenVerifier.
+var supportedAlgorithms = map[string]bool{
+	RS256: true,
+	RS384: true,
+	RS512: true,
+	ES256: true,
+	ES384: true,
+	ES512: true,
+	PS256: true,
+	PS384: true,
+	PS512: true,
+}
+
+// ProviderConfig allows creating providers when discovery isn't supported. It's
+// generally easier to use NewProvider directly.
+type ProviderConfig struct {
+	// IssuerURL is the identity of the provider, and the string it uses to sign
+	// ID tokens with. For example "https://accounts.google.com". This value MUST
+	// match ID tokens exactly.
+	IssuerURL string
+	// AuthURL is the endpoint used by the provider to support the OAuth 2.0
+	// authorization endpoint.
+	AuthURL string
+	// TokenURL is the endpoint used by the provider to support the OAuth 2.0
+	// token endpoint.
+	TokenURL string
+	// UserInfoURL is the endpoint used by the provider to support the OpenID
+	// Connect UserInfo flow.
+	//
+	// https://openid.net/specs/openid-connect-core-1_0.html#UserInfo
+	UserInfoURL string
+	// JWKSURL is the endpoint used by the provider to advertise public keys to
+	// verify issued ID tokens. This endpoint is polled as new keys are made
+	// available.
+	JWKSURL string
+
+	// Algorithms, if provided, indicate a list of JWT algorithms allowed to sign
+	// ID tokens. If not provided, this defaults to the algorithms advertised by
+	// the JWK endpoint, then the set of algorithms supported by this package.
+	Algorithms []string
+}
+
+// NewProvider initializes a provider from a set of endpoints, rather than
+// through discovery.
+func (p *ProviderConfig) NewProvider(ctx context.Context) *Provider {
+	return &Provider{
+		issuer:       p.IssuerURL,
+		authURL:      p.AuthURL,
+		tokenURL:     p.TokenURL,
+		userInfoURL:  p.UserInfoURL,
+		algorithms:   p.Algorithms,
+		remoteKeySet: NewRemoteKeySet(cloneContext(ctx), p.JWKSURL),
+	}
+}
+
+// NewProvider uses the OpenID Connect discovery mechanism to construct a Provider.
+//
+// The issuer is the URL identifier for the service. For example: "https://accounts.google.com"
+// or "https://login.salesforce.com".
+func NewProvider(ctx context.Context, issuer string) (*Provider, error) {
+	wellKnown := strings.TrimSuffix(issuer, "/") + "/.well-known/openid-configuration"
+	req, err := http.NewRequest("GET", wellKnown, nil)
+	if err != nil {
+		return nil, err
+	}
+	resp, err := doRequest(ctx, req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("unable to read response body: %v", err)
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("%s: %s", resp.Status, body)
+	}
+
+	var p providerJSON
+	err = unmarshalResp(resp, body, &p)
+	if err != nil {
+		return nil, fmt.Errorf("oidc: failed to decode provider discovery object: %v", err)
+	}
+
+	issuerURL, skipIssuerValidation := ctx.Value(issuerURLKey).(string)
+	if !skipIssuerValidation {
+		issuerURL = issuer
+	}
+	if p.Issuer != issuerURL && !skipIssuerValidation {
+		return nil, fmt.Errorf("oidc: issuer did not match the issuer returned by provider, expected %q got %q", issuer, p.Issuer)
+	}
+	var algs []string
+	for _, a := range p.Algorithms {
+		if supportedAlgorithms[a] {
+			algs = append(algs, a)
+		}
+	}
+	return &Provider{
+		issuer:       issuerURL,
+		authURL:      p.AuthURL,
+		tokenURL:     p.TokenURL,
+		userInfoURL:  p.UserInfoURL,
+		algorithms:   algs,
+		rawClaims:    body,
+		remoteKeySet: NewRemoteKeySet(cloneContext(ctx), p.JWKSURL),
+	}, nil
+}
+
+// Claims unmarshals raw fields returned by the server during discovery.
+//
+//    var claims struct {
+//        ScopesSupported []string `json:"scopes_supported"`
+//        ClaimsSupported []string `json:"claims_supported"`
+//    }
+//
+//    if err := provider.Claims(&claims); err != nil {
+//        // handle unmarshaling error
+//    }
+//
+// For a list of fields defined by the OpenID Connect spec see:
+// https://openid.net/specs/openid-connect-discovery-1_0.html#ProviderMetadata
+func (p *Provider) Claims(v interface{}) error {
+	if p.rawClaims == nil {
+		return errors.New("oidc: claims not set")
+	}
+	return json.Unmarshal(p.rawClaims, v)
+}
+
+// Endpoint returns the OAuth2 auth and token endpoints for the given provider.
+func (p *Provider) Endpoint() oauth2.Endpoint {
+	return oauth2.Endpoint{AuthURL: p.authURL, TokenURL: p.tokenURL}
+}
+
+// UserInfo represents the OpenID Connect userinfo claims.
+type UserInfo struct {
+	Subject       string `json:"sub"`
+	Profile       string `json:"profile"`
+	Email         string `json:"email"`
+	EmailVerified bool   `json:"email_verified"`
+
+	claims []byte
+}
+
+type userInfoRaw struct {
+	Subject string `json:"sub"`
+	Profile string `json:"profile"`
+	Email   string `json:"email"`
+	// Handle providers that return email_verified as a string
+	// https://forums.aws.amazon.com/thread.jspa?messageID=949441&#949441 and
+	// https://discuss.elastic.co/t/openid-error-after-authenticating-against-aws-cognito/206018/11
+	EmailVerified stringAsBool `json:"email_verified"`
+}
+
+// Claims unmarshals the raw JSON object claims into the provided object.
+func (u *UserInfo) Claims(v interface{}) error {
+	if u.claims == nil {
+		return errors.New("oidc: claims not set")
+	}
+	return json.Unmarshal(u.claims, v)
+}
+
+// UserInfo uses the token source to query the provider's user info endpoint.
+func (p *Provider) UserInfo(ctx context.Context, tokenSource oauth2.TokenSource) (*UserInfo, error) {
+	if p.userInfoURL == "" {
+		return nil, errors.New("oidc: user info endpoint is not supported by this provider")
+	}
+
+	req, err := http.NewRequest("GET", p.userInfoURL, nil)
+	if err != nil {
+		return nil, fmt.Errorf("oidc: create GET request: %v", err)
+	}
+
+	token, err := tokenSource.Token()
+	if err != nil {
+		return nil, fmt.Errorf("oidc: get access token: %v", err)
+	}
+	token.SetAuthHeader(req)
+
+	resp, err := doRequest(ctx, req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("%s: %s", resp.Status, body)
+	}
+
+	ct := resp.Header.Get("Content-Type")
+	mediaType, _, parseErr := mime.ParseMediaType(ct)
+	if parseErr == nil && mediaType == "application/jwt" {
+		payload, err := p.remoteKeySet.VerifySignature(ctx, string(body))
+		if err != nil {
+			return nil, fmt.Errorf("oidc: invalid userinfo jwt signature %v", err)
+		}
+		body = payload
+	}
+
+	var userInfo userInfoRaw
+	if err := json.Unmarshal(body, &userInfo); err != nil {
+		return nil, fmt.Errorf("oidc: failed to decode userinfo: %v", err)
+	}
+	return &UserInfo{
+		Subject:       userInfo.Subject,
+		Profile:       userInfo.Profile,
+		Email:         userInfo.Email,
+		EmailVerified: bool(userInfo.EmailVerified),
+		claims:        body,
+	}, nil
+}
+
+// IDToken is an OpenID Connect extension that provides a predictable representation
+// of an authorization event.
+//
+// The ID Token only holds fields OpenID Connect requires. To access additional
+// claims returned by the server, use the Claims method.
+type IDToken struct {
+	// The URL of the server which issued this token. OpenID Connect
+	// requires this value always be identical to the URL used for
+	// initial discovery.
+	//
+	// Note: Because of a known issue with Google Accounts' implementation
+	// this value may differ when using Google.
+	//
+	// See: https://developers.google.com/identity/protocols/OpenIDConnect#obtainuserinfo
+	Issuer string
+
+	// The client ID, or set of client IDs, that this token is issued for. For
+	// common uses, this is the client that initialized the auth flow.
+	//
+	// This package ensures the audience contains an expected value.
+	Audience []string
+
+	// A unique string which identifies the end user.
+	Subject string
+
+	// Expiry of the token. Ths package will not process tokens that have
+	// expired unless that validation is explicitly turned off.
+	Expiry time.Time
+	// When the token was issued by the provider.
+	IssuedAt time.Time
+
+	// Initial nonce provided during the authentication redirect.
+	//
+	// This package does NOT provided verification on the value of this field
+	// and it's the user's responsibility to ensure it contains a valid value.
+	Nonce string
+
+	// at_hash claim, if set in the ID token. Callers can verify an access token
+	// that corresponds to the ID token using the VerifyAccessToken method.
+	AccessTokenHash string
+
+	// signature algorithm used for ID token, needed to compute a verification hash of an
+	// access token
+	sigAlgorithm string
+
+	// Raw payload of the id_token.
+	claims []byte
+
+	// Map of distributed claim names to claim sources
+	distributedClaims map[string]claimSource
+}
+
+// Claims unmarshals the raw JSON payload of the ID Token into a provided struct.
+//
+//		idToken, err := idTokenVerifier.Verify(rawIDToken)
+//		if err != nil {
+//			// handle error
+//		}
+//		var claims struct {
+//			Email         string `json:"email"`
+//			EmailVerified bool   `json:"email_verified"`
+//		}
+//		if err := idToken.Claims(&claims); err != nil {
+//			// handle error
+//		}
+//
+func (i *IDToken) Claims(v interface{}) error {
+	if i.claims == nil {
+		return errors.New("oidc: claims not set")
+	}
+	return json.Unmarshal(i.claims, v)
+}
+
+// VerifyAccessToken verifies that the hash of the access token that corresponds to the iD token
+// matches the hash in the id token. It returns an error if the hashes  don't match.
+// It is the caller's responsibility to ensure that the optional access token hash is present for the ID token
+// before calling this method. See https://openid.net/specs/openid-connect-core-1_0.html#CodeIDToken
+func (i *IDToken) VerifyAccessToken(accessToken string) error {
+	if i.AccessTokenHash == "" {
+		return errNoAtHash
+	}
+	var h hash.Hash
+	switch i.sigAlgorithm {
+	case RS256, ES256, PS256:
+		h = sha256.New()
+	case RS384, ES384, PS384:
+		h = sha512.New384()
+	case RS512, ES512, PS512:
+		h = sha512.New()
+	default:
+		return fmt.Errorf("oidc: unsupported signing algorithm %q", i.sigAlgorithm)
+	}
+	h.Write([]byte(accessToken)) // hash documents that Write will never return an error
+	sum := h.Sum(nil)[:h.Size()/2]
+	actual := base64.RawURLEncoding.EncodeToString(sum)
+	if actual != i.AccessTokenHash {
+		return errInvalidAtHash
+	}
+	return nil
+}
+
+type idToken struct {
+	Issuer       string                 `json:"iss"`
+	Subject      string                 `json:"sub"`
+	Audience     audience               `json:"aud"`
+	Expiry       jsonTime               `json:"exp"`
+	IssuedAt     jsonTime               `json:"iat"`
+	NotBefore    *jsonTime              `json:"nbf"`
+	Nonce        string                 `json:"nonce"`
+	AtHash       string                 `json:"at_hash"`
+	ClaimNames   map[string]string      `json:"_claim_names"`
+	ClaimSources map[string]claimSource `json:"_claim_sources"`
+}
+
+type claimSource struct {
+	Endpoint    string `json:"endpoint"`
+	AccessToken string `json:"access_token"`
+}
+
+type stringAsBool bool
+
+func (sb *stringAsBool) UnmarshalJSON(b []byte) error {
+	switch string(b) {
+	case "true", `"true"`:
+		*sb = true
+	case "false", `"false"`:
+		*sb = false
+	default:
+		return errors.New("invalid value for boolean")
+	}
+	return nil
+}
+
+type audience []string
+
+func (a *audience) UnmarshalJSON(b []byte) error {
+	var s string
+	if json.Unmarshal(b, &s) == nil {
+		*a = audience{s}
+		return nil
+	}
+	var auds []string
+	if err := json.Unmarshal(b, &auds); err != nil {
+		return err
+	}
+	*a = auds
+	return nil
+}
+
+type jsonTime time.Time
+
+func (j *jsonTime) UnmarshalJSON(b []byte) error {
+	var n json.Number
+	if err := json.Unmarshal(b, &n); err != nil {
+		return err
+	}
+	var unix int64
+
+	if t, err := n.Int64(); err == nil {
+		unix = t
+	} else {
+		f, err := n.Float64()
+		if err != nil {
+			return err
+		}
+		unix = int64(f)
+	}
+	*j = jsonTime(time.Unix(unix, 0))
+	return nil
+}
+
+func unmarshalResp(r *http.Response, body []byte, v interface{}) error {
+	err := json.Unmarshal(body, &v)
+	if err == nil {
+		return nil
+	}
+	ct := r.Header.Get("Content-Type")
+	mediaType, _, parseErr := mime.ParseMediaType(ct)
+	if parseErr == nil && mediaType == "application/json" {
+		return fmt.Errorf("got Content-Type = application/json, but could not unmarshal as JSON: %v", err)
+	}
+	return fmt.Errorf("expected Content-Type = application/json, got %q: %v", ct, err)
+}

vendor/github.com/coreos/go-oidc/v3/oidc/verify.go

@@ -0,0 +1,344 @@
+package oidc
+
+import (
+	"bytes"
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"strings"
+	"time"
+
+	"golang.org/x/oauth2"
+	jose "gopkg.in/square/go-jose.v2"
+)
+
+const (
+	issuerGoogleAccounts         = "https://accounts.google.com"
+	issuerGoogleAccountsNoScheme = "accounts.google.com"
+)
+
+// TokenExpiredError indicates that Verify failed because the token was expired. This
+// error does NOT indicate that the token is not also invalid for other reasons. Other
+// checks might have failed if the expiration check had not failed.
+type TokenExpiredError struct {
+	// Expiry is the time when the token expired.
+	Expiry time.Time
+}
+
+func (e *TokenExpiredError) Error() string {
+	return fmt.Sprintf("oidc: token is expired (Token Expiry: %v)", e.Expiry)
+}
+
+// KeySet is a set of publc JSON Web Keys that can be used to validate the signature
+// of JSON web tokens. This is expected to be backed by a remote key set through
+// provider metadata discovery or an in-memory set of keys delivered out-of-band.
+type KeySet interface {
+	// VerifySignature parses the JSON web token, verifies the signature, and returns
+	// the raw payload. Header and claim fields are validated by other parts of the
+	// package. For example, the KeySet does not need to check values such as signature
+	// algorithm, issuer, and audience since the IDTokenVerifier validates these values
+	// independently.
+	//
+	// If VerifySignature makes HTTP requests to verify the token, it's expected to
+	// use any HTTP client associated with the context through ClientContext.
+	VerifySignature(ctx context.Context, jwt string) (payload []byte, err error)
+}
+
+// IDTokenVerifier provides verification for ID Tokens.
+type IDTokenVerifier struct {
+	keySet KeySet
+	config *Config
+	issuer string
+}
+
+// NewVerifier returns a verifier manually constructed from a key set and issuer URL.
+//
+// It's easier to use provider discovery to construct an IDTokenVerifier than creating
+// one directly. This method is intended to be used with provider that don't support
+// metadata discovery, or avoiding round trips when the key set URL is already known.
+//
+// This constructor can be used to create a verifier directly using the issuer URL and
+// JSON Web Key Set URL without using discovery:
+//
+//		keySet := oidc.NewRemoteKeySet(ctx, "https://www.googleapis.com/oauth2/v3/certs")
+//		verifier := oidc.NewVerifier("https://accounts.google.com", keySet, config)
+//
+// Or a static key set (e.g. for testing):
+//
+//		keySet := &oidc.StaticKeySet{PublicKeys: []crypto.PublicKey{pub1, pub2}}
+//		verifier := oidc.NewVerifier("https://accounts.google.com", keySet, config)
+//
+func NewVerifier(issuerURL string, keySet KeySet, config *Config) *IDTokenVerifier {
+	return &IDTokenVerifier{keySet: keySet, config: config, issuer: issuerURL}
+}
+
+// Config is the configuration for an IDTokenVerifier.
+type Config struct {
+	// Expected audience of the token. For a majority of the cases this is expected to be
+	// the ID of the client that initialized the login flow. It may occasionally differ if
+	// the provider supports the authorizing party (azp) claim.
+	//
+	// If not provided, users must explicitly set SkipClientIDCheck.
+	ClientID string
+	// If specified, only this set of algorithms may be used to sign the JWT.
+	//
+	// If the IDTokenVerifier is created from a provider with (*Provider).Verifier, this
+	// defaults to the set of algorithms the provider supports. Otherwise this values
+	// defaults to RS256.
+	SupportedSigningAlgs []string
+
+	// If true, no ClientID check performed. Must be true if ClientID field is empty.
+	SkipClientIDCheck bool
+	// If true, token expiry is not checked.
+	SkipExpiryCheck bool
+
+	// SkipIssuerCheck is intended for specialized cases where the the caller wishes to
+	// defer issuer validation. When enabled, callers MUST independently verify the Token's
+	// Issuer is a known good value.
+	//
+	// Mismatched issuers often indicate client mis-configuration. If mismatches are
+	// unexpected, evaluate if the provided issuer URL is incorrect instead of enabling
+	// this option.
+	SkipIssuerCheck bool
+
+	// Time function to check Token expiry. Defaults to time.Now
+	Now func() time.Time
+
+	// InsecureSkipSignatureCheck causes this package to skip JWT signature validation.
+	// It's intended for special cases where providers (such as Azure), use the "none"
+	// algorithm.
+	//
+	// This option can only be enabled safely when the ID Token is received directly
+	// from the provider after the token exchange.
+	//
+	// This option MUST NOT be used when receiving an ID Token from sources other
+	// than the token endpoint.
+	InsecureSkipSignatureCheck bool
+}
+
+// Verifier returns an IDTokenVerifier that uses the provider's key set to verify JWTs.
+func (p *Provider) Verifier(config *Config) *IDTokenVerifier {
+	if len(config.SupportedSigningAlgs) == 0 && len(p.algorithms) > 0 {
+		// Make a copy so we don't modify the config values.
+		cp := &Config{}
+		*cp = *config
+		cp.SupportedSigningAlgs = p.algorithms
+		config = cp
+	}
+	return NewVerifier(p.issuer, p.remoteKeySet, config)
+}
+
+func parseJWT(p string) ([]byte, error) {
+	parts := strings.Split(p, ".")
+	if len(parts) < 2 {
+		return nil, fmt.Errorf("oidc: malformed jwt, expected 3 parts got %d", len(parts))
+	}
+	payload, err := base64.RawURLEncoding.DecodeString(parts[1])
+	if err != nil {
+		return nil, fmt.Errorf("oidc: malformed jwt payload: %v", err)
+	}
+	return payload, nil
+}
+
+func contains(sli []string, ele string) bool {
+	for _, s := range sli {
+		if s == ele {
+			return true
+		}
+	}
+	return false
+}
+
+// Returns the Claims from the distributed JWT token
+func resolveDistributedClaim(ctx context.Context, verifier *IDTokenVerifier, src claimSource) ([]byte, error) {
+	req, err := http.NewRequest("GET", src.Endpoint, nil)
+	if err != nil {
+		return nil, fmt.Errorf("malformed request: %v", err)
+	}
+	if src.AccessToken != "" {
+		req.Header.Set("Authorization", "Bearer "+src.AccessToken)
+	}
+
+	resp, err := doRequest(ctx, req)
+	if err != nil {
+		return nil, fmt.Errorf("oidc: Request to endpoint failed: %v", err)
+	}
+	defer resp.Body.Close()
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("unable to read response body: %v", err)
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("oidc: request failed: %v", resp.StatusCode)
+	}
+
+	token, err := verifier.Verify(ctx, string(body))
+	if err != nil {
+		return nil, fmt.Errorf("malformed response body: %v", err)
+	}
+
+	return token.claims, nil
+}
+
+// Verify parses a raw ID Token, verifies it's been signed by the provider, performs
+// any additional checks depending on the Config, and returns the payload.
+//
+// Verify does NOT do nonce validation, which is the callers responsibility.
+//
+// See: https://openid.net/specs/openid-connect-core-1_0.html#IDTokenValidation
+//
+//    oauth2Token, err := oauth2Config.Exchange(ctx, r.URL.Query().Get("code"))
+//    if err != nil {
+//        // handle error
+//    }
+//
+//    // Extract the ID Token from oauth2 token.
+//    rawIDToken, ok := oauth2Token.Extra("id_token").(string)
+//    if !ok {
+//        // handle error
+//    }
+//
+//    token, err := verifier.Verify(ctx, rawIDToken)
+//
+func (v *IDTokenVerifier) Verify(ctx context.Context, rawIDToken string) (*IDToken, error) {
+	// Throw out tokens with invalid claims before trying to verify the token. This lets
+	// us do cheap checks before possibly re-syncing keys.
+	payload, err := parseJWT(rawIDToken)
+	if err != nil {
+		return nil, fmt.Errorf("oidc: malformed jwt: %v", err)
+	}
+	var token idToken
+	if err := json.Unmarshal(payload, &token); err != nil {
+		return nil, fmt.Errorf("oidc: failed to unmarshal claims: %v", err)
+	}
+
+	distributedClaims := make(map[string]claimSource)
+
+	//step through the token to map claim names to claim sources"
+	for cn, src := range token.ClaimNames {
+		if src == "" {
+			return nil, fmt.Errorf("oidc: failed to obtain source from claim name")
+		}
+		s, ok := token.ClaimSources[src]
+		if !ok {
+			return nil, fmt.Errorf("oidc: source does not exist")
+		}
+		distributedClaims[cn] = s
+	}
+
+	t := &IDToken{
+		Issuer:            token.Issuer,
+		Subject:           token.Subject,
+		Audience:          []string(token.Audience),
+		Expiry:            time.Time(token.Expiry),
+		IssuedAt:          time.Time(token.IssuedAt),
+		Nonce:             token.Nonce,
+		AccessTokenHash:   token.AtHash,
+		claims:            payload,
+		distributedClaims: distributedClaims,
+	}
+
+	// Check issuer.
+	if !v.config.SkipIssuerCheck && t.Issuer != v.issuer {
+		// Google sometimes returns "accounts.google.com" as the issuer claim instead of
+		// the required "https://accounts.google.com". Detect this case and allow it only
+		// for Google.
+		//
+		// We will not add hooks to let other providers go off spec like this.
+		if !(v.issuer == issuerGoogleAccounts && t.Issuer == issuerGoogleAccountsNoScheme) {
+			return nil, fmt.Errorf("oidc: id token issued by a different provider, expected %q got %q", v.issuer, t.Issuer)
+		}
+	}
+
+	// If a client ID has been provided, make sure it's part of the audience. SkipClientIDCheck must be true if ClientID is empty.
+	//
+	// This check DOES NOT ensure that the ClientID is the party to which the ID Token was issued (i.e. Authorized party).
+	if !v.config.SkipClientIDCheck {
+		if v.config.ClientID != "" {
+			if !contains(t.Audience, v.config.ClientID) {
+				return nil, fmt.Errorf("oidc: expected audience %q got %q", v.config.ClientID, t.Audience)
+			}
+		} else {
+			return nil, fmt.Errorf("oidc: invalid configuration, clientID must be provided or SkipClientIDCheck must be set")
+		}
+	}
+
+	// If a SkipExpiryCheck is false, make sure token is not expired.
+	if !v.config.SkipExpiryCheck {
+		now := time.Now
+		if v.config.Now != nil {
+			now = v.config.Now
+		}
+		nowTime := now()
+
+		if t.Expiry.Before(nowTime) {
+			return nil, &TokenExpiredError{Expiry: t.Expiry}
+		}
+
+		// If nbf claim is provided in token, ensure that it is indeed in the past.
+		if token.NotBefore != nil {
+			nbfTime := time.Time(*token.NotBefore)
+			// Set to 5 minutes since this is what other OpenID Connect providers do to deal with clock skew.
+			// https://github.com/AzureAD/azure-activedirectory-identitymodel-extensions-for-dotnet/blob/6.12.2/src/Microsoft.IdentityModel.Tokens/TokenValidationParameters.cs#L149-L153
+			leeway := 5 * time.Minute
+
+			if nowTime.Add(leeway).Before(nbfTime) {
+				return nil, fmt.Errorf("oidc: current time %v before the nbf (not before) time: %v", nowTime, nbfTime)
+			}
+		}
+	}
+
+	if v.config.InsecureSkipSignatureCheck {
+		return t, nil
+	}
+
+	jws, err := jose.ParseSigned(rawIDToken)
+	if err != nil {
+		return nil, fmt.Errorf("oidc: malformed jwt: %v", err)
+	}
+
+	switch len(jws.Signatures) {
+	case 0:
+		return nil, fmt.Errorf("oidc: id token not signed")
+	case 1:
+	default:
+		return nil, fmt.Errorf("oidc: multiple signatures on id token not supported")
+	}
+
+	sig := jws.Signatures[0]
+	supportedSigAlgs := v.config.SupportedSigningAlgs
+	if len(supportedSigAlgs) == 0 {
+		supportedSigAlgs = []string{RS256}
+	}
+
+	if !contains(supportedSigAlgs, sig.Header.Algorithm) {
+		return nil, fmt.Errorf("oidc: id token signed with unsupported algorithm, expected %q got %q", supportedSigAlgs, sig.Header.Algorithm)
+	}
+
+	t.sigAlgorithm = sig.Header.Algorithm
+
+	ctx = context.WithValue(ctx, parsedJWTKey, jws)
+	gotPayload, err := v.keySet.VerifySignature(ctx, rawIDToken)
+	if err != nil {
+		return nil, fmt.Errorf("failed to verify signature: %v", err)
+	}
+
+	// Ensure that the payload returned by the square actually matches the payload parsed earlier.
+	if !bytes.Equal(gotPayload, payload) {
+		return nil, errors.New("oidc: internal error, payload parsed did not match previous payload")
+	}
+
+	return t, nil
+}
+
+// Nonce returns an auth code option which requires the ID Token created by the
+// OpenID Connect provider to contain the specified nonce.
+func Nonce(nonce string) oauth2.AuthCodeOption {
+	return oauth2.SetAuthURLParam("nonce", nonce)
+}

vendor/github.com/golang/protobuf/jsonpb/decode.go

@@ -0,0 +1,524 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package jsonpb
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"math"
+	"reflect"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/golang/protobuf/proto"
+	"google.golang.org/protobuf/encoding/protojson"
+	protoV2 "google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/reflect/protoreflect"
+	"google.golang.org/protobuf/reflect/protoregistry"
+)
+
+const wrapJSONUnmarshalV2 = false
+
+// UnmarshalNext unmarshals the next JSON object from d into m.
+func UnmarshalNext(d *json.Decoder, m proto.Message) error {
+	return new(Unmarshaler).UnmarshalNext(d, m)
+}
+
+// Unmarshal unmarshals a JSON object from r into m.
+func Unmarshal(r io.Reader, m proto.Message) error {
+	return new(Unmarshaler).Unmarshal(r, m)
+}
+
+// UnmarshalString unmarshals a JSON object from s into m.
+func UnmarshalString(s string, m proto.Message) error {
+	return new(Unmarshaler).Unmarshal(strings.NewReader(s), m)
+}
+
+// Unmarshaler is a configurable object for converting from a JSON
+// representation to a protocol buffer object.
+type Unmarshaler struct {
+	// AllowUnknownFields specifies whether to allow messages to contain
+	// unknown JSON fields, as opposed to failing to unmarshal.
+	AllowUnknownFields bool
+
+	// AnyResolver is used to resolve the google.protobuf.Any well-known type.
+	// If unset, the global registry is used by default.
+	AnyResolver AnyResolver
+}
+
+// JSONPBUnmarshaler is implemented by protobuf messages that customize the way
+// they are unmarshaled from JSON. Messages that implement this should also
+// implement JSONPBMarshaler so that the custom format can be produced.
+//
+// The JSON unmarshaling must follow the JSON to proto specification:
+//	https://developers.google.com/protocol-buffers/docs/proto3#json
+//
+// Deprecated: Custom types should implement protobuf reflection instead.
+type JSONPBUnmarshaler interface {
+	UnmarshalJSONPB(*Unmarshaler, []byte) error
+}
+
+// Unmarshal unmarshals a JSON object from r into m.
+func (u *Unmarshaler) Unmarshal(r io.Reader, m proto.Message) error {
+	return u.UnmarshalNext(json.NewDecoder(r), m)
+}
+
+// UnmarshalNext unmarshals the next JSON object from d into m.
+func (u *Unmarshaler) UnmarshalNext(d *json.Decoder, m proto.Message) error {
+	if m == nil {
+		return errors.New("invalid nil message")
+	}
+
+	// Parse the next JSON object from the stream.
+	raw := json.RawMessage{}
+	if err := d.Decode(&raw); err != nil {
+		return err
+	}
+
+	// Check for custom unmarshalers first since they may not properly
+	// implement protobuf reflection that the logic below relies on.
+	if jsu, ok := m.(JSONPBUnmarshaler); ok {
+		return jsu.UnmarshalJSONPB(u, raw)
+	}
+
+	mr := proto.MessageReflect(m)
+
+	// NOTE: For historical reasons, a top-level null is treated as a noop.
+	// This is incorrect, but kept for compatibility.
+	if string(raw) == "null" && mr.Descriptor().FullName() != "google.protobuf.Value" {
+		return nil
+	}
+
+	if wrapJSONUnmarshalV2 {
+		// NOTE: If input message is non-empty, we need to preserve merge semantics
+		// of the old jsonpb implementation. These semantics are not supported by
+		// the protobuf JSON specification.
+		isEmpty := true
+		mr.Range(func(protoreflect.FieldDescriptor, protoreflect.Value) bool {
+			isEmpty = false // at least one iteration implies non-empty
+			return false
+		})
+		if !isEmpty {
+			// Perform unmarshaling into a newly allocated, empty message.
+			mr = mr.New()
+
+			// Use a defer to copy all unmarshaled fields into the original message.
+			dst := proto.MessageReflect(m)
+			defer mr.Range(func(fd protoreflect.FieldDescriptor, v protoreflect.Value) bool {
+				dst.Set(fd, v)
+				return true
+			})
+		}
+
+		// Unmarshal using the v2 JSON unmarshaler.
+		opts := protojson.UnmarshalOptions{
+			DiscardUnknown: u.AllowUnknownFields,
+		}
+		if u.AnyResolver != nil {
+			opts.Resolver = anyResolver{u.AnyResolver}
+		}
+		return opts.Unmarshal(raw, mr.Interface())
+	} else {
+		if err := u.unmarshalMessage(mr, raw); err != nil {
+			return err
+		}
+		return protoV2.CheckInitialized(mr.Interface())
+	}
+}
+
+func (u *Unmarshaler) unmarshalMessage(m protoreflect.Message, in []byte) error {
+	md := m.Descriptor()
+	fds := md.Fields()
+
+	if jsu, ok := proto.MessageV1(m.Interface()).(JSONPBUnmarshaler); ok {
+		return jsu.UnmarshalJSONPB(u, in)
+	}
+
+	if string(in) == "null" && md.FullName() != "google.protobuf.Value" {
+		return nil
+	}
+
+	switch wellKnownType(md.FullName()) {
+	case "Any":
+		var jsonObject map[string]json.RawMessage
+		if err := json.Unmarshal(in, &jsonObject); err != nil {
+			return err
+		}
+
+		rawTypeURL, ok := jsonObject["@type"]
+		if !ok {
+			return errors.New("Any JSON doesn't have '@type'")
+		}
+		typeURL, err := unquoteString(string(rawTypeURL))
+		if err != nil {
+			return fmt.Errorf("can't unmarshal Any's '@type': %q", rawTypeURL)
+		}
+		m.Set(fds.ByNumber(1), protoreflect.ValueOfString(typeURL))
+
+		var m2 protoreflect.Message
+		if u.AnyResolver != nil {
+			mi, err := u.AnyResolver.Resolve(typeURL)
+			if err != nil {
+				return err
+			}
+			m2 = proto.MessageReflect(mi)
+		} else {
+			mt, err := protoregistry.GlobalTypes.FindMessageByURL(typeURL)
+			if err != nil {
+				if err == protoregistry.NotFound {
+					return fmt.Errorf("could not resolve Any message type: %v", typeURL)
+				}
+				return err
+			}
+			m2 = mt.New()
+		}
+
+		if wellKnownType(m2.Descriptor().FullName()) != "" {
+			rawValue, ok := jsonObject["value"]
+			if !ok {
+				return errors.New("Any JSON doesn't have 'value'")
+			}
+			if err := u.unmarshalMessage(m2, rawValue); err != nil {
+				return fmt.Errorf("can't unmarshal Any nested proto %v: %v", typeURL, err)
+			}
+		} else {
+			delete(jsonObject, "@type")
+			rawJSON, err := json.Marshal(jsonObject)
+			if err != nil {
+				return fmt.Errorf("can't generate JSON for Any's nested proto to be unmarshaled: %v", err)
+			}
+			if err = u.unmarshalMessage(m2, rawJSON); err != nil {
+				return fmt.Errorf("can't unmarshal Any nested proto %v: %v", typeURL, err)
+			}
+		}
+
+		rawWire, err := protoV2.Marshal(m2.Interface())
+		if err != nil {
+			return fmt.Errorf("can't marshal proto %v into Any.Value: %v", typeURL, err)
+		}
+		m.Set(fds.ByNumber(2), protoreflect.ValueOfBytes(rawWire))
+		return nil
+	case "BoolValue", "BytesValue", "StringValue",
+		"Int32Value", "UInt32Value", "FloatValue",
+		"Int64Value", "UInt64Value", "DoubleValue":
+		fd := fds.ByNumber(1)
+		v, err := u.unmarshalValue(m.NewField(fd), in, fd)
+		if err != nil {
+			return err
+		}
+		m.Set(fd, v)
+		return nil
+	case "Duration":
+		v, err := unquoteString(string(in))
+		if err != nil {
+			return err
+		}
+		d, err := time.ParseDuration(v)
+		if err != nil {
+			return fmt.Errorf("bad Duration: %v", err)
+		}
+
+		sec := d.Nanoseconds() / 1e9
+		nsec := d.Nanoseconds() % 1e9
+		m.Set(fds.ByNumber(1), protoreflect.ValueOfInt64(int64(sec)))
+		m.Set(fds.ByNumber(2), protoreflect.ValueOfInt32(int32(nsec)))
+		return nil
+	case "Timestamp":
+		v, err := unquoteString(string(in))
+		if err != nil {
+			return err
+		}
+		t, err := time.Parse(time.RFC3339Nano, v)
+		if err != nil {
+			return fmt.Errorf("bad Timestamp: %v", err)
+		}
+
+		sec := t.Unix()
+		nsec := t.Nanosecond()
+		m.Set(fds.ByNumber(1), protoreflect.ValueOfInt64(int64(sec)))
+		m.Set(fds.ByNumber(2), protoreflect.ValueOfInt32(int32(nsec)))
+		return nil
+	case "Value":
+		switch {
+		case string(in) == "null":
+			m.Set(fds.ByNumber(1), protoreflect.ValueOfEnum(0))
+		case string(in) == "true":
+			m.Set(fds.ByNumber(4), protoreflect.ValueOfBool(true))
+		case string(in) == "false":
+			m.Set(fds.ByNumber(4), protoreflect.ValueOfBool(false))
+		case hasPrefixAndSuffix('"', in, '"'):
+			s, err := unquoteString(string(in))
+			if err != nil {
+				return fmt.Errorf("unrecognized type for Value %q", in)
+			}
+			m.Set(fds.ByNumber(3), protoreflect.ValueOfString(s))
+		case hasPrefixAndSuffix('[', in, ']'):
+			v := m.Mutable(fds.ByNumber(6))
+			return u.unmarshalMessage(v.Message(), in)
+		case hasPrefixAndSuffix('{', in, '}'):
+			v := m.Mutable(fds.ByNumber(5))
+			return u.unmarshalMessage(v.Message(), in)
+		default:
+			f, err := strconv.ParseFloat(string(in), 0)
+			if err != nil {
+				return fmt.Errorf("unrecognized type for Value %q", in)
+			}
+			m.Set(fds.ByNumber(2), protoreflect.ValueOfFloat64(f))
+		}
+		return nil
+	case "ListValue":
+		var jsonArray []json.RawMessage
+		if err := json.Unmarshal(in, &jsonArray); err != nil {
+			return fmt.Errorf("bad ListValue: %v", err)
+		}
+
+		lv := m.Mutable(fds.ByNumber(1)).List()
+		for _, raw := range jsonArray {
+			ve := lv.NewElement()
+			if err := u.unmarshalMessage(ve.Message(), raw); err != nil {
+				return err
+			}
+			lv.Append(ve)
+		}
+		return nil
+	case "Struct":
+		var jsonObject map[string]json.RawMessage
+		if err := json.Unmarshal(in, &jsonObject); err != nil {
+			return fmt.Errorf("bad StructValue: %v", err)
+		}
+
+		mv := m.Mutable(fds.ByNumber(1)).Map()
+		for key, raw := range jsonObject {
+			kv := protoreflect.ValueOf(key).MapKey()
+			vv := mv.NewValue()
+			if err := u.unmarshalMessage(vv.Message(), raw); err != nil {
+				return fmt.Errorf("bad value in StructValue for key %q: %v", key, err)
+			}
+			mv.Set(kv, vv)
+		}
+		return nil
+	}
+
+	var jsonObject map[string]json.RawMessage
+	if err := json.Unmarshal(in, &jsonObject); err != nil {
+		return err
+	}
+
+	// Handle known fields.
+	for i := 0; i < fds.Len(); i++ {
+		fd := fds.Get(i)
+		if fd.IsWeak() && fd.Message().IsPlaceholder() {
+			continue //  weak reference is not linked in
+		}
+
+		// Search for any raw JSON value associated with this field.
+		var raw json.RawMessage
+		name := string(fd.Name())
+		if fd.Kind() == protoreflect.GroupKind {
+			name = string(fd.Message().Name())
+		}
+		if v, ok := jsonObject[name]; ok {
+			delete(jsonObject, name)
+			raw = v
+		}
+		name = string(fd.JSONName())
+		if v, ok := jsonObject[name]; ok {
+			delete(jsonObject, name)
+			raw = v
+		}
+
+		field := m.NewField(fd)
+		// Unmarshal the field value.
+		if raw == nil || (string(raw) == "null" && !isSingularWellKnownValue(fd) && !isSingularJSONPBUnmarshaler(field, fd)) {
+			continue
+		}
+		v, err := u.unmarshalValue(field, raw, fd)
+		if err != nil {
+			return err
+		}
+		m.Set(fd, v)
+	}
+
+	// Handle extension fields.
+	for name, raw := range jsonObject {
+		if !strings.HasPrefix(name, "[") || !strings.HasSuffix(name, "]") {
+			continue
+		}
+
+		// Resolve the extension field by name.
+		xname := protoreflect.FullName(name[len("[") : len(name)-len("]")])
+		xt, _ := protoregistry.GlobalTypes.FindExtensionByName(xname)
+		if xt == nil && isMessageSet(md) {
+			xt, _ = protoregistry.GlobalTypes.FindExtensionByName(xname.Append("message_set_extension"))
+		}
+		if xt == nil {
+			continue
+		}
+		delete(jsonObject, name)
+		fd := xt.TypeDescriptor()
+		if fd.ContainingMessage().FullName() != m.Descriptor().FullName() {
+			return fmt.Errorf("extension field %q does not extend message %q", xname, m.Descriptor().FullName())
+		}
+
+		field := m.NewField(fd)
+		// Unmarshal the field value.
+		if raw == nil || (string(raw) == "null" && !isSingularWellKnownValue(fd) && !isSingularJSONPBUnmarshaler(field, fd)) {
+			continue
+		}
+		v, err := u.unmarshalValue(field, raw, fd)
+		if err != nil {
+			return err
+		}
+		m.Set(fd, v)
+	}
+
+	if !u.AllowUnknownFields && len(jsonObject) > 0 {
+		for name := range jsonObject {
+			return fmt.Errorf("unknown field %q in %v", name, md.FullName())
+		}
+	}
+	return nil
+}
+
+func isSingularWellKnownValue(fd protoreflect.FieldDescriptor) bool {
+	if md := fd.Message(); md != nil {
+		return md.FullName() == "google.protobuf.Value" && fd.Cardinality() != protoreflect.Repeated
+	}
+	return false
+}
+
+func isSingularJSONPBUnmarshaler(v protoreflect.Value, fd protoreflect.FieldDescriptor) bool {
+	if fd.Message() != nil && fd.Cardinality() != protoreflect.Repeated {
+		_, ok := proto.MessageV1(v.Interface()).(JSONPBUnmarshaler)
+		return ok
+	}
+	return false
+}
+
+func (u *Unmarshaler) unmarshalValue(v protoreflect.Value, in []byte, fd protoreflect.FieldDescriptor) (protoreflect.Value, error) {
+	switch {
+	case fd.IsList():
+		var jsonArray []json.RawMessage
+		if err := json.Unmarshal(in, &jsonArray); err != nil {
+			return v, err
+		}
+		lv := v.List()
+		for _, raw := range jsonArray {
+			ve, err := u.unmarshalSingularValue(lv.NewElement(), raw, fd)
+			if err != nil {
+				return v, err
+			}
+			lv.Append(ve)
+		}
+		return v, nil
+	case fd.IsMap():
+		var jsonObject map[string]json.RawMessage
+		if err := json.Unmarshal(in, &jsonObject); err != nil {
+			return v, err
+		}
+		kfd := fd.MapKey()
+		vfd := fd.MapValue()
+		mv := v.Map()
+		for key, raw := range jsonObject {
+			var kv protoreflect.MapKey
+			if kfd.Kind() == protoreflect.StringKind {
+				kv = protoreflect.ValueOf(key).MapKey()
+			} else {
+				v, err := u.unmarshalSingularValue(kfd.Default(), []byte(key), kfd)
+				if err != nil {
+					return v, err
+				}
+				kv = v.MapKey()
+			}
+
+			vv, err := u.unmarshalSingularValue(mv.NewValue(), raw, vfd)
+			if err != nil {
+				return v, err
+			}
+			mv.Set(kv, vv)
+		}
+		return v, nil
+	default:
+		return u.unmarshalSingularValue(v, in, fd)
+	}
+}
+
+var nonFinite = map[string]float64{
+	`"NaN"`:       math.NaN(),
+	`"Infinity"`:  math.Inf(+1),
+	`"-Infinity"`: math.Inf(-1),
+}
+
+func (u *Unmarshaler) unmarshalSingularValue(v protoreflect.Value, in []byte, fd protoreflect.FieldDescriptor) (protoreflect.Value, error) {
+	switch fd.Kind() {
+	case protoreflect.BoolKind:
+		return unmarshalValue(in, new(bool))
+	case protoreflect.Int32Kind, protoreflect.Sint32Kind, protoreflect.Sfixed32Kind:
+		return unmarshalValue(trimQuote(in), new(int32))
+	case protoreflect.Int64Kind, protoreflect.Sint64Kind, protoreflect.Sfixed64Kind:
+		return unmarshalValue(trimQuote(in), new(int64))
+	case protoreflect.Uint32Kind, protoreflect.Fixed32Kind:
+		return unmarshalValue(trimQuote(in), new(uint32))
+	case protoreflect.Uint64Kind, protoreflect.Fixed64Kind:
+		return unmarshalValue(trimQuote(in), new(uint64))
+	case protoreflect.FloatKind:
+		if f, ok := nonFinite[string(in)]; ok {
+			return protoreflect.ValueOfFloat32(float32(f)), nil
+		}
+		return unmarshalValue(trimQuote(in), new(float32))
+	case protoreflect.DoubleKind:
+		if f, ok := nonFinite[string(in)]; ok {
+			return protoreflect.ValueOfFloat64(float64(f)), nil
+		}
+		return unmarshalValue(trimQuote(in), new(float64))
+	case protoreflect.StringKind:
+		return unmarshalValue(in, new(string))
+	case protoreflect.BytesKind:
+		return unmarshalValue(in, new([]byte))
+	case protoreflect.EnumKind:
+		if hasPrefixAndSuffix('"', in, '"') {
+			vd := fd.Enum().Values().ByName(protoreflect.Name(trimQuote(in)))
+			if vd == nil {
+				return v, fmt.Errorf("unknown value %q for enum %s", in, fd.Enum().FullName())
+			}
+			return protoreflect.ValueOfEnum(vd.Number()), nil
+		}
+		return unmarshalValue(in, new(protoreflect.EnumNumber))
+	case protoreflect.MessageKind, protoreflect.GroupKind:
+		err := u.unmarshalMessage(v.Message(), in)
+		return v, err
+	default:
+		panic(fmt.Sprintf("invalid kind %v", fd.Kind()))
+	}
+}
+
+func unmarshalValue(in []byte, v interface{}) (protoreflect.Value, error) {
+	err := json.Unmarshal(in, v)
+	return protoreflect.ValueOf(reflect.ValueOf(v).Elem().Interface()), err
+}
+
+func unquoteString(in string) (out string, err error) {
+	err = json.Unmarshal([]byte(in), &out)
+	return out, err
+}
+
+func hasPrefixAndSuffix(prefix byte, in []byte, suffix byte) bool {
+	if len(in) >= 2 && in[0] == prefix && in[len(in)-1] == suffix {
+		return true
+	}
+	return false
+}
+
+// trimQuote is like unquoteString but simply strips surrounding quotes.
+// This is incorrect, but is behavior done by the legacy implementation.
+func trimQuote(in []byte) []byte {
+	if len(in) >= 2 && in[0] == '"' && in[len(in)-1] == '"' {
+		in = in[1 : len(in)-1]
+	}
+	return in
+}

vendor/github.com/golang/protobuf/jsonpb/encode.go

@@ -0,0 +1,559 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package jsonpb
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"math"
+	"reflect"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/golang/protobuf/proto"
+	"google.golang.org/protobuf/encoding/protojson"
+	protoV2 "google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/reflect/protoreflect"
+	"google.golang.org/protobuf/reflect/protoregistry"
+)
+
+const wrapJSONMarshalV2 = false
+
+// Marshaler is a configurable object for marshaling protocol buffer messages
+// to the specified JSON representation.
+type Marshaler struct {
+	// OrigName specifies whether to use the original protobuf name for fields.
+	OrigName bool
+
+	// EnumsAsInts specifies whether to render enum values as integers,
+	// as opposed to string values.
+	EnumsAsInts bool
+
+	// EmitDefaults specifies whether to render fields with zero values.
+	EmitDefaults bool
+
+	// Indent controls whether the output is compact or not.
+	// If empty, the output is compact JSON. Otherwise, every JSON object
+	// entry and JSON array value will be on its own line.
+	// Each line will be preceded by repeated copies of Indent, where the
+	// number of copies is the current indentation depth.
+	Indent string
+
+	// AnyResolver is used to resolve the google.protobuf.Any well-known type.
+	// If unset, the global registry is used by default.
+	AnyResolver AnyResolver
+}
+
+// JSONPBMarshaler is implemented by protobuf messages that customize the
+// way they are marshaled to JSON. Messages that implement this should also
+// implement JSONPBUnmarshaler so that the custom format can be parsed.
+//
+// The JSON marshaling must follow the proto to JSON specification:
+//	https://developers.google.com/protocol-buffers/docs/proto3#json
+//
+// Deprecated: Custom types should implement protobuf reflection instead.
+type JSONPBMarshaler interface {
+	MarshalJSONPB(*Marshaler) ([]byte, error)
+}
+
+// Marshal serializes a protobuf message as JSON into w.
+func (jm *Marshaler) Marshal(w io.Writer, m proto.Message) error {
+	b, err := jm.marshal(m)
+	if len(b) > 0 {
+		if _, err := w.Write(b); err != nil {
+			return err
+		}
+	}
+	return err
+}
+
+// MarshalToString serializes a protobuf message as JSON in string form.
+func (jm *Marshaler) MarshalToString(m proto.Message) (string, error) {
+	b, err := jm.marshal(m)
+	if err != nil {
+		return "", err
+	}
+	return string(b), nil
+}
+
+func (jm *Marshaler) marshal(m proto.Message) ([]byte, error) {
+	v := reflect.ValueOf(m)
+	if m == nil || (v.Kind() == reflect.Ptr && v.IsNil()) {
+		return nil, errors.New("Marshal called with nil")
+	}
+
+	// Check for custom marshalers first since they may not properly
+	// implement protobuf reflection that the logic below relies on.
+	if jsm, ok := m.(JSONPBMarshaler); ok {
+		return jsm.MarshalJSONPB(jm)
+	}
+
+	if wrapJSONMarshalV2 {
+		opts := protojson.MarshalOptions{
+			UseProtoNames:   jm.OrigName,
+			UseEnumNumbers:  jm.EnumsAsInts,
+			EmitUnpopulated: jm.EmitDefaults,
+			Indent:          jm.Indent,
+		}
+		if jm.AnyResolver != nil {
+			opts.Resolver = anyResolver{jm.AnyResolver}
+		}
+		return opts.Marshal(proto.MessageReflect(m).Interface())
+	} else {
+		// Check for unpopulated required fields first.
+		m2 := proto.MessageReflect(m)
+		if err := protoV2.CheckInitialized(m2.Interface()); err != nil {
+			return nil, err
+		}
+
+		w := jsonWriter{Marshaler: jm}
+		err := w.marshalMessage(m2, "", "")
+		return w.buf, err
+	}
+}
+
+type jsonWriter struct {
+	*Marshaler
+	buf []byte
+}
+
+func (w *jsonWriter) write(s string) {
+	w.buf = append(w.buf, s...)
+}
+
+func (w *jsonWriter) marshalMessage(m protoreflect.Message, indent, typeURL string) error {
+	if jsm, ok := proto.MessageV1(m.Interface()).(JSONPBMarshaler); ok {
+		b, err := jsm.MarshalJSONPB(w.Marshaler)
+		if err != nil {
+			return err
+		}
+		if typeURL != "" {
+			// we are marshaling this object to an Any type
+			var js map[string]*json.RawMessage
+			if err = json.Unmarshal(b, &js); err != nil {
+				return fmt.Errorf("type %T produced invalid JSON: %v", m.Interface(), err)
+			}
+			turl, err := json.Marshal(typeURL)
+			if err != nil {
+				return fmt.Errorf("failed to marshal type URL %q to JSON: %v", typeURL, err)
+			}
+			js["@type"] = (*json.RawMessage)(&turl)
+			if b, err = json.Marshal(js); err != nil {
+				return err
+			}
+		}
+		w.write(string(b))
+		return nil
+	}
+
+	md := m.Descriptor()
+	fds := md.Fields()
+
+	// Handle well-known types.
+	const secondInNanos = int64(time.Second / time.Nanosecond)
+	switch wellKnownType(md.FullName()) {
+	case "Any":
+		return w.marshalAny(m, indent)
+	case "BoolValue", "BytesValue", "StringValue",
+		"Int32Value", "UInt32Value", "FloatValue",
+		"Int64Value", "UInt64Value", "DoubleValue":
+		fd := fds.ByNumber(1)
+		return w.marshalValue(fd, m.Get(fd), indent)
+	case "Duration":
+		const maxSecondsInDuration = 315576000000
+		// "Generated output always contains 0, 3, 6, or 9 fractional digits,
+		//  depending on required precision."
+		s := m.Get(fds.ByNumber(1)).Int()
+		ns := m.Get(fds.ByNumber(2)).Int()
+		if s < -maxSecondsInDuration || s > maxSecondsInDuration {
+			return fmt.Errorf("seconds out of range %v", s)
+		}
+		if ns <= -secondInNanos || ns >= secondInNanos {
+			return fmt.Errorf("ns out of range (%v, %v)", -secondInNanos, secondInNanos)
+		}
+		if (s > 0 && ns < 0) || (s < 0 && ns > 0) {
+			return errors.New("signs of seconds and nanos do not match")
+		}
+		var sign string
+		if s < 0 || ns < 0 {
+			sign, s, ns = "-", -1*s, -1*ns
+		}
+		x := fmt.Sprintf("%s%d.%09d", sign, s, ns)
+		x = strings.TrimSuffix(x, "000")
+		x = strings.TrimSuffix(x, "000")
+		x = strings.TrimSuffix(x, ".000")
+		w.write(fmt.Sprintf(`"%vs"`, x))
+		return nil
+	case "Timestamp":
+		// "RFC 3339, where generated output will always be Z-normalized
+		//  and uses 0, 3, 6 or 9 fractional digits."
+		s := m.Get(fds.ByNumber(1)).Int()
+		ns := m.Get(fds.ByNumber(2)).Int()
+		if ns < 0 || ns >= secondInNanos {
+			return fmt.Errorf("ns out of range [0, %v)", secondInNanos)
+		}
+		t := time.Unix(s, ns).UTC()
+		// time.RFC3339Nano isn't exactly right (we need to get 3/6/9 fractional digits).
+		x := t.Format("2006-01-02T15:04:05.000000000")
+		x = strings.TrimSuffix(x, "000")
+		x = strings.TrimSuffix(x, "000")
+		x = strings.TrimSuffix(x, ".000")
+		w.write(fmt.Sprintf(`"%vZ"`, x))
+		return nil
+	case "Value":
+		// JSON value; which is a null, number, string, bool, object, or array.
+		od := md.Oneofs().Get(0)
+		fd := m.WhichOneof(od)
+		if fd == nil {
+			return errors.New("nil Value")
+		}
+		return w.marshalValue(fd, m.Get(fd), indent)
+	case "Struct", "ListValue":
+		// JSON object or array.
+		fd := fds.ByNumber(1)
+		return w.marshalValue(fd, m.Get(fd), indent)
+	}
+
+	w.write("{")
+	if w.Indent != "" {
+		w.write("\n")
+	}
+
+	firstField := true
+	if typeURL != "" {
+		if err := w.marshalTypeURL(indent, typeURL); err != nil {
+			return err
+		}
+		firstField = false
+	}
+
+	for i := 0; i < fds.Len(); {
+		fd := fds.Get(i)
+		if od := fd.ContainingOneof(); od != nil {
+			fd = m.WhichOneof(od)
+			i += od.Fields().Len()
+			if fd == nil {
+				continue
+			}
+		} else {
+			i++
+		}
+
+		v := m.Get(fd)
+
+		if !m.Has(fd) {
+			if !w.EmitDefaults || fd.ContainingOneof() != nil {
+				continue
+			}
+			if fd.Cardinality() != protoreflect.Repeated && (fd.Message() != nil || fd.Syntax() == protoreflect.Proto2) {
+				v = protoreflect.Value{} // use "null" for singular messages or proto2 scalars
+			}
+		}
+
+		if !firstField {
+			w.writeComma()
+		}
+		if err := w.marshalField(fd, v, indent); err != nil {
+			return err
+		}
+		firstField = false
+	}
+
+	// Handle proto2 extensions.
+	if md.ExtensionRanges().Len() > 0 {
+		// Collect a sorted list of all extension descriptor and values.
+		type ext struct {
+			desc protoreflect.FieldDescriptor
+			val  protoreflect.Value
+		}
+		var exts []ext
+		m.Range(func(fd protoreflect.FieldDescriptor, v protoreflect.Value) bool {
+			if fd.IsExtension() {
+				exts = append(exts, ext{fd, v})
+			}
+			return true
+		})
+		sort.Slice(exts, func(i, j int) bool {
+			return exts[i].desc.Number() < exts[j].desc.Number()
+		})
+
+		for _, ext := range exts {
+			if !firstField {
+				w.writeComma()
+			}
+			if err := w.marshalField(ext.desc, ext.val, indent); err != nil {
+				return err
+			}
+			firstField = false
+		}
+	}
+
+	if w.Indent != "" {
+		w.write("\n")
+		w.write(indent)
+	}
+	w.write("}")
+	return nil
+}
+
+func (w *jsonWriter) writeComma() {
+	if w.Indent != "" {
+		w.write(",\n")
+	} else {
+		w.write(",")
+	}
+}
+
+func (w *jsonWriter) marshalAny(m protoreflect.Message, indent string) error {
+	// "If the Any contains a value that has a special JSON mapping,
+	//  it will be converted as follows: {"@type": xxx, "value": yyy}.
+	//  Otherwise, the value will be converted into a JSON object,
+	//  and the "@type" field will be inserted to indicate the actual data type."
+	md := m.Descriptor()
+	typeURL := m.Get(md.Fields().ByNumber(1)).String()
+	rawVal := m.Get(md.Fields().ByNumber(2)).Bytes()
+
+	var m2 protoreflect.Message
+	if w.AnyResolver != nil {
+		mi, err := w.AnyResolver.Resolve(typeURL)
+		if err != nil {
+			return err
+		}
+		m2 = proto.MessageReflect(mi)
+	} else {
+		mt, err := protoregistry.GlobalTypes.FindMessageByURL(typeURL)
+		if err != nil {
+			return err
+		}
+		m2 = mt.New()
+	}
+
+	if err := protoV2.Unmarshal(rawVal, m2.Interface()); err != nil {
+		return err
+	}
+
+	if wellKnownType(m2.Descriptor().FullName()) == "" {
+		return w.marshalMessage(m2, indent, typeURL)
+	}
+
+	w.write("{")
+	if w.Indent != "" {
+		w.write("\n")
+	}
+	if err := w.marshalTypeURL(indent, typeURL); err != nil {
+		return err
+	}
+	w.writeComma()
+	if w.Indent != "" {
+		w.write(indent)
+		w.write(w.Indent)
+		w.write(`"value": `)
+	} else {
+		w.write(`"value":`)
+	}
+	if err := w.marshalMessage(m2, indent+w.Indent, ""); err != nil {
+		return err
+	}
+	if w.Indent != "" {
+		w.write("\n")
+		w.write(indent)
+	}
+	w.write("}")
+	return nil
+}
+
+func (w *jsonWriter) marshalTypeURL(indent, typeURL string) error {
+	if w.Indent != "" {
+		w.write(indent)
+		w.write(w.Indent)
+	}
+	w.write(`"@type":`)
+	if w.Indent != "" {
+		w.write(" ")
+	}
+	b, err := json.Marshal(typeURL)
+	if err != nil {
+		return err
+	}
+	w.write(string(b))
+	return nil
+}
+
+// marshalField writes field description and value to the Writer.
+func (w *jsonWriter) marshalField(fd protoreflect.FieldDescriptor, v protoreflect.Value, indent string) error {
+	if w.Indent != "" {
+		w.write(indent)
+		w.write(w.Indent)
+	}
+	w.write(`"`)
+	switch {
+	case fd.IsExtension():
+		// For message set, use the fname of the message as the extension name.
+		name := string(fd.FullName())
+		if isMessageSet(fd.ContainingMessage()) {
+			name = strings.TrimSuffix(name, ".message_set_extension")
+		}
+
+		w.write("[" + name + "]")
+	case w.OrigName:
+		name := string(fd.Name())
+		if fd.Kind() == protoreflect.GroupKind {
+			name = string(fd.Message().Name())
+		}
+		w.write(name)
+	default:
+		w.write(string(fd.JSONName()))
+	}
+	w.write(`":`)
+	if w.Indent != "" {
+		w.write(" ")
+	}
+	return w.marshalValue(fd, v, indent)
+}
+
+func (w *jsonWriter) marshalValue(fd protoreflect.FieldDescriptor, v protoreflect.Value, indent string) error {
+	switch {
+	case fd.IsList():
+		w.write("[")
+		comma := ""
+		lv := v.List()
+		for i := 0; i < lv.Len(); i++ {
+			w.write(comma)
+			if w.Indent != "" {
+				w.write("\n")
+				w.write(indent)
+				w.write(w.Indent)
+				w.write(w.Indent)
+			}
+			if err := w.marshalSingularValue(fd, lv.Get(i), indent+w.Indent); err != nil {
+				return err
+			}
+			comma = ","
+		}
+		if w.Indent != "" {
+			w.write("\n")
+			w.write(indent)
+			w.write(w.Indent)
+		}
+		w.write("]")
+		return nil
+	case fd.IsMap():
+		kfd := fd.MapKey()
+		vfd := fd.MapValue()
+		mv := v.Map()
+
+		// Collect a sorted list of all map keys and values.
+		type entry struct{ key, val protoreflect.Value }
+		var entries []entry
+		mv.Range(func(k protoreflect.MapKey, v protoreflect.Value) bool {
+			entries = append(entries, entry{k.Value(), v})
+			return true
+		})
+		sort.Slice(entries, func(i, j int) bool {
+			switch kfd.Kind() {
+			case protoreflect.BoolKind:
+				return !entries[i].key.Bool() && entries[j].key.Bool()
+			case protoreflect.Int32Kind, protoreflect.Sint32Kind, protoreflect.Sfixed32Kind, protoreflect.Int64Kind, protoreflect.Sint64Kind, protoreflect.Sfixed64Kind:
+				return entries[i].key.Int() < entries[j].key.Int()
+			case protoreflect.Uint32Kind, protoreflect.Fixed32Kind, protoreflect.Uint64Kind, protoreflect.Fixed64Kind:
+				return entries[i].key.Uint() < entries[j].key.Uint()
+			case protoreflect.StringKind:
+				return entries[i].key.String() < entries[j].key.String()
+			default:
+				panic("invalid kind")
+			}
+		})
+
+		w.write(`{`)
+		comma := ""
+		for _, entry := range entries {
+			w.write(comma)
+			if w.Indent != "" {
+				w.write("\n")
+				w.write(indent)
+				w.write(w.Indent)
+				w.write(w.Indent)
+			}
+
+			s := fmt.Sprint(entry.key.Interface())
+			b, err := json.Marshal(s)
+			if err != nil {
+				return err
+			}
+			w.write(string(b))
+
+			w.write(`:`)
+			if w.Indent != "" {
+				w.write(` `)
+			}
+
+			if err := w.marshalSingularValue(vfd, entry.val, indent+w.Indent); err != nil {
+				return err
+			}
+			comma = ","
+		}
+		if w.Indent != "" {
+			w.write("\n")
+			w.write(indent)
+			w.write(w.Indent)
+		}
+		w.write(`}`)
+		return nil
+	default:
+		return w.marshalSingularValue(fd, v, indent)
+	}
+}
+
+func (w *jsonWriter) marshalSingularValue(fd protoreflect.FieldDescriptor, v protoreflect.Value, indent string) error {
+	switch {
+	case !v.IsValid():
+		w.write("null")
+		return nil
+	case fd.Message() != nil:
+		return w.marshalMessage(v.Message(), indent+w.Indent, "")
+	case fd.Enum() != nil:
+		if fd.Enum().FullName() == "google.protobuf.NullValue" {
+			w.write("null")
+			return nil
+		}
+
+		vd := fd.Enum().Values().ByNumber(v.Enum())
+		if vd == nil || w.EnumsAsInts {
+			w.write(strconv.Itoa(int(v.Enum())))
+		} else {
+			w.write(`"` + string(vd.Name()) + `"`)
+		}
+		return nil
+	default:
+		switch v.Interface().(type) {
+		case float32, float64:
+			switch {
+			case math.IsInf(v.Float(), +1):
+				w.write(`"Infinity"`)
+				return nil
+			case math.IsInf(v.Float(), -1):
+				w.write(`"-Infinity"`)
+				return nil
+			case math.IsNaN(v.Float()):
+				w.write(`"NaN"`)
+				return nil
+			}
+		case int64, uint64:
+			w.write(fmt.Sprintf(`"%d"`, v.Interface()))
+			return nil
+		}
+
+		b, err := json.Marshal(v.Interface())
+		if err != nil {
+			return err
+		}
+		w.write(string(b))
+		return nil
+	}
+}

vendor/github.com/golang/protobuf/jsonpb/json.go

@@ -0,0 +1,69 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package jsonpb provides functionality to marshal and unmarshal between a
+// protocol buffer message and JSON. It follows the specification at
+// https://developers.google.com/protocol-buffers/docs/proto3#json.
+//
+// Do not rely on the default behavior of the standard encoding/json package
+// when called on generated message types as it does not operate correctly.
+//
+// Deprecated: Use the "google.golang.org/protobuf/encoding/protojson"
+// package instead.
+package jsonpb
+
+import (
+	"github.com/golang/protobuf/proto"
+	"google.golang.org/protobuf/reflect/protoreflect"
+	"google.golang.org/protobuf/reflect/protoregistry"
+	"google.golang.org/protobuf/runtime/protoimpl"
+)
+
+// AnyResolver takes a type URL, present in an Any message,
+// and resolves it into an instance of the associated message.
+type AnyResolver interface {
+	Resolve(typeURL string) (proto.Message, error)
+}
+
+type anyResolver struct{ AnyResolver }
+
+func (r anyResolver) FindMessageByName(message protoreflect.FullName) (protoreflect.MessageType, error) {
+	return r.FindMessageByURL(string(message))
+}
+
+func (r anyResolver) FindMessageByURL(url string) (protoreflect.MessageType, error) {
+	m, err := r.Resolve(url)
+	if err != nil {
+		return nil, err
+	}
+	return protoimpl.X.MessageTypeOf(m), nil
+}
+
+func (r anyResolver) FindExtensionByName(field protoreflect.FullName) (protoreflect.ExtensionType, error) {
+	return protoregistry.GlobalTypes.FindExtensionByName(field)
+}
+
+func (r anyResolver) FindExtensionByNumber(message protoreflect.FullName, field protoreflect.FieldNumber) (protoreflect.ExtensionType, error) {
+	return protoregistry.GlobalTypes.FindExtensionByNumber(message, field)
+}
+
+func wellKnownType(s protoreflect.FullName) string {
+	if s.Parent() == "google.protobuf" {
+		switch s.Name() {
+		case "Empty", "Any",
+			"BoolValue", "BytesValue", "StringValue",
+			"Int32Value", "UInt32Value", "FloatValue",
+			"Int64Value", "UInt64Value", "DoubleValue",
+			"Duration", "Timestamp",
+			"NullValue", "Struct", "Value", "ListValue":
+			return string(s.Name())
+		}
+	}
+	return ""
+}
+
+func isMessageSet(md protoreflect.MessageDescriptor) bool {
+	ms, ok := md.(interface{ IsMessageSet() bool })
+	return ok && ms.IsMessageSet()
+}

vendor/github.com/marten-seemann/qtls-go1-18/handshake_server.go

@@ -270,7 +270,7 @@ func (hs *serverHandshakeState) processClientHello() error {
 
 	hs.ecdheOk = supportsECDHE(c.config, hs.clientHello.supportedCurves, hs.clientHello.supportedPoints)
 
-	if hs.ecdheOk {
+	if hs.ecdheOk && len(hs.clientHello.supportedPoints) > 0 {
 		// Although omitting the ec_point_formats extension is permitted, some
 		// old OpenSSL version will refuse to handshake if not present.
 		//
@@ -351,6 +351,13 @@ func supportsECDHE(c *config, supportedCurves []CurveID, supportedPoints []uint8
 			break
 		}
 	}
+	// Per RFC 8422, Section 5.1.2, if the Supported Point Formats extension is
+	// missing, uncompressed points are supported. If supportedPoints is empty,
+	// the extension must be missing, as an empty extension body is rejected by
+	// the parser. See https://go.dev/issue/49126.
+	if len(supportedPoints) == 0 {
+		supportsPointFormat = true
+	}
 
 	return supportsCurve && supportsPointFormat
 }

vendor/github.com/marten-seemann/qtls-go1-19/handshake_server.go

@@ -270,7 +270,7 @@ func (hs *serverHandshakeState) processClientHello() error {
 
 	hs.ecdheOk = supportsECDHE(c.config, hs.clientHello.supportedCurves, hs.clientHello.supportedPoints)
 
-	if hs.ecdheOk {
+	if hs.ecdheOk && len(hs.clientHello.supportedPoints) > 0 {
 		// Although omitting the ec_point_formats extension is permitted, some
 		// old OpenSSL version will refuse to handshake if not present.
 		//
@@ -351,6 +351,13 @@ func supportsECDHE(c *config, supportedCurves []CurveID, supportedPoints []uint8
 			break
 		}
 	}
+	// Per RFC 8422, Section 5.1.2, if the Supported Point Formats extension is
+	// missing, uncompressed points are supported. If supportedPoints is empty,
+	// the extension must be missing, as an empty extension body is rejected by
+	// the parser. See https://go.dev/issue/49126.
+	if len(supportedPoints) == 0 {
+		supportsPointFormat = true
+	}
 
 	return supportsCurve && supportsPointFormat
 }

vendor/golang.org/x/crypto/chacha20/chacha_generic.go

@@ -12,7 +12,7 @@ import (
 	"errors"
 	"math/bits"
 
-	"golang.org/x/crypto/internal/subtle"
+	"golang.org/x/crypto/internal/alias"
 )
 
 const (
@@ -189,7 +189,7 @@ func (s *Cipher) XORKeyStream(dst, src []byte) {
 		panic("chacha20: output smaller than input")
 	}
 	dst = dst[:len(src)]
-	if subtle.InexactOverlap(dst, src) {
+	if alias.InexactOverlap(dst, src) {
 		panic("chacha20: invalid buffer overlap")
 	}