Release 2022.11.0

This PR changes protocol initialization of the other N connections to be
the same as the one we know the initial tunnel connected with. This is
so we homogenize connections and not lead to some connections being
QUIC-able and the others not.

There's also an improvement to the connection registered log so we know
what protocol every individual connection connected with from the
cloudflared side.

.docker-images

@@ -2,8 +2,7 @@ images:
   - name: cloudflared
     dockerfile: Dockerfile.$ARCH
     context: .
-    versions: 
-      - latest
+    version_file: versions
     registries:
     - name: docker.io/cloudflare
       user: env:DOCKER_USER

Dockerfile

@@ -7,8 +7,6 @@ ENV GO111MODULE=on \
     TARGET_GOOS=${TARGET_GOOS} \
     TARGET_GOARCH=${TARGET_GOARCH}
     
-LABEL org.opencontainers.image.source="https://github.com/cloudflare/cloudflared"
-
 WORKDIR /go/src/github.com/cloudflare/cloudflared/
 
 # copy our sources into the builder image
@@ -20,6 +18,8 @@ RUN make cloudflared
 # use a distroless base image with glibc
 FROM gcr.io/distroless/base-debian11:nonroot
 
+LABEL org.opencontainers.image.source="https://github.com/cloudflare/cloudflared"
+
 # copy our compiled binary
 COPY --from=builder --chown=nonroot /go/src/github.com/cloudflare/cloudflared/cloudflared /usr/local/bin/
 

Dockerfile.amd64

@@ -3,8 +3,6 @@ FROM golang:1.19 as builder
 ENV GO111MODULE=on \
     CGO_ENABLED=0 
     
-LABEL org.opencontainers.image.source="https://github.com/cloudflare/cloudflared"
-
 WORKDIR /go/src/github.com/cloudflare/cloudflared/
 
 # copy our sources into the builder image
@@ -16,6 +14,8 @@ RUN GOOS=linux GOARCH=amd64 make cloudflared
 # use a distroless base image with glibc
 FROM gcr.io/distroless/base-debian11:nonroot
 
+LABEL org.opencontainers.image.source="https://github.com/cloudflare/cloudflared"
+
 # copy our compiled binary
 COPY --from=builder --chown=nonroot /go/src/github.com/cloudflare/cloudflared/cloudflared /usr/local/bin/
 

Dockerfile.arm64

@@ -3,8 +3,6 @@ FROM golang:1.19 as builder
 ENV GO111MODULE=on \
     CGO_ENABLED=0 
     
-LABEL org.opencontainers.image.source="https://github.com/cloudflare/cloudflared"
-
 WORKDIR /go/src/github.com/cloudflare/cloudflared/
 
 # copy our sources into the builder image
@@ -16,6 +14,8 @@ RUN GOOS=linux GOARCH=arm64 make cloudflared
 # use a distroless base image with glibc
 FROM gcr.io/distroless/base-debian11:nonroot-arm64
 
+LABEL org.opencontainers.image.source="https://github.com/cloudflare/cloudflared"
+
 # copy our compiled binary
 COPY --from=builder --chown=nonroot /go/src/github.com/cloudflare/cloudflared/cloudflared /usr/local/bin/
 

Makefile

@@ -131,6 +131,10 @@ endif
 container:
 	docker build --build-arg=TARGET_ARCH=$(TARGET_ARCH) --build-arg=TARGET_OS=$(TARGET_OS) -t cloudflare/cloudflared-$(TARGET_OS)-$(TARGET_ARCH):"$(VERSION)" .
 
+.PHONY: generate-docker-version
+generate-docker-version:
+	echo latest $(VERSION) > versions
+
 .PHONY: test
 test: vet
 ifndef CI
@@ -146,7 +150,7 @@ test-ssh-server:
 	docker-compose -f ssh_server_tests/docker-compose.yml up
 
 cloudflared.1: cloudflared_man_template
-	cat cloudflared_man_template | sed -e 's/\$${VERSION}/$(VERSION)/; s/\$${DATE}/$(DATE)/' > cloudflared.1
+	sed -e 's/\$${VERSION}/$(VERSION)/; s/\$${DATE}/$(DATE)/' cloudflared_man_template > cloudflared.1
 
 install: cloudflared cloudflared.1
 	mkdir -p $(DESTDIR)$(INSTALL_BINDIR) $(DESTDIR)$(INSTALL_MANDIR)
@@ -293,8 +297,8 @@ quic-deps:
 
 .PHONY: vet
 vet:
-	go vet -v -mod=vendor ./...
+	go vet -v -mod=vendor github.com/cloudflare/cloudflared/...
 
-.PHONY: goimports
-goimports:
-	for d in $$(go list -mod=readonly -f '{{.Dir}}' -a ./... | fgrep -v tunnelrpc) ; do goimports -format-only -local github.com/cloudflare/cloudflared -w $$d ; done
+.PHONY: fmt
+fmt:
+	goimports -l -w -local github.com/cloudflare/cloudflared $$(go list -mod=vendor -f '{{.Dir}}' -a ./... | fgrep -v tunnelrpc)

README.md

@@ -25,12 +25,13 @@ routing), but for legacy reasons this requirement is still necessary:
 
 ## Installing `cloudflared`
 
-Downloads are available as standalone binaries, a Docker image, and Debian, RPM, and Homebrew packages. You can also find releases here on the `cloudflared` GitHub repository.
+Downloads are available as standalone binaries, a Docker image, and Debian, RPM, and Homebrew packages. You can also find releases [here](https://github.com/cloudflare/cloudflared/releases) on the `cloudflared` GitHub repository.
 
 * You can [install on macOS](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/install-and-setup/installation#macos) via Homebrew or by downloading the [latest Darwin amd64 release](https://github.com/cloudflare/cloudflared/releases)
 * Binaries, Debian, and RPM packages for Linux [can be found here](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/install-and-setup/installation#linux)
 * A Docker image of `cloudflared` is [available on DockerHub](https://hub.docker.com/r/cloudflare/cloudflared)
 * You can install on Windows machines with the [steps here](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/install-and-setup/installation#windows)
+* Build from source with the [instructions here](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/install-and-setup/installation#build-from-source)
 
 User documentation for Cloudflare Tunnel can be found at https://developers.cloudflare.com/cloudflare-one/connections/connect-apps
 

RELEASE_NOTES

@@ -1,3 +1,47 @@
+2022.11.0
+- 2022-11-16 Revert "TUN-6935: Cloudflared should use APIToken instead of serviceKey"
+- 2022-11-16 TUN-6929: Use same protocol for other connections as first one
+- 2022-11-14 TUN-6941: Reduce log level to debug when failing to proxy ICMP reply
+- 2022-11-14 TUN-6935: Cloudflared should use APIToken instead of serviceKey
+- 2022-11-14 TUN-6935: Cloudflared should use APIToken instead of serviceKey
+- 2022-11-11 TUN-6937: Bump golang.org/x/* packages to new release tags
+- 2022-11-10 ZTC-234: macOS tests
+- 2022-11-09 TUN-6927: Refactor validate access configuration to allow empty audTags only
+- 2022-11-08 ZTC-234: Replace ICMP funnels when ingress connection changes
+- 2022-11-04 TUN-6917: Bump go to 1.19.3
+- 2022-11-02 Issue #574: Better ssh config for short-lived cert (#763)
+- 2022-10-28 TUN-6898: Fix bug handling IPv6 based ingresses with missing port
+- 2022-10-28 TUN-6898: Refactor addPortIfMissing
+2022.10.3
+- 2022-10-24 TUN-6871: Add default feature to cloudflared to support EOF on QUIC connections
+- 2022-10-19 TUN-6876: Fix flaky TestTraceICMPRouterEcho by taking account request span can return before reply
+- 2022-10-18 TUN-6867: Clear spans right after they are serialized to avoid returning duplicate spans
+2022.10.2
+- 2022-10-18 TUN-6869: Fix Makefile complaining about missing GO packages
+- 2022-10-18 TUN-6864: Don't reuse port in quic unit tests
+- 2022-10-18 TUN-6868: Return left padded tracing ID when tracing identity is converted to string
+
+2022.10.1
+- 2022-10-16 TUN-6861: Trace ICMP on Windows
+- 2022-10-15 TUN-6860: Send access configuration keys to the edge
+- 2022-10-14 TUN-6858: Trace ICMP reply
+- 2022-10-13 TUN-6855: Add DatagramV2Type for IP packet with trace and tracing spans
+- 2022-10-13 TUN-6856: Refactor to lay foundation for tracing ICMP
+- 2022-10-13 TUN-6604: Trace icmp echo request on Linux and Darwin
+- 2022-10-12 Fix log message (#591)
+- 2022-10-12 TUN-6853: Reuse source port when connecting to the edge for quic connections
+- 2022-10-11 TUN-6829: Allow user of datagramsession to control logging level of errors
+- 2022-10-10 RTG-2276 Update qtls and go mod tidy
+- 2022-10-05 Add post-quantum flag to quick tunnel
+- 2022-10-05 TUN-6823: Update github release message to pull from KV
+- 2022-10-04 TUN-6825: Fix cloudflared:version images require arch hyphens
+- 2022-10-03 TUN-6806: Add ingress rule number to log when filtering due to middlware handler
+- 2022-08-17 Label correct container
+- 2022-08-16 Fix typo in help text for `cloudflared tunnel route lb`
+- 2022-07-18 drop usage of cat when sed is invoked to generate the manpage
+- 2021-03-15 update-build-readme
+- 2021-03-15 fix link
+
 2022.10.0
 - 2022-09-30 TUN-6755: Remove unused publish functions
 - 2022-09-30 TUN-6813: Only proxy ICMP packets when warp-routing is enabled

carrier/carrier.go

@@ -1,6 +1,6 @@
-//Package carrier provides a WebSocket proxy to carry or proxy a connection
-//from the local client to the edge. See it as a wrapper around any protocol
-//that it packages up in a WebSocket connection to the edge.
+// Package carrier provides a WebSocket proxy to carry or proxy a connection
+// from the local client to the edge. See it as a wrapper around any protocol
+// that it packages up in a WebSocket connection to the edge.
 package carrier
 
 import (

certutil/certutil.go

@@ -1,25 +1,21 @@
 package certutil
 
 import (
-	"crypto/x509"
 	"encoding/json"
 	"encoding/pem"
 	"fmt"
-	"strings"
 )
 
 type namedTunnelToken struct {
-	ZoneID     string `json:"zoneID"`
-	AccountID  string `json:"accountID"`
-	ServiceKey string `json:"serviceKey"`
+	ZoneID    string `json:"zoneID"`
+	AccountID string `json:"accountID"`
+	APIToken  string `json:"apiToken"`
 }
 
 type OriginCert struct {
-	PrivateKey interface{}
-	Cert       *x509.Certificate
-	ZoneID     string
-	ServiceKey string
-	AccountID  string
+	ZoneID    string
+	APIToken  string
+	AccountID string
 }
 
 func DecodeOriginCert(blocks []byte) (*OriginCert, error) {
@@ -33,29 +29,11 @@ func DecodeOriginCert(blocks []byte) (*OriginCert, error) {
 			break
 		}
 		switch block.Type {
-		case "PRIVATE KEY":
-			if originCert.PrivateKey != nil {
-				return nil, fmt.Errorf("Found multiple private key in the certificate")
-			}
-			// RSA private key
-			privateKey, err := x509.ParsePKCS8PrivateKey(block.Bytes)
-			if err != nil {
-				return nil, fmt.Errorf("Cannot parse private key")
-			}
-			originCert.PrivateKey = privateKey
-		case "CERTIFICATE":
-			if originCert.Cert != nil {
-				return nil, fmt.Errorf("Found multiple certificates in the certificate")
-			}
-			cert, err := x509.ParseCertificates(block.Bytes)
-			if err != nil {
-				return nil, fmt.Errorf("Cannot parse certificate")
-			} else if len(cert) > 1 {
-				return nil, fmt.Errorf("Found multiple certificates in the certificate")
-			}
-			originCert.Cert = cert[0]
-		case "WARP TOKEN", "ARGO TUNNEL TOKEN":
-			if originCert.ZoneID != "" || originCert.ServiceKey != "" {
+		case "PRIVATE KEY", "CERTIFICATE":
+			// this is for legacy purposes.
+			break
+		case "ARGO TUNNEL TOKEN":
+			if originCert.ZoneID != "" || originCert.APIToken != "" {
 				return nil, fmt.Errorf("Found multiple tokens in the certificate")
 			}
 			// The token is a string,
@@ -63,18 +41,8 @@ func DecodeOriginCert(blocks []byte) (*OriginCert, error) {
 			ntt := namedTunnelToken{}
 			if err := json.Unmarshal(block.Bytes, &ntt); err == nil {
 				originCert.ZoneID = ntt.ZoneID
-				originCert.ServiceKey = ntt.ServiceKey
+				originCert.APIToken = ntt.APIToken
 				originCert.AccountID = ntt.AccountID
-			} else {
-				// Try the older format, where the zoneID and service key are separated by
-				// a new line character
-				token := string(block.Bytes)
-				s := strings.Split(token, "\n")
-				if len(s) != 2 {
-					return nil, fmt.Errorf("Cannot parse token")
-				}
-				originCert.ZoneID = s[0]
-				originCert.ServiceKey = s[1]
 			}
 		default:
 			return nil, fmt.Errorf("Unknown block %s in the certificate", block.Type)
@@ -82,11 +50,7 @@ func DecodeOriginCert(blocks []byte) (*OriginCert, error) {
 		block, rest = pem.Decode(rest)
 	}
 
-	if originCert.PrivateKey == nil {
-		return nil, fmt.Errorf("Missing private key in the certificate")
-	} else if originCert.Cert == nil {
-		return nil, fmt.Errorf("Missing certificate in the certificate")
-	} else if originCert.ZoneID == "" || originCert.ServiceKey == "" {
+	if originCert.ZoneID == "" || originCert.APIToken == "" {
 		return nil, fmt.Errorf("Missing token in the certificate")
 	}
 

certutil/certutil_test.go

@@ -13,49 +13,33 @@ func TestLoadOriginCert(t *testing.T) {
 	assert.Equal(t, fmt.Errorf("Cannot decode empty certificate"), err)
 	assert.Nil(t, cert)
 
-	blocks, err := ioutil.ReadFile("test-cert-no-key.pem")
-	assert.Nil(t, err)
-	cert, err = DecodeOriginCert(blocks)
-	assert.Equal(t, fmt.Errorf("Missing private key in the certificate"), err)
-	assert.Nil(t, cert)
-
-	blocks, err = ioutil.ReadFile("test-cert-two-certificates.pem")
-	assert.Nil(t, err)
-	cert, err = DecodeOriginCert(blocks)
-	assert.Equal(t, fmt.Errorf("Found multiple certificates in the certificate"), err)
-	assert.Nil(t, cert)
-
-	blocks, err = ioutil.ReadFile("test-cert-unknown-block.pem")
+	blocks, err := ioutil.ReadFile("test-cert-unknown-block.pem")
 	assert.Nil(t, err)
 	cert, err = DecodeOriginCert(blocks)
 	assert.Equal(t, fmt.Errorf("Unknown block RSA PRIVATE KEY in the certificate"), err)
 	assert.Nil(t, cert)
-
-	blocks, err = ioutil.ReadFile("test-cert.pem")
-	assert.Nil(t, err)
-	cert, err = DecodeOriginCert(blocks)
-	assert.Nil(t, err)
-	assert.NotNil(t, cert)
-	assert.Equal(t, "7b0a4d77dfb881c1a3b7d61ea9443e19", cert.ZoneID)
-	key := "v1.0-58bd4f9e28f7b3c28e05a35ff3e80ab4fd9644ef3fece537eb0d12e2e9258217-183442fbb0bbdb3e571558fec9b5589ebd77aafc87498ee3f09f64a4ad79ffe8791edbae08b36c1d8f1d70a8670de56922dff92b15d214a524f4ebfa1958859e-7ce80f79921312a6022c5d25e2d380f82ceaefe3fbdc43dd13b080e3ef1e26f7"
-	assert.Equal(t, key, cert.ServiceKey)
 }
 
-func TestNewlineArgoTunnelToken(t *testing.T) {
-	ArgoTunnelTokenTest(t, "test-argo-tunnel-cert.pem")
+func TestJSONArgoTunnelTokenEmpty(t *testing.T) {
+	cert, err := DecodeOriginCert([]byte{})
+	blocks, err := ioutil.ReadFile("test-cert-no-token.pem")
+	assert.Nil(t, err)
+	cert, err = DecodeOriginCert(blocks)
+	assert.Equal(t, fmt.Errorf("Missing token in the certificate"), err)
+	assert.Nil(t, cert)
 }
 
 func TestJSONArgoTunnelToken(t *testing.T) {
 	// The given cert's Argo Tunnel Token was generated by base64 encoding this JSON:
 	// {
 	// "zoneID": "7b0a4d77dfb881c1a3b7d61ea9443e19",
-	// "serviceKey": "test-service-key",
+	// "apiToken": "test-service-key",
 	// "accountID": "abcdabcdabcdabcd1234567890abcdef"
 	// }
-	ArgoTunnelTokenTest(t, "test-argo-tunnel-cert-json.pem")
+	CloudflareTunnelTokenTest(t, "test-cloudflare-tunnel-cert-json.pem")
 }
 
-func ArgoTunnelTokenTest(t *testing.T, path string) {
+func CloudflareTunnelTokenTest(t *testing.T, path string) {
 	blocks, err := ioutil.ReadFile(path)
 	assert.Nil(t, err)
 	cert, err := DecodeOriginCert(blocks)
@@ -63,5 +47,5 @@ func ArgoTunnelTokenTest(t *testing.T, path string) {
 	assert.NotNil(t, cert)
 	assert.Equal(t, "7b0a4d77dfb881c1a3b7d61ea9443e19", cert.ZoneID)
 	key := "test-service-key"
-	assert.Equal(t, key, cert.ServiceKey)
+	assert.Equal(t, key, cert.APIToken)
 }

certutil/test-cert-no-key.pem

@@ -1,33 +0,0 @@
------BEGIN CERTIFICATE-----
-MIID+jCCA6CgAwIBAgIUJhFxUKEGvTRc3CjCok6dbPGH/P4wCgYIKoZIzj0EAwIw
-gagxCzAJBgNVBAYTAlVTMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMTgwNgYD
-VQQLEy9DbG91ZEZsYXJlIE9yaWdpbiBTU0wgRUNDIENlcnRpZmljYXRlIEF1dGhv
-cml0eTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzETMBEGA1UECBMKQ2FsaWZvcm5p
-YTEXMBUGA1UEAxMOKGRldiB1c2Ugb25seSkwHhcNMTcxMDEzMTM1OTAwWhcNMzIx
-MDA5MTM1OTAwWjBiMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMR0wGwYDVQQL
-ExRDbG91ZEZsYXJlIE9yaWdpbiBDQTEmMCQGA1UEAxMdQ2xvdWRGbGFyZSBPcmln
-aW4gQ2VydGlmaWNhdGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCf
-GswL16Fz9Ei3sAg5AmBizoN2nZdyXHP8T57UxUMcrlJXEEXCVS5RR4m9l+EmK0ng
-6yHR1H5oX1Lg1WKyXgWwr0whwmdTD+qWFJW2M8HyefyBKLrsGPuxw4CVYT0h72bx
-tG0uyrXYh7Mtz0lHjGV90qrFpq5o0jx0sLbDlDvpFPbIO58uYzKG4Sn2VTC4rOyX
-PE6SuDvMHIeX6Ekw4wSVQ9eTbksLQqTyxSqM3zp2ygc56SjGjy1nGQT8ZBGFzSbZ
-AzNOxVKrUsySx7LzZVl+zCGCPlQwaYLKObKXadZJmrqSFmErC5jcbVgBz7oJQOgl
-HJ2n0sMcZ+Ja1Y649mPVAgMBAAGjggEgMIIBHDAOBgNVHQ8BAf8EBAMCBaAwEwYD
-VR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUzA6f2Ajq
-zhX67c6piY2a1uTiUkwwHwYDVR0jBBgwFoAU2qfBlqxKMZnf0QeTeYiMelfqJfgw
-RAYIKwYBBQUHAQEEODA2MDQGCCsGAQUFBzABhihodHRwOi8vb2NzcC5jbG91ZGZs
-YXJlLmNvbS9vcmlnaW5fZWNjX2NhMCMGA1UdEQQcMBqCDCouYXJub2xkLmNvbYIK
-YXJub2xkLmNvbTA8BgNVHR8ENTAzMDGgL6AthitodHRwOi8vY3JsLmNsb3VkZmxh
-cmUuY29tL29yaWdpbl9lY2NfY2EuY3JsMAoGCCqGSM49BAMCA0gAMEUCIDV7HoMj
-K5rShE/l+90YAOzHC89OH/wUz3I5KYOFuehoAiEA8e92aIf9XBkr0K6EvFCiSsD+
-x+Yo/cL8fGfVpPt4UM8=
------END CERTIFICATE-----
------BEGIN WARP TOKEN-----
-N2IwYTRkNzdkZmI4ODFjMWEzYjdkNjFlYTk0NDNlMTkKdjEuMC01OGJkNGY5ZTI4
-ZjdiM2MyOGUwNWEzNWZmM2U4MGFiNGZkOTY0NGVmM2ZlY2U1MzdlYjBkMTJlMmU5
-MjU4MjE3LTE4MzQ0MmZiYjBiYmRiM2U1NzE1NThmZWM5YjU1ODllYmQ3N2FhZmM4
-NzQ5OGVlM2YwOWY2NGE0YWQ3OWZmZTg3OTFlZGJhZTA4YjM2YzFkOGYxZDcwYTg2
-NzBkZTU2OTIyZGZmOTJiMTVkMjE0YTUyNGY0ZWJmYTE5NTg4NTllLTdjZTgwZjc5
-OTIxMzEyYTYwMjJjNWQyNWUyZDM4MGY4MmNlYWVmZTNmYmRjNDNkZDEzYjA4MGUz
-ZWYxZTI2Zjc=
------END WARP TOKEN-----

certutil/test-cert-no-token.pem

@@ -51,7 +51,6 @@ K5rShE/l+90YAOzHC89OH/wUz3I5KYOFuehoAiEA8e92aIf9XBkr0K6EvFCiSsD+
 x+Yo/cL8fGfVpPt4UM8=
 -----END CERTIFICATE-----
 -----BEGIN ARGO TUNNEL TOKEN-----
-eyJ6b25lSUQiOiAiN2IwYTRkNzdkZmI4ODFjMWEzYjdkNjFlYTk0NDNlMTkiLCAi
-c2VydmljZUtleSI6ICJ0ZXN0LXNlcnZpY2Uta2V5IiwgImFjY291bnRJRCI6ICJh
-YmNkYWJjZGFiY2RhYmNkMTIzNDU2Nzg5MGFiY2RlZiJ9
+eyJ6b25lSUQiOiAiN2IwYTRkNzdkZmI4ODFjMWEzYjdkNjFlYTk0NDNlMTkiLCAiYWNjb3VudElE
+IjogImFiY2RhYmNkYWJjZGFiY2QxMjM0NTY3ODkwYWJjZGVmIn0=
 -----END ARGO TUNNEL TOKEN-----

certutil/test-cert-two-certificates.pem

@@ -1,85 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQCfGswL16Fz9Ei3
-sAg5AmBizoN2nZdyXHP8T57UxUMcrlJXEEXCVS5RR4m9l+EmK0ng6yHR1H5oX1Lg
-1WKyXgWwr0whwmdTD+qWFJW2M8HyefyBKLrsGPuxw4CVYT0h72bxtG0uyrXYh7Mt
-z0lHjGV90qrFpq5o0jx0sLbDlDvpFPbIO58uYzKG4Sn2VTC4rOyXPE6SuDvMHIeX
-6Ekw4wSVQ9eTbksLQqTyxSqM3zp2ygc56SjGjy1nGQT8ZBGFzSbZAzNOxVKrUsyS
-x7LzZVl+zCGCPlQwaYLKObKXadZJmrqSFmErC5jcbVgBz7oJQOglHJ2n0sMcZ+Ja
-1Y649mPVAgMBAAECggEAEbPF0ah9fH0IzTU/CPbIeh3flyY8GDuMpR1HvwUurSWB
-IFI9bLyVAXKb8vYP1TMaTnXi5qmFof+/JShgyZc3+1tZtWTfoaiC8Y1bRfE2yk+D
-xmwddhDmijYGG7i8uEaeddSdFEh2GKAqkbV/QgBvN2Nl4EVmIOAJXXNe9l5LFyjy
-sR10aNVJRYV1FahrCTwZ3SovHP4d4AUvHh/3FFZDukHc37CFA0+CcR4uehp5yedi
-2UdqaszXqunFo/3h+Tn9dW2C7gTTZx4+mfyaws3p3YOmdYArXvpejxHIc0FGwLBm
-sb9K7wGVUiF0Bt0ch+C1mdYrCaFNHnPuDswjmm3FwQKBgQDYtxOwwSLA6ZyppozX
-Doyx9a7PhiMHCFKSdVB4l8rpK545a+AmpG6LRScTtBsMTHBhT3IQ3QPWlVm1AhjF
-AvXMa1rOeaGbCbDn1xqEoEVPtj4tys8eTfyWmtU73jWTFauOt4/xpf/urEpg91xj
-m+Gl/8qgBrpm5rQxV5Y4MysRlQKBgQC78jzzlhocXGNvw0wT/K2NsknyeoZXqpIE
-QYL60FMl4geZn6w9hwxaL1r+g/tUjTnpBPQtS1r2Ed2gXby5zspN1g/PW8U3t3to
-P7zHIJ/sLBXrCh5RJko3hUgGhDNOOCIQj4IaKUfvHYvEIbIxlyI0vdsXsgXgMuQ8
-pb9Yifn5QQKBgQCmGu0EtYQlyOlDP10EGSrN3Dm45l9CrKZdi326cN4eCkikSoLs
-G2x/YumouItiydP5QiNzuXOPrbmse4bwumwb2s0nJSMw6iSmDsFMlmuJxW2zO5e0
-6qGH7fUyhgcaTanJIfk6hrm7/mKkH/S4hGpYCc8NCRsmc/35M+D4AoAoYQKBgQC0
-LWpZaxDlF30MbAHHN3l6We2iU+vup0sMYXGb2ZOcwa/fir+ozIr++l8VmJmdWTan
-OWSM96zgMghx8Os4hhJTxF+rvqK242OfcVsc2x31X94zUaP2z+peh5uhA6Pb3Nxr
-W+iyA9k+Vujiwhr+h5D3VvtvH++aG6/KpGtoCf5nAQKBgQDXX2+d7bd5CLNLLFNd
-M2i4QoOFcSKIG+v4SuvgEJHgG8vGvxh2qlSxnMWuPV+7/1P5ATLqDj1PlKms+BNR
-y7sc5AT9PclkL3Y9MNzOu0LXyBkGYcl8M0EQfLv9VPbWT+NXiMg/O2CHiT02pAAz
-uQicoQq3yzeQh20wtrtaXzTNmA==
------END PRIVATE KEY-----
------BEGIN CERTIFICATE-----
-MIID+jCCA6CgAwIBAgIUJhFxUKEGvTRc3CjCok6dbPGH/P4wCgYIKoZIzj0EAwIw
-gagxCzAJBgNVBAYTAlVTMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMTgwNgYD
-VQQLEy9DbG91ZEZsYXJlIE9yaWdpbiBTU0wgRUNDIENlcnRpZmljYXRlIEF1dGhv
-cml0eTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzETMBEGA1UECBMKQ2FsaWZvcm5p
-YTEXMBUGA1UEAxMOKGRldiB1c2Ugb25seSkwHhcNMTcxMDEzMTM1OTAwWhcNMzIx
-MDA5MTM1OTAwWjBiMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMR0wGwYDVQQL
-ExRDbG91ZEZsYXJlIE9yaWdpbiBDQTEmMCQGA1UEAxMdQ2xvdWRGbGFyZSBPcmln
-aW4gQ2VydGlmaWNhdGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCf
-GswL16Fz9Ei3sAg5AmBizoN2nZdyXHP8T57UxUMcrlJXEEXCVS5RR4m9l+EmK0ng
-6yHR1H5oX1Lg1WKyXgWwr0whwmdTD+qWFJW2M8HyefyBKLrsGPuxw4CVYT0h72bx
-tG0uyrXYh7Mtz0lHjGV90qrFpq5o0jx0sLbDlDvpFPbIO58uYzKG4Sn2VTC4rOyX
-PE6SuDvMHIeX6Ekw4wSVQ9eTbksLQqTyxSqM3zp2ygc56SjGjy1nGQT8ZBGFzSbZ
-AzNOxVKrUsySx7LzZVl+zCGCPlQwaYLKObKXadZJmrqSFmErC5jcbVgBz7oJQOgl
-HJ2n0sMcZ+Ja1Y649mPVAgMBAAGjggEgMIIBHDAOBgNVHQ8BAf8EBAMCBaAwEwYD
-VR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUzA6f2Ajq
-zhX67c6piY2a1uTiUkwwHwYDVR0jBBgwFoAU2qfBlqxKMZnf0QeTeYiMelfqJfgw
-RAYIKwYBBQUHAQEEODA2MDQGCCsGAQUFBzABhihodHRwOi8vb2NzcC5jbG91ZGZs
-YXJlLmNvbS9vcmlnaW5fZWNjX2NhMCMGA1UdEQQcMBqCDCouYXJub2xkLmNvbYIK
-YXJub2xkLmNvbTA8BgNVHR8ENTAzMDGgL6AthitodHRwOi8vY3JsLmNsb3VkZmxh
-cmUuY29tL29yaWdpbl9lY2NfY2EuY3JsMAoGCCqGSM49BAMCA0gAMEUCIDV7HoMj
-K5rShE/l+90YAOzHC89OH/wUz3I5KYOFuehoAiEA8e92aIf9XBkr0K6EvFCiSsD+
-x+Yo/cL8fGfVpPt4UM8=
------END CERTIFICATE-----
------BEGIN CERTIFICATE-----
-MIID+jCCA6CgAwIBAgIUJhFxUKEGvTRc3CjCok6dbPGH/P4wCgYIKoZIzj0EAwIw
-gagxCzAJBgNVBAYTAlVTMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMTgwNgYD
-VQQLEy9DbG91ZEZsYXJlIE9yaWdpbiBTU0wgRUNDIENlcnRpZmljYXRlIEF1dGhv
-cml0eTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzETMBEGA1UECBMKQ2FsaWZvcm5p
-YTEXMBUGA1UEAxMOKGRldiB1c2Ugb25seSkwHhcNMTcxMDEzMTM1OTAwWhcNMzIx
-MDA5MTM1OTAwWjBiMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMR0wGwYDVQQL
-ExRDbG91ZEZsYXJlIE9yaWdpbiBDQTEmMCQGA1UEAxMdQ2xvdWRGbGFyZSBPcmln
-aW4gQ2VydGlmaWNhdGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCf
-GswL16Fz9Ei3sAg5AmBizoN2nZdyXHP8T57UxUMcrlJXEEXCVS5RR4m9l+EmK0ng
-6yHR1H5oX1Lg1WKyXgWwr0whwmdTD+qWFJW2M8HyefyBKLrsGPuxw4CVYT0h72bx
-tG0uyrXYh7Mtz0lHjGV90qrFpq5o0jx0sLbDlDvpFPbIO58uYzKG4Sn2VTC4rOyX
-PE6SuDvMHIeX6Ekw4wSVQ9eTbksLQqTyxSqM3zp2ygc56SjGjy1nGQT8ZBGFzSbZ
-AzNOxVKrUsySx7LzZVl+zCGCPlQwaYLKObKXadZJmrqSFmErC5jcbVgBz7oJQOgl
-HJ2n0sMcZ+Ja1Y649mPVAgMBAAGjggEgMIIBHDAOBgNVHQ8BAf8EBAMCBaAwEwYD
-VR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUzA6f2Ajq
-zhX67c6piY2a1uTiUkwwHwYDVR0jBBgwFoAU2qfBlqxKMZnf0QeTeYiMelfqJfgw
-RAYIKwYBBQUHAQEEODA2MDQGCCsGAQUFBzABhihodHRwOi8vb2NzcC5jbG91ZGZs
-YXJlLmNvbS9vcmlnaW5fZWNjX2NhMCMGA1UdEQQcMBqCDCouYXJub2xkLmNvbYIK
-YXJub2xkLmNvbTA8BgNVHR8ENTAzMDGgL6AthitodHRwOi8vY3JsLmNsb3VkZmxh
-cmUuY29tL29yaWdpbl9lY2NfY2EuY3JsMAoGCCqGSM49BAMCA0gAMEUCIDV7HoMj
-K5rShE/l+90YAOzHC89OH/wUz3I5KYOFuehoAiEA8e92aIf9XBkr0K6EvFCiSsD+
-x+Yo/cL8fGfVpPt4UM8=
------END CERTIFICATE-----
------BEGIN WARP TOKEN-----
-N2IwYTRkNzdkZmI4ODFjMWEzYjdkNjFlYTk0NDNlMTkKdjEuMC01OGJkNGY5ZTI4
-ZjdiM2MyOGUwNWEzNWZmM2U4MGFiNGZkOTY0NGVmM2ZlY2U1MzdlYjBkMTJlMmU5
-MjU4MjE3LTE4MzQ0MmZiYjBiYmRiM2U1NzE1NThmZWM5YjU1ODllYmQ3N2FhZmM4
-NzQ5OGVlM2YwOWY2NGE0YWQ3OWZmZTg3OTFlZGJhZTA4YjM2YzFkOGYxZDcwYTg2
-NzBkZTU2OTIyZGZmOTJiMTVkMjE0YTUyNGY0ZWJmYTE5NTg4NTllLTdjZTgwZjc5
-OTIxMzEyYTYwMjJjNWQyNWUyZDM4MGY4MmNlYWVmZTNmYmRjNDNkZDEzYjA4MGUz
-ZWYxZTI2Zjc=
------END WARP TOKEN-----

certutil/test-cert-unknown-block.pem

@@ -50,7 +50,7 @@ cmUuY29tL29yaWdpbl9lY2NfY2EuY3JsMAoGCCqGSM49BAMCA0gAMEUCIDV7HoMj
 K5rShE/l+90YAOzHC89OH/wUz3I5KYOFuehoAiEA8e92aIf9XBkr0K6EvFCiSsD+
 x+Yo/cL8fGfVpPt4UM8=
 -----END CERTIFICATE-----
------BEGIN WARP TOKEN-----
+-----BEGIN ARGO TUNNEL TOKEN-----
 N2IwYTRkNzdkZmI4ODFjMWEzYjdkNjFlYTk0NDNlMTkKdjEuMC01OGJkNGY5ZTI4
 ZjdiM2MyOGUwNWEzNWZmM2U4MGFiNGZkOTY0NGVmM2ZlY2U1MzdlYjBkMTJlMmU5
 MjU4MjE3LTE4MzQ0MmZiYjBiYmRiM2U1NzE1NThmZWM5YjU1ODllYmQ3N2FhZmM4
@@ -58,7 +58,7 @@ NzQ5OGVlM2YwOWY2NGE0YWQ3OWZmZTg3OTFlZGJhZTA4YjM2YzFkOGYxZDcwYTg2
 NzBkZTU2OTIyZGZmOTJiMTVkMjE0YTUyNGY0ZWJmYTE5NTg4NTllLTdjZTgwZjc5
 OTIxMzEyYTYwMjJjNWQyNWUyZDM4MGY4MmNlYWVmZTNmYmRjNDNkZDEzYjA4MGUz
 ZWYxZTI2Zjc=
------END WARP TOKEN-----
+-----END ARGO TUNNEL TOKEN-----
 -----BEGIN RSA PRIVATE KEY-----
 MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQCfGswL16Fz9Ei3
 sAg5AmBizoN2nZdyXHP8T57UxUMcrlJXEEXCVS5RR4m9l+EmK0ng6yHR1H5oX1Lg

certutil/test-cert.pem

@@ -1,61 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQCfGswL16Fz9Ei3
-sAg5AmBizoN2nZdyXHP8T57UxUMcrlJXEEXCVS5RR4m9l+EmK0ng6yHR1H5oX1Lg
-1WKyXgWwr0whwmdTD+qWFJW2M8HyefyBKLrsGPuxw4CVYT0h72bxtG0uyrXYh7Mt
-z0lHjGV90qrFpq5o0jx0sLbDlDvpFPbIO58uYzKG4Sn2VTC4rOyXPE6SuDvMHIeX
-6Ekw4wSVQ9eTbksLQqTyxSqM3zp2ygc56SjGjy1nGQT8ZBGFzSbZAzNOxVKrUsyS
-x7LzZVl+zCGCPlQwaYLKObKXadZJmrqSFmErC5jcbVgBz7oJQOglHJ2n0sMcZ+Ja
-1Y649mPVAgMBAAECggEAEbPF0ah9fH0IzTU/CPbIeh3flyY8GDuMpR1HvwUurSWB
-IFI9bLyVAXKb8vYP1TMaTnXi5qmFof+/JShgyZc3+1tZtWTfoaiC8Y1bRfE2yk+D
-xmwddhDmijYGG7i8uEaeddSdFEh2GKAqkbV/QgBvN2Nl4EVmIOAJXXNe9l5LFyjy
-sR10aNVJRYV1FahrCTwZ3SovHP4d4AUvHh/3FFZDukHc37CFA0+CcR4uehp5yedi
-2UdqaszXqunFo/3h+Tn9dW2C7gTTZx4+mfyaws3p3YOmdYArXvpejxHIc0FGwLBm
-sb9K7wGVUiF0Bt0ch+C1mdYrCaFNHnPuDswjmm3FwQKBgQDYtxOwwSLA6ZyppozX
-Doyx9a7PhiMHCFKSdVB4l8rpK545a+AmpG6LRScTtBsMTHBhT3IQ3QPWlVm1AhjF
-AvXMa1rOeaGbCbDn1xqEoEVPtj4tys8eTfyWmtU73jWTFauOt4/xpf/urEpg91xj
-m+Gl/8qgBrpm5rQxV5Y4MysRlQKBgQC78jzzlhocXGNvw0wT/K2NsknyeoZXqpIE
-QYL60FMl4geZn6w9hwxaL1r+g/tUjTnpBPQtS1r2Ed2gXby5zspN1g/PW8U3t3to
-P7zHIJ/sLBXrCh5RJko3hUgGhDNOOCIQj4IaKUfvHYvEIbIxlyI0vdsXsgXgMuQ8
-pb9Yifn5QQKBgQCmGu0EtYQlyOlDP10EGSrN3Dm45l9CrKZdi326cN4eCkikSoLs
-G2x/YumouItiydP5QiNzuXOPrbmse4bwumwb2s0nJSMw6iSmDsFMlmuJxW2zO5e0
-6qGH7fUyhgcaTanJIfk6hrm7/mKkH/S4hGpYCc8NCRsmc/35M+D4AoAoYQKBgQC0
-LWpZaxDlF30MbAHHN3l6We2iU+vup0sMYXGb2ZOcwa/fir+ozIr++l8VmJmdWTan
-OWSM96zgMghx8Os4hhJTxF+rvqK242OfcVsc2x31X94zUaP2z+peh5uhA6Pb3Nxr
-W+iyA9k+Vujiwhr+h5D3VvtvH++aG6/KpGtoCf5nAQKBgQDXX2+d7bd5CLNLLFNd
-M2i4QoOFcSKIG+v4SuvgEJHgG8vGvxh2qlSxnMWuPV+7/1P5ATLqDj1PlKms+BNR
-y7sc5AT9PclkL3Y9MNzOu0LXyBkGYcl8M0EQfLv9VPbWT+NXiMg/O2CHiT02pAAz
-uQicoQq3yzeQh20wtrtaXzTNmA==
------END PRIVATE KEY-----
------BEGIN CERTIFICATE-----
-MIID+jCCA6CgAwIBAgIUJhFxUKEGvTRc3CjCok6dbPGH/P4wCgYIKoZIzj0EAwIw
-gagxCzAJBgNVBAYTAlVTMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMTgwNgYD
-VQQLEy9DbG91ZEZsYXJlIE9yaWdpbiBTU0wgRUNDIENlcnRpZmljYXRlIEF1dGhv
-cml0eTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzETMBEGA1UECBMKQ2FsaWZvcm5p
-YTEXMBUGA1UEAxMOKGRldiB1c2Ugb25seSkwHhcNMTcxMDEzMTM1OTAwWhcNMzIx
-MDA5MTM1OTAwWjBiMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMR0wGwYDVQQL
-ExRDbG91ZEZsYXJlIE9yaWdpbiBDQTEmMCQGA1UEAxMdQ2xvdWRGbGFyZSBPcmln
-aW4gQ2VydGlmaWNhdGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCf
-GswL16Fz9Ei3sAg5AmBizoN2nZdyXHP8T57UxUMcrlJXEEXCVS5RR4m9l+EmK0ng
-6yHR1H5oX1Lg1WKyXgWwr0whwmdTD+qWFJW2M8HyefyBKLrsGPuxw4CVYT0h72bx
-tG0uyrXYh7Mtz0lHjGV90qrFpq5o0jx0sLbDlDvpFPbIO58uYzKG4Sn2VTC4rOyX
-PE6SuDvMHIeX6Ekw4wSVQ9eTbksLQqTyxSqM3zp2ygc56SjGjy1nGQT8ZBGFzSbZ
-AzNOxVKrUsySx7LzZVl+zCGCPlQwaYLKObKXadZJmrqSFmErC5jcbVgBz7oJQOgl
-HJ2n0sMcZ+Ja1Y649mPVAgMBAAGjggEgMIIBHDAOBgNVHQ8BAf8EBAMCBaAwEwYD
-VR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUzA6f2Ajq
-zhX67c6piY2a1uTiUkwwHwYDVR0jBBgwFoAU2qfBlqxKMZnf0QeTeYiMelfqJfgw
-RAYIKwYBBQUHAQEEODA2MDQGCCsGAQUFBzABhihodHRwOi8vb2NzcC5jbG91ZGZs
-YXJlLmNvbS9vcmlnaW5fZWNjX2NhMCMGA1UdEQQcMBqCDCouYXJub2xkLmNvbYIK
-YXJub2xkLmNvbTA8BgNVHR8ENTAzMDGgL6AthitodHRwOi8vY3JsLmNsb3VkZmxh
-cmUuY29tL29yaWdpbl9lY2NfY2EuY3JsMAoGCCqGSM49BAMCA0gAMEUCIDV7HoMj
-K5rShE/l+90YAOzHC89OH/wUz3I5KYOFuehoAiEA8e92aIf9XBkr0K6EvFCiSsD+
-x+Yo/cL8fGfVpPt4UM8=
------END CERTIFICATE-----
------BEGIN WARP TOKEN-----
-N2IwYTRkNzdkZmI4ODFjMWEzYjdkNjFlYTk0NDNlMTkKdjEuMC01OGJkNGY5ZTI4
-ZjdiM2MyOGUwNWEzNWZmM2U4MGFiNGZkOTY0NGVmM2ZlY2U1MzdlYjBkMTJlMmU5
-MjU4MjE3LTE4MzQ0MmZiYjBiYmRiM2U1NzE1NThmZWM5YjU1ODllYmQ3N2FhZmM4
-NzQ5OGVlM2YwOWY2NGE0YWQ3OWZmZTg3OTFlZGJhZTA4YjM2YzFkOGYxZDcwYTg2
-NzBkZTU2OTIyZGZmOTJiMTVkMjE0YTUyNGY0ZWJmYTE5NTg4NTllLTdjZTgwZjc5
-OTIxMzEyYTYwMjJjNWQyNWUyZDM4MGY4MmNlYWVmZTNmYmRjNDNkZDEzYjA4MGUz
-ZWYxZTI2Zjc=
------END WARP TOKEN-----

certutil/test-cloudflare-tunnel-cert-json.pem

@@ -51,6 +51,7 @@ K5rShE/l+90YAOzHC89OH/wUz3I5KYOFuehoAiEA8e92aIf9XBkr0K6EvFCiSsD+
 x+Yo/cL8fGfVpPt4UM8=
 -----END CERTIFICATE-----
 -----BEGIN ARGO TUNNEL TOKEN-----
-N2IwYTRkNzdkZmI4ODFjMWEzYjdkNjFlYTk0NDNlMTkKdGVzdC1zZXJ2aWNlLWtl
-eQ==
+eyJ6b25lSUQiOiAiN2IwYTRkNzdkZmI4ODFjMWEzYjdkNjFlYTk0NDNlMTkiLCAiYXBpVG9rZW4i
+OiAidGVzdC1zZXJ2aWNlLWtleSIsICJhY2NvdW50SUQiOiAiYWJjZGFiY2RhYmNkYWJjZDEyMzQ1
+Njc4OTBhYmNkZWYifQ==
 -----END ARGO TUNNEL TOKEN-----

cfapi/base_client.go

@@ -104,7 +104,7 @@ func (r *RESTClient) sendRequest(method string, url url.URL, body interface{}) (
 	if bodyReader != nil {
 		req.Header.Set("Content-Type", jsonContentType)
 	}
-	req.Header.Add("X-Auth-User-Service-Key", r.authToken)
+	req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", r.authToken))
 	req.Header.Add("Accept", "application/json;version=1")
 	return r.client.Do(req)
 }

cfsetup.yaml

@@ -1,23 +1,29 @@
-pinned_go: &pinned_go go=1.18.6-1
-pinned_go_fips: &pinned_go_fips go-boring=1.18.6-1
+pinned_go: &pinned_go go=1.19.3-1
+pinned_go_fips: &pinned_go_fips go-boring=1.19.3-1
 
 build_dir: &build_dir /cfsetup_build
 default-flavor: bullseye
 stretch: &stretch
   build:
     build_dir: *build_dir
-    builddeps:
+    builddeps: &build_deps
       - *pinned_go
       - build-essential
+      - gotest-to-teamcity
+    pre-cache: &build_pre_cache
+      - export GOCACHE=/cfsetup_build/.cache/go-build
+      - go install golang.org/x/tools/cmd/goimports@latest
     post-cache:
       - export GOOS=linux
       - export GOARCH=amd64
       - make cloudflared
   build-fips:
     build_dir: *build_dir
-    builddeps:
+    builddeps: &build_deps_fips
       - *pinned_go_fips
       - build-essential
+      - gotest-to-teamcity
+    pre-cache: *build_pre_cache
     post-cache:
       - export GOOS=linux
       - export GOARCH=amd64
@@ -78,6 +84,13 @@ stretch: &stretch
       # same logic as above, but for FIPS packages only
       - ./build-packages-fips.sh
       - make github-release-built-pkgs
+  generate-versions-file:
+    build_dir: *build_dir
+    builddeps: 
+      - *pinned_go
+      - build-essential
+    post-cache:
+      - make generate-docker-version
   build-deb:
     build_dir: *build_dir
     builddeps: &build_deb_deps
@@ -156,12 +169,8 @@ stretch: &stretch
       - make github-windows-upload
   test:
     build_dir: *build_dir
-    builddeps:
-      - *pinned_go
-      - build-essential
-      - gotest-to-teamcity
-    pre-cache: &test_pre_cache
-      - go install golang.org/x/tools/cmd/goimports@latest
+    builddeps: *build_deps
+    pre-cache: *build_pre_cache
     post-cache:
       - export GOOS=linux
       - export GOARCH=amd64
@@ -170,11 +179,8 @@ stretch: &stretch
       - make test | gotest-to-teamcity
   test-fips:
     build_dir: *build_dir
-    builddeps:
-      - *pinned_go_fips
-      - build-essential
-      - gotest-to-teamcity
-    pre-cache: *test_pre_cache
+    builddeps: *build_deps_fips
+    pre-cache: *build_pre_cache
     post-cache:
       - export GOOS=linux
       - export GOARCH=amd64

cmd/cloudflared/access/cmd.go

@@ -37,16 +37,13 @@ const (
 	sshConfigTemplate  = `
 Add to your {{.Home}}/.ssh/config:
 
-Host {{.Hostname}}
 {{- if .ShortLivedCerts}}
-  ProxyCommand bash -c '{{.Cloudflared}} access ssh-gen --hostname %h; ssh -tt %r@cfpipe-{{.Hostname}} >&2 <&1'
-
-Host cfpipe-{{.Hostname}}
-  HostName {{.Hostname}}
+Match host {{.Hostname}} exec "{{.Cloudflared}} access ssh-gen --hostname %h"
   ProxyCommand {{.Cloudflared}} access ssh --hostname %h
-  IdentityFile ~/.cloudflared/{{.Hostname}}-cf_key
-  CertificateFile ~/.cloudflared/{{.Hostname}}-cf_key-cert.pub
+  IdentityFile ~/.cloudflared/%h-cf_key
+  CertificateFile ~/.cloudflared/%h-cf_key-cert.pub
 {{- else}}
+Host {{.Hostname}}
   ProxyCommand {{.Cloudflared}} access ssh --hostname %h
 {{end}}
 `

cmd/cloudflared/tunnel/cmd.go

@@ -665,6 +665,13 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			EnvVars: []string{"TUNNEL_MAX_FETCH_SIZE"},
 			Hidden:  true,
 		}),
+		altsrc.NewBoolFlag(&cli.BoolFlag{
+			Name:    "post-quantum",
+			Usage:   "When given creates an experimental post-quantum secure tunnel",
+			Aliases: []string{"pq"},
+			EnvVars: []string{"TUNNEL_POST_QUANTUM"},
+			Hidden:  FipsEnabled,
+		}),
 		selectProtocolFlag,
 		overwriteDNSFlag,
 	}...)

cmd/cloudflared/tunnel/configuration.go

@@ -22,7 +22,6 @@ import (
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
 	"github.com/cloudflare/cloudflared/edgediscovery/allregions"
-	"github.com/cloudflare/cloudflared/packet"
 
 	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/connection"
@@ -47,7 +46,7 @@ var (
 	LogFieldHostname = "hostname"
 
 	secretFlags     = [2]*altsrc.StringFlag{credentialsContentsFlag, tunnelTokenFlag}
-	defaultFeatures = []string{supervisor.FeatureAllowRemoteConfig, supervisor.FeatureSerializedHeaders, supervisor.FeatureDatagramV2}
+	defaultFeatures = []string{supervisor.FeatureAllowRemoteConfig, supervisor.FeatureSerializedHeaders, supervisor.FeatureDatagramV2, supervisor.FeatureQUICSupportEOF}
 
 	configFlags = []string{"autoupdate-freq", "no-autoupdate", "retries", "protocol", "loglevel", "transport-loglevel", "origincert", "metrics", "metrics-update-freq", "edge-ip-version"}
 )
@@ -463,7 +462,7 @@ func parseConfigIPVersion(version string) (v allregions.ConfigIPVersion, err err
 	return
 }
 
-func newPacketConfig(c *cli.Context, logger *zerolog.Logger) (*packet.GlobalRouterConfig, error) {
+func newPacketConfig(c *cli.Context, logger *zerolog.Logger) (*ingress.GlobalRouterConfig, error) {
 	ipv4Src, err := determineICMPv4Src(c.String("icmpv4-src"), logger)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to determine IPv4 source address for ICMP proxy")
@@ -484,7 +483,7 @@ func newPacketConfig(c *cli.Context, logger *zerolog.Logger) (*packet.GlobalRout
 	if err != nil {
 		return nil, err
 	}
-	return &packet.GlobalRouterConfig{
+	return &ingress.GlobalRouterConfig{
 		ICMPRouter: icmpRouter,
 		IPv4Src:    ipv4Src,
 		IPv6Src:    ipv6Src,

cmd/cloudflared/tunnel/subcommand_context.go

@@ -74,7 +74,7 @@ func (sc *subcommandContext) client() (cfapi.Client, error) {
 		sc.c.String("api-url"),
 		credential.cert.AccountID,
 		credential.cert.ZoneID,
-		credential.cert.ServiceKey,
+		credential.cert.APIToken,
 		userAgent,
 		sc.log,
 	)

cmd/cloudflared/tunnel/subcommands.go

@@ -821,7 +821,7 @@ Further information about managing Cloudflare WARP traffic to your tunnel is ava
 				Name:        "lb",
 				Action:      cliutil.ConfiguredAction(routeLbCommand),
 				Usage:       "Use this tunnel as a load balancer origin, creating pool and load balancer if necessary",
-				UsageText:   "cloudflared tunnel route dns [TUNNEL] [HOSTNAME] [LB-POOL]",
+				UsageText:   "cloudflared tunnel route lb [TUNNEL] [HOSTNAME] [LB-POOL]",
 				Description: `Creates Load Balancer with an origin pool that points to the tunnel.`,
 			},
 			buildRouteIPSubcommand(),

connection/control.go

@@ -85,7 +85,7 @@ func (c *controlStream) ServeControlStream(
 		return err
 	}
 
-	c.observer.logServerInfo(c.connIndex, registrationDetails.Location, c.edgeAddress, fmt.Sprintf("Connection %s registered", registrationDetails.UUID))
+	c.observer.logServerInfo(c.connIndex, registrationDetails.Location, c.edgeAddress, fmt.Sprintf("Connection %s registered with protocol: %s", registrationDetails.UUID, c.protocol))
 	c.observer.sendConnectedEvent(c.connIndex, c.protocol, registrationDetails.Location)
 	c.connectedFuse.Connected()
 

connection/quic.go

@@ -10,6 +10,7 @@ import (
 	"net/netip"
 	"strconv"
 	"strings"
+	"sync"
 	"sync/atomic"
 	"time"
 
@@ -42,6 +43,11 @@ const (
 	demuxChanCapacity = 16
 )
 
+var (
+	portForConnIndex = make(map[uint8]int, 0)
+	portMapMutex     sync.Mutex
+)
+
 // QUICConnection represents the type that facilitates Proxying via QUIC streams.
 type QUICConnection struct {
 	session      quic.Connection
@@ -51,7 +57,7 @@ type QUICConnection struct {
 	sessionManager datagramsession.Manager
 	// datagramMuxer mux/demux datagrams from quic connection
 	datagramMuxer        *quicpogs.DatagramMuxerV2
-	packetRouter         *packet.Router
+	packetRouter         *ingress.PacketRouter
 	controlStreamHandler ControlStreamHandler
 	connOptions          *tunnelpogs.ConnectionOptions
 }
@@ -60,22 +66,34 @@ type QUICConnection struct {
 func NewQUICConnection(
 	quicConfig *quic.Config,
 	edgeAddr net.Addr,
+	connIndex uint8,
 	tlsConfig *tls.Config,
 	orchestrator Orchestrator,
 	connOptions *tunnelpogs.ConnectionOptions,
 	controlStreamHandler ControlStreamHandler,
 	logger *zerolog.Logger,
-	packetRouterConfig *packet.GlobalRouterConfig,
+	packetRouterConfig *ingress.GlobalRouterConfig,
 ) (*QUICConnection, error) {
-	session, err := quic.DialAddr(edgeAddr.String(), tlsConfig, quicConfig)
+	udpConn, err := createUDPConnForConnIndex(connIndex, logger)
+	if err != nil {
+		return nil, err
+	}
+
+	session, err := quic.Dial(udpConn, edgeAddr, edgeAddr.String(), tlsConfig, quicConfig)
 	if err != nil {
 		return nil, &EdgeQuicDialError{Cause: err}
 	}
 
+	// wrap the session, so that the UDPConn is closed after session is closed.
+	session = &wrapCloseableConnQuicConnection{
+		session,
+		udpConn,
+	}
+
 	sessionDemuxChan := make(chan *packet.Session, demuxChanCapacity)
 	datagramMuxer := quicpogs.NewDatagramMuxerV2(session, logger, sessionDemuxChan)
 	sessionManager := datagramsession.NewManager(logger, datagramMuxer.SendToSession, sessionDemuxChan)
-	packetRouter := packet.NewRouter(packetRouterConfig, datagramMuxer, &returnPipe{muxer: datagramMuxer}, logger, orchestrator.WarpRoutingEnabled)
+	packetRouter := ingress.NewPacketRouter(packetRouterConfig, datagramMuxer, logger, orchestrator.WarpRoutingEnabled)
 
 	return &QUICConnection{
 		session:              session,
@@ -480,15 +498,69 @@ func (np *nopCloserReadWriter) Close() error {
 	return nil
 }
 
-// returnPipe wraps DatagramMuxerV2 to satisfy the packet.FunnelUniPipe interface
-type returnPipe struct {
+// muxerWrapper wraps DatagramMuxerV2 to satisfy the packet.FunnelUniPipe interface
+type muxerWrapper struct {
 	muxer *quicpogs.DatagramMuxerV2
 }
 
-func (rp *returnPipe) SendPacket(dst netip.Addr, pk packet.RawPacket) error {
-	return rp.muxer.SendPacket(pk)
+func (rp *muxerWrapper) SendPacket(dst netip.Addr, pk packet.RawPacket) error {
+	return rp.muxer.SendPacket(quicpogs.RawPacket(pk))
 }
 
-func (rp *returnPipe) Close() error {
+func (rp *muxerWrapper) ReceivePacket(ctx context.Context) (packet.RawPacket, error) {
+	pk, err := rp.muxer.ReceivePacket(ctx)
+	if err != nil {
+		return packet.RawPacket{}, err
+	}
+	rawPacket, ok := pk.(quicpogs.RawPacket)
+	if ok {
+		return packet.RawPacket(rawPacket), nil
+	}
+	return packet.RawPacket{}, fmt.Errorf("unexpected packet type %+v", pk)
+}
+
+func (rp *muxerWrapper) Close() error {
 	return nil
 }
+
+func createUDPConnForConnIndex(connIndex uint8, logger *zerolog.Logger) (*net.UDPConn, error) {
+	portMapMutex.Lock()
+	defer portMapMutex.Unlock()
+
+	// if port was not set yet, it will be zero, so bind will randomly allocate one.
+	if port, ok := portForConnIndex[connIndex]; ok {
+		udpConn, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.IPv4zero, Port: port})
+		// if there wasn't an error, or if port was 0 (independently of error or not, just return)
+		if err == nil {
+			return udpConn, nil
+		} else {
+			logger.Debug().Err(err).Msgf("Unable to reuse port %d for connIndex %d. Falling back to random allocation.", port, connIndex)
+		}
+	}
+
+	// if we reached here, then there was an error or port as not been allocated it.
+	udpConn, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.IPv4zero, Port: 0})
+	if err == nil {
+		udpAddr, ok := (udpConn.LocalAddr()).(*net.UDPAddr)
+		if !ok {
+			return nil, fmt.Errorf("unable to cast to udpConn")
+		}
+		portForConnIndex[connIndex] = udpAddr.Port
+	} else {
+		delete(portForConnIndex, connIndex)
+	}
+
+	return udpConn, err
+}
+
+type wrapCloseableConnQuicConnection struct {
+	quic.Connection
+	udpConn *net.UDPConn
+}
+
+func (w *wrapCloseableConnQuicConnection) CloseWithError(errorCode quic.ApplicationErrorCode, reason string) error {
+	err := w.Connection.CloseWithError(errorCode, reason)
+	w.udpConn.Close()
+
+	return err
+}

connection/quic_test.go

@@ -11,7 +11,6 @@ import (
 	"net/url"
 	"os"
 	"strings"
-	"sync"
 	"testing"
 	"time"
 
@@ -142,24 +141,33 @@ func TestQUICServer(t *testing.T) {
 		},
 	}
 
-	for _, test := range tests {
+	for i, test := range tests {
+		test := test // capture range variable
 		t.Run(test.desc, func(t *testing.T) {
 			ctx, cancel := context.WithCancel(context.Background())
-			var wg sync.WaitGroup
-			wg.Add(1)
+
+			quicListener, err := quic.Listen(udpListener, testTLSServerConfig, testQUICConfig)
+			require.NoError(t, err)
+
+			serverDone := make(chan struct{})
 			go func() {
-				defer wg.Done()
 				quicServer(
-					t, udpListener, testTLSServerConfig, testQUICConfig,
-					test.dest, test.connectionType, test.metadata, test.message, test.expectedResponse,
+					ctx, t, quicListener, test.dest, test.connectionType, test.metadata, test.message, test.expectedResponse,
 				)
+				close(serverDone)
 			}()
 
-			qc := testQUICConnection(udpListener.LocalAddr(), t)
-			go qc.Serve(ctx)
+			qc := testQUICConnection(udpListener.LocalAddr(), t, uint8(i))
 
-			wg.Wait()
+			connDone := make(chan struct{})
+			go func() {
+				qc.Serve(ctx)
+				close(connDone)
+			}()
+
+			<-serverDone
 			cancel()
+			<-connDone
 		})
 	}
 }
@@ -177,23 +185,16 @@ func (fakeControlStream) IsStopped() bool {
 }
 
 func quicServer(
+	ctx context.Context,
 	t *testing.T,
-	conn net.PacketConn,
-	tlsConf *tls.Config,
-	config *quic.Config,
+	listener quic.Listener,
 	dest string,
 	connectionType quicpogs.ConnectionType,
 	metadata []quicpogs.Metadata,
 	message []byte,
 	expectedResponse []byte,
 ) {
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	earlyListener, err := quic.Listen(conn, tlsConf, config)
-	require.NoError(t, err)
-
-	session, err := earlyListener.Accept(ctx)
+	session, err := listener.Accept(ctx)
 	require.NoError(t, err)
 
 	quicStream, err := session.OpenStreamSync(context.Background())
@@ -482,6 +483,7 @@ func TestBuildHTTPRequest(t *testing.T) {
 
 	log := zerolog.Nop()
 	for _, test := range tests {
+		test := test // capture range variable
 		t.Run(test.name, func(t *testing.T) {
 			req, err := buildHTTPRequest(context.Background(), test.connectRequest, test.body, &log)
 			assert.NoError(t, err)
@@ -519,7 +521,8 @@ func TestServeUDPSession(t *testing.T) {
 		edgeQUICSessionChan <- edgeQUICSession
 	}()
 
-	qc := testQUICConnection(val, t)
+	// Random index to avoid reusing port
+	qc := testQUICConnection(val, t, 28)
 	go qc.Serve(ctx)
 
 	edgeQUICSession := <-edgeQUICSessionChan
@@ -567,6 +570,37 @@ func TestNopCloserReadWriterCloseAfterEOF(t *testing.T) {
 	require.Equal(t, err, io.EOF)
 }
 
+func TestCreateUDPConnReuseSourcePort(t *testing.T) {
+	logger := zerolog.Nop()
+	conn, err := createUDPConnForConnIndex(0, &logger)
+	require.NoError(t, err)
+
+	getPortFunc := func(conn *net.UDPConn) int {
+		addr := conn.LocalAddr().(*net.UDPAddr)
+		return addr.Port
+	}
+
+	initialPort := getPortFunc(conn)
+
+	// close conn
+	conn.Close()
+
+	// should get the same port as before.
+	conn, err = createUDPConnForConnIndex(0, &logger)
+	require.NoError(t, err)
+	require.Equal(t, initialPort, getPortFunc(conn))
+
+	// new index, should get a different port
+	conn1, err := createUDPConnForConnIndex(1, &logger)
+	require.NoError(t, err)
+	require.NotEqual(t, initialPort, getPortFunc(conn1))
+
+	// not closing the conn and trying to obtain a new conn for same index should give a different random port
+	conn, err = createUDPConnForConnIndex(0, &logger)
+	require.NoError(t, err)
+	require.NotEqual(t, initialPort, getPortFunc(conn))
+}
+
 func serveSession(ctx context.Context, qc *QUICConnection, edgeQUICSession quic.Connection, closeType closeReason, expectedReason string, t *testing.T) {
 	var (
 		payload = []byte(t.Name())
@@ -672,7 +706,7 @@ func (s mockSessionRPCServer) UnregisterUdpSession(ctx context.Context, sessionI
 	return nil
 }
 
-func testQUICConnection(udpListenerAddr net.Addr, t *testing.T) *QUICConnection {
+func testQUICConnection(udpListenerAddr net.Addr, t *testing.T, index uint8) *QUICConnection {
 	tlsClientConfig := &tls.Config{
 		InsecureSkipVerify: true,
 		NextProtos:         []string{"argotunnel"},
@@ -682,6 +716,7 @@ func testQUICConnection(udpListenerAddr net.Addr, t *testing.T) *QUICConnection
 	qc, err := NewQUICConnection(
 		testQUICConfig,
 		udpListenerAddr,
+		index,
 		tlsClientConfig,
 		&mockOrchestrator{originProxy: &mockOriginProxyWithRequest{}},
 		&tunnelpogs.ConnectionOptions{},

datagramsession/session.go

@@ -24,6 +24,13 @@ func SessionIdleErr(timeout time.Duration) error {
 
 type transportSender func(session *packet.Session) error
 
+// ErrVithVariableSeverity are errors that have variable severity
+type ErrVithVariableSeverity interface {
+	error
+	// LogLevel return the severity of this error
+	LogLevel() zerolog.Level
+}
+
 // Session is a bidirectional pipe of datagrams between transport and dstConn
 // Destination can be a connection with origin or with eyeball
 // When the destination is origin:
@@ -53,7 +60,11 @@ func (s *Session) Serve(ctx context.Context, closeAfterIdle time.Duration) (clos
 				if errors.Is(err, net.ErrClosed) {
 					s.log.Debug().Msg("Destination connection closed")
 				} else {
-					s.log.Error().Err(err).Msg("Failed to send session payload from destination to transport")
+					level := zerolog.ErrorLevel
+					if variableErr, ok := err.(ErrVithVariableSeverity); ok {
+						level = variableErr.LogLevel()
+					}
+					s.log.WithLevel(level).Err(err).Msg("Failed to send session payload from destination to transport")
 				}
 				if closeSession {
 					s.closeChan <- err

dev.Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.18 as builder
+FROM golang:1.19 as builder
 ENV GO111MODULE=on \
     CGO_ENABLED=0
 WORKDIR /go/src/github.com/cloudflare/cloudflared/

edgediscovery/allregions/discovery.go

@@ -71,11 +71,14 @@ type EdgeAddr struct {
 // If the call to net.LookupSRV fails, try to fall back to DoT from Cloudflare directly.
 //
 // Note: Instead of DoT, we could also have used DoH. Either of these:
-//     - directly via the JSON API (https://1.1.1.1/dns-query?ct=application/dns-json&name=_origintunneld._tcp.argotunnel.com&type=srv)
-//     - indirectly via `tunneldns.NewUpstreamHTTPS()`
+//   - directly via the JSON API (https://1.1.1.1/dns-query?ct=application/dns-json&name=_origintunneld._tcp.argotunnel.com&type=srv)
+//   - indirectly via `tunneldns.NewUpstreamHTTPS()`
+//
 // But both of these cases miss out on a key feature from the stdlib:
-//     "The returned records are sorted by priority and randomized by weight within a priority."
-//     (https://golang.org/pkg/net/#Resolver.LookupSRV)
+//
+//	"The returned records are sorted by priority and randomized by weight within a priority."
+//	(https://golang.org/pkg/net/#Resolver.LookupSRV)
+//
 // Does this matter? I don't know. It may someday. Let's use DoT so we don't need to worry about it.
 // See also: Go feature request for stdlib-supported DoH: https://github.com/golang/go/issues/27552
 var fallbackLookupSRV = lookupSRVWithDOT

fmt-check.sh

@@ -2,12 +2,12 @@
 
 set -e -o pipefail
 
-OUTPUT=$(for d in $(go list -mod=vendor -f '{{.Dir}}' -a ./... | fgrep -v tunnelrpc) ; do goimports -format-only -local github.com/cloudflare/cloudflared -d $d ; done)
+OUTPUT=$(goimports -l -d -local github.com/cloudflare/cloudflared $(go list -mod=vendor -f '{{.Dir}}' -a ./... | fgrep -v tunnelrpc))
 
 if [ -n "$OUTPUT" ] ; then
   PAGER=$(which colordiff || echo cat)
   echo
-  echo "Code formatting issues found, use 'goimports -format-only -local github.com/cloudflare/cloudflared' to correct them"
+  echo "Code formatting issues found, use 'make fmt' to correct them"
   echo
   echo "$OUTPUT" | $PAGER
   exit 1

github_message.py

@@ -1,29 +1,56 @@
 #!/usr/bin/python3
 """
-Creates Github Releases Notes with content hashes
+Create Github Releases Notes with binary checksums from Workers KV
 """
 
 import argparse
 import logging
 import os
-import hashlib
-import glob
+import requests
 
-from github import Github, GithubException, UnknownObjectException
+from github import Github, UnknownObjectException
 
 FORMAT = "%(levelname)s - %(asctime)s: %(message)s"
-logging.basicConfig(format=FORMAT)
+logging.basicConfig(format=FORMAT, level=logging.INFO)
 
 CLOUDFLARED_REPO = os.environ.get("GITHUB_REPO", "cloudflare/cloudflared")
 GITHUB_CONFLICT_CODE = "already_exists"
+BASE_KV_URL = 'https://api.cloudflare.com/client/v4/accounts/'
 
-def get_sha256(filename):
-    """ get the sha256 of a file """
-    sha256_hash = hashlib.sha256()
-    with open(filename,"rb") as f:
-        for byte_block in iter(lambda: f.read(4096),b""):
-            sha256_hash.update(byte_block)
-        return sha256_hash.hexdigest()
+
+def kv_get_keys(prefix, account, namespace, api_token):
+    """ get the KV keys for a given prefix """
+    response = requests.get(
+        BASE_KV_URL + account + "/storage/kv/namespaces/" +
+        namespace + "/keys" + "?prefix=" + prefix,
+        headers={
+            "Content-Type": "application/json",
+            "Authorization": "Bearer " + api_token,
+        },
+    )
+    if response.status_code != 200:
+        jsonResponse = response.json()
+        errors = jsonResponse["errors"]
+        if len(errors) > 0:
+            raise Exception("failed to get checksums: {0}", errors[0])
+    return response.json()["result"]
+
+
+def kv_get_value(key, account, namespace, api_token):
+    """ get the KV value for a provided key """
+    response = requests.get(
+        BASE_KV_URL + account + "/storage/kv/namespaces/" + namespace + "/values/" + key,
+        headers={
+            "Content-Type": "application/json",
+            "Authorization": "Bearer " + api_token,
+        },
+    )
+    if response.status_code != 200:
+        jsonResponse = response.json()
+        errors = jsonResponse["errors"]
+        if len(errors) > 0:
+            raise Exception("failed to get checksums: {0}", errors[0])
+    return response.text
 
 
 def update_or_add_message(msg, name, sha):
@@ -42,6 +69,7 @@ def update_or_add_message(msg, name, sha):
         return '{0}{1}```'.format(msg[:back], new_text)
     return '{0} \n### SHA256 Checksums:\n```\n{1}```'.format(msg, new_text)
 
+
 def get_release(repo, version):
     """ Get a Github Release matching the version tag. """
     try:
@@ -55,11 +83,20 @@ def get_release(repo, version):
 def parse_args():
     """ Parse and validate args """
     parser = argparse.ArgumentParser(
-        description="Creates Github Releases and uploads assets."
+        description="Updates a Github Release with checksums from KV"
     )
     parser.add_argument(
         "--api-key", default=os.environ.get("API_KEY"), help="Github API key"
     )
+    parser.add_argument(
+        "--kv-namespace-id", default=os.environ.get("KV_NAMESPACE"), help="workers KV namespace id"
+    )
+    parser.add_argument(
+        "--kv-account-id", default=os.environ.get("KV_ACCOUNT"), help="workers KV account id"
+    )
+    parser.add_argument(
+        "--kv-api-token", default=os.environ.get("KV_API_TOKEN"), help="workers KV API Token"
+    )
     parser.add_argument(
         "--release-version",
         metavar="version",
@@ -80,6 +117,18 @@ def parse_args():
         logging.error("Missing API key")
         is_valid = False
 
+    if not args.kv_namespace_id:
+        logging.error("Missing KV namespace id")
+        is_valid = False
+
+    if not args.kv_account_id:
+        logging.error("Missing KV account id")
+        is_valid = False
+
+    if not args.kv_api_token:
+        logging.error("Missing KV API token")
+        is_valid = False
+
     if is_valid:
         return args
 
@@ -95,16 +144,20 @@ def main():
         repo = client.get_repo(CLOUDFLARED_REPO)
         release = get_release(repo, args.release_version)
 
-        msg = release.body
+        msg = ""
 
-        for filepath in glob.glob("artifacts/*"):
-            pkg_hash = get_sha256(filepath)
-            # add the sha256 of the new artifact to the release message body
-            name = os.path.basename(filepath)
-            msg = update_or_add_message(msg, name, pkg_hash)
+        prefix = f"update_{args.release_version}_"
+        keys = kv_get_keys(prefix, args.kv_account_id,
+                           args.kv_namespace_id, args.kv_api_token)
+        for key in [k["name"] for k in keys]:
+            checksum = kv_get_value(
+                key, args.kv_account_id, args.kv_namespace_id, args.kv_api_token)
+            binary_name = key[len(prefix):]
+            msg = update_or_add_message(msg, binary_name, checksum)
 
         if args.dry_run:
-            logging.info("Skipping asset upload because of dry-run")
+            logging.info("Skipping release message update because of dry-run")
+            logging.info(f"Github message:\n{msg}")
             return
 
         # update the release body text
@@ -115,4 +168,4 @@ def main():
         exit(1)
 
 
-main()
+main()

go.mod

@@ -1,11 +1,12 @@
 module github.com/cloudflare/cloudflared
 
-go 1.18
+go 1.19
 
 require (
 	github.com/cloudflare/brotli-go v0.0.0-20191101163834-d34379f7ff93
 	github.com/cloudflare/golibs v0.0.0-20170913112048-333127dbecfc
 	github.com/coredns/coredns v1.8.7
+	github.com/coreos/go-oidc/v3 v3.4.0
 	github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf
 	github.com/facebookgo/grace v0.0.0-20180706040059-75cf19382434
 	github.com/fsnotify/fsnotify v1.4.9
@@ -34,11 +35,11 @@ require (
 	go.opentelemetry.io/otel/trace v1.6.3
 	go.opentelemetry.io/proto/otlp v0.15.0
 	go.uber.org/automaxprocs v1.4.0
-	golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa
-	golang.org/x/net v0.0.0-20220909164309-bea034e7d591
-	golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f
-	golang.org/x/sys v0.0.0-20220808155132-1c4a2a72c664
-	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211
+	golang.org/x/crypto v0.2.0
+	golang.org/x/net v0.2.0
+	golang.org/x/sync v0.1.0
+	golang.org/x/sys v0.2.0
+	golang.org/x/term v0.2.0
 	google.golang.org/protobuf v1.28.0
 	gopkg.in/coreos/go-oidc.v2 v2.2.1
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
@@ -56,7 +57,6 @@ require (
 	github.com/cheekybits/genny v1.0.0 // indirect
 	github.com/cloudflare/circl v1.2.1-0.20220809205628-0a9554f37a47 // indirect
 	github.com/coredns/caddy v1.1.1 // indirect
-	github.com/coreos/go-oidc/v3 v3.4.0 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 // indirect
@@ -89,11 +89,10 @@ require (
 	github.com/prometheus/common v0.32.1 // indirect
 	github.com/prometheus/procfs v0.7.3 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
-	golang.org/x/mod v0.4.2 // indirect
+	golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
 	golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094 // indirect
-	golang.org/x/text v0.3.7 // indirect
-	golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2 // indirect
-	golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect
+	golang.org/x/text v0.4.0 // indirect
+	golang.org/x/tools v0.1.12 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90 // indirect
 	google.golang.org/grpc v1.47.0 // indirect
@@ -113,8 +112,8 @@ replace gopkg.in/yaml.v3 => gopkg.in/yaml.v3 v3.0.1
 // Post-quantum tunnel RTG-1339
 replace (
 	// branch go1.18
-	github.com/marten-seemann/qtls-go1-18 => github.com/cloudflare/qtls-pq v0.0.0-20220824105406-fb955667e0af
+	github.com/marten-seemann/qtls-go1-18 => github.com/cloudflare/qtls-pq v0.0.0-20221010110824-0053225e48b2
 
 	// branch go1.19
-	github.com/marten-seemann/qtls-go1-19 => github.com/cloudflare/qtls-pq v0.0.0-20220824104809-96561a41e0af
+	github.com/marten-seemann/qtls-go1-19 => github.com/cloudflare/qtls-pq v0.0.0-20221010110800-4f3769902fe0
 )

go.sum

@@ -132,10 +132,10 @@ github.com/cloudflare/circl v1.2.1-0.20220809205628-0a9554f37a47 h1:YzpECHxZ9TzO
 github.com/cloudflare/circl v1.2.1-0.20220809205628-0a9554f37a47/go.mod h1:qhx8gBILsYlbam7h09SvHDSkjpe3TfLA7b/z4rxJvkE=
 github.com/cloudflare/golibs v0.0.0-20170913112048-333127dbecfc h1:Dvk3ySBsOm5EviLx6VCyILnafPcQinXGP5jbTdHUJgE=
 github.com/cloudflare/golibs v0.0.0-20170913112048-333127dbecfc/go.mod h1:HlgKKR8V5a1wroIDDIz3/A+T+9Janfq+7n1P5sEFdi0=
-github.com/cloudflare/qtls-pq v0.0.0-20220824104809-96561a41e0af h1:JMpOQAaXjRRBkUX73fTNe9mConJLFl6FsIp9fHdLm7Y=
-github.com/cloudflare/qtls-pq v0.0.0-20220824104809-96561a41e0af/go.mod h1:aIsWqC0WXyUiUxBl/RfxAjDyWE9CCLqvSMnCMTd/+bc=
-github.com/cloudflare/qtls-pq v0.0.0-20220824105406-fb955667e0af h1:bhCmedjwrOSyzLtHVeQ+KhimcNTSfs0P5T7kbRQS+gA=
-github.com/cloudflare/qtls-pq v0.0.0-20220824105406-fb955667e0af/go.mod h1:mW0BgKFFDAiSmOdUwoORtjo0V2vqw5QzVYRtKQqw/Jg=
+github.com/cloudflare/qtls-pq v0.0.0-20221010110800-4f3769902fe0 h1:LEsjEHfKnIJEJU9QEIPVRuslxpBu+2kG2DXhxpkGT+o=
+github.com/cloudflare/qtls-pq v0.0.0-20221010110800-4f3769902fe0/go.mod h1:aIsWqC0WXyUiUxBl/RfxAjDyWE9CCLqvSMnCMTd/+bc=
+github.com/cloudflare/qtls-pq v0.0.0-20221010110824-0053225e48b2 h1:ErNoeVNqFXV+emlf4gY7Ms7/0DbQ8PT2UFxNyWBc51Q=
+github.com/cloudflare/qtls-pq v0.0.0-20221010110824-0053225e48b2/go.mod h1:mW0BgKFFDAiSmOdUwoORtjo0V2vqw5QzVYRtKQqw/Jg=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
@@ -644,8 +644,9 @@ golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20210920023735-84f357641f63/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa h1:zuSxTR4o9y82ebqCUJYNGJbGPo6sKVl54f/TVDObg1c=
 golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.2.0 h1:BRXPfhNivWL5Yq0BGQ39a2sW6t44aODpfxkWjYdzewE=
+golang.org/x/crypto v0.2.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -681,8 +682,9 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -741,8 +743,9 @@ golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su
 golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.0.0-20220909164309-bea034e7d591 h1:D0B/7al0LLrVC8aWF4+oxpv/m8bc7ViFfVS8/gXGdqI=
-golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
+golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
+golang.org/x/net v0.2.0 h1:sZfSu1wtKLGlWI4ZZayP0ck9Y73K1ynO6gqzTdBVdPU=
+golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -779,8 +782,9 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f h1:Ax0t5p6N38Ga0dThY21weqDEyz2oklo4IvDkpigvkD8=
 golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -864,12 +868,14 @@ golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220808155132-1c4a2a72c664 h1:v1W7bwXHsnLLloWYTVEdvGvA7BHMeBYsPcF0GLDxIRs=
 golang.org/x/sys v0.0.0-20220808155132-1c4a2a72c664/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A=
+golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.2.0 h1:z85xZCsEl7bi/KwbNADeBYoOP0++7W1ipu+aGnpwzRM=
+golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -878,8 +884,9 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg=
+golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -944,15 +951,15 @@ golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
-golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2 h1:BonxutuHCTL0rBDnZlKjpGIQFTjyUVTexFOdWkB6Fg0=
 golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
-golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f h1:uF6paiQQebLeSXkrTqHqz0MXhXXS1KgF41eUdBNvxK0=
 golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
 google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
 google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=

ingress/config.go

@@ -427,12 +427,19 @@ func (defaults *OriginRequestConfig) setHttp2Origin(overrides config.OriginReque
 	}
 }
 
+func (defaults *OriginRequestConfig) setAccess(overrides config.OriginRequestConfig) {
+	if val := overrides.Access; val != nil {
+		defaults.Access = *val
+	}
+}
+
 // SetConfig gets config for the requests that cloudflared sends to origins.
 // Each field has a setter method which sets a value for the field by trying to find:
-//   1. The user config for this rule
-//   2. The user config for the overall ingress config
-//   3. Defaults chosen by the cloudflared team
-//   4. Golang zero values for that type
+//  1. The user config for this rule
+//  2. The user config for the overall ingress config
+//  3. Defaults chosen by the cloudflared team
+//  4. Golang zero values for that type
+//
 // If an earlier option isn't set, it will try the next option down.
 func setConfig(defaults OriginRequestConfig, overrides config.OriginRequestConfig) OriginRequestConfig {
 	cfg := defaults
@@ -453,6 +460,8 @@ func setConfig(defaults OriginRequestConfig, overrides config.OriginRequestConfi
 	cfg.setProxyType(overrides)
 	cfg.setIPRules(overrides)
 	cfg.setHttp2Origin(overrides)
+	cfg.setAccess(overrides)
+
 	return cfg
 }
 
@@ -463,6 +472,7 @@ func ConvertToRawOriginConfig(c OriginRequestConfig) config.OriginRequestConfig
 	var keepAliveConnections *int
 	var keepAliveTimeout *config.CustomDuration
 	var proxyAddress *string
+	var access *config.AccessConfig
 
 	if c.ConnectTimeout != defaultHTTPConnectTimeout {
 		connectTimeout = &c.ConnectTimeout
@@ -482,6 +492,9 @@ func ConvertToRawOriginConfig(c OriginRequestConfig) config.OriginRequestConfig
 	if c.ProxyAddress != defaultProxyAddress {
 		proxyAddress = &c.ProxyAddress
 	}
+	if c.Access.Required {
+		access = &c.Access
+	}
 
 	return config.OriginRequestConfig{
 		ConnectTimeout:         connectTimeout,
@@ -501,6 +514,7 @@ func ConvertToRawOriginConfig(c OriginRequestConfig) config.OriginRequestConfig
 		ProxyType:              emptyStringToNil(c.ProxyType),
 		IPRules:                convertToRawIPRules(c.IPRules),
 		Http2Origin:            defaultBoolToNil(c.Http2Origin),
+		Access:                 access,
 	}
 }
 

ingress/icmp_darwin.go

@@ -11,16 +11,17 @@ import (
 	"context"
 	"fmt"
 	"math"
-	"net"
 	"net/netip"
 	"strconv"
 	"sync"
 	"time"
 
 	"github.com/rs/zerolog"
+	"go.opentelemetry.io/otel/attribute"
 	"golang.org/x/net/icmp"
 
 	"github.com/cloudflare/cloudflared/packet"
+	"github.com/cloudflare/cloudflared/tracing"
 )
 
 type icmpProxy struct {
@@ -129,44 +130,53 @@ func newICMPProxy(listenIP netip.Addr, zone string, logger *zerolog.Logger, idle
 	}, nil
 }
 
-func (ip *icmpProxy) Request(pk *packet.ICMP, responder packet.FunnelUniPipe) error {
-	if pk == nil {
-		return errPacketNil
-	}
+func (ip *icmpProxy) Request(ctx context.Context, pk *packet.ICMP, responder *packetResponder) error {
+	ctx, span := responder.requestSpan(ctx, pk)
+	defer responder.exportSpan()
+
 	originalEcho, err := getICMPEcho(pk.Message)
 	if err != nil {
+		tracing.EndWithErrorStatus(span, err)
 		return err
 	}
+	span.SetAttributes(
+		attribute.Int("originalEchoID", originalEcho.ID),
+		attribute.Int("seq", originalEcho.Seq),
+	)
 	echoIDTrackerKey := flow3Tuple{
 		srcIP:          pk.Src,
 		dstIP:          pk.Dst,
 		originalEchoID: originalEcho.ID,
 	}
-	// TODO: TUN-6744 assign unique flow per (src, echo ID)
 	assignedEchoID, success := ip.echoIDTracker.getOrAssign(echoIDTrackerKey)
 	if !success {
-		return fmt.Errorf("failed to assign unique echo ID")
+		err := fmt.Errorf("failed to assign unique echo ID")
+		tracing.EndWithErrorStatus(span, err)
+		return err
 	}
+	span.SetAttributes(attribute.Int("assignedEchoID", int(assignedEchoID)))
+
+	shouldReplaceFunnelFunc := createShouldReplaceFunnelFunc(ip.logger, responder.datagramMuxer, pk, originalEcho.ID)
 	newFunnelFunc := func() (packet.Funnel, error) {
 		originalEcho, err := getICMPEcho(pk.Message)
 		if err != nil {
 			return nil, err
 		}
-		originSender := originSender{
-			conn:             ip.conn,
-			echoIDTracker:    ip.echoIDTracker,
-			echoIDTrackerKey: echoIDTrackerKey,
-			assignedEchoID:   assignedEchoID,
+		closeCallback := func() error {
+			ip.echoIDTracker.release(echoIDTrackerKey, assignedEchoID)
+			return nil
 		}
-		icmpFlow := newICMPEchoFlow(pk.Src, &originSender, responder, int(assignedEchoID), originalEcho.ID, ip.encoder)
+		icmpFlow := newICMPEchoFlow(pk.Src, closeCallback, ip.conn, responder, int(assignedEchoID), originalEcho.ID, ip.encoder)
 		return icmpFlow, nil
 	}
 	funnelID := echoFunnelID(assignedEchoID)
-	funnel, isNew, err := ip.srcFunnelTracker.GetOrRegister(funnelID, newFunnelFunc)
+	funnel, isNew, err := ip.srcFunnelTracker.GetOrRegister(funnelID, shouldReplaceFunnelFunc, newFunnelFunc)
 	if err != nil {
+		tracing.EndWithErrorStatus(span, err)
 		return err
 	}
 	if isNew {
+		span.SetAttributes(attribute.Bool("newFlow", true))
 		ip.logger.Debug().
 			Str("src", pk.Src.String()).
 			Str("dst", pk.Dst.String()).
@@ -176,9 +186,16 @@ func (ip *icmpProxy) Request(pk *packet.ICMP, responder packet.FunnelUniPipe) er
 	}
 	icmpFlow, err := toICMPEchoFlow(funnel)
 	if err != nil {
+		tracing.EndWithErrorStatus(span, err)
 		return err
 	}
-	return icmpFlow.sendToDst(pk.Dst, pk.Message)
+	err = icmpFlow.sendToDst(pk.Dst, pk.Message)
+	if err != nil {
+		tracing.EndWithErrorStatus(span, err)
+		return err
+	}
+	tracing.End(span)
+	return nil
 }
 
 // Serve listens for responses to the requests until context is done
@@ -202,7 +219,7 @@ func (ip *icmpProxy) Serve(ctx context.Context) error {
 			ip.logger.Debug().Err(err).Str("dst", from.String()).Msg("Failed to parse ICMP reply, continue to parse as full packet")
 			// In unit test, we found out when the listener listens on 0.0.0.0, the socket reads the full packet after
 			// the second reply
-			if err := ip.handleFullPacket(icmpDecoder, buf[:n]); err != nil {
+			if err := ip.handleFullPacket(ctx, icmpDecoder, buf[:n]); err != nil {
 				ip.logger.Debug().Err(err).Str("dst", from.String()).Msg("Failed to parse ICMP reply as full packet")
 			}
 			continue
@@ -211,14 +228,14 @@ func (ip *icmpProxy) Serve(ctx context.Context) error {
 			ip.logger.Debug().Str("dst", from.String()).Msgf("Drop ICMP %s from reply", reply.msg.Type)
 			continue
 		}
-		if err := ip.sendReply(reply); err != nil {
-			ip.logger.Error().Err(err).Str("dst", from.String()).Msg("Failed to send ICMP reply")
+		if err := ip.sendReply(ctx, reply); err != nil {
+			ip.logger.Debug().Err(err).Str("dst", from.String()).Msg("Failed to send ICMP reply")
 			continue
 		}
 	}
 }
 
-func (ip *icmpProxy) handleFullPacket(decoder *packet.ICMPDecoder, rawPacket []byte) error {
+func (ip *icmpProxy) handleFullPacket(ctx context.Context, decoder *packet.ICMPDecoder, rawPacket []byte) error {
 	icmpPacket, err := decoder.Decode(packet.RawPacket{Data: rawPacket})
 	if err != nil {
 		return err
@@ -232,13 +249,13 @@ func (ip *icmpProxy) handleFullPacket(decoder *packet.ICMPDecoder, rawPacket []b
 		msg:  icmpPacket.Message,
 		echo: echo,
 	}
-	if ip.sendReply(&reply); err != nil {
+	if ip.sendReply(ctx, &reply); err != nil {
 		return err
 	}
 	return nil
 }
 
-func (ip *icmpProxy) sendReply(reply *echoReply) error {
+func (ip *icmpProxy) sendReply(ctx context.Context, reply *echoReply) error {
 	funnelID := echoFunnelID(reply.echo.ID)
 	funnel, ok := ip.srcFunnelTracker.Get(funnelID)
 	if !ok {
@@ -248,25 +265,19 @@ func (ip *icmpProxy) sendReply(reply *echoReply) error {
 	if err != nil {
 		return err
 	}
-	return icmpFlow.returnToSrc(reply)
-}
 
-// originSender wraps icmp.PacketConn to implement packet.FunnelUniPipe interface
-type originSender struct {
-	conn             *icmp.PacketConn
-	echoIDTracker    *echoIDTracker
-	echoIDTrackerKey flow3Tuple
-	assignedEchoID   uint16
-}
+	_, span := icmpFlow.responder.replySpan(ctx, ip.logger)
+	defer icmpFlow.responder.exportSpan()
 
-func (os *originSender) SendPacket(dst netip.Addr, pk packet.RawPacket) error {
-	_, err := os.conn.WriteTo(pk.Data, &net.UDPAddr{
-		IP: dst.AsSlice(),
-	})
-	return err
-}
-
-func (os *originSender) Close() error {
-	os.echoIDTracker.release(os.echoIDTrackerKey, os.assignedEchoID)
+	span.SetAttributes(
+		attribute.String("dst", reply.from.String()),
+		attribute.Int("echoID", reply.echo.ID),
+		attribute.Int("seq", reply.echo.Seq),
+		attribute.Int("originalEchoID", icmpFlow.originalEchoID),
+	)
+	if err := icmpFlow.returnToSrc(reply); err != nil {
+		tracing.EndWithErrorStatus(span, err)
+	}
+	tracing.End(span)
 	return nil
 }

ingress/icmp_darwin_test.go

@@ -8,6 +8,8 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/require"
+
+	"github.com/cloudflare/cloudflared/packet"
 )
 
 func TestSingleEchoIDTracker(t *testing.T) {
@@ -119,3 +121,9 @@ func (eit *echoIDTracker) get(key flow3Tuple) (id uint16, exist bool) {
 	id, exists := eit.mapping[key]
 	return id, exists
 }
+
+func getFunnel(t *testing.T, proxy *icmpProxy, tuple flow3Tuple) (packet.Funnel, bool) {
+	assignedEchoID, success := proxy.echoIDTracker.getOrAssign(tuple)
+	require.True(t, success)
+	return proxy.srcFunnelTracker.Get(echoFunnelID(assignedEchoID))
+}

ingress/icmp_generic.go

@@ -18,7 +18,7 @@ var errICMPProxyNotImplemented = fmt.Errorf("ICMP proxy is not implemented on %s
 
 type icmpProxy struct{}
 
-func (ip icmpProxy) Request(pk *packet.ICMP, responder packet.FunnelUniPipe) error {
+func (ip icmpProxy) Request(ctx context.Context, pk *packet.ICMP, responder *packetResponder) error {
 	return errICMPProxyNotImplemented
 }
 

ingress/icmp_linux.go

@@ -19,9 +19,10 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
-	"golang.org/x/net/icmp"
+	"go.opentelemetry.io/otel/attribute"
 
 	"github.com/cloudflare/cloudflared/packet"
+	"github.com/cloudflare/cloudflared/tracing"
 )
 
 const (
@@ -97,29 +98,39 @@ func checkInPingGroup() error {
 	return fmt.Errorf("did not find group range in %s", pingGroupPath)
 }
 
-func (ip *icmpProxy) Request(pk *packet.ICMP, responder packet.FunnelUniPipe) error {
-	if pk == nil {
-		return errPacketNil
-	}
+func (ip *icmpProxy) Request(ctx context.Context, pk *packet.ICMP, responder *packetResponder) error {
+	ctx, span := responder.requestSpan(ctx, pk)
+	defer responder.exportSpan()
+
 	originalEcho, err := getICMPEcho(pk.Message)
 	if err != nil {
+		tracing.EndWithErrorStatus(span, err)
 		return err
 	}
-	newConnChan := make(chan *icmp.PacketConn, 1)
+	span.SetAttributes(
+		attribute.Int("originalEchoID", originalEcho.ID),
+		attribute.Int("seq", originalEcho.Seq),
+	)
+
+	shouldReplaceFunnelFunc := createShouldReplaceFunnelFunc(ip.logger, responder.datagramMuxer, pk, originalEcho.ID)
 	newFunnelFunc := func() (packet.Funnel, error) {
 		conn, err := newICMPConn(ip.listenIP, ip.ipv6Zone)
 		if err != nil {
+			tracing.EndWithErrorStatus(span, err)
 			return nil, errors.Wrap(err, "failed to open ICMP socket")
 		}
 		ip.logger.Debug().Msgf("Opened ICMP socket listen on %s", conn.LocalAddr())
-		newConnChan <- conn
+		closeCallback := func() error {
+			return conn.Close()
+		}
 		localUDPAddr, ok := conn.LocalAddr().(*net.UDPAddr)
 		if !ok {
 			return nil, fmt.Errorf("ICMP listener address %s is not net.UDPAddr", conn.LocalAddr())
 		}
-		originSender := originSender{conn: conn}
+		span.SetAttributes(attribute.Int("port", localUDPAddr.Port))
+
 		echoID := localUDPAddr.Port
-		icmpFlow := newICMPEchoFlow(pk.Src, &originSender, responder, echoID, originalEcho.ID, packet.NewEncoder())
+		icmpFlow := newICMPEchoFlow(pk.Src, closeCallback, conn, responder, echoID, originalEcho.ID, packet.NewEncoder())
 		return icmpFlow, nil
 	}
 	funnelID := flow3Tuple{
@@ -127,24 +138,26 @@ func (ip *icmpProxy) Request(pk *packet.ICMP, responder packet.FunnelUniPipe) er
 		dstIP:          pk.Dst,
 		originalEchoID: originalEcho.ID,
 	}
-	funnel, isNew, err := ip.srcFunnelTracker.GetOrRegister(funnelID, newFunnelFunc)
+	funnel, isNew, err := ip.srcFunnelTracker.GetOrRegister(funnelID, shouldReplaceFunnelFunc, newFunnelFunc)
 	if err != nil {
+		tracing.EndWithErrorStatus(span, err)
 		return err
 	}
 	icmpFlow, err := toICMPEchoFlow(funnel)
 	if err != nil {
+		tracing.EndWithErrorStatus(span, err)
 		return err
 	}
 	if isNew {
+		span.SetAttributes(attribute.Bool("newFlow", true))
 		ip.logger.Debug().
 			Str("src", pk.Src.String()).
 			Str("dst", pk.Dst.String()).
 			Int("originalEchoID", originalEcho.ID).
 			Msg("New flow")
-		conn := <-newConnChan
 		go func() {
 			defer ip.srcFunnelTracker.Unregister(funnelID, icmpFlow)
-			if err := ip.listenResponse(icmpFlow, conn); err != nil {
+			if err := ip.listenResponse(ctx, icmpFlow); err != nil {
 				ip.logger.Debug().Err(err).
 					Str("src", pk.Src.String()).
 					Str("dst", pk.Dst.String()).
@@ -154,8 +167,10 @@ func (ip *icmpProxy) Request(pk *packet.ICMP, responder packet.FunnelUniPipe) er
 		}()
 	}
 	if err := icmpFlow.sendToDst(pk.Dst, pk.Message); err != nil {
+		tracing.EndWithErrorStatus(span, err)
 		return errors.Wrap(err, "failed to send ICMP echo request")
 	}
+	tracing.End(span)
 	return nil
 }
 
@@ -164,43 +179,53 @@ func (ip *icmpProxy) Serve(ctx context.Context) error {
 	return ctx.Err()
 }
 
-func (ip *icmpProxy) listenResponse(flow *icmpEchoFlow, conn *icmp.PacketConn) error {
+func (ip *icmpProxy) listenResponse(ctx context.Context, flow *icmpEchoFlow) error {
 	buf := make([]byte, mtu)
 	for {
-		n, from, err := conn.ReadFrom(buf)
-		if err != nil {
+		retryable, err := ip.handleResponse(ctx, flow, buf)
+		if err != nil && !retryable {
 			return err
 		}
-		reply, err := parseReply(from, buf[:n])
-		if err != nil {
-			ip.logger.Error().Err(err).Str("dst", from.String()).Msg("Failed to parse ICMP reply")
-			continue
-		}
-		if !isEchoReply(reply.msg) {
-			ip.logger.Debug().Str("dst", from.String()).Msgf("Drop ICMP %s from reply", reply.msg.Type)
-			continue
-		}
-		if err := flow.returnToSrc(reply); err != nil {
-			ip.logger.Err(err).Str("dst", from.String()).Msg("Failed to send ICMP reply")
-			continue
-		}
 	}
 }
 
-// originSender wraps icmp.PacketConn to implement packet.FunnelUniPipe interface
-type originSender struct {
-	conn *icmp.PacketConn
-}
+func (ip *icmpProxy) handleResponse(ctx context.Context, flow *icmpEchoFlow, buf []byte) (retryableErr bool, err error) {
+	_, span := flow.responder.replySpan(ctx, ip.logger)
+	defer flow.responder.exportSpan()
 
-func (os *originSender) SendPacket(dst netip.Addr, pk packet.RawPacket) error {
-	_, err := os.conn.WriteTo(pk.Data, &net.UDPAddr{
-		IP: dst.AsSlice(),
-	})
-	return err
-}
+	span.SetAttributes(
+		attribute.Int("originalEchoID", flow.originalEchoID),
+	)
 
-func (os *originSender) Close() error {
-	return os.conn.Close()
+	n, from, err := flow.originConn.ReadFrom(buf)
+	if err != nil {
+		tracing.EndWithErrorStatus(span, err)
+		return false, err
+	}
+	reply, err := parseReply(from, buf[:n])
+	if err != nil {
+		ip.logger.Error().Err(err).Str("dst", from.String()).Msg("Failed to parse ICMP reply")
+		tracing.EndWithErrorStatus(span, err)
+		return true, err
+	}
+	if !isEchoReply(reply.msg) {
+		err := fmt.Errorf("Expect ICMP echo reply, got %s", reply.msg.Type)
+		ip.logger.Debug().Str("dst", from.String()).Msgf("Drop ICMP %s from reply", reply.msg.Type)
+		tracing.EndWithErrorStatus(span, err)
+		return true, err
+	}
+	span.SetAttributes(
+		attribute.String("dst", reply.from.String()),
+		attribute.Int("echoID", reply.echo.ID),
+		attribute.Int("seq", reply.echo.Seq),
+	)
+	if err := flow.returnToSrc(reply); err != nil {
+		ip.logger.Debug().Err(err).Str("dst", from.String()).Msg("Failed to send ICMP reply")
+		tracing.EndWithErrorStatus(span, err)
+		return true, err
+	}
+	tracing.End(span)
+	return true, nil
 }
 
 // Only linux uses flow3Tuple as FunnelID

ingress/icmp_linux_test.go

@@ -0,0 +1,13 @@
+//go:build linux
+
+package ingress
+
+import (
+	"testing"
+
+	"github.com/cloudflare/cloudflared/packet"
+)
+
+func getFunnel(t *testing.T, proxy *icmpProxy, tuple flow3Tuple) (packet.Funnel, bool) {
+	return proxy.srcFunnelTracker.Get(tuple)
+}

ingress/icmp_posix.go

@@ -10,6 +10,7 @@ import (
 	"net/netip"
 
 	"github.com/google/gopacket/layers"
+	"github.com/rs/zerolog"
 	"golang.org/x/net/icmp"
 
 	"github.com/cloudflare/cloudflared/packet"
@@ -43,24 +44,54 @@ type flow3Tuple struct {
 
 // icmpEchoFlow implements the packet.Funnel interface.
 type icmpEchoFlow struct {
-	*packet.RawPacketFunnel
+	*packet.ActivityTracker
+	closeCallback  func() error
+	src            netip.Addr
+	originConn     *icmp.PacketConn
+	responder      *packetResponder
 	assignedEchoID int
 	originalEchoID int
 	// it's up to the user to ensure respEncoder is not used concurrently
 	respEncoder *packet.Encoder
 }
 
-func newICMPEchoFlow(src netip.Addr, sendPipe, returnPipe packet.FunnelUniPipe, assignedEchoID, originalEchoID int, respEncoder *packet.Encoder) *icmpEchoFlow {
+func newICMPEchoFlow(src netip.Addr, closeCallback func() error, originConn *icmp.PacketConn, responder *packetResponder, assignedEchoID, originalEchoID int, respEncoder *packet.Encoder) *icmpEchoFlow {
 	return &icmpEchoFlow{
-		RawPacketFunnel: packet.NewRawPacketFunnel(src, sendPipe, returnPipe),
+		ActivityTracker: packet.NewActivityTracker(),
+		closeCallback:   closeCallback,
+		src:             src,
+		originConn:      originConn,
+		responder:       responder,
 		assignedEchoID:  assignedEchoID,
 		originalEchoID:  originalEchoID,
 		respEncoder:     respEncoder,
 	}
 }
 
+func (ief *icmpEchoFlow) Equal(other packet.Funnel) bool {
+	otherICMPFlow, ok := other.(*icmpEchoFlow)
+	if !ok {
+		return false
+	}
+	if otherICMPFlow.src != ief.src {
+		return false
+	}
+	if otherICMPFlow.originalEchoID != ief.originalEchoID {
+		return false
+	}
+	if otherICMPFlow.assignedEchoID != ief.assignedEchoID {
+		return false
+	}
+	return true
+}
+
+func (ief *icmpEchoFlow) Close() error {
+	return ief.closeCallback()
+}
+
 // sendToDst rewrites the echo ID to the one assigned to this flow
 func (ief *icmpEchoFlow) sendToDst(dst netip.Addr, msg *icmp.Message) error {
+	ief.UpdateLastActive()
 	originalEcho, err := getICMPEcho(msg)
 	if err != nil {
 		return err
@@ -80,17 +111,21 @@ func (ief *icmpEchoFlow) sendToDst(dst netip.Addr, msg *icmp.Message) error {
 	if err != nil {
 		return err
 	}
-	return ief.SendToDst(dst, packet.RawPacket{Data: serializedPacket})
+	_, err = ief.originConn.WriteTo(serializedPacket, &net.UDPAddr{
+		IP: dst.AsSlice(),
+	})
+	return err
 }
 
 // returnToSrc rewrites the echo ID to the original echo ID from the eyeball
 func (ief *icmpEchoFlow) returnToSrc(reply *echoReply) error {
+	ief.UpdateLastActive()
 	reply.echo.ID = ief.originalEchoID
 	reply.msg.Body = reply.echo
 	pk := packet.ICMP{
 		IP: &packet.IP{
 			Src:      reply.from,
-			Dst:      ief.Src,
+			Dst:      ief.src,
 			Protocol: layers.IPProtocol(reply.msg.Type.Protocol()),
 			TTL:      packet.DefaultTTL,
 		},
@@ -100,7 +135,7 @@ func (ief *icmpEchoFlow) returnToSrc(reply *echoReply) error {
 	if err != nil {
 		return err
 	}
-	return ief.ReturnToSrc(serializedPacket)
+	return ief.responder.returnPacket(serializedPacket)
 }
 
 type echoReply struct {
@@ -140,3 +175,30 @@ func toICMPEchoFlow(funnel packet.Funnel) (*icmpEchoFlow, error) {
 	}
 	return icmpFlow, nil
 }
+
+func createShouldReplaceFunnelFunc(logger *zerolog.Logger, muxer muxer, pk *packet.ICMP, originalEchoID int) func(packet.Funnel) bool {
+	return func(existing packet.Funnel) bool {
+		existingFlow, err := toICMPEchoFlow(existing)
+		if err != nil {
+			logger.Err(err).
+				Str("src", pk.Src.String()).
+				Str("dst", pk.Dst.String()).
+				Int("originalEchoID", originalEchoID).
+				Msg("Funnel of wrong type found")
+			return true
+		}
+		// Each quic connection should have a unique muxer.
+		// If the existing flow has a different muxer, there's a new quic connection where return packets should be
+		// routed. Otherwise, return packets will be send to the first observed incoming connection, rather than the
+		// most recently observed connection.
+		if existingFlow.responder.datagramMuxer != muxer {
+			logger.Debug().
+				Str("src", pk.Src.String()).
+				Str("dst", pk.Dst.String()).
+				Int("originalEchoID", originalEchoID).
+				Msg("Replacing funnel with new responder")
+			return true
+		}
+		return false
+	}
+}

ingress/icmp_posix_test.go

@@ -52,24 +52,88 @@ func TestFunnelIdleTimeout(t *testing.T) {
 			},
 		},
 	}
-	responder := echoFlowResponder{
-		decoder:  packet.NewICMPDecoder(),
-		respChan: make(chan []byte),
+	muxer := newMockMuxer(0)
+	responder := packetResponder{
+		datagramMuxer: muxer,
 	}
-	require.NoError(t, proxy.Request(&pk, &responder))
-	responder.validate(t, &pk)
+	require.NoError(t, proxy.Request(ctx, &pk, &responder))
+	validateEchoFlow(t, <-muxer.cfdToEdge, &pk)
 
 	// Send second request, should reuse the funnel
-	require.NoError(t, proxy.Request(&pk, nil))
-	responder.validate(t, &pk)
+	require.NoError(t, proxy.Request(ctx, &pk, &packetResponder{
+		datagramMuxer: muxer,
+	}))
+	validateEchoFlow(t, <-muxer.cfdToEdge, &pk)
 
 	time.Sleep(idleTimeout * 2)
-	newResponder := echoFlowResponder{
-		decoder:  packet.NewICMPDecoder(),
-		respChan: make(chan []byte),
+	newMuxer := newMockMuxer(0)
+	newResponder := packetResponder{
+		datagramMuxer: newMuxer,
 	}
-	require.NoError(t, proxy.Request(&pk, &newResponder))
-	newResponder.validate(t, &pk)
+	require.NoError(t, proxy.Request(ctx, &pk, &newResponder))
+	validateEchoFlow(t, <-newMuxer.cfdToEdge, &pk)
+
+	cancel()
+	<-proxyDone
+}
+
+func TestReuseFunnel(t *testing.T) {
+	const (
+		idleTimeout = time.Second
+		echoID      = 42573
+		startSeq    = 8129
+	)
+	logger := zerolog.New(os.Stderr)
+	proxy, err := newICMPProxy(localhostIP, "", &logger, idleTimeout)
+	require.NoError(t, err)
+
+	ctx, cancel := context.WithCancel(context.Background())
+
+	proxyDone := make(chan struct{})
+	go func() {
+		proxy.Serve(ctx)
+		close(proxyDone)
+	}()
+
+	// Send a packet to register the flow
+	pk := packet.ICMP{
+		IP: &packet.IP{
+			Src:      localhostIP,
+			Dst:      localhostIP,
+			Protocol: layers.IPProtocolICMPv4,
+		},
+		Message: &icmp.Message{
+			Type: ipv4.ICMPTypeEcho,
+			Code: 0,
+			Body: &icmp.Echo{
+				ID:   echoID,
+				Seq:  startSeq,
+				Data: []byte(t.Name()),
+			},
+		},
+	}
+	tuple := flow3Tuple{
+		srcIP:          pk.Src,
+		dstIP:          pk.Dst,
+		originalEchoID: echoID,
+	}
+	muxer := newMockMuxer(0)
+	responder := packetResponder{
+		datagramMuxer: muxer,
+	}
+	require.NoError(t, proxy.Request(ctx, &pk, &responder))
+	validateEchoFlow(t, <-muxer.cfdToEdge, &pk)
+	funnel1, found := getFunnel(t, proxy, tuple)
+	require.True(t, found)
+
+	// Send second request, should reuse the funnel
+	require.NoError(t, proxy.Request(ctx, &pk, &packetResponder{
+		datagramMuxer: muxer,
+	}))
+	validateEchoFlow(t, <-muxer.cfdToEdge, &pk)
+	funnel2, found := getFunnel(t, proxy, tuple)
+	require.True(t, found)
+	require.Equal(t, funnel1, funnel2)
 
 	cancel()
 	<-proxyDone

ingress/icmp_windows.go

@@ -21,11 +21,13 @@ import (
 	"github.com/google/gopacket/layers"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
+	"go.opentelemetry.io/otel/attribute"
 	"golang.org/x/net/icmp"
 	"golang.org/x/net/ipv4"
 	"golang.org/x/net/ipv6"
 
 	"github.com/cloudflare/cloudflared/packet"
+	"github.com/cloudflare/cloudflared/tracing"
 )
 
 const (
@@ -265,34 +267,51 @@ func (ip *icmpProxy) Serve(ctx context.Context) error {
 // Request sends an ICMP echo request and wait for a reply or timeout.
 // The async version of Win32 APIs take a callback whose memory is not garbage collected, so we use the synchronous version.
 // It's possible that a slow request will block other requests, so we set the timeout to only 1s.
-func (ip *icmpProxy) Request(pk *packet.ICMP, responder packet.FunnelUniPipe) error {
-	if pk == nil {
-		return errPacketNil
-	}
+func (ip *icmpProxy) Request(ctx context.Context, pk *packet.ICMP, responder *packetResponder) error {
 	defer func() {
 		if r := recover(); r != nil {
 			ip.logger.Error().Interface("error", r).Msgf("Recover panic from sending icmp request/response, error %s", debug.Stack())
 		}
 	}()
 
+	_, requestSpan := responder.requestSpan(ctx, pk)
+	defer responder.exportSpan()
+
 	echo, err := getICMPEcho(pk.Message)
 	if err != nil {
 		return err
 	}
-	respData, err := ip.icmpEchoRoundtrip(pk.Dst, echo)
+	requestSpan.SetAttributes(
+		attribute.Int("originalEchoID", echo.ID),
+		attribute.Int("seq", echo.Seq),
+	)
+
+	resp, err := ip.icmpEchoRoundtrip(pk.Dst, echo)
 	if err != nil {
 		ip.logger.Err(err).Msg("ICMP echo roundtrip failed")
+		tracing.EndWithErrorStatus(requestSpan, err)
 		return err
 	}
+	tracing.End(requestSpan)
+	responder.exportSpan()
 
-	err = ip.handleEchoReply(pk, echo, respData, responder)
+	_, replySpan := responder.replySpan(ctx, ip.logger)
+	replySpan.SetAttributes(
+		attribute.Int("originalEchoID", echo.ID),
+		attribute.Int("seq", echo.Seq),
+		attribute.Int64("rtt", int64(resp.rtt())),
+		attribute.String("status", resp.status().String()),
+	)
+	err = ip.handleEchoReply(pk, echo, resp, responder)
 	if err != nil {
+		tracing.EndWithErrorStatus(replySpan, err)
 		return errors.Wrap(err, "failed to handle ICMP echo reply")
 	}
+	tracing.End(replySpan)
 	return nil
 }
 
-func (ip *icmpProxy) handleEchoReply(request *packet.ICMP, echoReq *icmp.Echo, data []byte, responder packet.FunnelUniPipe) error {
+func (ip *icmpProxy) handleEchoReply(request *packet.ICMP, echoReq *icmp.Echo, resp echoResp, responder *packetResponder) error {
 	var replyType icmp.Type
 	if request.Dst.Is4() {
 		replyType = ipv4.ICMPTypeEchoReply
@@ -313,7 +332,7 @@ func (ip *icmpProxy) handleEchoReply(request *packet.ICMP, echoReq *icmp.Echo, d
 			Body: &icmp.Echo{
 				ID:   echoReq.ID,
 				Seq:  echoReq.Seq,
-				Data: data,
+				Data: resp.payload(),
 			},
 		},
 	}
@@ -331,19 +350,20 @@ func (ip *icmpProxy) handleEchoReply(request *packet.ICMP, echoReq *icmp.Echo, d
 	if err != nil {
 		return err
 	}
-	return responder.SendPacket(request.Src, serializedPacket)
+	return responder.returnPacket(serializedPacket)
 }
 
-func (ip *icmpProxy) icmpEchoRoundtrip(dst netip.Addr, echo *icmp.Echo) ([]byte, error) {
+func (ip *icmpProxy) icmpEchoRoundtrip(dst netip.Addr, echo *icmp.Echo) (echoResp, error) {
 	if dst.Is6() {
 		if ip.srcSocketAddr == nil {
 			return nil, fmt.Errorf("cannot send ICMPv6 using ICMPv4 proxy")
 		}
 		resp, err := ip.icmp6SendEcho(dst, echo)
 		if err != nil {
+
 			return nil, errors.Wrap(err, "failed to send/receive ICMPv6 echo")
 		}
-		return resp.data, nil
+		return resp, nil
 	}
 	if ip.srcSocketAddr != nil {
 		return nil, fmt.Errorf("cannot send ICMPv4 using ICMPv6 proxy")
@@ -352,26 +372,26 @@ func (ip *icmpProxy) icmpEchoRoundtrip(dst netip.Addr, echo *icmp.Echo) ([]byte,
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to send/receive ICMPv4 echo")
 	}
-	return resp.data, nil
+	return resp, nil
 }
 
 /*
-	Wrapper to call https://docs.microsoft.com/en-us/windows/win32/api/icmpapi/nf-icmpapi-icmpsendecho
-	Parameters:
-	- IcmpHandle: Handle created by IcmpCreateFile
-	- DestinationAddress: IPv4 in the form of https://docs.microsoft.com/en-us/windows/win32/api/inaddr/ns-inaddr-in_addr#syntax
-	- RequestData: A pointer to echo data
-	- RequestSize: Number of bytes in buffer pointed by echo data
-	- RequestOptions: IP header options
-	- ReplyBuffer: A pointer to the buffer for echoReply, options and data
-	- ReplySize: Number of bytes allocated for ReplyBuffer
-	- Timeout: Timeout in milliseconds to wait for a reply
-	Returns:
-	- the number of replies in uint32 https://docs.microsoft.com/en-us/windows/win32/api/icmpapi/nf-icmpapi-icmpsendecho#return-value
-	To retain the reference allocated objects, conversion from pointer to uintptr must happen as arguments to the
-	syscall function
+Wrapper to call https://docs.microsoft.com/en-us/windows/win32/api/icmpapi/nf-icmpapi-icmpsendecho
+Parameters:
+- IcmpHandle: Handle created by IcmpCreateFile
+- DestinationAddress: IPv4 in the form of https://docs.microsoft.com/en-us/windows/win32/api/inaddr/ns-inaddr-in_addr#syntax
+- RequestData: A pointer to echo data
+- RequestSize: Number of bytes in buffer pointed by echo data
+- RequestOptions: IP header options
+- ReplyBuffer: A pointer to the buffer for echoReply, options and data
+- ReplySize: Number of bytes allocated for ReplyBuffer
+- Timeout: Timeout in milliseconds to wait for a reply
+Returns:
+- the number of replies in uint32 https://docs.microsoft.com/en-us/windows/win32/api/icmpapi/nf-icmpapi-icmpsendecho#return-value
+To retain the reference allocated objects, conversion from pointer to uintptr must happen as arguments to the
+syscall function
 */
-func (ip *icmpProxy) icmpSendEcho(dst netip.Addr, echo *icmp.Echo) (*echoResp, error) {
+func (ip *icmpProxy) icmpSendEcho(dst netip.Addr, echo *icmp.Echo) (*echoV4Resp, error) {
 	dataSize := len(echo.Data)
 	replySize := echoReplySize + uintptr(dataSize)
 	replyBuf := make([]byte, replySize)
@@ -399,7 +419,7 @@ func (ip *icmpProxy) icmpSendEcho(dst netip.Addr, echo *icmp.Echo) (*echoResp, e
 	} else if replyCount > 1 {
 		ip.logger.Warn().Msgf("Received %d ICMP echo replies, only sending 1 back", replyCount)
 	}
-	return newEchoResp(replyBuf)
+	return newEchoV4Resp(replyBuf)
 }
 
 // Third definition of https://docs.microsoft.com/en-us/windows/win32/api/inaddr/ns-inaddr-in_addr#syntax is address in uint32
@@ -411,12 +431,30 @@ func inAddrV4(ip netip.Addr) (uint32, error) {
 	return endian.Uint32(v4[:]), nil
 }
 
-type echoResp struct {
+type echoResp interface {
+	status() ipStatus
+	rtt() uint32
+	payload() []byte
+}
+
+type echoV4Resp struct {
 	reply *echoReply
 	data  []byte
 }
 
-func newEchoResp(replyBuf []byte) (*echoResp, error) {
+func (r *echoV4Resp) status() ipStatus {
+	return r.reply.Status
+}
+
+func (r *echoV4Resp) rtt() uint32 {
+	return r.reply.RoundTripTime
+}
+
+func (r *echoV4Resp) payload() []byte {
+	return r.data
+}
+
+func newEchoV4Resp(replyBuf []byte) (*echoV4Resp, error) {
 	if len(replyBuf) == 0 {
 		return nil, fmt.Errorf("reply buffer is empty")
 	}
@@ -430,7 +468,7 @@ func newEchoResp(replyBuf []byte) (*echoResp, error) {
 	if dataBufStart < int(echoReplySize) {
 		return nil, fmt.Errorf("reply buffer size %d is too small to hold data of size %d", len(replyBuf), int(reply.DataSize))
 	}
-	return &echoResp{
+	return &echoV4Resp{
 		reply: &reply,
 		data:  replyBuf[dataBufStart:],
 	}, nil
@@ -502,6 +540,18 @@ type echoV6Resp struct {
 	data  []byte
 }
 
+func (r *echoV6Resp) status() ipStatus {
+	return r.reply.Status
+}
+
+func (r *echoV6Resp) rtt() uint32 {
+	return r.reply.RoundTripTime
+}
+
+func (r *echoV6Resp) payload() []byte {
+	return r.data
+}
+
 func newEchoV6Resp(replyBuf []byte, dataSize int) (*echoV6Resp, error) {
 	if len(replyBuf) == 0 {
 		return nil, fmt.Errorf("reply buffer is empty")

ingress/icmp_windows_test.go

@@ -59,7 +59,7 @@ func TestParseEchoReply(t *testing.T) {
 	}
 
 	for _, test := range tests {
-		resp, err := newEchoResp(test.replyBuf)
+		resp, err := newEchoV4Resp(test.replyBuf)
 		if test.expectedReply == nil {
 			require.Error(t, err)
 			require.Nil(t, resp)
@@ -125,7 +125,7 @@ func TestParseEchoV6Reply(t *testing.T) {
 	}
 }
 
-//  TestSendEchoErrors makes sure icmpSendEcho handles error cases
+// TestSendEchoErrors makes sure icmpSendEcho handles error cases
 func TestSendEchoErrors(t *testing.T) {
 	testSendEchoErrors(t, netip.IPv4Unspecified())
 	testSendEchoErrors(t, netip.IPv6Unspecified())

ingress/ingress.go

@@ -175,18 +175,10 @@ func validateAccessConfiguration(cfg *config.AccessConfig) error {
 		return nil
 	}
 
-	// It is possible to set `required:true` and not have these two configured yet.
-	// But if one of them is configured, we'd validate for correctness.
-	if len(cfg.AudTag) == 0 && cfg.TeamName == "" {
-		return nil
-	}
-
-	if len(cfg.AudTag) == 0 {
-		return errors.New("access audtag cannot be empty")
-	}
-
-	if cfg.TeamName == "" {
-		return errors.New("access.TeamName cannot be blank")
+	// we allow for an initial setup where user can force Access but not configure the rest of the keys.
+	// however, if the user specified audTags but forgot teamName, we should alert it.
+	if cfg.TeamName == "" && len(cfg.AudTag) > 0 {
+		return errors.New("access.TeamName cannot be blank when access.audTags are present")
 	}
 
 	return nil

ingress/ingress_test.go

@@ -674,6 +674,46 @@ ingress:
 	}
 }
 
+func TestParseAccessConfig(t *testing.T) {
+	tests := []struct {
+		name        string
+		cfg         config.AccessConfig
+		expectError bool
+	}{
+		{
+			name:        "Config required with teamName only",
+			cfg:         config.AccessConfig{Required: true, TeamName: "team"},
+			expectError: false,
+		},
+		{
+			name:        "required false",
+			cfg:         config.AccessConfig{Required: false},
+			expectError: false,
+		},
+		{
+			name:        "required true but empty config",
+			cfg:         config.AccessConfig{Required: true},
+			expectError: false,
+		},
+		{
+			name:        "complete config",
+			cfg:         config.AccessConfig{Required: true, TeamName: "team", AudTag: []string{"a"}},
+			expectError: false,
+		},
+		{
+			name:        "required true with audTags but no teamName",
+			cfg:         config.AccessConfig{Required: true, AudTag: []string{"a"}},
+			expectError: true,
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			err := validateAccessConfiguration(&test.cfg)
+			require.Equal(t, err != nil, test.expectError)
+		})
+	}
+}
+
 func MustReadIngress(s string) *config.Configuration {
 	var conf config.Configuration
 	err := yaml.Unmarshal([]byte(s), &conf)

ingress/origin_icmp_proxy.go

@@ -37,7 +37,9 @@ func NewICMPRouter(ipv4Addr, ipv6Addr netip.Addr, ipv6Zone string, logger *zerol
 	ipv4Proxy, ipv4Err := newICMPProxy(ipv4Addr, "", logger, funnelIdleTimeout)
 	ipv6Proxy, ipv6Err := newICMPProxy(ipv6Addr, ipv6Zone, logger, funnelIdleTimeout)
 	if ipv4Err != nil && ipv6Err != nil {
-		return nil, fmt.Errorf("cannot create ICMPv4 proxy: %v nor ICMPv6 proxy: %v", ipv4Err, ipv6Err)
+		err := fmt.Errorf("cannot create ICMPv4 proxy: %v nor ICMPv6 proxy: %v", ipv4Err, ipv6Err)
+		logger.Debug().Err(err).Msg("ICMP proxy feature is disabled")
+		return nil, err
 	}
 	if ipv4Err != nil {
 		logger.Debug().Err(ipv4Err).Msg("failed to create ICMPv4 proxy, only ICMPv6 proxy is created")
@@ -73,15 +75,18 @@ func (ir *icmpRouter) Serve(ctx context.Context) error {
 	return fmt.Errorf("ICMPv4 proxy and ICMPv6 proxy are both nil")
 }
 
-func (ir *icmpRouter) Request(pk *packet.ICMP, responder packet.FunnelUniPipe) error {
+func (ir *icmpRouter) Request(ctx context.Context, pk *packet.ICMP, responder *packetResponder) error {
+	if pk == nil {
+		return errPacketNil
+	}
 	if pk.Dst.Is4() {
 		if ir.ipv4Proxy != nil {
-			return ir.ipv4Proxy.Request(pk, responder)
+			return ir.ipv4Proxy.Request(ctx, pk, responder)
 		}
 		return fmt.Errorf("ICMPv4 proxy was not instantiated")
 	}
 	if ir.ipv6Proxy != nil {
-		return ir.ipv6Proxy.Request(pk, responder)
+		return ir.ipv6Proxy.Request(ctx, pk, responder)
 	}
 	return fmt.Errorf("ICMPv6 proxy was not instantiated")
 }

ingress/origin_icmp_proxy_test.go

@@ -1,6 +1,7 @@
 package ingress
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"net"
@@ -17,6 +18,8 @@ import (
 	"golang.org/x/net/ipv6"
 
 	"github.com/cloudflare/cloudflared/packet"
+	quicpogs "github.com/cloudflare/cloudflared/quic"
+	"github.com/cloudflare/cloudflared/tracing"
 )
 
 var (
@@ -52,9 +55,9 @@ func testICMPRouterEcho(t *testing.T, sendIPv4 bool) {
 		close(proxyDone)
 	}()
 
-	responder := echoFlowResponder{
-		decoder:  packet.NewICMPDecoder(),
-		respChan: make(chan []byte, 1),
+	muxer := newMockMuxer(1)
+	responder := packetResponder{
+		datagramMuxer: muxer,
 	}
 
 	protocol := layers.IPProtocolICMPv6
@@ -90,14 +93,113 @@ func testICMPRouterEcho(t *testing.T, sendIPv4 bool) {
 					},
 				},
 			}
-			require.NoError(t, router.Request(&pk, &responder))
-			responder.validate(t, &pk)
+			require.NoError(t, router.Request(ctx, &pk, &responder))
+			validateEchoFlow(t, <-muxer.cfdToEdge, &pk)
 		}
 	}
 	cancel()
 	<-proxyDone
 }
 
+func TestTraceICMPRouterEcho(t *testing.T) {
+	tracingCtx := "ec31ad8a01fde11fdcabe2efdce36873:52726f6cabc144f5:0:1"
+
+	router, err := NewICMPRouter(localhostIP, localhostIPv6, "", &noopLogger)
+	require.NoError(t, err)
+
+	proxyDone := make(chan struct{})
+	ctx, cancel := context.WithCancel(context.Background())
+	go func() {
+		router.Serve(ctx)
+		close(proxyDone)
+	}()
+
+	// Buffer 3 packets, request span, reply span and reply
+	muxer := newMockMuxer(3)
+	tracingIdentity, err := tracing.NewIdentity(tracingCtx)
+	require.NoError(t, err)
+	serializedIdentity, err := tracingIdentity.MarshalBinary()
+	require.NoError(t, err)
+
+	responder := packetResponder{
+		datagramMuxer:      muxer,
+		tracedCtx:          tracing.NewTracedContext(ctx, tracingIdentity.String(), &noopLogger),
+		serializedIdentity: serializedIdentity,
+	}
+
+	echo := &icmp.Echo{
+		ID:   12910,
+		Seq:  182,
+		Data: []byte(t.Name()),
+	}
+	pk := packet.ICMP{
+		IP: &packet.IP{
+			Src:      localhostIP,
+			Dst:      localhostIP,
+			Protocol: layers.IPProtocolICMPv4,
+			TTL:      packet.DefaultTTL,
+		},
+		Message: &icmp.Message{
+			Type: ipv4.ICMPTypeEcho,
+			Code: 0,
+			Body: echo,
+		},
+	}
+
+	require.NoError(t, router.Request(ctx, &pk, &responder))
+	firstPK := <-muxer.cfdToEdge
+	var requestSpan *quicpogs.TracingSpanPacket
+	// The order of receiving reply or request span is not deterministic
+	switch firstPK.Type() {
+	case quicpogs.DatagramTypeIP:
+		// reply packet
+		validateEchoFlow(t, firstPK, &pk)
+	case quicpogs.DatagramTypeTracingSpan:
+		// Request span
+		requestSpan = firstPK.(*quicpogs.TracingSpanPacket)
+		require.NotEmpty(t, requestSpan.Spans)
+		require.True(t, bytes.Equal(serializedIdentity, requestSpan.TracingIdentity))
+	default:
+		panic(fmt.Sprintf("received unexpected packet type %d", firstPK.Type()))
+	}
+
+	secondPK := <-muxer.cfdToEdge
+	if requestSpan != nil {
+		// If first packet is request span, second packet should be the reply
+		validateEchoFlow(t, secondPK, &pk)
+	} else {
+		requestSpan = secondPK.(*quicpogs.TracingSpanPacket)
+		require.NotEmpty(t, requestSpan.Spans)
+		require.True(t, bytes.Equal(serializedIdentity, requestSpan.TracingIdentity))
+	}
+
+	// Reply span
+	thirdPacket := <-muxer.cfdToEdge
+	replySpan, ok := thirdPacket.(*quicpogs.TracingSpanPacket)
+	require.True(t, ok)
+	require.NotEmpty(t, replySpan.Spans)
+	require.True(t, bytes.Equal(serializedIdentity, replySpan.TracingIdentity))
+	require.False(t, bytes.Equal(requestSpan.Spans, replySpan.Spans))
+
+	echo.Seq++
+	pk.Body = echo
+	// Only first request for a flow is traced. The edge will not send tracing context for the second request
+	newResponder := packetResponder{
+		datagramMuxer: muxer,
+	}
+	require.NoError(t, router.Request(ctx, &pk, &newResponder))
+	validateEchoFlow(t, <-muxer.cfdToEdge, &pk)
+
+	select {
+	case receivedPacket := <-muxer.cfdToEdge:
+		panic(fmt.Sprintf("Receive unexpected packet %+v", receivedPacket))
+	default:
+	}
+
+	cancel()
+	<-proxyDone
+}
+
 // TestConcurrentRequests makes sure icmpRouter can send concurrent requests to the same destination with different
 // echo ID. This simulates concurrent ping to the same destination.
 func TestConcurrentRequestsToSameDst(t *testing.T) {
@@ -123,9 +225,10 @@ func TestConcurrentRequestsToSameDst(t *testing.T) {
 		echoID := 38451 + i
 		go func() {
 			defer wg.Done()
-			responder := echoFlowResponder{
-				decoder:  packet.NewICMPDecoder(),
-				respChan: make(chan []byte, 1),
+
+			muxer := newMockMuxer(1)
+			responder := packetResponder{
+				datagramMuxer: muxer,
 			}
 			for seq := 0; seq < endSeq; seq++ {
 				pk := &packet.ICMP{
@@ -145,15 +248,15 @@ func TestConcurrentRequestsToSameDst(t *testing.T) {
 						},
 					},
 				}
-				require.NoError(t, router.Request(pk, &responder))
-				responder.validate(t, pk)
+				require.NoError(t, router.Request(ctx, pk, &responder))
+				validateEchoFlow(t, <-muxer.cfdToEdge, pk)
 			}
 		}()
 		go func() {
 			defer wg.Done()
-			responder := echoFlowResponder{
-				decoder:  packet.NewICMPDecoder(),
-				respChan: make(chan []byte, 1),
+			muxer := newMockMuxer(1)
+			responder := packetResponder{
+				datagramMuxer: muxer,
 			}
 			for seq := 0; seq < endSeq; seq++ {
 				pk := &packet.ICMP{
@@ -173,8 +276,8 @@ func TestConcurrentRequestsToSameDst(t *testing.T) {
 						},
 					},
 				}
-				require.NoError(t, router.Request(pk, &responder))
-				responder.validate(t, pk)
+				require.NoError(t, router.Request(ctx, pk, &responder))
+				validateEchoFlow(t, <-muxer.cfdToEdge, pk)
 			}
 		}()
 	}
@@ -241,9 +344,9 @@ func testICMPRouterRejectNotEcho(t *testing.T, srcDstIP netip.Addr, msgs []icmp.
 	router, err := NewICMPRouter(localhostIP, localhostIPv6, "", &noopLogger)
 	require.NoError(t, err)
 
-	responder := echoFlowResponder{
-		decoder:  packet.NewICMPDecoder(),
-		respChan: make(chan []byte),
+	muxer := newMockMuxer(1)
+	responder := packetResponder{
+		datagramMuxer: muxer,
 	}
 	protocol := layers.IPProtocolICMPv4
 	if srcDstIP.Is6() {
@@ -259,36 +362,14 @@ func testICMPRouterRejectNotEcho(t *testing.T, srcDstIP netip.Addr, msgs []icmp.
 			},
 			Message: &m,
 		}
-		require.Error(t, router.Request(&pk, &responder))
+		require.Error(t, router.Request(context.Background(), &pk, &responder))
 	}
 }
 
-type echoFlowResponder struct {
-	lock     sync.Mutex
-	decoder  *packet.ICMPDecoder
-	respChan chan []byte
-}
-
-func (efr *echoFlowResponder) SendPacket(dst netip.Addr, pk packet.RawPacket) error {
-	efr.lock.Lock()
-	defer efr.lock.Unlock()
-	copiedPacket := make([]byte, len(pk.Data))
-	copy(copiedPacket, pk.Data)
-	efr.respChan <- copiedPacket
-	return nil
-}
-
-func (efr *echoFlowResponder) Close() error {
-	efr.lock.Lock()
-	defer efr.lock.Unlock()
-	close(efr.respChan)
-	return nil
-}
-
-func (efr *echoFlowResponder) validate(t *testing.T, echoReq *packet.ICMP) {
-	pk := <-efr.respChan
-	decoded, err := efr.decoder.Decode(packet.RawPacket{Data: pk})
-	require.NoError(t, err)
+func validateEchoFlow(t *testing.T, pk quicpogs.Packet, echoReq *packet.ICMP) {
+	decoder := packet.NewICMPDecoder()
+	decoded, err := decoder.Decode(packet.RawPacket{Data: pk.Payload()})
+	require.NoError(t, err, pk)
 	require.Equal(t, decoded.Src, echoReq.Dst)
 	require.Equal(t, decoded.Dst, echoReq.Src)
 	require.Equal(t, echoReq.Protocol, decoded.Protocol)

ingress/origin_service.go

@@ -9,6 +9,7 @@ import (
 	"net"
 	"net/http"
 	"net/url"
+	"strconv"
 	"time"
 
 	"github.com/pkg/errors"
@@ -155,8 +156,10 @@ func newSocksProxyOverWSService(accessPolicy *ipaccess.Policy) *socksProxyOverWS
 }
 
 func addPortIfMissing(uri *url.URL, port int) {
+	hostname := uri.Hostname()
+
 	if uri.Port() == "" {
-		uri.Host = fmt.Sprintf("%s:%d", uri.Hostname(), port)
+		uri.Host = net.JoinHostPort(hostname, strconv.FormatInt(int64(port), 10))
 	}
 }
 

ingress/origin_service_test.go

@@ -0,0 +1,29 @@
+package ingress
+
+import (
+	"net/url"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestAddPortIfMissing(t *testing.T) {
+	testCases := []struct {
+		input    string
+		expected string
+	}{
+		{"ssh://[::1]", "[::1]:22"},
+		{"ssh://[::1]:38", "[::1]:38"},
+		{"ssh://abc:38", "abc:38"},
+		{"ssh://127.0.0.1:38", "127.0.0.1:38"},
+		{"ssh://127.0.0.1", "127.0.0.1:22"},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.input, func(t *testing.T) {
+			url1, _ := url.Parse(tc.input)
+			addPortIfMissing(url1, 22)
+			require.Equal(t, tc.expected, url1.Host)
+		})
+	}
+}

ingress/packet_router.go

@@ -0,0 +1,184 @@
+package ingress
+
+import (
+	"context"
+	"fmt"
+	"net/netip"
+
+	"github.com/rs/zerolog"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
+
+	"github.com/cloudflare/cloudflared/packet"
+	quicpogs "github.com/cloudflare/cloudflared/quic"
+	"github.com/cloudflare/cloudflared/tracing"
+)
+
+// Upstream of raw packets
+type muxer interface {
+	SendPacket(pk quicpogs.Packet) error
+	// ReceivePacket waits for the next raw packet from upstream
+	ReceivePacket(ctx context.Context) (quicpogs.Packet, error)
+}
+
+// PacketRouter routes packets between Upstream and ICMPRouter. Currently it rejects all other type of ICMP packets
+type PacketRouter struct {
+	globalConfig           *GlobalRouterConfig
+	muxer                  muxer
+	logger                 *zerolog.Logger
+	checkRouterEnabledFunc func() bool
+	icmpDecoder            *packet.ICMPDecoder
+	encoder                *packet.Encoder
+}
+
+// GlobalRouterConfig is the configuration shared by all instance of Router.
+type GlobalRouterConfig struct {
+	ICMPRouter *icmpRouter
+	IPv4Src    netip.Addr
+	IPv6Src    netip.Addr
+	Zone       string
+}
+
+// NewPacketRouter creates a PacketRouter that handles ICMP packets. Packets are read from muxer but dropped if globalConfig is nil.
+func NewPacketRouter(globalConfig *GlobalRouterConfig, muxer muxer, logger *zerolog.Logger, checkRouterEnabledFunc func() bool) *PacketRouter {
+	return &PacketRouter{
+		globalConfig:           globalConfig,
+		muxer:                  muxer,
+		logger:                 logger,
+		checkRouterEnabledFunc: checkRouterEnabledFunc,
+		icmpDecoder:            packet.NewICMPDecoder(),
+		encoder:                packet.NewEncoder(),
+	}
+}
+
+func (r *PacketRouter) Serve(ctx context.Context) error {
+	for {
+		rawPacket, responder, err := r.nextPacket(ctx)
+		if err != nil {
+			return err
+		}
+		r.handlePacket(ctx, rawPacket, responder)
+	}
+}
+
+func (r *PacketRouter) nextPacket(ctx context.Context) (packet.RawPacket, *packetResponder, error) {
+	pk, err := r.muxer.ReceivePacket(ctx)
+	if err != nil {
+		return packet.RawPacket{}, nil, err
+	}
+	responder := &packetResponder{
+		datagramMuxer: r.muxer,
+	}
+	switch pk.Type() {
+	case quicpogs.DatagramTypeIP:
+		return packet.RawPacket{Data: pk.Payload()}, responder, nil
+	case quicpogs.DatagramTypeIPWithTrace:
+		var identity tracing.Identity
+		if err := identity.UnmarshalBinary(pk.Metadata()); err != nil {
+			r.logger.Err(err).Bytes("tracingIdentity", pk.Metadata()).Msg("Failed to unmarshal tracing identity")
+		} else {
+			responder.tracedCtx = tracing.NewTracedContext(ctx, identity.String(), r.logger)
+			responder.serializedIdentity = pk.Metadata()
+		}
+		return packet.RawPacket{Data: pk.Payload()}, responder, nil
+	default:
+		return packet.RawPacket{}, nil, fmt.Errorf("unexpected datagram type %d", pk.Type())
+	}
+}
+
+func (r *PacketRouter) handlePacket(ctx context.Context, rawPacket packet.RawPacket, responder *packetResponder) {
+	// ICMP Proxy feature is disabled, drop packets
+	if r.globalConfig == nil {
+		return
+	}
+
+	if enabled := r.checkRouterEnabledFunc(); !enabled {
+		return
+	}
+
+	icmpPacket, err := r.icmpDecoder.Decode(rawPacket)
+	if err != nil {
+		r.logger.Err(err).Msg("Failed to decode ICMP packet from quic datagram")
+		return
+	}
+
+	if icmpPacket.TTL <= 1 {
+		if err := r.sendTTLExceedMsg(ctx, icmpPacket, rawPacket, r.encoder); err != nil {
+			r.logger.Err(err).Msg("Failed to return ICMP TTL exceed error")
+		}
+		return
+	}
+	icmpPacket.TTL--
+
+	if err := r.globalConfig.ICMPRouter.Request(ctx, icmpPacket, responder); err != nil {
+		r.logger.Err(err).
+			Str("src", icmpPacket.Src.String()).
+			Str("dst", icmpPacket.Dst.String()).
+			Interface("type", icmpPacket.Type).
+			Msg("Failed to send ICMP packet")
+	}
+}
+
+func (r *PacketRouter) sendTTLExceedMsg(ctx context.Context, pk *packet.ICMP, rawPacket packet.RawPacket, encoder *packet.Encoder) error {
+	var srcIP netip.Addr
+	if pk.Dst.Is4() {
+		srcIP = r.globalConfig.IPv4Src
+	} else {
+		srcIP = r.globalConfig.IPv6Src
+	}
+	ttlExceedPacket := packet.NewICMPTTLExceedPacket(pk.IP, rawPacket, srcIP)
+
+	encodedTTLExceed, err := encoder.Encode(ttlExceedPacket)
+	if err != nil {
+		return err
+	}
+	return r.muxer.SendPacket(quicpogs.RawPacket(encodedTTLExceed))
+}
+
+// packetResponder should not be used concurrently. This assumption is upheld because reply packets are ready one-by-one
+type packetResponder struct {
+	datagramMuxer      muxer
+	tracedCtx          *tracing.TracedContext
+	serializedIdentity []byte
+	// hadReply tracks if there has been any reply for this flow
+	hadReply bool
+}
+
+func (pr *packetResponder) tracingEnabled() bool {
+	return pr.tracedCtx != nil
+}
+
+func (pr *packetResponder) returnPacket(rawPacket packet.RawPacket) error {
+	pr.hadReply = true
+	return pr.datagramMuxer.SendPacket(quicpogs.RawPacket(rawPacket))
+}
+
+func (pr *packetResponder) requestSpan(ctx context.Context, pk *packet.ICMP) (context.Context, trace.Span) {
+	if !pr.tracingEnabled() {
+		return ctx, tracing.NewNoopSpan()
+	}
+	return pr.tracedCtx.Tracer().Start(pr.tracedCtx, "icmp-echo-request", trace.WithAttributes(
+		attribute.String("src", pk.Src.String()),
+		attribute.String("dst", pk.Dst.String()),
+	))
+}
+
+func (pr *packetResponder) replySpan(ctx context.Context, logger *zerolog.Logger) (context.Context, trace.Span) {
+	if !pr.tracingEnabled() || pr.hadReply {
+		return ctx, tracing.NewNoopSpan()
+	}
+	return pr.tracedCtx.Tracer().Start(pr.tracedCtx, "icmp-echo-reply")
+}
+
+func (pr *packetResponder) exportSpan() {
+	if !pr.tracingEnabled() {
+		return
+	}
+	spans := pr.tracedCtx.GetProtoSpans()
+	if len(spans) > 0 {
+		pr.datagramMuxer.SendPacket(&quicpogs.TracingSpanPacket{
+			Spans:           spans,
+			TracingIdentity: pr.serializedIdentity,
+		})
+	}
+}

ingress/packet_router_test.go

@@ -1,4 +1,4 @@
-package packet
+package ingress
 
 import (
 	"bytes"
@@ -10,32 +10,28 @@ import (
 	"time"
 
 	"github.com/google/gopacket/layers"
-	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/require"
 	"golang.org/x/net/icmp"
 	"golang.org/x/net/ipv4"
 	"golang.org/x/net/ipv6"
+
+	"github.com/cloudflare/cloudflared/packet"
+	quicpogs "github.com/cloudflare/cloudflared/quic"
 )
 
 var (
-	noopLogger   = zerolog.Nop()
 	packetConfig = &GlobalRouterConfig{
-		ICMPRouter: &mockICMPRouter{},
+		ICMPRouter: nil,
 		IPv4Src:    netip.MustParseAddr("172.16.0.1"),
 		IPv6Src:    netip.MustParseAddr("fd51:2391:523:f4ee::1"),
 	}
 )
 
 func TestRouterReturnTTLExceed(t *testing.T) {
-	upstream := &mockUpstream{
-		source: make(chan RawPacket),
-	}
-	returnPipe := &mockFunnelUniPipe{
-		uniPipe: make(chan RawPacket),
-	}
+	muxer := newMockMuxer(0)
 	routerEnabled := &routerEnabledChecker{}
 	routerEnabled.set(true)
-	router := NewRouter(packetConfig, upstream, returnPipe, &noopLogger, routerEnabled.isEnabled)
+	router := NewPacketRouter(packetConfig, muxer, &noopLogger, routerEnabled.isEnabled)
 	ctx, cancel := context.WithCancel(context.Background())
 	routerStopped := make(chan struct{})
 	go func() {
@@ -43,8 +39,8 @@ func TestRouterReturnTTLExceed(t *testing.T) {
 		close(routerStopped)
 	}()
 
-	pk := ICMP{
-		IP: &IP{
+	pk := packet.ICMP{
+		IP: &packet.IP{
 			Src:      netip.MustParseAddr("192.168.1.1"),
 			Dst:      netip.MustParseAddr("10.0.0.1"),
 			Protocol: layers.IPProtocolICMPv4,
@@ -60,9 +56,9 @@ func TestRouterReturnTTLExceed(t *testing.T) {
 			},
 		},
 	}
-	assertTTLExceed(t, &pk, router.globalConfig.IPv4Src, upstream, returnPipe)
-	pk = ICMP{
-		IP: &IP{
+	assertTTLExceed(t, &pk, router.globalConfig.IPv4Src, muxer)
+	pk = packet.ICMP{
+		IP: &packet.IP{
 			Src:      netip.MustParseAddr("fd51:2391:523:f4ee::1"),
 			Dst:      netip.MustParseAddr("fd51:2391:697:f4ee::2"),
 			Protocol: layers.IPProtocolICMPv6,
@@ -78,21 +74,16 @@ func TestRouterReturnTTLExceed(t *testing.T) {
 			},
 		},
 	}
-	assertTTLExceed(t, &pk, router.globalConfig.IPv6Src, upstream, returnPipe)
+	assertTTLExceed(t, &pk, router.globalConfig.IPv6Src, muxer)
 
 	cancel()
 	<-routerStopped
 }
 
 func TestRouterCheckEnabled(t *testing.T) {
-	upstream := &mockUpstream{
-		source: make(chan RawPacket),
-	}
-	returnPipe := &mockFunnelUniPipe{
-		uniPipe: make(chan RawPacket),
-	}
+	muxer := newMockMuxer(0)
 	routerEnabled := &routerEnabledChecker{}
-	router := NewRouter(packetConfig, upstream, returnPipe, &noopLogger, routerEnabled.isEnabled)
+	router := NewPacketRouter(packetConfig, muxer, &noopLogger, routerEnabled.isEnabled)
 	ctx, cancel := context.WithCancel(context.Background())
 	routerStopped := make(chan struct{})
 	go func() {
@@ -100,8 +91,8 @@ func TestRouterCheckEnabled(t *testing.T) {
 		close(routerStopped)
 	}()
 
-	pk := ICMP{
-		IP: &IP{
+	pk := packet.ICMP{
+		IP: &packet.IP{
 			Src:      netip.MustParseAddr("192.168.1.1"),
 			Dst:      netip.MustParseAddr("10.0.0.1"),
 			Protocol: layers.IPProtocolICMPv4,
@@ -119,23 +110,28 @@ func TestRouterCheckEnabled(t *testing.T) {
 	}
 
 	// router is disabled
-	require.NoError(t, upstream.send(&pk))
+	encoder := packet.NewEncoder()
+	encodedPacket, err := encoder.Encode(&pk)
+	require.NoError(t, err)
+	sendPacket := quicpogs.RawPacket(encodedPacket)
+
+	muxer.edgeToCfd <- sendPacket
 	select {
 	case <-time.After(time.Millisecond * 10):
-	case <-returnPipe.uniPipe:
+	case <-muxer.cfdToEdge:
 		t.Error("Unexpected reply when router is disabled")
 	}
 	routerEnabled.set(true)
 	// router is enabled, expects reply
-	require.NoError(t, upstream.send(&pk))
-	<-returnPipe.uniPipe
+	muxer.edgeToCfd <- sendPacket
+	<-muxer.cfdToEdge
 
 	routerEnabled.set(false)
 	// router is disabled
-	require.NoError(t, upstream.send(&pk))
+	muxer.edgeToCfd <- sendPacket
 	select {
 	case <-time.After(time.Millisecond * 10):
-	case <-returnPipe.uniPipe:
+	case <-muxer.cfdToEdge:
 		t.Error("Unexpected reply when router is disabled")
 	}
 
@@ -143,66 +139,88 @@ func TestRouterCheckEnabled(t *testing.T) {
 	<-routerStopped
 }
 
-func assertTTLExceed(t *testing.T, originalPacket *ICMP, expectedSrc netip.Addr, upstream *mockUpstream, returnPipe *mockFunnelUniPipe) {
-	encoder := NewEncoder()
+func assertTTLExceed(t *testing.T, originalPacket *packet.ICMP, expectedSrc netip.Addr, muxer *mockMuxer) {
+	encoder := packet.NewEncoder()
 	rawPacket, err := encoder.Encode(originalPacket)
 	require.NoError(t, err)
-	upstream.source <- rawPacket
+	muxer.edgeToCfd <- quicpogs.RawPacket(rawPacket)
 
-	resp := <-returnPipe.uniPipe
-	decoder := NewICMPDecoder()
-	decoded, err := decoder.Decode(resp)
+	resp := <-muxer.cfdToEdge
+	decoder := packet.NewICMPDecoder()
+	decoded, err := decoder.Decode(packet.RawPacket(resp.(quicpogs.RawPacket)))
 	require.NoError(t, err)
 
 	require.Equal(t, expectedSrc, decoded.Src)
 	require.Equal(t, originalPacket.Src, decoded.Dst)
 	require.Equal(t, originalPacket.Protocol, decoded.Protocol)
-	require.Equal(t, DefaultTTL, decoded.TTL)
+	require.Equal(t, packet.DefaultTTL, decoded.TTL)
 	if originalPacket.Dst.Is4() {
 		require.Equal(t, ipv4.ICMPTypeTimeExceeded, decoded.Type)
 	} else {
 		require.Equal(t, ipv6.ICMPTypeTimeExceeded, decoded.Type)
 	}
 	require.Equal(t, 0, decoded.Code)
-	assertICMPChecksum(t, decoded)
 	timeExceed, ok := decoded.Body.(*icmp.TimeExceeded)
 	require.True(t, ok)
 	require.True(t, bytes.Equal(rawPacket.Data, timeExceed.Data))
 }
 
-type mockUpstream struct {
-	source chan RawPacket
+type mockMuxer struct {
+	cfdToEdge chan quicpogs.Packet
+	edgeToCfd chan quicpogs.Packet
 }
 
-func (ms *mockUpstream) send(pk Packet) error {
-	encoder := NewEncoder()
-	rawPacket, err := encoder.Encode(pk)
-	if err != nil {
-		return err
+func newMockMuxer(capacity int) *mockMuxer {
+	return &mockMuxer{
+		cfdToEdge: make(chan quicpogs.Packet, capacity),
+		edgeToCfd: make(chan quicpogs.Packet, capacity),
 	}
-	ms.source <- rawPacket
+}
+
+// Copy packet, because icmpProxy expects the encoder buffer to be reusable after the packet is sent
+func (mm *mockMuxer) SendPacket(pk quicpogs.Packet) error {
+	payload := pk.Payload()
+	copiedPayload := make([]byte, len(payload))
+	copy(copiedPayload, payload)
+
+	metadata := pk.Metadata()
+	copiedMetadata := make([]byte, len(metadata))
+	copy(copiedMetadata, metadata)
+
+	var copiedPacket quicpogs.Packet
+	switch pk.Type() {
+	case quicpogs.DatagramTypeIP:
+		copiedPacket = quicpogs.RawPacket(packet.RawPacket{
+			Data: copiedPayload,
+		})
+	case quicpogs.DatagramTypeIPWithTrace:
+		copiedPacket = &quicpogs.TracedPacket{
+			Packet: packet.RawPacket{
+				Data: copiedPayload,
+			},
+			TracingIdentity: copiedMetadata,
+		}
+	case quicpogs.DatagramTypeTracingSpan:
+		copiedPacket = &quicpogs.TracingSpanPacket{
+			Spans:           copiedPayload,
+			TracingIdentity: copiedMetadata,
+		}
+	default:
+		return fmt.Errorf("unexpected metadata type %d", pk.Type())
+	}
+	mm.cfdToEdge <- copiedPacket
 	return nil
 }
 
-func (ms *mockUpstream) ReceivePacket(ctx context.Context) (RawPacket, error) {
+func (mm *mockMuxer) ReceivePacket(ctx context.Context) (quicpogs.Packet, error) {
 	select {
 	case <-ctx.Done():
-		return RawPacket{}, ctx.Err()
-	case pk := <-ms.source:
+		return nil, ctx.Err()
+	case pk := <-mm.edgeToCfd:
 		return pk, nil
 	}
 }
 
-type mockICMPRouter struct{}
-
-func (mir mockICMPRouter) Serve(ctx context.Context) error {
-	return fmt.Errorf("Serve not implemented by mockICMPRouter")
-}
-
-func (mir mockICMPRouter) Request(pk *ICMP, responder FunnelUniPipe) error {
-	return fmt.Errorf("Request not implemented by mockICMPRouter")
-}
-
 type routerEnabledChecker struct {
 	enabled uint32
 }

packet/funnel.go

@@ -16,11 +16,9 @@ var (
 
 // Funnel is an abstraction to pipe from 1 src to 1 or more destinations
 type Funnel interface {
-	// SendToDst sends a raw packet to a destination
-	SendToDst(dst netip.Addr, pk RawPacket) error
-	// ReturnToSrc returns a raw packet to the source
-	ReturnToSrc(pk RawPacket) error
-	// LastActive returns the last time SendToDst or ReturnToSrc is called
+	// Updates the last time traffic went through this funnel
+	UpdateLastActive()
+	// LastActive returns the last time there is traffic through this funnel
 	LastActive() time.Time
 	// Close closes the funnel. Further call to SendToDst or ReturnToSrc should return an error
 	Close() error
@@ -36,73 +34,26 @@ type FunnelUniPipe interface {
 	Close() error
 }
 
-// RawPacketFunnel is an implementation of Funnel that sends raw packets. It can be embedded in other structs to
-// satisfy the Funnel interface.
-type RawPacketFunnel struct {
-	Src netip.Addr
+type ActivityTracker struct {
 	// last active unix time. Unit is seconds
 	lastActive int64
-	sendPipe   FunnelUniPipe
-	returnPipe FunnelUniPipe
 }
 
-func NewRawPacketFunnel(src netip.Addr, sendPipe, returnPipe FunnelUniPipe) *RawPacketFunnel {
-	return &RawPacketFunnel{
-		Src:        src,
+func NewActivityTracker() *ActivityTracker {
+	return &ActivityTracker{
 		lastActive: time.Now().Unix(),
-		sendPipe:   sendPipe,
-		returnPipe: returnPipe,
 	}
 }
 
-func (rpf *RawPacketFunnel) SendToDst(dst netip.Addr, pk RawPacket) error {
-	rpf.updateLastActive()
-	return rpf.sendPipe.SendPacket(dst, pk)
+func (at *ActivityTracker) UpdateLastActive() {
+	atomic.StoreInt64(&at.lastActive, time.Now().Unix())
 }
 
-func (rpf *RawPacketFunnel) ReturnToSrc(pk RawPacket) error {
-	rpf.updateLastActive()
-	return rpf.returnPipe.SendPacket(rpf.Src, pk)
-}
-
-func (rpf *RawPacketFunnel) updateLastActive() {
-	atomic.StoreInt64(&rpf.lastActive, time.Now().Unix())
-}
-
-func (rpf *RawPacketFunnel) LastActive() time.Time {
-	lastActive := atomic.LoadInt64(&rpf.lastActive)
+func (at *ActivityTracker) LastActive() time.Time {
+	lastActive := atomic.LoadInt64(&at.lastActive)
 	return time.Unix(lastActive, 0)
 }
 
-func (rpf *RawPacketFunnel) Close() error {
-	sendPipeErr := rpf.sendPipe.Close()
-	returnPipeErr := rpf.returnPipe.Close()
-	if sendPipeErr != nil {
-		return sendPipeErr
-	}
-	if returnPipeErr != nil {
-		return returnPipeErr
-	}
-	return nil
-}
-
-func (rpf *RawPacketFunnel) Equal(other Funnel) bool {
-	otherRawFunnel, ok := other.(*RawPacketFunnel)
-	if !ok {
-		return false
-	}
-	if rpf.Src != otherRawFunnel.Src {
-		return false
-	}
-	if rpf.sendPipe != otherRawFunnel.sendPipe {
-		return false
-	}
-	if rpf.returnPipe != otherRawFunnel.returnPipe {
-		return false
-	}
-	return true
-}
-
 // FunnelID represents a key type that can be used by FunnelTracker
 type FunnelID interface {
 	// Type returns the name of the type that implements the FunnelID
@@ -157,13 +108,23 @@ func (ft *FunnelTracker) Get(id FunnelID) (Funnel, bool) {
 	return funnel, ok
 }
 
-// Registers a funnel. It replaces the current funnel.
-func (ft *FunnelTracker) GetOrRegister(id FunnelID, newFunnelFunc func() (Funnel, error)) (funnel Funnel, new bool, err error) {
+// Registers a funnel. If the `id` is already registered and `shouldReplaceFunc` returns true, it closes and replaces
+// the current funnel. If `newFunnelFunc` returns an error, the `id` will remain unregistered, even if it was registered
+// when calling this function.
+func (ft *FunnelTracker) GetOrRegister(
+	id FunnelID,
+	shouldReplaceFunc func(Funnel) bool,
+	newFunnelFunc func() (Funnel, error),
+) (funnel Funnel, new bool, err error) {
 	ft.lock.Lock()
 	defer ft.lock.Unlock()
 	currentFunnel, exists := ft.funnels[id]
 	if exists {
-		return currentFunnel, false, nil
+		if !shouldReplaceFunc(currentFunnel) {
+			return currentFunnel, false, nil
+		}
+		currentFunnel.Close()
+		delete(ft.funnels, id)
 	}
 	newFunnel, err := newFunnelFunc()
 	if err != nil {
@@ -173,7 +134,7 @@ func (ft *FunnelTracker) GetOrRegister(id FunnelID, newFunnelFunc func() (Funnel
 	return newFunnel, true, nil
 }
 
-// Unregisters a funnel if the funnel equals to the current funnel
+// Unregisters and closes a funnel if the funnel equals to the current funnel
 func (ft *FunnelTracker) Unregister(id FunnelID, funnel Funnel) (deleted bool) {
 	ft.lock.Lock()
 	defer ft.lock.Unlock()

packet/funnel_test.go

@@ -1,6 +1,13 @@
 package packet
 
-import "net/netip"
+import (
+	"fmt"
+	"net/netip"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
 
 type mockFunnelUniPipe struct {
 	uniPipe chan RawPacket
@@ -14,3 +21,118 @@ func (mfui *mockFunnelUniPipe) SendPacket(dst netip.Addr, pk RawPacket) error {
 func (mfui *mockFunnelUniPipe) Close() error {
 	return nil
 }
+
+func TestFunnelRegistration(t *testing.T) {
+	id := testFunnelID{"id1"}
+	funnelErr := fmt.Errorf("expected error")
+	newFunnelFuncErr := func() (Funnel, error) { return nil, funnelErr }
+	newFunnelFuncUncalled := func() (Funnel, error) {
+		require.FailNow(t, "a new funnel should not be created")
+		panic("unreached")
+	}
+	funnel1, newFunnelFunc1 := newFunnelAndFunc("funnel1")
+	funnel2, newFunnelFunc2 := newFunnelAndFunc("funnel2")
+
+	ft := NewFunnelTracker()
+	// Register funnel1
+	funnel, new, err := ft.GetOrRegister(id, shouldReplaceFalse, newFunnelFunc1)
+	require.NoError(t, err)
+	require.True(t, new)
+	require.Equal(t, funnel1, funnel)
+	// Register funnel, no replace
+	funnel, new, err = ft.GetOrRegister(id, shouldReplaceFalse, newFunnelFuncUncalled)
+	require.NoError(t, err)
+	require.False(t, new)
+	require.Equal(t, funnel1, funnel)
+	// Register funnel2, replace
+	funnel, new, err = ft.GetOrRegister(id, shouldReplaceTrue, newFunnelFunc2)
+	require.NoError(t, err)
+	require.True(t, new)
+	require.Equal(t, funnel2, funnel)
+	require.True(t, funnel1.closed)
+	// Register funnel error, replace
+	funnel, new, err = ft.GetOrRegister(id, shouldReplaceTrue, newFunnelFuncErr)
+	require.ErrorIs(t, err, funnelErr)
+	require.False(t, new)
+	require.Nil(t, funnel)
+	require.True(t, funnel2.closed)
+}
+
+func TestFunnelUnregister(t *testing.T) {
+	id := testFunnelID{"id1"}
+	funnel1, newFunnelFunc1 := newFunnelAndFunc("funnel1")
+	funnel2, newFunnelFunc2 := newFunnelAndFunc("funnel2")
+	funnel3, newFunnelFunc3 := newFunnelAndFunc("funnel3")
+
+	ft := NewFunnelTracker()
+	// Register & unregister
+	_, _, err := ft.GetOrRegister(id, shouldReplaceFalse, newFunnelFunc1)
+	require.NoError(t, err)
+	require.True(t, ft.Unregister(id, funnel1))
+	require.True(t, funnel1.closed)
+	require.True(t, ft.Unregister(id, funnel1))
+	// Register, replace, and unregister
+	_, _, err = ft.GetOrRegister(id, shouldReplaceFalse, newFunnelFunc2)
+	require.NoError(t, err)
+	_, _, err = ft.GetOrRegister(id, shouldReplaceTrue, newFunnelFunc3)
+	require.NoError(t, err)
+	require.True(t, funnel2.closed)
+	require.False(t, ft.Unregister(id, funnel2))
+	require.True(t, ft.Unregister(id, funnel3))
+	require.True(t, funnel3.closed)
+}
+
+func shouldReplaceFalse(_ Funnel) bool {
+	return false
+}
+
+func shouldReplaceTrue(_ Funnel) bool {
+	return true
+}
+
+func newFunnelAndFunc(id string) (*testFunnel, func() (Funnel, error)) {
+	funnel := newTestFunnel(id)
+	funnelFunc := func() (Funnel, error) {
+		return funnel, nil
+	}
+	return funnel, funnelFunc
+}
+
+type testFunnelID struct {
+	id string
+}
+
+func (t testFunnelID) Type() string {
+	return "testFunnelID"
+}
+
+func (t testFunnelID) String() string {
+	return t.id
+}
+
+type testFunnel struct {
+	id     string
+	closed bool
+}
+
+func newTestFunnel(id string) *testFunnel {
+	return &testFunnel{
+		id,
+		false,
+	}
+}
+
+func (tf *testFunnel) Close() error {
+	tf.closed = true
+	return nil
+}
+
+func (tf *testFunnel) Equal(other Funnel) bool {
+	return tf.id == other.(*testFunnel).id
+}
+
+func (tf *testFunnel) LastActive() time.Time {
+	return time.Now()
+}
+
+func (tf *testFunnel) UpdateLastActive() {}

packet/router.go

@@ -1,108 +0,0 @@
-package packet
-
-import (
-	"context"
-	"net/netip"
-
-	"github.com/rs/zerolog"
-)
-
-// ICMPRouter sends ICMP messages and listens for their responses
-type ICMPRouter interface {
-	// Serve starts listening for responses to the requests until context is done
-	Serve(ctx context.Context) error
-	// Request sends an ICMP message. Implementations should not modify pk after the function returns.
-	Request(pk *ICMP, responder FunnelUniPipe) error
-}
-
-// Upstream of raw packets
-type Upstream interface {
-	// ReceivePacket waits for the next raw packet from upstream
-	ReceivePacket(ctx context.Context) (RawPacket, error)
-}
-
-// Router routes packets between Upstream and ICMPRouter. Currently it rejects all other type of ICMP packets
-type Router struct {
-	upstream               Upstream
-	returnPipe             FunnelUniPipe
-	globalConfig           *GlobalRouterConfig
-	logger                 *zerolog.Logger
-	checkRouterEnabledFunc func() bool
-}
-
-// GlobalRouterConfig is the configuration shared by all instance of Router.
-type GlobalRouterConfig struct {
-	ICMPRouter ICMPRouter
-	IPv4Src    netip.Addr
-	IPv6Src    netip.Addr
-	Zone       string
-}
-
-func NewRouter(globalConfig *GlobalRouterConfig, upstream Upstream, returnPipe FunnelUniPipe, logger *zerolog.Logger, checkRouterEnabledFunc func() bool) *Router {
-	return &Router{
-		upstream:               upstream,
-		returnPipe:             returnPipe,
-		globalConfig:           globalConfig,
-		logger:                 logger,
-		checkRouterEnabledFunc: checkRouterEnabledFunc,
-	}
-}
-
-func (r *Router) Serve(ctx context.Context) error {
-	icmpDecoder := NewICMPDecoder()
-	encoder := NewEncoder()
-	for {
-		rawPacket, err := r.upstream.ReceivePacket(ctx)
-		if err != nil {
-			return err
-		}
-
-		// Drop packets if ICMPRouter wasn't created
-		if r.globalConfig == nil {
-			continue
-		}
-
-		if enabled := r.checkRouterEnabledFunc(); !enabled {
-			continue
-		}
-
-		icmpPacket, err := icmpDecoder.Decode(rawPacket)
-		if err != nil {
-			r.logger.Err(err).Msg("Failed to decode ICMP packet from quic datagram")
-			continue
-		}
-
-		if icmpPacket.TTL <= 1 {
-			if err := r.sendTTLExceedMsg(icmpPacket, rawPacket, encoder); err != nil {
-				r.logger.Err(err).Msg("Failed to return ICMP TTL exceed error")
-			}
-			continue
-		}
-		icmpPacket.TTL--
-
-		if err := r.globalConfig.ICMPRouter.Request(icmpPacket, r.returnPipe); err != nil {
-			r.logger.Err(err).
-				Str("src", icmpPacket.Src.String()).
-				Str("dst", icmpPacket.Dst.String()).
-				Interface("type", icmpPacket.Type).
-				Msg("Failed to send ICMP packet")
-			continue
-		}
-	}
-}
-
-func (r *Router) sendTTLExceedMsg(pk *ICMP, rawPacket RawPacket, encoder *Encoder) error {
-	var srcIP netip.Addr
-	if pk.Dst.Is4() {
-		srcIP = r.globalConfig.IPv4Src
-	} else {
-		srcIP = r.globalConfig.IPv6Src
-	}
-	ttlExceedPacket := NewICMPTTLExceedPacket(pk.IP, rawPacket, srcIP)
-
-	encodedTTLExceed, err := encoder.Encode(ttlExceedPacket)
-	if err != nil {
-		return err
-	}
-	return r.returnPipe.SendPacket(pk.Src, encodedTTLExceed)
-}

proxy/proxy.go

@@ -103,7 +103,8 @@ func (p *Proxy) ProxyHTTP(
 	ruleSpan.End()
 	if err, applied := p.applyIngressMiddleware(rule, req, w); err != nil {
 		if applied {
-			p.log.Error().Msg(err.Error())
+			rule, srv := ruleField(p.ingressRules, ruleNum)
+			p.logRequestError(err, cfRay, "", rule, srv)
 			return nil
 		}
 		return err

quic/datagram.go

@@ -113,9 +113,12 @@ func extractSessionID(b []byte) (uuid.UUID, []byte, error) {
 // SuffixSessionID appends the session ID at the end of the payload. Suffix is more performant than prefix because
 // the payload slice might already have enough capacity to append the session ID at the end
 func SuffixSessionID(sessionID uuid.UUID, b []byte) ([]byte, error) {
-	if len(b)+len(sessionID) > MaxDatagramFrameSize {
+	return suffixMetadata(b, sessionID[:])
+}
+
+func suffixMetadata(payload, metadata []byte) ([]byte, error) {
+	if len(payload)+len(metadata) > MaxDatagramFrameSize {
 		return nil, fmt.Errorf("datagram size exceed %d", MaxDatagramFrameSize)
 	}
-	b = append(b, sessionID[:]...)
-	return b, nil
+	return append(payload, metadata...), nil
 }

quic/datagram_test.go

@@ -1,6 +1,7 @@
 package quic
 
 import (
+	"bytes"
 	"context"
 	"crypto/rand"
 	"crypto/rsa"
@@ -23,6 +24,7 @@ import (
 	"golang.org/x/sync/errgroup"
 
 	"github.com/cloudflare/cloudflared/packet"
+	"github.com/cloudflare/cloudflared/tracing"
 )
 
 var (
@@ -121,6 +123,15 @@ func testDatagram(t *testing.T, version uint8, sessionToPayloads []*packet.Sessi
 
 	logger := zerolog.Nop()
 
+	tracingIdentity, err := tracing.NewIdentity("ec31ad8a01fde11fdcabe2efdce36873:52726f6cabc144f5:0:1")
+	require.NoError(t, err)
+	serializedTracingID, err := tracingIdentity.MarshalBinary()
+	require.NoError(t, err)
+	tracingSpan := &TracingSpanPacket{
+		Spans:           []byte("tracing"),
+		TracingIdentity: serializedTracingID,
+	}
+
 	errGroup, ctx := errgroup.WithContext(context.Background())
 	// Run edge side of datagram muxer
 	errGroup.Go(func() error {
@@ -140,18 +151,17 @@ func testDatagram(t *testing.T, version uint8, sessionToPayloads []*packet.Sessi
 			muxer := NewDatagramMuxerV2(quicSession, &logger, sessionDemuxChan)
 			muxer.ServeReceive(ctx)
 
-			icmpDecoder := packet.NewICMPDecoder()
 			for _, pk := range packets {
 				received, err := muxer.ReceivePacket(ctx)
 				require.NoError(t, err)
-
-				receivedICMP, err := icmpDecoder.Decode(received)
+				validateIPPacket(t, received, &pk)
+				received, err = muxer.ReceivePacket(ctx)
 				require.NoError(t, err)
-				require.Equal(t, pk.IP, receivedICMP.IP)
-				require.Equal(t, pk.Type, receivedICMP.Type)
-				require.Equal(t, pk.Code, receivedICMP.Code)
-				require.Equal(t, pk.Body, receivedICMP.Body)
+				validateIPPacketWithTracing(t, received, &pk, serializedTracingID)
 			}
+			received, err := muxer.ReceivePacket(ctx)
+			require.NoError(t, err)
+			validateTracingSpans(t, received, tracingSpan)
 		default:
 			return fmt.Errorf("unknown datagram version %d", version)
 		}
@@ -188,10 +198,15 @@ func testDatagram(t *testing.T, version uint8, sessionToPayloads []*packet.Sessi
 			for _, pk := range packets {
 				encodedPacket, err := encoder.Encode(&pk)
 				require.NoError(t, err)
-				require.NoError(t, muxerV2.SendPacket(encodedPacket))
+				require.NoError(t, muxerV2.SendPacket(RawPacket(encodedPacket)))
+				require.NoError(t, muxerV2.SendPacket(&TracedPacket{
+					Packet:          encodedPacket,
+					TracingIdentity: serializedTracingID,
+				}))
 			}
+			require.NoError(t, muxerV2.SendPacket(tracingSpan))
 			// Payload larger than transport MTU, should not be sent
-			require.Error(t, muxerV2.SendPacket(packet.RawPacket{
+			require.Error(t, muxerV2.SendPacket(RawPacket{
 				Data: largePayload,
 			}))
 			muxer = muxerV2
@@ -217,6 +232,38 @@ func testDatagram(t *testing.T, version uint8, sessionToPayloads []*packet.Sessi
 	require.NoError(t, errGroup.Wait())
 }
 
+func validateIPPacket(t *testing.T, receivedPacket Packet, expectedICMP *packet.ICMP) {
+	require.Equal(t, DatagramTypeIP, receivedPacket.Type())
+	rawPacket := receivedPacket.(RawPacket)
+	decoder := packet.NewICMPDecoder()
+	receivedICMP, err := decoder.Decode(packet.RawPacket(rawPacket))
+	require.NoError(t, err)
+	validateICMP(t, expectedICMP, receivedICMP)
+}
+
+func validateIPPacketWithTracing(t *testing.T, receivedPacket Packet, expectedICMP *packet.ICMP, serializedTracingID []byte) {
+	require.Equal(t, DatagramTypeIPWithTrace, receivedPacket.Type())
+	tracedPacket := receivedPacket.(*TracedPacket)
+	decoder := packet.NewICMPDecoder()
+	receivedICMP, err := decoder.Decode(tracedPacket.Packet)
+	require.NoError(t, err)
+	validateICMP(t, expectedICMP, receivedICMP)
+	require.True(t, bytes.Equal(tracedPacket.TracingIdentity, serializedTracingID))
+}
+
+func validateICMP(t *testing.T, expected, actual *packet.ICMP) {
+	require.Equal(t, expected.IP, actual.IP)
+	require.Equal(t, expected.Type, actual.Type)
+	require.Equal(t, expected.Code, actual.Code)
+	require.Equal(t, expected.Body, actual.Body)
+}
+
+func validateTracingSpans(t *testing.T, receivedPacket Packet, expectedSpan *TracingSpanPacket) {
+	require.Equal(t, DatagramTypeTracingSpan, receivedPacket.Type())
+	tracingSpans := receivedPacket.(*TracingSpanPacket)
+	require.Equal(t, tracingSpans, expectedSpan)
+}
+
 func newQUICListener(t *testing.T, config *quic.Config) quic.Listener {
 	// Create a simple tls config.
 	tlsConfig := generateTLSConfig()

quic/datagramv2.go

@@ -9,15 +9,28 @@ import (
 	"github.com/rs/zerolog"
 
 	"github.com/cloudflare/cloudflared/packet"
+	"github.com/cloudflare/cloudflared/tracing"
 )
 
 type DatagramV2Type byte
 
 const (
+	// UDP payload
 	DatagramTypeUDP DatagramV2Type = iota
+	// Full IP packet
 	DatagramTypeIP
+	// DatagramTypeIP + tracing ID
+	DatagramTypeIPWithTrace
+	// Tracing spans in protobuf format
+	DatagramTypeTracingSpan
 )
 
+type Packet interface {
+	Type() DatagramV2Type
+	Payload() []byte
+	Metadata() []byte
+}
+
 const (
 	typeIDLen = 1
 	// Same as sessionDemuxChan capacity
@@ -41,7 +54,7 @@ type DatagramMuxerV2 struct {
 	session          quic.Connection
 	logger           *zerolog.Logger
 	sessionDemuxChan chan<- *packet.Session
-	packetDemuxChan  chan packet.RawPacket
+	packetDemuxChan  chan Packet
 }
 
 func NewDatagramMuxerV2(
@@ -54,7 +67,7 @@ func NewDatagramMuxerV2(
 		session:          quicSession,
 		logger:           &logger,
 		sessionDemuxChan: sessionDemuxChan,
-		packetDemuxChan:  make(chan packet.RawPacket, packetChanCapacity),
+		packetDemuxChan:  make(chan Packet, packetChanCapacity),
 	}
 }
 
@@ -79,14 +92,19 @@ func (dm *DatagramMuxerV2) SendToSession(session *packet.Session) error {
 	return nil
 }
 
-// SendPacket suffix the datagram type to the packet. The other end of the QUIC connection can demultiplex by parsing
-// the payload as IP and look at the source and destination.
-func (dm *DatagramMuxerV2) SendPacket(pk packet.RawPacket) error {
-	payloadWithVersion, err := SuffixType(pk.Data, DatagramTypeIP)
+// SendPacket sends a packet with datagram version in the suffix. If ctx is a TracedContext, it adds the tracing
+// context between payload and datagram version.
+// The other end of the QUIC connection can demultiplex by parsing the payload as IP and look at the source and destination.
+func (dm *DatagramMuxerV2) SendPacket(pk Packet) error {
+	payloadWithMetadata, err := suffixMetadata(pk.Payload(), pk.Metadata())
+	if err != nil {
+		return err
+	}
+	payloadWithMetadataAndType, err := SuffixType(payloadWithMetadata, pk.Type())
 	if err != nil {
 		return errors.Wrap(err, "Failed to suffix datagram type, it will be dropped")
 	}
-	if err := dm.session.SendMessage(payloadWithVersion); err != nil {
+	if err := dm.session.SendMessage(payloadWithMetadataAndType); err != nil {
 		return errors.Wrap(err, "Failed to send datagram back to edge")
 	}
 	return nil
@@ -108,10 +126,10 @@ func (dm *DatagramMuxerV2) ServeReceive(ctx context.Context) error {
 	}
 }
 
-func (dm *DatagramMuxerV2) ReceivePacket(ctx context.Context) (packet.RawPacket, error) {
+func (dm *DatagramMuxerV2) ReceivePacket(ctx context.Context) (pk Packet, err error) {
 	select {
 	case <-ctx.Done():
-		return packet.RawPacket{}, ctx.Err()
+		return nil, ctx.Err()
 	case pk := <-dm.packetDemuxChan:
 		return pk, nil
 	}
@@ -126,10 +144,8 @@ func (dm *DatagramMuxerV2) demux(ctx context.Context, msgWithType []byte) error
 	switch msgType {
 	case DatagramTypeUDP:
 		return dm.handleSession(ctx, msg)
-	case DatagramTypeIP:
-		return dm.handlePacket(ctx, msg)
 	default:
-		return fmt.Errorf("Unexpected datagram type %d", msgType)
+		return dm.handlePacket(ctx, msg, msgType)
 	}
 }
 
@@ -150,13 +166,93 @@ func (dm *DatagramMuxerV2) handleSession(ctx context.Context, session []byte) er
 	}
 }
 
-func (dm *DatagramMuxerV2) handlePacket(ctx context.Context, pk []byte) error {
+func (dm *DatagramMuxerV2) handlePacket(ctx context.Context, pk []byte, msgType DatagramV2Type) error {
+	var demuxedPacket Packet
+	switch msgType {
+	case DatagramTypeIP:
+		demuxedPacket = RawPacket(packet.RawPacket{Data: pk})
+	case DatagramTypeIPWithTrace:
+		tracingIdentity, payload, err := extractTracingIdentity(pk)
+		if err != nil {
+			return err
+		}
+		demuxedPacket = &TracedPacket{
+			Packet:          packet.RawPacket{Data: payload},
+			TracingIdentity: tracingIdentity,
+		}
+	case DatagramTypeTracingSpan:
+		tracingIdentity, spans, err := extractTracingIdentity(pk)
+		if err != nil {
+			return err
+		}
+		demuxedPacket = &TracingSpanPacket{
+			Spans:           spans,
+			TracingIdentity: tracingIdentity,
+		}
+	default:
+		return fmt.Errorf("Unexpected datagram type %d", msgType)
+	}
 	select {
 	case <-ctx.Done():
 		return ctx.Err()
-	case dm.packetDemuxChan <- packet.RawPacket{
-		Data: pk,
-	}:
+	case dm.packetDemuxChan <- demuxedPacket:
 		return nil
 	}
 }
+
+func extractTracingIdentity(pk []byte) (tracingIdentity []byte, payload []byte, err error) {
+	if len(pk) < tracing.IdentityLength {
+		return nil, nil, fmt.Errorf("packet with tracing context should have at least %d bytes, got %v", tracing.IdentityLength, pk)
+	}
+	tracingIdentity = pk[len(pk)-tracing.IdentityLength:]
+	payload = pk[:len(pk)-tracing.IdentityLength]
+	return tracingIdentity, payload, nil
+}
+
+type RawPacket packet.RawPacket
+
+func (rw RawPacket) Type() DatagramV2Type {
+	return DatagramTypeIP
+}
+
+func (rw RawPacket) Payload() []byte {
+	return rw.Data
+}
+
+func (rw RawPacket) Metadata() []byte {
+	return []byte{}
+}
+
+type TracedPacket struct {
+	Packet          packet.RawPacket
+	TracingIdentity []byte
+}
+
+func (tp *TracedPacket) Type() DatagramV2Type {
+	return DatagramTypeIPWithTrace
+}
+
+func (tp *TracedPacket) Payload() []byte {
+	return tp.Packet.Data
+}
+
+func (tp *TracedPacket) Metadata() []byte {
+	return tp.TracingIdentity
+}
+
+type TracingSpanPacket struct {
+	Spans           []byte
+	TracingIdentity []byte
+}
+
+func (tsp *TracingSpanPacket) Type() DatagramV2Type {
+	return DatagramTypeTracingSpan
+}
+
+func (tsp *TracingSpanPacket) Payload() []byte {
+	return tsp.Spans
+}
+
+func (tsp *TracingSpanPacket) Metadata() []byte {
+	return tsp.TracingIdentity
+}

supervisor/supervisor.go

@@ -45,7 +45,7 @@ type Supervisor struct {
 	config                  *TunnelConfig
 	orchestrator            *orchestration.Orchestrator
 	edgeIPs                 *edgediscovery.Edge
-	edgeTunnelServer        *EdgeTunnelServer
+	edgeTunnelServer        TunnelServer
 	tunnelErrors            chan tunnelError
 	tunnelsConnecting       map[int]chan struct{}
 	tunnelsProtocolFallback map[int]*protocolFallback
@@ -285,7 +285,8 @@ func (s *Supervisor) initialize(
 	for i := 1; i < s.config.HAConnections; i++ {
 		s.tunnelsProtocolFallback[i] = &protocolFallback{
 			retry.BackoffHandler{MaxRetries: s.config.Retries, RetryForever: true},
-			s.config.ProtocolSelector.Current(),
+			// Set the protocol we know the first tunnel connected with.
+			s.tunnelsProtocolFallback[0].protocol,
 			false,
 		}
 		go s.startTunnel(ctx, i, s.newConnectedTunnelSignal(i))

supervisor/supervisor_test.go

@@ -0,0 +1,74 @@
+package supervisor
+
+import (
+	"context"
+	"testing"
+
+	"github.com/rs/zerolog"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/cloudflare/cloudflared/connection"
+	"github.com/cloudflare/cloudflared/edgediscovery"
+	"github.com/cloudflare/cloudflared/edgediscovery/allregions"
+	"github.com/cloudflare/cloudflared/signal"
+)
+
+type mockProtocolSelector struct {
+	protocols []connection.Protocol
+	index     int
+}
+
+func (m *mockProtocolSelector) Current() connection.Protocol {
+	return m.protocols[m.index]
+}
+
+func (m *mockProtocolSelector) Fallback() (connection.Protocol, bool) {
+	m.index++
+	if m.index == len(m.protocols) {
+		return m.protocols[len(m.protocols)-1], false
+	}
+
+	return m.protocols[m.index], true
+}
+
+type mockEdgeTunnelServer struct {
+	config *TunnelConfig
+}
+
+func (m *mockEdgeTunnelServer) Serve(ctx context.Context, connIndex uint8, protocolFallback *protocolFallback, connectedSignal *signal.Signal) error {
+	// This is to mock the first connection falling back because of connectivity issues.
+	protocolFallback.protocol, _ = m.config.ProtocolSelector.Fallback()
+	connectedSignal.Notify()
+	return nil
+}
+
+// Test to check if initialize sets all the different connections to the same protocol should the first
+// tunnel fall back.
+func Test_Initialize_Same_Protocol(t *testing.T) {
+	edgeIPs, err := edgediscovery.ResolveEdge(&zerolog.Logger{}, "us", allregions.Auto)
+	assert.Nil(t, err)
+	s := Supervisor{
+		edgeIPs: edgeIPs,
+		config: &TunnelConfig{
+			ProtocolSelector: &mockProtocolSelector{protocols: []connection.Protocol{connection.QUIC, connection.HTTP2, connection.H2mux}},
+		},
+		tunnelsProtocolFallback: make(map[int]*protocolFallback),
+		edgeTunnelServer: &mockEdgeTunnelServer{
+			config: &TunnelConfig{
+				ProtocolSelector: &mockProtocolSelector{protocols: []connection.Protocol{connection.QUIC, connection.HTTP2, connection.H2mux}},
+			},
+		},
+	}
+
+	ctx := context.Background()
+	connectedSignal := signal.New(make(chan struct{}))
+	s.initialize(ctx, connectedSignal)
+
+	// Make sure we fell back to http2 as the mock Serve is wont to do.
+	assert.Equal(t, s.tunnelsProtocolFallback[0].protocol, connection.HTTP2)
+
+	// Ensure all the protocols we set to try are the same as what the first tunnel has fallen back to.
+	for _, protocolFallback := range s.tunnelsProtocolFallback {
+		assert.Equal(t, protocolFallback.protocol, s.tunnelsProtocolFallback[0].protocol)
+	}
+}

supervisor/tunnel.go

@@ -20,8 +20,8 @@ import (
 	"github.com/cloudflare/cloudflared/edgediscovery"
 	"github.com/cloudflare/cloudflared/edgediscovery/allregions"
 	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/ingress"
 	"github.com/cloudflare/cloudflared/orchestration"
-	"github.com/cloudflare/cloudflared/packet"
 	quicpogs "github.com/cloudflare/cloudflared/quic"
 	"github.com/cloudflare/cloudflared/retry"
 	"github.com/cloudflare/cloudflared/signal"
@@ -37,6 +37,7 @@ const (
 	FeatureAllowRemoteConfig = "allow_remote_config"
 	FeatureDatagramV2        = "support_datagram_v2"
 	FeaturePostQuantum       = "postquantum"
+	FeatureQUICSupportEOF    = "support_quic_eof"
 )
 
 type TunnelConfig struct {
@@ -70,7 +71,7 @@ type TunnelConfig struct {
 	MuxerConfig      *connection.MuxerConfig
 	ProtocolSelector connection.ProtocolSelector
 	EdgeTLSConfigs   map[connection.Protocol]*tls.Config
-	PacketConfig     *packet.GlobalRouterConfig
+	PacketConfig     *ingress.GlobalRouterConfig
 }
 
 func (c *TunnelConfig) registrationOptions(connectionID uint8, OriginLocalIP string, uuid uuid.UUID) *tunnelpogs.RegistrationOptions {
@@ -205,6 +206,10 @@ type EdgeTunnelServer struct {
 	connAwareLogger *ConnAwareLogger
 }
 
+type TunnelServer interface {
+	Serve(ctx context.Context, connIndex uint8, protocolFallback *protocolFallback, connectedSignal *signal.Signal) error
+}
+
 func (e *EdgeTunnelServer) Serve(ctx context.Context, connIndex uint8, protocolFallback *protocolFallback, connectedSignal *signal.Signal) error {
 	haConnections.Inc()
 	defer haConnections.Dec()
@@ -254,8 +259,9 @@ func (e *EdgeTunnelServer) Serve(ctx context.Context, connIndex uint8, protocolF
 		if !ok {
 			return err
 		}
+
 		e.config.Observer.SendReconnect(connIndex)
-		connLog.Logger().Info().Msgf("Retrying connection in up to %s seconds", duration)
+		connLog.Logger().Info().Msgf("Retrying connection in up to %s", duration)
 	}
 
 	// Check if the connection error was from an IP issue with the host or
@@ -656,6 +662,7 @@ func (e *EdgeTunnelServer) serveQUIC(
 	quicConn, err := connection.NewQUICConnection(
 		quicConfig,
 		edgeAddr,
+		connIndex,
 		tlsConfig,
 		e.orchestrator,
 		connOptions,

token/encrypt.go

@@ -3,26 +3,26 @@
 // tldr is it uses Elliptic Curves (Curve25519) for the keys, XSalsa20 and Poly1305 for encryption.
 // You can read more here https://godoc.org/golang.org/x/crypto/nacl/box.
 //
-//		msg := []byte("super safe message.")
-//		alice, err := NewEncrypter("alice_priv_key.pem", "alice_pub_key.pem")
-//		if err != nil {
-//			log.Fatal(err)
-//		}
+//	msg := []byte("super safe message.")
+//	alice, err := NewEncrypter("alice_priv_key.pem", "alice_pub_key.pem")
+//	if err != nil {
+//		log.Fatal(err)
+//	}
 //
-//		bob, err := NewEncrypter("bob_priv_key.pem", "bob_pub_key.pem")
-//		if err != nil {
-//			log.Fatal(err)
-//		}
-//		encrypted, err := alice.Encrypt(msg, bob.PublicKey())
-//		if err != nil {
-//			log.Fatal(err)
-//		}
+//	bob, err := NewEncrypter("bob_priv_key.pem", "bob_pub_key.pem")
+//	if err != nil {
+//		log.Fatal(err)
+//	}
+//	encrypted, err := alice.Encrypt(msg, bob.PublicKey())
+//	if err != nil {
+//		log.Fatal(err)
+//	}
 //
-//		data, err := bob.Decrypt(encrypted, alice.PublicKey())
-//		if err != nil {
-//			log.Fatal(err)
-//		}
-//		fmt.Println(string(data))
+//	data, err := bob.Decrypt(encrypted, alice.PublicKey())
+//	if err != nil {
+//		log.Fatal(err)
+//	}
+//	fmt.Println(string(data))
 package token
 
 import (

tracing/client.go

@@ -24,9 +24,9 @@ type InMemoryClient interface {
 	// Spans returns a copy of the list of in-memory stored spans as a base64
 	// encoded otlp protobuf string.
 	Spans() (string, error)
-	// ProtoSpans returns a copy of the list of in-memory stored spans as otlp
-	// protobuf byte array.
-	ProtoSpans() ([]byte, error)
+	// ExportProtoSpans returns a copy of the list of in-memory stored spans as otlp
+	// protobuf byte array and clears the in-memory spans.
+	ExportProtoSpans() ([]byte, error)
 }
 
 // InMemoryOtlpClient is a client implementation for otlptrace.Client
@@ -58,7 +58,7 @@ func (mc *InMemoryOtlpClient) UploadTraces(_ context.Context, protoSpans []*trac
 
 // Spans returns the list of in-memory stored spans as a base64 encoded otlp protobuf string.
 func (mc *InMemoryOtlpClient) Spans() (string, error) {
-	data, err := mc.ProtoSpans()
+	data, err := mc.ExportProtoSpans()
 	if err != nil {
 		return "", err
 	}
@@ -66,7 +66,7 @@ func (mc *InMemoryOtlpClient) Spans() (string, error) {
 }
 
 // ProtoSpans returns the list of in-memory stored spans as the protobuf byte array.
-func (mc *InMemoryOtlpClient) ProtoSpans() ([]byte, error) {
+func (mc *InMemoryOtlpClient) ExportProtoSpans() ([]byte, error) {
 	mc.mu.Lock()
 	defer mc.mu.Unlock()
 	if len(mc.spans) <= 0 {
@@ -75,7 +75,12 @@ func (mc *InMemoryOtlpClient) ProtoSpans() ([]byte, error) {
 	pbRequest := &coltracepb.ExportTraceServiceRequest{
 		ResourceSpans: mc.spans,
 	}
-	return proto.Marshal(pbRequest)
+	serializedSpans, err := proto.Marshal(pbRequest)
+	if err != nil {
+		return nil, err
+	}
+	mc.spans = make([]*tracepb.ResourceSpans, 0)
+	return serializedSpans, nil
 }
 
 // NoopOtlpClient is a client implementation for otlptrace.Client that does nothing
@@ -99,6 +104,8 @@ func (mc *NoopOtlpClient) Spans() (string, error) {
 }
 
 // Spans always returns no traces error
-func (mc *NoopOtlpClient) ProtoSpans() ([]byte, error) {
+func (mc *NoopOtlpClient) ExportProtoSpans() ([]byte, error) {
 	return nil, errNoopTracer
 }
+
+func (mc *NoopOtlpClient) ClearSpans() {}

tracing/identity.go

@@ -0,0 +1,110 @@
+package tracing
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"strconv"
+	"strings"
+)
+
+const (
+	// 16 bytes for tracing ID, 8 bytes for span ID and 1 byte for flags
+	IdentityLength = 16 + 8 + 1
+)
+
+type Identity struct {
+	// Based on https://www.jaegertracing.io/docs/1.36/client-libraries/#value
+	// parent span ID is always 0 for our case
+	traceIDUpper uint64
+	traceIDLower uint64
+	spanID       uint64
+	flags        uint8
+}
+
+// TODO: TUN-6604 Remove this. To reconstruct into Jaeger propagation format, convert tracingContext to tracing.Identity
+func (tc *Identity) String() string {
+	return fmt.Sprintf("%016x%016x:%x:0:%x", tc.traceIDUpper, tc.traceIDLower, tc.spanID, tc.flags)
+}
+
+func (tc *Identity) MarshalBinary() ([]byte, error) {
+	buf := bytes.NewBuffer(make([]byte, 0, IdentityLength))
+	for _, field := range []interface{}{
+		tc.traceIDUpper,
+		tc.traceIDLower,
+		tc.spanID,
+		tc.flags,
+	} {
+		if err := binary.Write(buf, binary.BigEndian, field); err != nil {
+			return nil, err
+		}
+	}
+	return buf.Bytes(), nil
+}
+
+func (tc *Identity) UnmarshalBinary(data []byte) error {
+	if len(data) < IdentityLength {
+		return fmt.Errorf("expect tracingContext to have at least %d bytes, got %d", IdentityLength, len(data))
+	}
+
+	buf := bytes.NewBuffer(data)
+	for _, field := range []interface{}{
+		&tc.traceIDUpper,
+		&tc.traceIDLower,
+		&tc.spanID,
+		&tc.flags,
+	} {
+		if err := binary.Read(buf, binary.BigEndian, field); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func NewIdentity(trace string) (*Identity, error) {
+	parts := strings.Split(trace, separator)
+	if len(parts) != 4 {
+		return nil, fmt.Errorf("trace '%s' doesn't have exactly 4 parts separated by %s", trace, separator)
+	}
+	const base = 16
+	tracingID, err := padTracingID(parts[0])
+	if err != nil {
+		return nil, err
+	}
+	traceIDUpper, err := strconv.ParseUint(tracingID[:16], base, 64)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse first 16 bytes of tracing ID as uint64, err: %w", err)
+	}
+	traceIDLower, err := strconv.ParseUint(tracingID[16:], base, 64)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse last 16 bytes of tracing ID as uint64, err: %w", err)
+	}
+	spanID, err := strconv.ParseUint(parts[1], base, 64)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse span ID as uint64, err: %w", err)
+	}
+	flags, err := strconv.ParseUint(parts[3], base, 8)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse flag as uint8, err: %w", err)
+	}
+	return &Identity{
+		traceIDUpper: traceIDUpper,
+		traceIDLower: traceIDLower,
+		spanID:       spanID,
+		flags:        uint8(flags),
+	}, nil
+}
+
+func padTracingID(tracingID string) (string, error) {
+	if len(tracingID) == 0 {
+		return "", fmt.Errorf("missing tracing ID")
+	}
+	if len(tracingID) == traceID128bitsWidth {
+		return tracingID, nil
+	}
+	// Correctly left pad the trace to a length of 32
+	left := traceID128bitsWidth - len(tracingID)
+	paddedTracingID := strings.Repeat("0", left) + tracingID
+	return paddedTracingID, nil
+}

tracing/identity_test.go

@@ -0,0 +1,67 @@
+package tracing
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestNewIdentity(t *testing.T) {
+	testCases := []struct {
+		testCase string
+		trace    string
+		expected string
+	}{
+		{
+			testCase: "full length trace",
+			trace:    "ec31ad8a01fde11fdcabe2efdce36873:52726f6cabc144f5:0:1",
+			expected: "ec31ad8a01fde11fdcabe2efdce36873:52726f6cabc144f5:0:1",
+		},
+		{
+			testCase: "short trace ID",
+			trace:    "ad8a01fde11fdcabe2efdce36873:52726f6cabc144f5:0:1",
+			expected: "0000ad8a01fde11fdcabe2efdce36873:52726f6cabc144f5:0:1",
+		},
+		{
+			testCase: "short trace ID with 0s in the middle",
+			trace:    "ad8a01fde11f000002efdce36873:52726f6cabc144f5:0:1",
+			expected: "0000ad8a01fde11f000002efdce36873:52726f6cabc144f5:0:1",
+		},
+		{
+			testCase: "short trace ID with 0s in the beginning and middle",
+			trace:    "001ad8a01fde11fdcabe2efdce36873:52726f6cabc144f5:0:1",
+			expected: "0001ad8a01fde11fdcabe2efdce36873:52726f6cabc144f5:0:1",
+		},
+		{
+			testCase: "no trace",
+			trace:    "",
+		},
+		{
+			testCase: "missing flags",
+			trace:    "ec31ad8a01fde11fdcabe2efdce36873:52726f6cabc144f5:0",
+		},
+		{
+			testCase: "missing separator",
+			trace:    "ec31ad8a01fde11fdcabe2efdce3687352726f6cabc144f501",
+		},
+	}
+
+	for _, testCase := range testCases {
+		identity, err := NewIdentity(testCase.trace)
+		if testCase.expected != "" {
+			require.NoError(t, err)
+			require.Equal(t, testCase.expected, identity.String())
+
+			serializedIdentity, err := identity.MarshalBinary()
+			require.NoError(t, err)
+			deserializedIdentity := new(Identity)
+			err = deserializedIdentity.UnmarshalBinary(serializedIdentity)
+			require.NoError(t, err)
+			require.Equal(t, identity, deserializedIdentity)
+
+		} else {
+			require.Error(t, err)
+			require.Nil(t, identity)
+		}
+	}
+}

tracing/tracing.go

@@ -157,7 +157,7 @@ func (cft *cfdTracer) GetSpans() (enc string) {
 
 // GetProtoSpans returns the spans as the otlp traces in protobuf byte array.
 func (cft *cfdTracer) GetProtoSpans() (proto []byte) {
-	proto, err := cft.exporter.ProtoSpans()
+	proto, err := cft.exporter.ExportProtoSpans()
 	switch err {
 	case nil:
 		break
@@ -246,7 +246,6 @@ func extractTraceFromString(ctx context.Context, trace string) (context.Context,
 		parts[0] = strings.Repeat("0", left) + parts[0]
 		trace = strings.Join(parts, separator)
 	}
-
 	// Override the 'cf-trace-id' as 'uber-trace-id' so the jaeger propagator can extract it.
 	traceHeader := map[string]string{TracerContextNameOverride: trace}
 	remoteCtx := otel.GetTextMapPropagator().Extract(ctx, propagation.MapCarrier(traceHeader))
@@ -277,6 +276,11 @@ func extractTrace(req *http.Request) (context.Context, bool) {
 	if traceHeader[TracerContextNameOverride] == "" {
 		return nil, false
 	}
+
 	remoteCtx := otel.GetTextMapPropagator().Extract(req.Context(), propagation.MapCarrier(traceHeader))
 	return remoteCtx, true
 }
+
+func NewNoopSpan() trace.Span {
+	return trace.SpanFromContext(nil)
+}

tunnelrpc/pogs/auth_outcome.go

@@ -40,7 +40,6 @@ func (ar AuthenticateResponse) Outcome() AuthOutcome {
 }
 
 // AuthOutcome is a programmer-friendly sum type denoting the possible outcomes of Authenticate.
-//go-sumtype:decl AuthOutcome
 type AuthOutcome interface {
 	isAuthOutcome()
 	// Serialize into an AuthenticateResponse which can be sent via Capnp

validation/validation.go

@@ -61,9 +61,13 @@ func ValidateHostname(hostname string) (string, error) {
 
 // ValidateUrl returns a validated version of `originUrl` with a scheme prepended (by default http://).
 // Note: when originUrl contains a scheme, the path is removed:
-//   ValidateUrl("https://localhost:8080/api/") => "https://localhost:8080"
+//
+//	ValidateUrl("https://localhost:8080/api/") => "https://localhost:8080"
+//
 // but when it does not, the path is preserved:
-//   ValidateUrl("localhost:8080/api/") => "http://localhost:8080/api/"
+//
+//	ValidateUrl("localhost:8080/api/") => "http://localhost:8080/api/"
+//
 // This is arguably a bug, but changing it might break some cloudflared users.
 func ValidateUrl(originUrl string) (*url.URL, error) {
 	urlStr, err := validateUrlString(originUrl)

vendor/github.com/marten-seemann/qtls-go1-18/handshake_server.go

@@ -270,7 +270,7 @@ func (hs *serverHandshakeState) processClientHello() error {
 
 	hs.ecdheOk = supportsECDHE(c.config, hs.clientHello.supportedCurves, hs.clientHello.supportedPoints)
 
-	if hs.ecdheOk {
+	if hs.ecdheOk && len(hs.clientHello.supportedPoints) > 0 {
 		// Although omitting the ec_point_formats extension is permitted, some
 		// old OpenSSL version will refuse to handshake if not present.
 		//
@@ -351,6 +351,13 @@ func supportsECDHE(c *config, supportedCurves []CurveID, supportedPoints []uint8
 			break
 		}
 	}
+	// Per RFC 8422, Section 5.1.2, if the Supported Point Formats extension is
+	// missing, uncompressed points are supported. If supportedPoints is empty,
+	// the extension must be missing, as an empty extension body is rejected by
+	// the parser. See https://go.dev/issue/49126.
+	if len(supportedPoints) == 0 {
+		supportsPointFormat = true
+	}
 
 	return supportsCurve && supportsPointFormat
 }

vendor/github.com/marten-seemann/qtls-go1-19/handshake_server.go

@@ -270,7 +270,7 @@ func (hs *serverHandshakeState) processClientHello() error {
 
 	hs.ecdheOk = supportsECDHE(c.config, hs.clientHello.supportedCurves, hs.clientHello.supportedPoints)
 
-	if hs.ecdheOk {
+	if hs.ecdheOk && len(hs.clientHello.supportedPoints) > 0 {
 		// Although omitting the ec_point_formats extension is permitted, some
 		// old OpenSSL version will refuse to handshake if not present.
 		//
@@ -351,6 +351,13 @@ func supportsECDHE(c *config, supportedCurves []CurveID, supportedPoints []uint8
 			break
 		}
 	}
+	// Per RFC 8422, Section 5.1.2, if the Supported Point Formats extension is
+	// missing, uncompressed points are supported. If supportedPoints is empty,
+	// the extension must be missing, as an empty extension body is rejected by
+	// the parser. See https://go.dev/issue/49126.
+	if len(supportedPoints) == 0 {
+		supportsPointFormat = true
+	}
 
 	return supportsCurve && supportsPointFormat
 }

vendor/golang.org/x/crypto/chacha20/chacha_generic.go

@@ -12,7 +12,7 @@ import (
 	"errors"
 	"math/bits"
 
-	"golang.org/x/crypto/internal/subtle"
+	"golang.org/x/crypto/internal/alias"
 )
 
 const (
@@ -189,7 +189,7 @@ func (s *Cipher) XORKeyStream(dst, src []byte) {
 		panic("chacha20: output smaller than input")
 	}
 	dst = dst[:len(src)]
-	if subtle.InexactOverlap(dst, src) {
+	if alias.InexactOverlap(dst, src) {
 		panic("chacha20: invalid buffer overlap")
 	}
 

vendor/golang.org/x/crypto/chacha20poly1305/chacha20poly1305_amd64.go

@@ -10,7 +10,7 @@ package chacha20poly1305
 import (
 	"encoding/binary"
 
-	"golang.org/x/crypto/internal/subtle"
+	"golang.org/x/crypto/internal/alias"
 	"golang.org/x/sys/cpu"
 )
 
@@ -56,7 +56,7 @@ func (c *chacha20poly1305) seal(dst, nonce, plaintext, additionalData []byte) []
 	setupState(&state, &c.key, nonce)
 
 	ret, out := sliceForAppend(dst, len(plaintext)+16)
-	if subtle.InexactOverlap(out, plaintext) {
+	if alias.InexactOverlap(out, plaintext) {
 		panic("chacha20poly1305: invalid buffer overlap")
 	}
 	chacha20Poly1305Seal(out[:], state[:], plaintext, additionalData)
@@ -73,7 +73,7 @@ func (c *chacha20poly1305) open(dst, nonce, ciphertext, additionalData []byte) (
 
 	ciphertext = ciphertext[:len(ciphertext)-16]
 	ret, out := sliceForAppend(dst, len(ciphertext))
-	if subtle.InexactOverlap(out, ciphertext) {
+	if alias.InexactOverlap(out, ciphertext) {
 		panic("chacha20poly1305: invalid buffer overlap")
 	}
 	if !chacha20Poly1305Open(out, state[:], ciphertext, additionalData) {

vendor/golang.org/x/crypto/chacha20poly1305/chacha20poly1305_generic.go

@@ -8,8 +8,8 @@ import (
 	"encoding/binary"
 
 	"golang.org/x/crypto/chacha20"
+	"golang.org/x/crypto/internal/alias"
 	"golang.org/x/crypto/internal/poly1305"
-	"golang.org/x/crypto/internal/subtle"
 )
 
 func writeWithPadding(p *poly1305.MAC, b []byte) {
@@ -30,7 +30,7 @@ func writeUint64(p *poly1305.MAC, n int) {
 func (c *chacha20poly1305) sealGeneric(dst, nonce, plaintext, additionalData []byte) []byte {
 	ret, out := sliceForAppend(dst, len(plaintext)+poly1305.TagSize)
 	ciphertext, tag := out[:len(plaintext)], out[len(plaintext):]
-	if subtle.InexactOverlap(out, plaintext) {
+	if alias.InexactOverlap(out, plaintext) {
 		panic("chacha20poly1305: invalid buffer overlap")
 	}
 
@@ -66,7 +66,7 @@ func (c *chacha20poly1305) openGeneric(dst, nonce, ciphertext, additionalData []
 	writeUint64(p, len(ciphertext))
 
 	ret, out := sliceForAppend(dst, len(ciphertext))
-	if subtle.InexactOverlap(out, ciphertext) {
+	if alias.InexactOverlap(out, ciphertext) {
 		panic("chacha20poly1305: invalid buffer overlap")
 	}
 	if !p.Verify(tag) {

vendor/golang.org/x/crypto/cryptobyte/builder.go

@@ -95,6 +95,11 @@ func (b *Builder) AddUint32(v uint32) {
 	b.add(byte(v>>24), byte(v>>16), byte(v>>8), byte(v))
 }
 
+// AddUint64 appends a big-endian, 64-bit value to the byte string.
+func (b *Builder) AddUint64(v uint64) {
+	b.add(byte(v>>56), byte(v>>48), byte(v>>40), byte(v>>32), byte(v>>24), byte(v>>16), byte(v>>8), byte(v))
+}
+
 // AddBytes appends a sequence of bytes to the byte string.
 func (b *Builder) AddBytes(v []byte) {
 	b.add(v...)

vendor/golang.org/x/crypto/cryptobyte/string.go

@@ -81,6 +81,17 @@ func (s *String) ReadUint32(out *uint32) bool {
 	return true
 }
 
+// ReadUint64 decodes a big-endian, 64-bit value into out and advances over it.
+// It reports whether the read was successful.
+func (s *String) ReadUint64(out *uint64) bool {
+	v := s.read(8)
+	if v == nil {
+		return false
+	}
+	*out = uint64(v[0])<<56 | uint64(v[1])<<48 | uint64(v[2])<<40 | uint64(v[3])<<32 | uint64(v[4])<<24 | uint64(v[5])<<16 | uint64(v[6])<<8 | uint64(v[7])
+	return true
+}
+
 func (s *String) readUnsigned(out *uint32, length int) bool {
 	v := s.read(length)
 	if v == nil {

vendor/golang.org/x/crypto/internal/alias/alias.go

@@ -5,9 +5,8 @@
 //go:build !purego
 // +build !purego
 
-// Package subtle implements functions that are often useful in cryptographic
-// code but require careful thought to use correctly.
-package subtle // import "golang.org/x/crypto/internal/subtle"
+// Package alias implements memory aliasing tests.
+package alias
 
 import "unsafe"
 

vendor/golang.org/x/crypto/internal/alias/alias_purego.go

@@ -5,9 +5,8 @@
 //go:build purego
 // +build purego
 
-// Package subtle implements functions that are often useful in cryptographic
-// code but require careful thought to use correctly.
-package subtle // import "golang.org/x/crypto/internal/subtle"
+// Package alias implements memory aliasing tests.
+package alias
 
 // This is the Google App Engine standard variant based on reflect
 // because the unsafe package and cgo are disallowed.

vendor/golang.org/x/crypto/nacl/secretbox/secretbox.go

@@ -35,8 +35,8 @@ This package is interoperable with NaCl: https://nacl.cr.yp.to/secretbox.html.
 package secretbox // import "golang.org/x/crypto/nacl/secretbox"
 
 import (
+	"golang.org/x/crypto/internal/alias"
 	"golang.org/x/crypto/internal/poly1305"
-	"golang.org/x/crypto/internal/subtle"
 	"golang.org/x/crypto/salsa20/salsa"
 )
 
@@ -88,7 +88,7 @@ func Seal(out, message []byte, nonce *[24]byte, key *[32]byte) []byte {
 	copy(poly1305Key[:], firstBlock[:])
 
 	ret, out := sliceForAppend(out, len(message)+poly1305.TagSize)
-	if subtle.AnyOverlap(out, message) {
+	if alias.AnyOverlap(out, message) {
 		panic("nacl: invalid buffer overlap")
 	}
 
@@ -147,7 +147,7 @@ func Open(out, box []byte, nonce *[24]byte, key *[32]byte) ([]byte, bool) {
 	}
 
 	ret, out := sliceForAppend(out, len(box)-Overhead)
-	if subtle.AnyOverlap(out, box) {
+	if alias.AnyOverlap(out, box) {
 		panic("nacl: invalid buffer overlap")
 	}
 

vendor/golang.org/x/crypto/salsa20/salsa/hsalsa20.go

@@ -5,6 +5,8 @@
 // Package salsa provides low-level access to functions in the Salsa family.
 package salsa // import "golang.org/x/crypto/salsa20/salsa"
 
+import "math/bits"
+
 // Sigma is the Salsa20 constant for 256-bit keys.
 var Sigma = [16]byte{'e', 'x', 'p', 'a', 'n', 'd', ' ', '3', '2', '-', 'b', 'y', 't', 'e', ' ', 'k'}
 
@@ -31,76 +33,76 @@ func HSalsa20(out *[32]byte, in *[16]byte, k *[32]byte, c *[16]byte) {
 
 	for i := 0; i < 20; i += 2 {
 		u := x0 + x12
-		x4 ^= u<<7 | u>>(32-7)
+		x4 ^= bits.RotateLeft32(u, 7)
 		u = x4 + x0
-		x8 ^= u<<9 | u>>(32-9)
+		x8 ^= bits.RotateLeft32(u, 9)
 		u = x8 + x4
-		x12 ^= u<<13 | u>>(32-13)
+		x12 ^= bits.RotateLeft32(u, 13)
 		u = x12 + x8
-		x0 ^= u<<18 | u>>(32-18)
+		x0 ^= bits.RotateLeft32(u, 18)
 
 		u = x5 + x1
-		x9 ^= u<<7 | u>>(32-7)
+		x9 ^= bits.RotateLeft32(u, 7)
 		u = x9 + x5
-		x13 ^= u<<9 | u>>(32-9)
+		x13 ^= bits.RotateLeft32(u, 9)
 		u = x13 + x9
-		x1 ^= u<<13 | u>>(32-13)
+		x1 ^= bits.RotateLeft32(u, 13)
 		u = x1 + x13
-		x5 ^= u<<18 | u>>(32-18)
+		x5 ^= bits.RotateLeft32(u, 18)
 
 		u = x10 + x6
-		x14 ^= u<<7 | u>>(32-7)
+		x14 ^= bits.RotateLeft32(u, 7)
 		u = x14 + x10
-		x2 ^= u<<9 | u>>(32-9)
+		x2 ^= bits.RotateLeft32(u, 9)
 		u = x2 + x14
-		x6 ^= u<<13 | u>>(32-13)
+		x6 ^= bits.RotateLeft32(u, 13)
 		u = x6 + x2
-		x10 ^= u<<18 | u>>(32-18)
+		x10 ^= bits.RotateLeft32(u, 18)
 
 		u = x15 + x11
-		x3 ^= u<<7 | u>>(32-7)
+		x3 ^= bits.RotateLeft32(u, 7)
 		u = x3 + x15
-		x7 ^= u<<9 | u>>(32-9)
+		x7 ^= bits.RotateLeft32(u, 9)
 		u = x7 + x3
-		x11 ^= u<<13 | u>>(32-13)
+		x11 ^= bits.RotateLeft32(u, 13)
 		u = x11 + x7
-		x15 ^= u<<18 | u>>(32-18)
+		x15 ^= bits.RotateLeft32(u, 18)
 
 		u = x0 + x3
-		x1 ^= u<<7 | u>>(32-7)
+		x1 ^= bits.RotateLeft32(u, 7)
 		u = x1 + x0
-		x2 ^= u<<9 | u>>(32-9)
+		x2 ^= bits.RotateLeft32(u, 9)
 		u = x2 + x1
-		x3 ^= u<<13 | u>>(32-13)
+		x3 ^= bits.RotateLeft32(u, 13)
 		u = x3 + x2
-		x0 ^= u<<18 | u>>(32-18)
+		x0 ^= bits.RotateLeft32(u, 18)
 
 		u = x5 + x4
-		x6 ^= u<<7 | u>>(32-7)
+		x6 ^= bits.RotateLeft32(u, 7)
 		u = x6 + x5
-		x7 ^= u<<9 | u>>(32-9)
+		x7 ^= bits.RotateLeft32(u, 9)
 		u = x7 + x6
-		x4 ^= u<<13 | u>>(32-13)
+		x4 ^= bits.RotateLeft32(u, 13)
 		u = x4 + x7
-		x5 ^= u<<18 | u>>(32-18)
+		x5 ^= bits.RotateLeft32(u, 18)
 
 		u = x10 + x9
-		x11 ^= u<<7 | u>>(32-7)
+		x11 ^= bits.RotateLeft32(u, 7)
 		u = x11 + x10
-		x8 ^= u<<9 | u>>(32-9)
+		x8 ^= bits.RotateLeft32(u, 9)
 		u = x8 + x11
-		x9 ^= u<<13 | u>>(32-13)
+		x9 ^= bits.RotateLeft32(u, 13)
 		u = x9 + x8
-		x10 ^= u<<18 | u>>(32-18)
+		x10 ^= bits.RotateLeft32(u, 18)
 
 		u = x15 + x14
-		x12 ^= u<<7 | u>>(32-7)
+		x12 ^= bits.RotateLeft32(u, 7)
 		u = x12 + x15
-		x13 ^= u<<9 | u>>(32-9)
+		x13 ^= bits.RotateLeft32(u, 9)
 		u = x13 + x12
-		x14 ^= u<<13 | u>>(32-13)
+		x14 ^= bits.RotateLeft32(u, 13)
 		u = x14 + x13
-		x15 ^= u<<18 | u>>(32-18)
+		x15 ^= bits.RotateLeft32(u, 18)
 	}
 	out[0] = byte(x0)
 	out[1] = byte(x0 >> 8)

vendor/golang.org/x/crypto/salsa20/salsa/salsa208.go

@@ -4,6 +4,8 @@
 
 package salsa
 
+import "math/bits"
+
 // Core208 applies the Salsa20/8 core function to the 64-byte array in and puts
 // the result into the 64-byte array out. The input and output may be the same array.
 func Core208(out *[64]byte, in *[64]byte) {
@@ -29,76 +31,76 @@ func Core208(out *[64]byte, in *[64]byte) {
 
 	for i := 0; i < 8; i += 2 {
 		u := x0 + x12
-		x4 ^= u<<7 | u>>(32-7)
+		x4 ^= bits.RotateLeft32(u, 7)
 		u = x4 + x0
-		x8 ^= u<<9 | u>>(32-9)
+		x8 ^= bits.RotateLeft32(u, 9)
 		u = x8 + x4
-		x12 ^= u<<13 | u>>(32-13)
+		x12 ^= bits.RotateLeft32(u, 13)
 		u = x12 + x8
-		x0 ^= u<<18 | u>>(32-18)
+		x0 ^= bits.RotateLeft32(u, 18)
 
 		u = x5 + x1
-		x9 ^= u<<7 | u>>(32-7)
+		x9 ^= bits.RotateLeft32(u, 7)
 		u = x9 + x5
-		x13 ^= u<<9 | u>>(32-9)
+		x13 ^= bits.RotateLeft32(u, 9)
 		u = x13 + x9
-		x1 ^= u<<13 | u>>(32-13)
+		x1 ^= bits.RotateLeft32(u, 13)
 		u = x1 + x13
-		x5 ^= u<<18 | u>>(32-18)
+		x5 ^= bits.RotateLeft32(u, 18)
 
 		u = x10 + x6
-		x14 ^= u<<7 | u>>(32-7)
+		x14 ^= bits.RotateLeft32(u, 7)
 		u = x14 + x10
-		x2 ^= u<<9 | u>>(32-9)
+		x2 ^= bits.RotateLeft32(u, 9)
 		u = x2 + x14
-		x6 ^= u<<13 | u>>(32-13)
+		x6 ^= bits.RotateLeft32(u, 13)
 		u = x6 + x2
-		x10 ^= u<<18 | u>>(32-18)
+		x10 ^= bits.RotateLeft32(u, 18)
 
 		u = x15 + x11
-		x3 ^= u<<7 | u>>(32-7)
+		x3 ^= bits.RotateLeft32(u, 7)
 		u = x3 + x15
-		x7 ^= u<<9 | u>>(32-9)
+		x7 ^= bits.RotateLeft32(u, 9)
 		u = x7 + x3
-		x11 ^= u<<13 | u>>(32-13)
+		x11 ^= bits.RotateLeft32(u, 13)
 		u = x11 + x7
-		x15 ^= u<<18 | u>>(32-18)
+		x15 ^= bits.RotateLeft32(u, 18)
 
 		u = x0 + x3
-		x1 ^= u<<7 | u>>(32-7)
+		x1 ^= bits.RotateLeft32(u, 7)
 		u = x1 + x0
-		x2 ^= u<<9 | u>>(32-9)
+		x2 ^= bits.RotateLeft32(u, 9)
 		u = x2 + x1
-		x3 ^= u<<13 | u>>(32-13)
+		x3 ^= bits.RotateLeft32(u, 13)
 		u = x3 + x2
-		x0 ^= u<<18 | u>>(32-18)
+		x0 ^= bits.RotateLeft32(u, 18)
 
 		u = x5 + x4
-		x6 ^= u<<7 | u>>(32-7)
+		x6 ^= bits.RotateLeft32(u, 7)
 		u = x6 + x5
-		x7 ^= u<<9 | u>>(32-9)
+		x7 ^= bits.RotateLeft32(u, 9)
 		u = x7 + x6
-		x4 ^= u<<13 | u>>(32-13)
+		x4 ^= bits.RotateLeft32(u, 13)
 		u = x4 + x7
-		x5 ^= u<<18 | u>>(32-18)
+		x5 ^= bits.RotateLeft32(u, 18)
 
 		u = x10 + x9
-		x11 ^= u<<7 | u>>(32-7)
+		x11 ^= bits.RotateLeft32(u, 7)
 		u = x11 + x10
-		x8 ^= u<<9 | u>>(32-9)
+		x8 ^= bits.RotateLeft32(u, 9)
 		u = x8 + x11
-		x9 ^= u<<13 | u>>(32-13)
+		x9 ^= bits.RotateLeft32(u, 13)
 		u = x9 + x8
-		x10 ^= u<<18 | u>>(32-18)
+		x10 ^= bits.RotateLeft32(u, 18)
 
 		u = x15 + x14
-		x12 ^= u<<7 | u>>(32-7)
+		x12 ^= bits.RotateLeft32(u, 7)
 		u = x12 + x15
-		x13 ^= u<<9 | u>>(32-9)
+		x13 ^= bits.RotateLeft32(u, 9)
 		u = x13 + x12
-		x14 ^= u<<13 | u>>(32-13)
+		x14 ^= bits.RotateLeft32(u, 13)
 		u = x14 + x13
-		x15 ^= u<<18 | u>>(32-18)
+		x15 ^= bits.RotateLeft32(u, 18)
 	}
 	x0 += j0
 	x1 += j1

vendor/golang.org/x/crypto/salsa20/salsa/salsa20_ref.go

@@ -4,6 +4,8 @@
 
 package salsa
 
+import "math/bits"
+
 const rounds = 20
 
 // core applies the Salsa20 core function to 16-byte input in, 32-byte key k,
@@ -31,76 +33,76 @@ func core(out *[64]byte, in *[16]byte, k *[32]byte, c *[16]byte) {
 
 	for i := 0; i < rounds; i += 2 {
 		u := x0 + x12
-		x4 ^= u<<7 | u>>(32-7)
+		x4 ^= bits.RotateLeft32(u, 7)
 		u = x4 + x0
-		x8 ^= u<<9 | u>>(32-9)
+		x8 ^= bits.RotateLeft32(u, 9)
 		u = x8 + x4
-		x12 ^= u<<13 | u>>(32-13)
+		x12 ^= bits.RotateLeft32(u, 13)
 		u = x12 + x8
-		x0 ^= u<<18 | u>>(32-18)
+		x0 ^= bits.RotateLeft32(u, 18)
 
 		u = x5 + x1
-		x9 ^= u<<7 | u>>(32-7)
+		x9 ^= bits.RotateLeft32(u, 7)
 		u = x9 + x5
-		x13 ^= u<<9 | u>>(32-9)
+		x13 ^= bits.RotateLeft32(u, 9)
 		u = x13 + x9
-		x1 ^= u<<13 | u>>(32-13)
+		x1 ^= bits.RotateLeft32(u, 13)
 		u = x1 + x13
-		x5 ^= u<<18 | u>>(32-18)
+		x5 ^= bits.RotateLeft32(u, 18)
 
 		u = x10 + x6
-		x14 ^= u<<7 | u>>(32-7)
+		x14 ^= bits.RotateLeft32(u, 7)
 		u = x14 + x10
-		x2 ^= u<<9 | u>>(32-9)
+		x2 ^= bits.RotateLeft32(u, 9)
 		u = x2 + x14
-		x6 ^= u<<13 | u>>(32-13)
+		x6 ^= bits.RotateLeft32(u, 13)
 		u = x6 + x2
-		x10 ^= u<<18 | u>>(32-18)
+		x10 ^= bits.RotateLeft32(u, 18)
 
 		u = x15 + x11
-		x3 ^= u<<7 | u>>(32-7)
+		x3 ^= bits.RotateLeft32(u, 7)
 		u = x3 + x15
-		x7 ^= u<<9 | u>>(32-9)
+		x7 ^= bits.RotateLeft32(u, 9)
 		u = x7 + x3
-		x11 ^= u<<13 | u>>(32-13)
+		x11 ^= bits.RotateLeft32(u, 13)
 		u = x11 + x7
-		x15 ^= u<<18 | u>>(32-18)
+		x15 ^= bits.RotateLeft32(u, 18)
 
 		u = x0 + x3
-		x1 ^= u<<7 | u>>(32-7)
+		x1 ^= bits.RotateLeft32(u, 7)
 		u = x1 + x0
-		x2 ^= u<<9 | u>>(32-9)
+		x2 ^= bits.RotateLeft32(u, 9)
 		u = x2 + x1
-		x3 ^= u<<13 | u>>(32-13)
+		x3 ^= bits.RotateLeft32(u, 13)
 		u = x3 + x2
-		x0 ^= u<<18 | u>>(32-18)
+		x0 ^= bits.RotateLeft32(u, 18)
 
 		u = x5 + x4
-		x6 ^= u<<7 | u>>(32-7)
+		x6 ^= bits.RotateLeft32(u, 7)
 		u = x6 + x5
-		x7 ^= u<<9 | u>>(32-9)
+		x7 ^= bits.RotateLeft32(u, 9)
 		u = x7 + x6
-		x4 ^= u<<13 | u>>(32-13)
+		x4 ^= bits.RotateLeft32(u, 13)
 		u = x4 + x7
-		x5 ^= u<<18 | u>>(32-18)
+		x5 ^= bits.RotateLeft32(u, 18)
 
 		u = x10 + x9
-		x11 ^= u<<7 | u>>(32-7)
+		x11 ^= bits.RotateLeft32(u, 7)
 		u = x11 + x10
-		x8 ^= u<<9 | u>>(32-9)
+		x8 ^= bits.RotateLeft32(u, 9)
 		u = x8 + x11
-		x9 ^= u<<13 | u>>(32-13)
+		x9 ^= bits.RotateLeft32(u, 13)
 		u = x9 + x8
-		x10 ^= u<<18 | u>>(32-18)
+		x10 ^= bits.RotateLeft32(u, 18)
 
 		u = x15 + x14
-		x12 ^= u<<7 | u>>(32-7)
+		x12 ^= bits.RotateLeft32(u, 7)
 		u = x12 + x15
-		x13 ^= u<<9 | u>>(32-9)
+		x13 ^= bits.RotateLeft32(u, 9)
 		u = x13 + x12
-		x14 ^= u<<13 | u>>(32-13)
+		x14 ^= bits.RotateLeft32(u, 13)
 		u = x14 + x13
-		x15 ^= u<<18 | u>>(32-18)
+		x15 ^= bits.RotateLeft32(u, 18)
 	}
 	x0 += j0
 	x1 += j1

vendor/golang.org/x/crypto/ssh/certs.go

@@ -251,7 +251,7 @@ type algorithmOpenSSHCertSigner struct {
 // private key is held by signer. It returns an error if the public key in cert
 // doesn't match the key used by signer.
 func NewCertSigner(cert *Certificate, signer Signer) (Signer, error) {
-	if bytes.Compare(cert.Key.Marshal(), signer.PublicKey().Marshal()) != 0 {
+	if !bytes.Equal(cert.Key.Marshal(), signer.PublicKey().Marshal()) {
 		return nil, errors.New("ssh: signer and cert have different public key")
 	}
 

vendor/golang.org/x/crypto/ssh/cipher.go

@@ -15,7 +15,6 @@ import (
 	"fmt"
 	"hash"
 	"io"
-	"io/ioutil"
 
 	"golang.org/x/crypto/chacha20"
 	"golang.org/x/crypto/internal/poly1305"
@@ -97,13 +96,13 @@ func streamCipherMode(skip int, createFunc func(key, iv []byte) (cipher.Stream,
 // are not supported and will not be negotiated, even if explicitly requested in
 // ClientConfig.Crypto.Ciphers.
 var cipherModes = map[string]*cipherMode{
-	// Ciphers from RFC4344, which introduced many CTR-based ciphers. Algorithms
+	// Ciphers from RFC 4344, which introduced many CTR-based ciphers. Algorithms
 	// are defined in the order specified in the RFC.
 	"aes128-ctr": {16, aes.BlockSize, streamCipherMode(0, newAESCTR)},
 	"aes192-ctr": {24, aes.BlockSize, streamCipherMode(0, newAESCTR)},
 	"aes256-ctr": {32, aes.BlockSize, streamCipherMode(0, newAESCTR)},
 
-	// Ciphers from RFC4345, which introduces security-improved arcfour ciphers.
+	// Ciphers from RFC 4345, which introduces security-improved arcfour ciphers.
 	// They are defined in the order specified in the RFC.
 	"arcfour128": {16, 0, streamCipherMode(1536, newRC4)},
 	"arcfour256": {32, 0, streamCipherMode(1536, newRC4)},
@@ -111,7 +110,7 @@ var cipherModes = map[string]*cipherMode{
 	// Cipher defined in RFC 4253, which describes SSH Transport Layer Protocol.
 	// Note that this cipher is not safe, as stated in RFC 4253: "Arcfour (and
 	// RC4) has problems with weak keys, and should be used with caution."
-	// RFC4345 introduces improved versions of Arcfour.
+	// RFC 4345 introduces improved versions of Arcfour.
 	"arcfour": {16, 0, streamCipherMode(0, newRC4)},
 
 	// AEAD ciphers
@@ -497,7 +496,7 @@ func (c *cbcCipher) readCipherPacket(seqNum uint32, r io.Reader) ([]byte, error)
 			// data, to make distinguishing between
 			// failing MAC and failing length check more
 			// difficult.
-			io.CopyN(ioutil.Discard, r, int64(c.oracleCamouflage))
+			io.CopyN(io.Discard, r, int64(c.oracleCamouflage))
 		}
 	}
 	return p, err
@@ -642,7 +641,7 @@ const chacha20Poly1305ID = "chacha20-poly1305@openssh.com"
 //
 //	https://tools.ietf.org/html/draft-josefsson-ssh-chacha20-poly1305-openssh-00
 //
-// the methods here also implement padding, which RFC4253 Section 6
+// the methods here also implement padding, which RFC 4253 Section 6
 // also requires of stream ciphers.
 type chacha20Poly1305Cipher struct {
 	lengthKey  [32]byte

vendor/golang.org/x/crypto/ssh/common.go

@@ -149,7 +149,7 @@ type directionAlgorithms struct {
 
 // rekeyBytes returns a rekeying intervals in bytes.
 func (a *directionAlgorithms) rekeyBytes() int64 {
-	// According to RFC4344 block ciphers should rekey after
+	// According to RFC 4344 block ciphers should rekey after
 	// 2^(BLOCKSIZE/4) blocks. For all AES flavors BLOCKSIZE is
 	// 128.
 	switch a.Cipher {
@@ -158,7 +158,7 @@ func (a *directionAlgorithms) rekeyBytes() int64 {
 
 	}
 
-	// For others, stick with RFC4253 recommendation to rekey after 1 Gb of data.
+	// For others, stick with RFC 4253 recommendation to rekey after 1 Gb of data.
 	return 1 << 30
 }
 

vendor/golang.org/x/crypto/ssh/connection.go

@@ -52,7 +52,7 @@ type Conn interface {
 
 	// SendRequest sends a global request, and returns the
 	// reply. If wantReply is true, it returns the response status
-	// and payload. See also RFC4254, section 4.
+	// and payload. See also RFC 4254, section 4.
 	SendRequest(name string, wantReply bool, payload []byte) (bool, []byte, error)
 
 	// OpenChannel tries to open an channel. If the request is

vendor/golang.org/x/crypto/ssh/keys.go

@@ -184,7 +184,7 @@ func ParseKnownHosts(in []byte) (marker string, hosts []string, pubKey PublicKey
 	return "", nil, nil, "", nil, io.EOF
 }
 
-// ParseAuthorizedKeys parses a public key from an authorized_keys
+// ParseAuthorizedKey parses a public key from an authorized_keys
 // file used in OpenSSH according to the sshd(8) manual page.
 func ParseAuthorizedKey(in []byte) (out PublicKey, comment string, options []string, rest []byte, err error) {
 	for len(in) > 0 {

vendor/golang.org/x/crypto/ssh/messages.go

@@ -68,7 +68,7 @@ type kexInitMsg struct {
 
 // See RFC 4253, section 8.
 
-// Diffie-Helman
+// Diffie-Hellman
 const msgKexDHInit = 30
 
 type kexDHInitMsg struct {

vendor/golang.org/x/crypto/ssh/server.go

@@ -68,8 +68,16 @@ type ServerConfig struct {
 
 	// NoClientAuth is true if clients are allowed to connect without
 	// authenticating.
+	// To determine NoClientAuth at runtime, set NoClientAuth to true
+	// and the optional NoClientAuthCallback to a non-nil value.
 	NoClientAuth bool
 
+	// NoClientAuthCallback, if non-nil, is called when a user
+	// attempts to authenticate with auth method "none".
+	// NoClientAuth must also be set to true for this be used, or
+	// this func is unused.
+	NoClientAuthCallback func(ConnMetadata) (*Permissions, error)
+
 	// MaxAuthTries specifies the maximum number of authentication attempts
 	// permitted per connection. If set to a negative number, the number of
 	// attempts are unlimited. If set to zero, the number of attempts are limited
@@ -455,7 +463,11 @@ userAuthLoop:
 		switch userAuthReq.Method {
 		case "none":
 			if config.NoClientAuth {
-				authErr = nil
+				if config.NoClientAuthCallback != nil {
+					perms, authErr = config.NoClientAuthCallback(s)
+				} else {
+					authErr = nil
+				}
 			}
 
 			// allow initial attempt of 'none' without penalty

vendor/golang.org/x/crypto/ssh/session.go

@@ -13,7 +13,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"sync"
 )
 
@@ -124,7 +123,7 @@ type Session struct {
 	// output and error.
 	//
 	// If either is nil, Run connects the corresponding file
-	// descriptor to an instance of ioutil.Discard. There is a
+	// descriptor to an instance of io.Discard. There is a
 	// fixed amount of buffering that is shared for the two streams.
 	// If either blocks it may eventually cause the remote
 	// command to block.
@@ -506,7 +505,7 @@ func (s *Session) stdout() {
 		return
 	}
 	if s.Stdout == nil {
-		s.Stdout = ioutil.Discard
+		s.Stdout = io.Discard
 	}
 	s.copyFuncs = append(s.copyFuncs, func() error {
 		_, err := io.Copy(s.Stdout, s.ch)
@@ -519,7 +518,7 @@ func (s *Session) stderr() {
 		return
 	}
 	if s.Stderr == nil {
-		s.Stderr = ioutil.Discard
+		s.Stderr = io.Discard
 	}
 	s.copyFuncs = append(s.copyFuncs, func() error {
 		_, err := io.Copy(s.Stderr, s.ch.Stderr())

vendor/golang.org/x/mod/internal/lazyregexp/lazyre.go

@@ -0,0 +1,78 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package lazyregexp is a thin wrapper over regexp, allowing the use of global
+// regexp variables without forcing them to be compiled at init.
+package lazyregexp
+
+import (
+	"os"
+	"regexp"
+	"strings"
+	"sync"
+)
+
+// Regexp is a wrapper around regexp.Regexp, where the underlying regexp will be
+// compiled the first time it is needed.
+type Regexp struct {
+	str  string
+	once sync.Once
+	rx   *regexp.Regexp
+}
+
+func (r *Regexp) re() *regexp.Regexp {
+	r.once.Do(r.build)
+	return r.rx
+}
+
+func (r *Regexp) build() {
+	r.rx = regexp.MustCompile(r.str)
+	r.str = ""
+}
+
+func (r *Regexp) FindSubmatch(s []byte) [][]byte {
+	return r.re().FindSubmatch(s)
+}
+
+func (r *Regexp) FindStringSubmatch(s string) []string {
+	return r.re().FindStringSubmatch(s)
+}
+
+func (r *Regexp) FindStringSubmatchIndex(s string) []int {
+	return r.re().FindStringSubmatchIndex(s)
+}
+
+func (r *Regexp) ReplaceAllString(src, repl string) string {
+	return r.re().ReplaceAllString(src, repl)
+}
+
+func (r *Regexp) FindString(s string) string {
+	return r.re().FindString(s)
+}
+
+func (r *Regexp) FindAllString(s string, n int) []string {
+	return r.re().FindAllString(s, n)
+}
+
+func (r *Regexp) MatchString(s string) bool {
+	return r.re().MatchString(s)
+}
+
+func (r *Regexp) SubexpNames() []string {
+	return r.re().SubexpNames()
+}
+
+var inTest = len(os.Args) > 0 && strings.HasSuffix(strings.TrimSuffix(os.Args[0], ".exe"), ".test")
+
+// New creates a new lazy regexp, delaying the compiling work until it is first
+// needed. If the code is being run as part of tests, the regexp compiling will
+// happen immediately.
+func New(str string) *Regexp {
+	lr := &Regexp{str: str}
+	if inTest {
+		// In tests, always compile the regexps early.
+		lr.re()
+	}
+	return lr
+}

vendor/golang.org/x/mod/module/module.go

@@ -15,7 +15,7 @@
 // but additional checking functions, most notably Check, verify that
 // a particular path, version pair is valid.
 //
-// Escaped Paths
+// # Escaped Paths
 //
 // Module paths appear as substrings of file system paths
 // (in the download cache) and of web server URLs in the proxy protocol.
@@ -55,7 +55,7 @@
 // Import paths have never allowed exclamation marks, so there is no
 // need to define how to escape a literal !.
 //
-// Unicode Restrictions
+// # Unicode Restrictions
 //
 // Today, paths are disallowed from using Unicode.
 //
@@ -102,9 +102,9 @@ import (
 	"strings"
 	"unicode"
 	"unicode/utf8"
+	"errors"
 
 	"golang.org/x/mod/semver"
-	errors "golang.org/x/xerrors"
 )
 
 // A Version (for clients, a module.Version) is defined by a module path and version pair.
@@ -192,6 +192,21 @@ func (e *InvalidVersionError) Error() string {
 
 func (e *InvalidVersionError) Unwrap() error { return e.Err }
 
+// An InvalidPathError indicates a module, import, or file path doesn't
+// satisfy all naming constraints. See CheckPath, CheckImportPath,
+// and CheckFilePath for specific restrictions.
+type InvalidPathError struct {
+	Kind string // "module", "import", or "file"
+	Path string
+	Err  error
+}
+
+func (e *InvalidPathError) Error() string {
+	return fmt.Sprintf("malformed %s path %q: %v", e.Kind, e.Path, e.Err)
+}
+
+func (e *InvalidPathError) Unwrap() error { return e.Err }
+
 // Check checks that a given module path, version pair is valid.
 // In addition to the path being a valid module path
 // and the version being a valid semantic version,
@@ -271,12 +286,7 @@ func fileNameOK(r rune) bool {
 		if '0' <= r && r <= '9' || 'A' <= r && r <= 'Z' || 'a' <= r && r <= 'z' {
 			return true
 		}
-		for i := 0; i < len(allowed); i++ {
-			if rune(allowed[i]) == r {
-				return true
-			}
-		}
-		return false
+		return strings.ContainsRune(allowed, r)
 	}
 	// It may be OK to add more ASCII punctuation here, but only carefully.
 	// For example Windows disallows < > \, and macOS disallows :, so we must not allow those.
@@ -296,30 +306,36 @@ func fileNameOK(r rune) bool {
 // this second requirement is replaced by a requirement that the path
 // follow the gopkg.in server's conventions.
 // Third, no path element may begin with a dot.
-func CheckPath(path string) error {
+func CheckPath(path string) (err error) {
+	defer func() {
+		if err != nil {
+			err = &InvalidPathError{Kind: "module", Path: path, Err: err}
+		}
+	}()
+
 	if err := checkPath(path, modulePath); err != nil {
-		return fmt.Errorf("malformed module path %q: %v", path, err)
+		return err
 	}
 	i := strings.Index(path, "/")
 	if i < 0 {
 		i = len(path)
 	}
 	if i == 0 {
-		return fmt.Errorf("malformed module path %q: leading slash", path)
+		return fmt.Errorf("leading slash")
 	}
 	if !strings.Contains(path[:i], ".") {
-		return fmt.Errorf("malformed module path %q: missing dot in first path element", path)
+		return fmt.Errorf("missing dot in first path element")
 	}
 	if path[0] == '-' {
-		return fmt.Errorf("malformed module path %q: leading dash in first path element", path)
+		return fmt.Errorf("leading dash in first path element")
 	}
 	for _, r := range path[:i] {
 		if !firstPathOK(r) {
-			return fmt.Errorf("malformed module path %q: invalid char %q in first path element", path, r)
+			return fmt.Errorf("invalid char %q in first path element", r)
 		}
 	}
 	if _, _, ok := SplitPathVersion(path); !ok {
-		return fmt.Errorf("malformed module path %q: invalid version", path)
+		return fmt.Errorf("invalid version")
 	}
 	return nil
 }
@@ -343,7 +359,7 @@ func CheckPath(path string) error {
 // subtleties of Unicode.
 func CheckImportPath(path string) error {
 	if err := checkPath(path, importPath); err != nil {
-		return fmt.Errorf("malformed import path %q: %v", path, err)
+		return &InvalidPathError{Kind: "import", Path: path, Err: err}
 	}
 	return nil
 }
@@ -358,12 +374,13 @@ const (
 	filePath
 )
 
-// checkPath checks that a general path is valid.
-// It returns an error describing why but not mentioning path.
-// Because these checks apply to both module paths and import paths,
-// the caller is expected to add the "malformed ___ path %q: " prefix.
-// fileName indicates whether the final element of the path is a file name
-// (as opposed to a directory name).
+// checkPath checks that a general path is valid. kind indicates what
+// specific constraints should be applied.
+//
+// checkPath returns an error describing why the path is not valid.
+// Because these checks apply to module, import, and file paths,
+// and because other checks may be applied, the caller is expected to wrap
+// this error with InvalidPathError.
 func checkPath(path string, kind pathKind) error {
 	if !utf8.ValidString(path) {
 		return fmt.Errorf("invalid UTF-8")
@@ -371,7 +388,7 @@ func checkPath(path string, kind pathKind) error {
 	if path == "" {
 		return fmt.Errorf("empty string")
 	}
-	if path[0] == '-' {
+	if path[0] == '-' && kind != filePath {
 		return fmt.Errorf("leading dash")
 	}
 	if strings.Contains(path, "//") {
@@ -477,7 +494,7 @@ func checkElem(elem string, kind pathKind) error {
 // subtleties of Unicode.
 func CheckFilePath(path string) error {
 	if err := checkPath(path, filePath); err != nil {
-		return fmt.Errorf("malformed file path %q: %v", path, err)
+		return &InvalidPathError{Kind: "file", Path: path, Err: err}
 	}
 	return nil
 }
@@ -781,6 +798,7 @@ func unescapeString(escaped string) (string, bool) {
 // GOPRIVATE environment variable, as described by 'go help module-private'.
 //
 // It ignores any empty or malformed patterns in the list.
+// Trailing slashes on patterns are ignored.
 func MatchPrefixPatterns(globs, target string) bool {
 	for globs != "" {
 		// Extract next non-empty glob in comma-separated list.
@@ -790,6 +808,7 @@ func MatchPrefixPatterns(globs, target string) bool {
 		} else {
 			glob, globs = globs, ""
 		}
+		glob = strings.TrimSuffix(glob, "/")
 		if glob == "" {
 			continue
 		}

vendor/golang.org/x/mod/module/pseudo.go

@@ -0,0 +1,250 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Pseudo-versions
+//
+// Code authors are expected to tag the revisions they want users to use,
+// including prereleases. However, not all authors tag versions at all,
+// and not all commits a user might want to try will have tags.
+// A pseudo-version is a version with a special form that allows us to
+// address an untagged commit and order that version with respect to
+// other versions we might encounter.
+//
+// A pseudo-version takes one of the general forms:
+//
+//	(1) vX.0.0-yyyymmddhhmmss-abcdef123456
+//	(2) vX.Y.(Z+1)-0.yyyymmddhhmmss-abcdef123456
+//	(3) vX.Y.(Z+1)-0.yyyymmddhhmmss-abcdef123456+incompatible
+//	(4) vX.Y.Z-pre.0.yyyymmddhhmmss-abcdef123456
+//	(5) vX.Y.Z-pre.0.yyyymmddhhmmss-abcdef123456+incompatible
+//
+// If there is no recently tagged version with the right major version vX,
+// then form (1) is used, creating a space of pseudo-versions at the bottom
+// of the vX version range, less than any tagged version, including the unlikely v0.0.0.
+//
+// If the most recent tagged version before the target commit is vX.Y.Z or vX.Y.Z+incompatible,
+// then the pseudo-version uses form (2) or (3), making it a prerelease for the next
+// possible semantic version after vX.Y.Z. The leading 0 segment in the prerelease string
+// ensures that the pseudo-version compares less than possible future explicit prereleases
+// like vX.Y.(Z+1)-rc1 or vX.Y.(Z+1)-1.
+//
+// If the most recent tagged version before the target commit is vX.Y.Z-pre or vX.Y.Z-pre+incompatible,
+// then the pseudo-version uses form (4) or (5), making it a slightly later prerelease.
+
+package module
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+
+	"golang.org/x/mod/internal/lazyregexp"
+	"golang.org/x/mod/semver"
+)
+
+var pseudoVersionRE = lazyregexp.New(`^v[0-9]+\.(0\.0-|\d+\.\d+-([^+]*\.)?0\.)\d{14}-[A-Za-z0-9]+(\+[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$`)
+
+const PseudoVersionTimestampFormat = "20060102150405"
+
+// PseudoVersion returns a pseudo-version for the given major version ("v1")
+// preexisting older tagged version ("" or "v1.2.3" or "v1.2.3-pre"), revision time,
+// and revision identifier (usually a 12-byte commit hash prefix).
+func PseudoVersion(major, older string, t time.Time, rev string) string {
+	if major == "" {
+		major = "v0"
+	}
+	segment := fmt.Sprintf("%s-%s", t.UTC().Format(PseudoVersionTimestampFormat), rev)
+	build := semver.Build(older)
+	older = semver.Canonical(older)
+	if older == "" {
+		return major + ".0.0-" + segment // form (1)
+	}
+	if semver.Prerelease(older) != "" {
+		return older + ".0." + segment + build // form (4), (5)
+	}
+
+	// Form (2), (3).
+	// Extract patch from vMAJOR.MINOR.PATCH
+	i := strings.LastIndex(older, ".") + 1
+	v, patch := older[:i], older[i:]
+
+	// Reassemble.
+	return v + incDecimal(patch) + "-0." + segment + build
+}
+
+// ZeroPseudoVersion returns a pseudo-version with a zero timestamp and
+// revision, which may be used as a placeholder.
+func ZeroPseudoVersion(major string) string {
+	return PseudoVersion(major, "", time.Time{}, "000000000000")
+}
+
+// incDecimal returns the decimal string incremented by 1.
+func incDecimal(decimal string) string {
+	// Scan right to left turning 9s to 0s until you find a digit to increment.
+	digits := []byte(decimal)
+	i := len(digits) - 1
+	for ; i >= 0 && digits[i] == '9'; i-- {
+		digits[i] = '0'
+	}
+	if i >= 0 {
+		digits[i]++
+	} else {
+		// digits is all zeros
+		digits[0] = '1'
+		digits = append(digits, '0')
+	}
+	return string(digits)
+}
+
+// decDecimal returns the decimal string decremented by 1, or the empty string
+// if the decimal is all zeroes.
+func decDecimal(decimal string) string {
+	// Scan right to left turning 0s to 9s until you find a digit to decrement.
+	digits := []byte(decimal)
+	i := len(digits) - 1
+	for ; i >= 0 && digits[i] == '0'; i-- {
+		digits[i] = '9'
+	}
+	if i < 0 {
+		// decimal is all zeros
+		return ""
+	}
+	if i == 0 && digits[i] == '1' && len(digits) > 1 {
+		digits = digits[1:]
+	} else {
+		digits[i]--
+	}
+	return string(digits)
+}
+
+// IsPseudoVersion reports whether v is a pseudo-version.
+func IsPseudoVersion(v string) bool {
+	return strings.Count(v, "-") >= 2 && semver.IsValid(v) && pseudoVersionRE.MatchString(v)
+}
+
+// IsZeroPseudoVersion returns whether v is a pseudo-version with a zero base,
+// timestamp, and revision, as returned by ZeroPseudoVersion.
+func IsZeroPseudoVersion(v string) bool {
+	return v == ZeroPseudoVersion(semver.Major(v))
+}
+
+// PseudoVersionTime returns the time stamp of the pseudo-version v.
+// It returns an error if v is not a pseudo-version or if the time stamp
+// embedded in the pseudo-version is not a valid time.
+func PseudoVersionTime(v string) (time.Time, error) {
+	_, timestamp, _, _, err := parsePseudoVersion(v)
+	if err != nil {
+		return time.Time{}, err
+	}
+	t, err := time.Parse("20060102150405", timestamp)
+	if err != nil {
+		return time.Time{}, &InvalidVersionError{
+			Version: v,
+			Pseudo:  true,
+			Err:     fmt.Errorf("malformed time %q", timestamp),
+		}
+	}
+	return t, nil
+}
+
+// PseudoVersionRev returns the revision identifier of the pseudo-version v.
+// It returns an error if v is not a pseudo-version.
+func PseudoVersionRev(v string) (rev string, err error) {
+	_, _, rev, _, err = parsePseudoVersion(v)
+	return
+}
+
+// PseudoVersionBase returns the canonical parent version, if any, upon which
+// the pseudo-version v is based.
+//
+// If v has no parent version (that is, if it is "vX.0.0-[…]"),
+// PseudoVersionBase returns the empty string and a nil error.
+func PseudoVersionBase(v string) (string, error) {
+	base, _, _, build, err := parsePseudoVersion(v)
+	if err != nil {
+		return "", err
+	}
+
+	switch pre := semver.Prerelease(base); pre {
+	case "":
+		// vX.0.0-yyyymmddhhmmss-abcdef123456 → ""
+		if build != "" {
+			// Pseudo-versions of the form vX.0.0-yyyymmddhhmmss-abcdef123456+incompatible
+			// are nonsensical: the "vX.0.0-" prefix implies that there is no parent tag,
+			// but the "+incompatible" suffix implies that the major version of
+			// the parent tag is not compatible with the module's import path.
+			//
+			// There are a few such entries in the index generated by proxy.golang.org,
+			// but we believe those entries were generated by the proxy itself.
+			return "", &InvalidVersionError{
+				Version: v,
+				Pseudo:  true,
+				Err:     fmt.Errorf("lacks base version, but has build metadata %q", build),
+			}
+		}
+		return "", nil
+
+	case "-0":
+		// vX.Y.(Z+1)-0.yyyymmddhhmmss-abcdef123456 → vX.Y.Z
+		// vX.Y.(Z+1)-0.yyyymmddhhmmss-abcdef123456+incompatible → vX.Y.Z+incompatible
+		base = strings.TrimSuffix(base, pre)
+		i := strings.LastIndexByte(base, '.')
+		if i < 0 {
+			panic("base from parsePseudoVersion missing patch number: " + base)
+		}
+		patch := decDecimal(base[i+1:])
+		if patch == "" {
+			// vX.0.0-0 is invalid, but has been observed in the wild in the index
+			// generated by requests to proxy.golang.org.
+			//
+			// NOTE(bcmills): I cannot find a historical bug that accounts for
+			// pseudo-versions of this form, nor have I seen such versions in any
+			// actual go.mod files. If we find actual examples of this form and a
+			// reasonable theory of how they came into existence, it seems fine to
+			// treat them as equivalent to vX.0.0 (especially since the invalid
+			// pseudo-versions have lower precedence than the real ones). For now, we
+			// reject them.
+			return "", &InvalidVersionError{
+				Version: v,
+				Pseudo:  true,
+				Err:     fmt.Errorf("version before %s would have negative patch number", base),
+			}
+		}
+		return base[:i+1] + patch + build, nil
+
+	default:
+		// vX.Y.Z-pre.0.yyyymmddhhmmss-abcdef123456 → vX.Y.Z-pre
+		// vX.Y.Z-pre.0.yyyymmddhhmmss-abcdef123456+incompatible → vX.Y.Z-pre+incompatible
+		if !strings.HasSuffix(base, ".0") {
+			panic(`base from parsePseudoVersion missing ".0" before date: ` + base)
+		}
+		return strings.TrimSuffix(base, ".0") + build, nil
+	}
+}
+
+var errPseudoSyntax = errors.New("syntax error")
+
+func parsePseudoVersion(v string) (base, timestamp, rev, build string, err error) {
+	if !IsPseudoVersion(v) {
+		return "", "", "", "", &InvalidVersionError{
+			Version: v,
+			Pseudo:  true,
+			Err:     errPseudoSyntax,
+		}
+	}
+	build = semver.Build(v)
+	v = strings.TrimSuffix(v, build)
+	j := strings.LastIndex(v, "-")
+	v, rev = v[:j], v[j+1:]
+	i := strings.LastIndex(v, "-")
+	if j := strings.LastIndex(v, "."); j > i {
+		base = v[:j] // "vX.Y.Z-pre.0" or "vX.Y.(Z+1)-0"
+		timestamp = v[j+1:]
+	} else {
+		base = v[:i] // "vX.0.0"
+		timestamp = v[i+1:]
+	}
+	return base, timestamp, rev, build, nil
+}

vendor/golang.org/x/mod/semver/semver.go

@@ -22,6 +22,8 @@
 // as shorthands for vMAJOR.0.0 and vMAJOR.MINOR.0.
 package semver
 
+import "sort"
+
 // parsed returns the parsed form of a semantic version string.
 type parsed struct {
 	major      string
@@ -30,7 +32,6 @@ type parsed struct {
 	short      string
 	prerelease string
 	build      string
-	err        string
 }
 
 // IsValid reports whether v is a valid semantic version string.
@@ -150,14 +151,30 @@ func Max(v, w string) string {
 	return w
 }
 
+// ByVersion implements sort.Interface for sorting semantic version strings.
+type ByVersion []string
+
+func (vs ByVersion) Len() int      { return len(vs) }
+func (vs ByVersion) Swap(i, j int) { vs[i], vs[j] = vs[j], vs[i] }
+func (vs ByVersion) Less(i, j int) bool {
+	cmp := Compare(vs[i], vs[j])
+	if cmp != 0 {
+		return cmp < 0
+	}
+	return vs[i] < vs[j]
+}
+
+// Sort sorts a list of semantic version strings using ByVersion.
+func Sort(list []string) {
+	sort.Sort(ByVersion(list))
+}
+
 func parse(v string) (p parsed, ok bool) {
 	if v == "" || v[0] != 'v' {
-		p.err = "missing v prefix"
 		return
 	}
 	p.major, v, ok = parseInt(v[1:])
 	if !ok {
-		p.err = "bad major version"
 		return
 	}
 	if v == "" {
@@ -167,13 +184,11 @@ func parse(v string) (p parsed, ok bool) {
 		return
 	}
 	if v[0] != '.' {
-		p.err = "bad minor prefix"
 		ok = false
 		return
 	}
 	p.minor, v, ok = parseInt(v[1:])
 	if !ok {
-		p.err = "bad minor version"
 		return
 	}
 	if v == "" {
@@ -182,31 +197,26 @@ func parse(v string) (p parsed, ok bool) {
 		return
 	}
 	if v[0] != '.' {
-		p.err = "bad patch prefix"
 		ok = false
 		return
 	}
 	p.patch, v, ok = parseInt(v[1:])
 	if !ok {
-		p.err = "bad patch version"
 		return
 	}
 	if len(v) > 0 && v[0] == '-' {
 		p.prerelease, v, ok = parsePrerelease(v)
 		if !ok {
-			p.err = "bad prerelease"
 			return
 		}
 	}
 	if len(v) > 0 && v[0] == '+' {
 		p.build, v, ok = parseBuild(v)
 		if !ok {
-			p.err = "bad build"
 			return
 		}
 	}
 	if v != "" {
-		p.err = "junk on end"
 		ok = false
 		return
 	}

vendor/golang.org/x/net/http2/headermap.go

@@ -27,7 +27,14 @@ func buildCommonHeaderMaps() {
 		"accept-language",
 		"accept-ranges",
 		"age",
+		"access-control-allow-credentials",
+		"access-control-allow-headers",
+		"access-control-allow-methods",
 		"access-control-allow-origin",
+		"access-control-expose-headers",
+		"access-control-max-age",
+		"access-control-request-headers",
+		"access-control-request-method",
 		"allow",
 		"authorization",
 		"cache-control",
@@ -53,6 +60,7 @@ func buildCommonHeaderMaps() {
 		"link",
 		"location",
 		"max-forwards",
+		"origin",
 		"proxy-authenticate",
 		"proxy-authorization",
 		"range",
@@ -68,6 +76,8 @@ func buildCommonHeaderMaps() {
 		"vary",
 		"via",
 		"www-authenticate",
+		"x-forwarded-for",
+		"x-forwarded-proto",
 	}
 	commonLowerHeader = make(map[string]string, len(common))
 	commonCanonHeader = make(map[string]string, len(common))
@@ -85,3 +95,11 @@ func lowerHeader(v string) (lower string, ascii bool) {
 	}
 	return asciiToLower(v)
 }
+
+func canonicalHeader(v string) string {
+	buildCommonHeaderMapsOnce()
+	if s, ok := commonCanonHeader[v]; ok {
+		return s
+	}
+	return http.CanonicalHeaderKey(v)
+}

vendor/golang.org/x/net/http2/hpack/static_table.go

@@ -0,0 +1,188 @@
+// go generate gen.go
+// Code generated by the command above; DO NOT EDIT.
+
+package hpack
+
+var staticTable = &headerFieldTable{
+	evictCount: 0,
+	byName: map[string]uint64{
+		":authority":                  1,
+		":method":                     3,
+		":path":                       5,
+		":scheme":                     7,
+		":status":                     14,
+		"accept-charset":              15,
+		"accept-encoding":             16,
+		"accept-language":             17,
+		"accept-ranges":               18,
+		"accept":                      19,
+		"access-control-allow-origin": 20,
+		"age":                         21,
+		"allow":                       22,
+		"authorization":               23,
+		"cache-control":               24,
+		"content-disposition":         25,
+		"content-encoding":            26,
+		"content-language":            27,
+		"content-length":              28,
+		"content-location":            29,
+		"content-range":               30,
+		"content-type":                31,
+		"cookie":                      32,
+		"date":                        33,
+		"etag":                        34,
+		"expect":                      35,
+		"expires":                     36,
+		"from":                        37,
+		"host":                        38,
+		"if-match":                    39,
+		"if-modified-since":           40,
+		"if-none-match":               41,
+		"if-range":                    42,
+		"if-unmodified-since":         43,
+		"last-modified":               44,
+		"link":                        45,
+		"location":                    46,
+		"max-forwards":                47,
+		"proxy-authenticate":          48,
+		"proxy-authorization":         49,
+		"range":                       50,
+		"referer":                     51,
+		"refresh":                     52,
+		"retry-after":                 53,
+		"server":                      54,
+		"set-cookie":                  55,
+		"strict-transport-security":   56,
+		"transfer-encoding":           57,
+		"user-agent":                  58,
+		"vary":                        59,
+		"via":                         60,
+		"www-authenticate":            61,
+	},
+	byNameValue: map[pairNameValue]uint64{
+		{name: ":authority", value: ""}:                   1,
+		{name: ":method", value: "GET"}:                   2,
+		{name: ":method", value: "POST"}:                  3,
+		{name: ":path", value: "/"}:                       4,
+		{name: ":path", value: "/index.html"}:             5,
+		{name: ":scheme", value: "http"}:                  6,
+		{name: ":scheme", value: "https"}:                 7,
+		{name: ":status", value: "200"}:                   8,
+		{name: ":status", value: "204"}:                   9,
+		{name: ":status", value: "206"}:                   10,
+		{name: ":status", value: "304"}:                   11,
+		{name: ":status", value: "400"}:                   12,
+		{name: ":status", value: "404"}:                   13,
+		{name: ":status", value: "500"}:                   14,
+		{name: "accept-charset", value: ""}:               15,
+		{name: "accept-encoding", value: "gzip, deflate"}: 16,
+		{name: "accept-language", value: ""}:              17,
+		{name: "accept-ranges", value: ""}:                18,
+		{name: "accept", value: ""}:                       19,
+		{name: "access-control-allow-origin", value: ""}:  20,
+		{name: "age", value: ""}:                          21,
+		{name: "allow", value: ""}:                        22,
+		{name: "authorization", value: ""}:                23,
+		{name: "cache-control", value: ""}:                24,
+		{name: "content-disposition", value: ""}:          25,
+		{name: "content-encoding", value: ""}:             26,
+		{name: "content-language", value: ""}:             27,
+		{name: "content-length", value: ""}:               28,
+		{name: "content-location", value: ""}:             29,
+		{name: "content-range", value: ""}:                30,
+		{name: "content-type", value: ""}:                 31,
+		{name: "cookie", value: ""}:                       32,
+		{name: "date", value: ""}:                         33,
+		{name: "etag", value: ""}:                         34,
+		{name: "expect", value: ""}:                       35,
+		{name: "expires", value: ""}:                      36,
+		{name: "from", value: ""}:                         37,
+		{name: "host", value: ""}:                         38,
+		{name: "if-match", value: ""}:                     39,
+		{name: "if-modified-since", value: ""}:            40,
+		{name: "if-none-match", value: ""}:                41,
+		{name: "if-range", value: ""}:                     42,
+		{name: "if-unmodified-since", value: ""}:          43,
+		{name: "last-modified", value: ""}:                44,
+		{name: "link", value: ""}:                         45,
+		{name: "location", value: ""}:                     46,
+		{name: "max-forwards", value: ""}:                 47,
+		{name: "proxy-authenticate", value: ""}:           48,
+		{name: "proxy-authorization", value: ""}:          49,
+		{name: "range", value: ""}:                        50,
+		{name: "referer", value: ""}:                      51,
+		{name: "refresh", value: ""}:                      52,
+		{name: "retry-after", value: ""}:                  53,
+		{name: "server", value: ""}:                       54,
+		{name: "set-cookie", value: ""}:                   55,
+		{name: "strict-transport-security", value: ""}:    56,
+		{name: "transfer-encoding", value: ""}:            57,
+		{name: "user-agent", value: ""}:                   58,
+		{name: "vary", value: ""}:                         59,
+		{name: "via", value: ""}:                          60,
+		{name: "www-authenticate", value: ""}:             61,
+	},
+	ents: []HeaderField{
+		{Name: ":authority", Value: "", Sensitive: false},
+		{Name: ":method", Value: "GET", Sensitive: false},
+		{Name: ":method", Value: "POST", Sensitive: false},
+		{Name: ":path", Value: "/", Sensitive: false},
+		{Name: ":path", Value: "/index.html", Sensitive: false},
+		{Name: ":scheme", Value: "http", Sensitive: false},
+		{Name: ":scheme", Value: "https", Sensitive: false},
+		{Name: ":status", Value: "200", Sensitive: false},
+		{Name: ":status", Value: "204", Sensitive: false},
+		{Name: ":status", Value: "206", Sensitive: false},
+		{Name: ":status", Value: "304", Sensitive: false},
+		{Name: ":status", Value: "400", Sensitive: false},
+		{Name: ":status", Value: "404", Sensitive: false},
+		{Name: ":status", Value: "500", Sensitive: false},
+		{Name: "accept-charset", Value: "", Sensitive: false},
+		{Name: "accept-encoding", Value: "gzip, deflate", Sensitive: false},
+		{Name: "accept-language", Value: "", Sensitive: false},
+		{Name: "accept-ranges", Value: "", Sensitive: false},
+		{Name: "accept", Value: "", Sensitive: false},
+		{Name: "access-control-allow-origin", Value: "", Sensitive: false},
+		{Name: "age", Value: "", Sensitive: false},
+		{Name: "allow", Value: "", Sensitive: false},
+		{Name: "authorization", Value: "", Sensitive: false},
+		{Name: "cache-control", Value: "", Sensitive: false},
+		{Name: "content-disposition", Value: "", Sensitive: false},
+		{Name: "content-encoding", Value: "", Sensitive: false},
+		{Name: "content-language", Value: "", Sensitive: false},
+		{Name: "content-length", Value: "", Sensitive: false},
+		{Name: "content-location", Value: "", Sensitive: false},
+		{Name: "content-range", Value: "", Sensitive: false},
+		{Name: "content-type", Value: "", Sensitive: false},
+		{Name: "cookie", Value: "", Sensitive: false},
+		{Name: "date", Value: "", Sensitive: false},
+		{Name: "etag", Value: "", Sensitive: false},
+		{Name: "expect", Value: "", Sensitive: false},
+		{Name: "expires", Value: "", Sensitive: false},
+		{Name: "from", Value: "", Sensitive: false},
+		{Name: "host", Value: "", Sensitive: false},
+		{Name: "if-match", Value: "", Sensitive: false},
+		{Name: "if-modified-since", Value: "", Sensitive: false},
+		{Name: "if-none-match", Value: "", Sensitive: false},
+		{Name: "if-range", Value: "", Sensitive: false},
+		{Name: "if-unmodified-since", Value: "", Sensitive: false},
+		{Name: "last-modified", Value: "", Sensitive: false},
+		{Name: "link", Value: "", Sensitive: false},
+		{Name: "location", Value: "", Sensitive: false},
+		{Name: "max-forwards", Value: "", Sensitive: false},
+		{Name: "proxy-authenticate", Value: "", Sensitive: false},
+		{Name: "proxy-authorization", Value: "", Sensitive: false},
+		{Name: "range", Value: "", Sensitive: false},
+		{Name: "referer", Value: "", Sensitive: false},
+		{Name: "refresh", Value: "", Sensitive: false},
+		{Name: "retry-after", Value: "", Sensitive: false},
+		{Name: "server", Value: "", Sensitive: false},
+		{Name: "set-cookie", Value: "", Sensitive: false},
+		{Name: "strict-transport-security", Value: "", Sensitive: false},
+		{Name: "transfer-encoding", Value: "", Sensitive: false},
+		{Name: "user-agent", Value: "", Sensitive: false},
+		{Name: "vary", Value: "", Sensitive: false},
+		{Name: "via", Value: "", Sensitive: false},
+		{Name: "www-authenticate", Value: "", Sensitive: false},
+	},
+}

vendor/golang.org/x/net/http2/hpack/tables.go

@@ -96,8 +96,7 @@ func (t *headerFieldTable) evictOldest(n int) {
 // meaning t.ents is reversed for dynamic tables. Hence, when t is a dynamic
 // table, the return value i actually refers to the entry t.ents[t.len()-i].
 //
-// All tables are assumed to be a dynamic tables except for the global
-// staticTable pointer.
+// All tables are assumed to be a dynamic tables except for the global staticTable.
 //
 // See Section 2.3.3.
 func (t *headerFieldTable) search(f HeaderField) (i uint64, nameValueMatch bool) {
@@ -125,81 +124,6 @@ func (t *headerFieldTable) idToIndex(id uint64) uint64 {
 	return k + 1
 }
 
-// http://tools.ietf.org/html/draft-ietf-httpbis-header-compression-07#appendix-B
-var staticTable = newStaticTable()
-var staticTableEntries = [...]HeaderField{
-	{Name: ":authority"},
-	{Name: ":method", Value: "GET"},
-	{Name: ":method", Value: "POST"},
-	{Name: ":path", Value: "/"},
-	{Name: ":path", Value: "/index.html"},
-	{Name: ":scheme", Value: "http"},
-	{Name: ":scheme", Value: "https"},
-	{Name: ":status", Value: "200"},
-	{Name: ":status", Value: "204"},
-	{Name: ":status", Value: "206"},
-	{Name: ":status", Value: "304"},
-	{Name: ":status", Value: "400"},
-	{Name: ":status", Value: "404"},
-	{Name: ":status", Value: "500"},
-	{Name: "accept-charset"},
-	{Name: "accept-encoding", Value: "gzip, deflate"},
-	{Name: "accept-language"},
-	{Name: "accept-ranges"},
-	{Name: "accept"},
-	{Name: "access-control-allow-origin"},
-	{Name: "age"},
-	{Name: "allow"},
-	{Name: "authorization"},
-	{Name: "cache-control"},
-	{Name: "content-disposition"},
-	{Name: "content-encoding"},
-	{Name: "content-language"},
-	{Name: "content-length"},
-	{Name: "content-location"},
-	{Name: "content-range"},
-	{Name: "content-type"},
-	{Name: "cookie"},
-	{Name: "date"},
-	{Name: "etag"},
-	{Name: "expect"},
-	{Name: "expires"},
-	{Name: "from"},
-	{Name: "host"},
-	{Name: "if-match"},
-	{Name: "if-modified-since"},
-	{Name: "if-none-match"},
-	{Name: "if-range"},
-	{Name: "if-unmodified-since"},
-	{Name: "last-modified"},
-	{Name: "link"},
-	{Name: "location"},
-	{Name: "max-forwards"},
-	{Name: "proxy-authenticate"},
-	{Name: "proxy-authorization"},
-	{Name: "range"},
-	{Name: "referer"},
-	{Name: "refresh"},
-	{Name: "retry-after"},
-	{Name: "server"},
-	{Name: "set-cookie"},
-	{Name: "strict-transport-security"},
-	{Name: "transfer-encoding"},
-	{Name: "user-agent"},
-	{Name: "vary"},
-	{Name: "via"},
-	{Name: "www-authenticate"},
-}
-
-func newStaticTable() *headerFieldTable {
-	t := &headerFieldTable{}
-	t.init()
-	for _, e := range staticTableEntries[:] {
-		t.addEntry(e)
-	}
-	return t
-}
-
 var huffmanCodes = [256]uint32{
 	0x1ff8,
 	0x7fffd8,

vendor/golang.org/x/net/http2/server.go

@@ -143,7 +143,7 @@ type Server struct {
 }
 
 func (s *Server) initialConnRecvWindowSize() int32 {
-	if s.MaxUploadBufferPerConnection > initialWindowSize {
+	if s.MaxUploadBufferPerConnection >= initialWindowSize {
 		return s.MaxUploadBufferPerConnection
 	}
 	return 1 << 20
@@ -622,7 +622,9 @@ type stream struct {
 	resetQueued      bool        // RST_STREAM queued for write; set by sc.resetStream
 	gotTrailerHeader bool        // HEADER frame for trailers was seen
 	wroteHeaders     bool        // whether we wrote headers (not status 100)
+	readDeadline     *time.Timer // nil if unused
 	writeDeadline    *time.Timer // nil if unused
+	closeErr         error       // set before cw is closed
 
 	trailer    http.Header // accumulated trailers
 	reqTrailer http.Header // handler's Request.Trailer
@@ -948,6 +950,8 @@ func (sc *serverConn) serve() {
 				}
 			case *startPushRequest:
 				sc.startPush(v)
+			case func(*serverConn):
+				v(sc)
 			default:
 				panic(fmt.Sprintf("unexpected type %T", v))
 			}
@@ -1461,6 +1465,21 @@ func (sc *serverConn) processFrame(f Frame) error {
 		sc.sawFirstSettings = true
 	}
 
+	// Discard frames for streams initiated after the identified last
+	// stream sent in a GOAWAY, or all frames after sending an error.
+	// We still need to return connection-level flow control for DATA frames.
+	// RFC 9113 Section 6.8.
+	if sc.inGoAway && (sc.goAwayCode != ErrCodeNo || f.Header().StreamID > sc.maxClientStreamID) {
+
+		if f, ok := f.(*DataFrame); ok {
+			if sc.inflow.available() < int32(f.Length) {
+				return sc.countError("data_flow", streamError(f.Header().StreamID, ErrCodeFlowControl))
+			}
+			sc.sendWindowUpdate(nil, int(f.Length)) // conn-level
+		}
+		return nil
+	}
+
 	switch f := f.(type) {
 	case *SettingsFrame:
 		return sc.processSettings(f)
@@ -1503,9 +1522,6 @@ func (sc *serverConn) processPing(f *PingFrame) error {
 		// PROTOCOL_ERROR."
 		return sc.countError("ping_on_stream", ConnectionError(ErrCodeProtocol))
 	}
-	if sc.inGoAway && sc.goAwayCode != ErrCodeNo {
-		return nil
-	}
 	sc.writeFrame(FrameWriteRequest{write: writePingAck{f}})
 	return nil
 }
@@ -1567,6 +1583,9 @@ func (sc *serverConn) closeStream(st *stream, err error) {
 		panic(fmt.Sprintf("invariant; can't close stream in state %v", st.state))
 	}
 	st.state = stateClosed
+	if st.readDeadline != nil {
+		st.readDeadline.Stop()
+	}
 	if st.writeDeadline != nil {
 		st.writeDeadline.Stop()
 	}
@@ -1592,6 +1611,14 @@ func (sc *serverConn) closeStream(st *stream, err error) {
 
 		p.CloseWithError(err)
 	}
+	if e, ok := err.(StreamError); ok {
+		if e.Cause != nil {
+			err = e.Cause
+		} else {
+			err = errStreamClosed
+		}
+	}
+	st.closeErr = err
 	st.cw.Close() // signals Handler's CloseNotifier, unblocks writes, etc
 	sc.writeSched.CloseStream(st.id)
 }
@@ -1688,16 +1715,6 @@ func (sc *serverConn) processSettingInitialWindowSize(val uint32) error {
 func (sc *serverConn) processData(f *DataFrame) error {
 	sc.serveG.check()
 	id := f.Header().StreamID
-	if sc.inGoAway && (sc.goAwayCode != ErrCodeNo || id > sc.maxClientStreamID) {
-		// Discard all DATA frames if the GOAWAY is due to an
-		// error, or:
-		//
-		// Section 6.8: After sending a GOAWAY frame, the sender
-		// can discard frames for streams initiated by the
-		// receiver with identifiers higher than the identified
-		// last stream.
-		return nil
-	}
 
 	data := f.Data()
 	state, st := sc.state(id)
@@ -1840,19 +1857,27 @@ func (st *stream) copyTrailersToHandlerRequest() {
 	}
 }
 
+// onReadTimeout is run on its own goroutine (from time.AfterFunc)
+// when the stream's ReadTimeout has fired.
+func (st *stream) onReadTimeout() {
+	// Wrap the ErrDeadlineExceeded to avoid callers depending on us
+	// returning the bare error.
+	st.body.CloseWithError(fmt.Errorf("%w", os.ErrDeadlineExceeded))
+}
+
 // onWriteTimeout is run on its own goroutine (from time.AfterFunc)
 // when the stream's WriteTimeout has fired.
 func (st *stream) onWriteTimeout() {
-	st.sc.writeFrameFromHandler(FrameWriteRequest{write: streamError(st.id, ErrCodeInternal)})
+	st.sc.writeFrameFromHandler(FrameWriteRequest{write: StreamError{
+		StreamID: st.id,
+		Code:     ErrCodeInternal,
+		Cause:    os.ErrDeadlineExceeded,
+	}})
 }
 
 func (sc *serverConn) processHeaders(f *MetaHeadersFrame) error {
 	sc.serveG.check()
 	id := f.StreamID
-	if sc.inGoAway {
-		// Ignore.
-		return nil
-	}
 	// http://tools.ietf.org/html/rfc7540#section-5.1.1
 	// Streams initiated by a client MUST use odd-numbered stream
 	// identifiers. [...] An endpoint that receives an unexpected
@@ -1955,6 +1980,9 @@ func (sc *serverConn) processHeaders(f *MetaHeadersFrame) error {
 	// (in Go 1.8), though. That's a more sane option anyway.
 	if sc.hs.ReadTimeout != 0 {
 		sc.conn.SetReadDeadline(time.Time{})
+		if st.body != nil {
+			st.readDeadline = time.AfterFunc(sc.hs.ReadTimeout, st.onReadTimeout)
+		}
 	}
 
 	go sc.runHandler(rw, req, handler)
@@ -2023,9 +2051,6 @@ func (sc *serverConn) checkPriority(streamID uint32, p PriorityParam) error {
 }
 
 func (sc *serverConn) processPriority(f *PriorityFrame) error {
-	if sc.inGoAway {
-		return nil
-	}
 	if err := sc.checkPriority(f.StreamID, f.PriorityParam); err != nil {
 		return err
 	}
@@ -2099,12 +2124,6 @@ func (sc *serverConn) newWriterAndRequest(st *stream, f *MetaHeadersFrame) (*res
 		return nil, nil, sc.countError("bad_path_method", streamError(f.StreamID, ErrCodeProtocol))
 	}
 
-	bodyOpen := !f.StreamEnded()
-	if rp.method == "HEAD" && bodyOpen {
-		// HEAD requests can't have bodies
-		return nil, nil, sc.countError("head_body", streamError(f.StreamID, ErrCodeProtocol))
-	}
-
 	rp.header = make(http.Header)
 	for _, hf := range f.RegularFields() {
 		rp.header.Add(sc.canonicalHeader(hf.Name), hf.Value)
@@ -2117,6 +2136,7 @@ func (sc *serverConn) newWriterAndRequest(st *stream, f *MetaHeadersFrame) (*res
 	if err != nil {
 		return nil, nil, err
 	}
+	bodyOpen := !f.StreamEnded()
 	if bodyOpen {
 		if vv, ok := rp.header["Content-Length"]; ok {
 			if cl, err := strconv.ParseUint(vv[0], 10, 63); err == nil {
@@ -2346,7 +2366,7 @@ func (sc *serverConn) sendWindowUpdate(st *stream, n int) {
 	// a larger Read than this. Very unlikely, but we handle it here
 	// rather than elsewhere for now.
 	const maxUint31 = 1<<31 - 1
-	for n >= maxUint31 {
+	for n > maxUint31 {
 		sc.sendWindowUpdate32(st, maxUint31)
 		n -= maxUint31
 	}
@@ -2466,7 +2486,15 @@ type responseWriterState struct {
 
 type chunkWriter struct{ rws *responseWriterState }
 
-func (cw chunkWriter) Write(p []byte) (n int, err error) { return cw.rws.writeChunk(p) }
+func (cw chunkWriter) Write(p []byte) (n int, err error) {
+	n, err = cw.rws.writeChunk(p)
+	if err == errStreamClosed {
+		// If writing failed because the stream has been closed,
+		// return the reason it was closed.
+		err = cw.rws.stream.closeErr
+	}
+	return n, err
+}
 
 func (rws *responseWriterState) hasTrailers() bool { return len(rws.trailers) > 0 }
 
@@ -2505,6 +2533,10 @@ func (rws *responseWriterState) writeChunk(p []byte) (n int, err error) {
 		rws.writeHeader(200)
 	}
 
+	if rws.handlerDone {
+		rws.promoteUndeclaredTrailers()
+	}
+
 	isHeadResp := rws.req.Method == "HEAD"
 	if !rws.sentHeader {
 		rws.sentHeader = true
@@ -2576,10 +2608,6 @@ func (rws *responseWriterState) writeChunk(p []byte) (n int, err error) {
 		return 0, nil
 	}
 
-	if rws.handlerDone {
-		rws.promoteUndeclaredTrailers()
-	}
-
 	// only send trailers if they have actually been defined by the
 	// server handler.
 	hasNonemptyTrailers := rws.hasNonemptyTrailers()
@@ -2660,23 +2688,85 @@ func (rws *responseWriterState) promoteUndeclaredTrailers() {
 	}
 }
 
+func (w *responseWriter) SetReadDeadline(deadline time.Time) error {
+	st := w.rws.stream
+	if !deadline.IsZero() && deadline.Before(time.Now()) {
+		// If we're setting a deadline in the past, reset the stream immediately
+		// so writes after SetWriteDeadline returns will fail.
+		st.onReadTimeout()
+		return nil
+	}
+	w.rws.conn.sendServeMsg(func(sc *serverConn) {
+		if st.readDeadline != nil {
+			if !st.readDeadline.Stop() {
+				// Deadline already exceeded, or stream has been closed.
+				return
+			}
+		}
+		if deadline.IsZero() {
+			st.readDeadline = nil
+		} else if st.readDeadline == nil {
+			st.readDeadline = time.AfterFunc(deadline.Sub(time.Now()), st.onReadTimeout)
+		} else {
+			st.readDeadline.Reset(deadline.Sub(time.Now()))
+		}
+	})
+	return nil
+}
+
+func (w *responseWriter) SetWriteDeadline(deadline time.Time) error {
+	st := w.rws.stream
+	if !deadline.IsZero() && deadline.Before(time.Now()) {
+		// If we're setting a deadline in the past, reset the stream immediately
+		// so writes after SetWriteDeadline returns will fail.
+		st.onWriteTimeout()
+		return nil
+	}
+	w.rws.conn.sendServeMsg(func(sc *serverConn) {
+		if st.writeDeadline != nil {
+			if !st.writeDeadline.Stop() {
+				// Deadline already exceeded, or stream has been closed.
+				return
+			}
+		}
+		if deadline.IsZero() {
+			st.writeDeadline = nil
+		} else if st.writeDeadline == nil {
+			st.writeDeadline = time.AfterFunc(deadline.Sub(time.Now()), st.onWriteTimeout)
+		} else {
+			st.writeDeadline.Reset(deadline.Sub(time.Now()))
+		}
+	})
+	return nil
+}
+
 func (w *responseWriter) Flush() {
+	w.FlushError()
+}
+
+func (w *responseWriter) FlushError() error {
 	rws := w.rws
 	if rws == nil {
 		panic("Header called after Handler finished")
 	}
+	var err error
 	if rws.bw.Buffered() > 0 {
-		if err := rws.bw.Flush(); err != nil {
-			// Ignore the error. The frame writer already knows.
-			return
-		}
+		err = rws.bw.Flush()
 	} else {
 		// The bufio.Writer won't call chunkWriter.Write
 		// (writeChunk with zero bytes, so we have to do it
 		// ourselves to force the HTTP response header and/or
 		// final DATA frame (with END_STREAM) to be sent.
-		rws.writeChunk(nil)
+		_, err = chunkWriter{rws}.Write(nil)
+		if err == nil {
+			select {
+			case <-rws.stream.cw:
+				err = rws.stream.closeErr
+			default:
+			}
+		}
 	}
+	return err
 }
 
 func (w *responseWriter) CloseNotify() <-chan bool {

vendor/golang.org/x/net/http2/transport.go

@@ -16,6 +16,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/fs"
 	"log"
 	"math"
 	mathrand "math/rand"
@@ -258,7 +259,8 @@ func (t *Transport) initConnPool() {
 // HTTP/2 server.
 type ClientConn struct {
 	t             *Transport
-	tconn         net.Conn             // usually *tls.Conn, except specialized impls
+	tconn         net.Conn // usually *tls.Conn, except specialized impls
+	tconnClosed   bool
 	tlsState      *tls.ConnectionState // nil only for specialized impls
 	reused        uint32               // whether conn is being reused; atomic
 	singleUse     bool                 // whether being used for a single http.Request
@@ -344,8 +346,8 @@ type clientStream struct {
 	readErr     error // sticky read error; owned by transportResponseBody.Read
 
 	reqBody              io.ReadCloser
-	reqBodyContentLength int64 // -1 means unknown
-	reqBodyClosed        bool  // body has been closed; guarded by cc.mu
+	reqBodyContentLength int64         // -1 means unknown
+	reqBodyClosed        chan struct{} // guarded by cc.mu; non-nil on Close, closed when done
 
 	// owned by writeRequest:
 	sentEndStream bool // sent an END_STREAM flag to the peer
@@ -385,9 +387,8 @@ func (cs *clientStream) abortStreamLocked(err error) {
 		cs.abortErr = err
 		close(cs.abort)
 	})
-	if cs.reqBody != nil && !cs.reqBodyClosed {
-		cs.reqBody.Close()
-		cs.reqBodyClosed = true
+	if cs.reqBody != nil {
+		cs.closeReqBodyLocked()
 	}
 	// TODO(dneil): Clean up tests where cs.cc.cond is nil.
 	if cs.cc.cond != nil {
@@ -400,13 +401,24 @@ func (cs *clientStream) abortRequestBodyWrite() {
 	cc := cs.cc
 	cc.mu.Lock()
 	defer cc.mu.Unlock()
-	if cs.reqBody != nil && !cs.reqBodyClosed {
-		cs.reqBody.Close()
-		cs.reqBodyClosed = true
+	if cs.reqBody != nil && cs.reqBodyClosed == nil {
+		cs.closeReqBodyLocked()
 		cc.cond.Broadcast()
 	}
 }
 
+func (cs *clientStream) closeReqBodyLocked() {
+	if cs.reqBodyClosed != nil {
+		return
+	}
+	cs.reqBodyClosed = make(chan struct{})
+	reqBodyClosed := cs.reqBodyClosed
+	go func() {
+		cs.reqBody.Close()
+		close(reqBodyClosed)
+	}()
+}
+
 type stickyErrWriter struct {
 	conn    net.Conn
 	timeout time.Duration
@@ -490,6 +502,15 @@ func authorityAddr(scheme string, authority string) (addr string) {
 	return net.JoinHostPort(host, port)
 }
 
+var retryBackoffHook func(time.Duration) *time.Timer
+
+func backoffNewTimer(d time.Duration) *time.Timer {
+	if retryBackoffHook != nil {
+		return retryBackoffHook(d)
+	}
+	return time.NewTimer(d)
+}
+
 // RoundTripOpt is like RoundTrip, but takes options.
 func (t *Transport) RoundTripOpt(req *http.Request, opt RoundTripOpt) (*http.Response, error) {
 	if !(req.URL.Scheme == "https" || (req.URL.Scheme == "http" && t.AllowHTTP)) {
@@ -515,11 +536,14 @@ func (t *Transport) RoundTripOpt(req *http.Request, opt RoundTripOpt) (*http.Res
 				}
 				backoff := float64(uint(1) << (uint(retry) - 1))
 				backoff += backoff * (0.1 * mathrand.Float64())
+				d := time.Second * time.Duration(backoff)
+				timer := backoffNewTimer(d)
 				select {
-				case <-time.After(time.Second * time.Duration(backoff)):
+				case <-timer.C:
 					t.vlogf("RoundTrip retrying after failure: %v", err)
 					continue
 				case <-req.Context().Done():
+					timer.Stop()
 					err = req.Context().Err()
 				}
 			}
@@ -921,10 +945,10 @@ func (cc *ClientConn) onIdleTimeout() {
 	cc.closeIfIdle()
 }
 
-func (cc *ClientConn) closeConn() error {
+func (cc *ClientConn) closeConn() {
 	t := time.AfterFunc(250*time.Millisecond, cc.forceCloseConn)
 	defer t.Stop()
-	return cc.tconn.Close()
+	cc.tconn.Close()
 }
 
 // A tls.Conn.Close can hang for a long time if the peer is unresponsive.
@@ -990,7 +1014,8 @@ func (cc *ClientConn) Shutdown(ctx context.Context) error {
 	shutdownEnterWaitStateHook()
 	select {
 	case <-done:
-		return cc.closeConn()
+		cc.closeConn()
+		return nil
 	case <-ctx.Done():
 		cc.mu.Lock()
 		// Free the goroutine above
@@ -1027,7 +1052,7 @@ func (cc *ClientConn) sendGoAway() error {
 
 // closes the client connection immediately. In-flight requests are interrupted.
 // err is sent to streams.
-func (cc *ClientConn) closeForError(err error) error {
+func (cc *ClientConn) closeForError(err error) {
 	cc.mu.Lock()
 	cc.closed = true
 	for _, cs := range cc.streams {
@@ -1035,7 +1060,7 @@ func (cc *ClientConn) closeForError(err error) error {
 	}
 	cc.cond.Broadcast()
 	cc.mu.Unlock()
-	return cc.closeConn()
+	cc.closeConn()
 }
 
 // Close closes the client connection immediately.
@@ -1043,16 +1068,17 @@ func (cc *ClientConn) closeForError(err error) error {
 // In-flight requests are interrupted. For a graceful shutdown, use Shutdown instead.
 func (cc *ClientConn) Close() error {
 	err := errors.New("http2: client connection force closed via ClientConn.Close")
-	return cc.closeForError(err)
+	cc.closeForError(err)
+	return nil
 }
 
 // closes the client connection immediately. In-flight requests are interrupted.
-func (cc *ClientConn) closeForLostPing() error {
+func (cc *ClientConn) closeForLostPing() {
 	err := errors.New("http2: client connection lost")
 	if f := cc.t.CountError; f != nil {
 		f("conn_close_lost_ping")
 	}
-	return cc.closeForError(err)
+	cc.closeForError(err)
 }
 
 // errRequestCanceled is a copy of net/http's errRequestCanceled because it's not
@@ -1062,7 +1088,7 @@ var errRequestCanceled = errors.New("net/http: request canceled")
 func commaSeparatedTrailers(req *http.Request) (string, error) {
 	keys := make([]string, 0, len(req.Trailer))
 	for k := range req.Trailer {
-		k = http.CanonicalHeaderKey(k)
+		k = canonicalHeader(k)
 		switch k {
 		case "Transfer-Encoding", "Trailer", "Content-Length":
 			return "", fmt.Errorf("invalid Trailer key %q", k)
@@ -1430,11 +1456,19 @@ func (cs *clientStream) cleanupWriteRequest(err error) {
 	// and in multiple cases: server replies <=299 and >299
 	// while still writing request body
 	cc.mu.Lock()
+	mustCloseBody := false
+	if cs.reqBody != nil && cs.reqBodyClosed == nil {
+		mustCloseBody = true
+		cs.reqBodyClosed = make(chan struct{})
+	}
 	bodyClosed := cs.reqBodyClosed
-	cs.reqBodyClosed = true
 	cc.mu.Unlock()
-	if !bodyClosed && cs.reqBody != nil {
+	if mustCloseBody {
 		cs.reqBody.Close()
+		close(bodyClosed)
+	}
+	if bodyClosed != nil {
+		<-bodyClosed
 	}
 
 	if err != nil && cs.sentEndStream {
@@ -1591,7 +1625,7 @@ func (cs *clientStream) writeRequestBody(req *http.Request) (err error) {
 
 	var sawEOF bool
 	for !sawEOF {
-		n, err := body.Read(buf[:len(buf)])
+		n, err := body.Read(buf)
 		if hasContentLen {
 			remainLen -= int64(n)
 			if remainLen == 0 && err == nil {
@@ -1614,7 +1648,7 @@ func (cs *clientStream) writeRequestBody(req *http.Request) (err error) {
 		}
 		if err != nil {
 			cc.mu.Lock()
-			bodyClosed := cs.reqBodyClosed
+			bodyClosed := cs.reqBodyClosed != nil
 			cc.mu.Unlock()
 			switch {
 			case bodyClosed:
@@ -1709,7 +1743,7 @@ func (cs *clientStream) awaitFlowControl(maxBytes int) (taken int32, err error)
 		if cc.closed {
 			return 0, errClientConnClosed
 		}
-		if cs.reqBodyClosed {
+		if cs.reqBodyClosed != nil {
 			return 0, errStopReqBodyWrite
 		}
 		select {
@@ -1894,7 +1928,7 @@ func (cc *ClientConn) encodeHeaders(req *http.Request, addGzipHeader bool, trail
 
 	// Header list size is ok. Write the headers.
 	enumerateHeaders(func(name, value string) {
-		name, ascii := asciiToLower(name)
+		name, ascii := lowerHeader(name)
 		if !ascii {
 			// Skip writing invalid headers. Per RFC 7540, Section 8.1.2, header
 			// field names have to be ASCII characters (just as in HTTP/1.x).
@@ -1947,7 +1981,7 @@ func (cc *ClientConn) encodeTrailers(trailer http.Header) ([]byte, error) {
 	}
 
 	for k, vv := range trailer {
-		lowKey, ascii := asciiToLower(k)
+		lowKey, ascii := lowerHeader(k)
 		if !ascii {
 			// Skip writing invalid headers. Per RFC 7540, Section 8.1.2, header
 			// field names have to be ASCII characters (just as in HTTP/1.x).
@@ -2005,7 +2039,7 @@ func (cc *ClientConn) forgetStreamID(id uint32) {
 	// wake up RoundTrip if there is a pending request.
 	cc.cond.Broadcast()
 
-	closeOnIdle := cc.singleUse || cc.doNotReuse || cc.t.disableKeepAlives()
+	closeOnIdle := cc.singleUse || cc.doNotReuse || cc.t.disableKeepAlives() || cc.goAway != nil
 	if closeOnIdle && cc.streamsReserved == 0 && len(cc.streams) == 0 {
 		if VerboseLogs {
 			cc.vlogf("http2: Transport closing idle conn %p (forSingleUse=%v, maxStream=%v)", cc, cc.singleUse, cc.nextStreamID-2)
@@ -2081,6 +2115,7 @@ func (rl *clientConnReadLoop) cleanup() {
 		err = io.ErrUnexpectedEOF
 	}
 	cc.closed = true
+
 	for _, cs := range cc.streams {
 		select {
 		case <-cs.peerClosed:
@@ -2279,7 +2314,7 @@ func (rl *clientConnReadLoop) handleResponse(cs *clientStream, f *MetaHeadersFra
 		Status:     status + " " + http.StatusText(statusCode),
 	}
 	for _, hf := range regularFields {
-		key := http.CanonicalHeaderKey(hf.Name)
+		key := canonicalHeader(hf.Name)
 		if key == "Trailer" {
 			t := res.Trailer
 			if t == nil {
@@ -2287,7 +2322,7 @@ func (rl *clientConnReadLoop) handleResponse(cs *clientStream, f *MetaHeadersFra
 				res.Trailer = t
 			}
 			foreachHeaderElement(hf.Value, func(v string) {
-				t[http.CanonicalHeaderKey(v)] = nil
+				t[canonicalHeader(v)] = nil
 			})
 		} else {
 			vv := header[key]
@@ -2392,7 +2427,7 @@ func (rl *clientConnReadLoop) processTrailers(cs *clientStream, f *MetaHeadersFr
 
 	trailer := make(http.Header)
 	for _, hf := range f.RegularFields() {
-		key := http.CanonicalHeaderKey(hf.Name)
+		key := canonicalHeader(hf.Name)
 		trailer[key] = append(trailer[key], hf.Value)
 	}
 	cs.trailer = trailer
@@ -2674,7 +2709,6 @@ func (rl *clientConnReadLoop) processGoAway(f *GoAwayFrame) error {
 		if fn := cc.t.CountError; fn != nil {
 			fn("recv_goaway_" + f.ErrCode.stringToken())
 		}
-
 	}
 	cc.setGoAway(f)
 	return nil
@@ -2964,7 +2998,11 @@ func (gz *gzipReader) Read(p []byte) (n int, err error) {
 }
 
 func (gz *gzipReader) Close() error {
-	return gz.body.Close()
+	if err := gz.body.Close(); err != nil {
+		return err
+	}
+	gz.zerr = fs.ErrClosed
+	return nil
 }
 
 type errorReader struct{ err error }
@@ -3028,7 +3066,7 @@ func traceGotConn(req *http.Request, cc *ClientConn, reused bool) {
 	cc.mu.Lock()
 	ci.WasIdle = len(cc.streams) == 0 && reused
 	if ci.WasIdle && !cc.lastActive.IsZero() {
-		ci.IdleTime = time.Now().Sub(cc.lastActive)
+		ci.IdleTime = time.Since(cc.lastActive)
 	}
 	cc.mu.Unlock()
 

vendor/golang.org/x/net/internal/socket/mmsghdr_unix.go

@@ -172,7 +172,23 @@ type mmsgTmpsPool struct {
 }
 
 func (p *mmsgTmpsPool) Get() *mmsgTmps {
-	return p.p.Get().(*mmsgTmps)
+	m := p.p.Get().(*mmsgTmps)
+	// Clear fields up to the len (not the cap) of the slice,
+	// assuming that the previous caller only used that many elements.
+	for i := range m.packer.sockaddrs {
+		m.packer.sockaddrs[i] = 0
+	}
+	m.packer.sockaddrs = m.packer.sockaddrs[:0]
+	for i := range m.packer.vs {
+		m.packer.vs[i] = iovec{}
+	}
+	m.packer.vs = m.packer.vs[:0]
+	for i := range m.packer.hs {
+		m.packer.hs[i].Len = 0
+		m.packer.hs[i].Hdr = msghdr{}
+	}
+	m.packer.hs = m.packer.hs[:0]
+	return m
 }
 
 func (p *mmsgTmpsPool) Put(tmps *mmsgTmps) {

vendor/golang.org/x/net/internal/socket/msghdr_linux.go

@@ -17,9 +17,6 @@ func (h *msghdr) pack(vs []iovec, bs [][]byte, oob []byte, sa []byte) {
 	if sa != nil {
 		h.Name = (*byte)(unsafe.Pointer(&sa[0]))
 		h.Namelen = uint32(len(sa))
-	} else {
-		h.Name = nil
-		h.Namelen = 0
 	}
 }