Release 2022.10.2

cloudflared's Makefile uses `shell go env GOOS` to determine the
LOCAL_OS regardless of it being provided. We therefore need pinned_go as
a dependency to run docker-generate-versions.

.docker-images

@@ -1,10 +1,12 @@
 images:
   - name: cloudflared
-    dockerfile: Dockerfile
+    dockerfile: Dockerfile.$ARCH
     context: .
-    versions: 
-      - latest
+    version_file: versions
     registries:
     - name: docker.io/cloudflare
       user: env:DOCKER_USER
       password: env:DOCKER_PASSWORD
+    architectures:
+    - amd64
+    - arm64

.github/ISSUE_TEMPLATE/---bug-report.md

@@ -3,7 +3,6 @@ name: "\U0001F41B Bug report"
 about: Create a report to help us improve cloudflared
 title: "\U0001F41B"
 labels: 'Priority: Normal, Type: Bug'
-assignees: abelinkinbio
 
 ---
 
@@ -16,6 +15,10 @@ Steps to reproduce the behavior:
 2. Run '....'
 3. See error
 
+If it's an issue with Cloudflare Tunnel:
+4. Tunnel ID : 
+5. cloudflared config: 
+
 **Expected behavior**
 A clear and concise description of what you expected to happen.
 

.github/ISSUE_TEMPLATE/---documentation.md

@@ -3,7 +3,6 @@ name: "\U0001F4DD Documentation"
 about: Request new or updated documentation for cloudflared
 title: "\U0001F4DD"
 labels: 'Priority: Normal, Type: Documentation'
-assignees: abelinkinbio, ranbel
 
 ---
 

.github/ISSUE_TEMPLATE/---feature-request.md

@@ -3,7 +3,6 @@ name: "\U0001F4A1 Feature request"
 about: Suggest a feature or enhancement for cloudflared
 title: "\U0001F4A1"
 labels: 'Priority: Normal, Type: Feature Request'
-assignees: abelinkinbio, sssilver
 
 ---
 

.github/workflows/check.yaml

@@ -4,7 +4,7 @@ jobs:
   check:
     strategy:
       matrix:
-        go-version: [1.17.x]
+        go-version: [1.19.x]
         os: [ubuntu-latest, macos-latest, windows-latest]
     runs-on: ${{ matrix.os }}
     steps:

.teamcity/package-windows.sh

@@ -0,0 +1,17 @@
+VERSION=$(git describe --tags --always --match "[0-9][0-9][0-9][0-9].*.*")
+echo $VERSION
+
+export TARGET_OS=windows
+# This controls the directory the built artifacts go into
+export ARTIFACT_DIR=built_artifacts/
+mkdir -p $ARTIFACT_DIR
+windowsArchs=("amd64" "386")
+for arch in ${windowsArchs[@]}; do
+    export TARGET_ARCH=$arch
+    # Copy exe into final directory
+    cp ./artifacts/cloudflared-windows-$arch.exe $ARTIFACT_DIR/cloudflared-windows-$arch.exe
+    cp ./artifacts/cloudflared-windows-$arch.exe ./cloudflared.exe
+    make cloudflared-msi
+    # Copy msi into final directory
+    mv cloudflared-$VERSION-$arch.msi $ARTIFACT_DIR/cloudflared-windows-$arch.msi
+done

.teamcity/update-homebrew-core.sh

@@ -23,4 +23,4 @@ git config --global user.name "cloudflare-warp-bot"
 git config --global user.email "warp-bot@cloudflare.com"
 
 # bump formula pr
-brew bump-formula-pr cloudflared --version="$VERSION" --no-browse
+brew bump-formula-pr cloudflared --version="$VERSION" --no-browse --no-audit

CHANGES.md

@@ -1,3 +1,19 @@
+## 2022.9.0
+### New Features
+- cloudflared now rejects ingress rules with invalid http status codes for http_status.
+
+## 2022.8.1
+### New Features
+- cloudflared now remembers if it connected to a certain protocol successfully. If it did, it does not fall back to a lower
+  protocol on connection failures.
+
+## 2022.7.1
+### New Features
+- It is now possible to connect cloudflared tunnel to Cloudflare Global Network with IPv6. See `cloudflared tunnel --help` and look for `edge-ip-version` for more information. For now, the default behavior is to still connect with IPv4 only.
+
+### Bug Fixes
+- Several bug fixes related with QUIC transport (used between cloudflared tunnel and Cloudflare Global Network). Updating to this version is highly recommended.
+
 ## 2022.4.0
 ### Bug Fixes
 - `cloudflared tunnel run` no longer logs the Tunnel token or JSON credentials in clear text as those are the secret

Dockerfile

@@ -1,14 +1,12 @@
 # use a builder image for building cloudflare
 ARG TARGET_GOOS
 ARG TARGET_GOARCH
-FROM golang:1.17.1 as builder
+FROM golang:1.19 as builder
 ENV GO111MODULE=on \
     CGO_ENABLED=0 \
     TARGET_GOOS=${TARGET_GOOS} \
     TARGET_GOARCH=${TARGET_GOARCH}
     
-LABEL org.opencontainers.image.source="https://github.com/cloudflare/cloudflared"
-
 WORKDIR /go/src/github.com/cloudflare/cloudflared/
 
 # copy our sources into the builder image
@@ -18,7 +16,9 @@ COPY . .
 RUN make cloudflared
 
 # use a distroless base image with glibc
-FROM gcr.io/distroless/base-debian10:nonroot
+FROM gcr.io/distroless/base-debian11:nonroot
+
+LABEL org.opencontainers.image.source="https://github.com/cloudflare/cloudflared"
 
 # copy our compiled binary
 COPY --from=builder --chown=nonroot /go/src/github.com/cloudflare/cloudflared/cloudflared /usr/local/bin/

Dockerfile.amd64

@@ -0,0 +1,27 @@
+# use a builder image for building cloudflare
+FROM golang:1.19 as builder
+ENV GO111MODULE=on \
+    CGO_ENABLED=0 
+    
+WORKDIR /go/src/github.com/cloudflare/cloudflared/
+
+# copy our sources into the builder image
+COPY . .
+
+# compile cloudflared
+RUN GOOS=linux GOARCH=amd64 make cloudflared
+
+# use a distroless base image with glibc
+FROM gcr.io/distroless/base-debian11:nonroot
+
+LABEL org.opencontainers.image.source="https://github.com/cloudflare/cloudflared"
+
+# copy our compiled binary
+COPY --from=builder --chown=nonroot /go/src/github.com/cloudflare/cloudflared/cloudflared /usr/local/bin/
+
+# run as non-privileged user
+USER nonroot
+
+# command / entrypoint of container
+ENTRYPOINT ["cloudflared", "--no-autoupdate"]
+CMD ["version"]

Dockerfile.arm64

@@ -0,0 +1,27 @@
+# use a builder image for building cloudflare
+FROM golang:1.19 as builder
+ENV GO111MODULE=on \
+    CGO_ENABLED=0 
+    
+WORKDIR /go/src/github.com/cloudflare/cloudflared/
+
+# copy our sources into the builder image
+COPY . .
+
+# compile cloudflared
+RUN GOOS=linux GOARCH=arm64 make cloudflared
+
+# use a distroless base image with glibc
+FROM gcr.io/distroless/base-debian11:nonroot-arm64
+
+LABEL org.opencontainers.image.source="https://github.com/cloudflare/cloudflared"
+
+# copy our compiled binary
+COPY --from=builder --chown=nonroot /go/src/github.com/cloudflare/cloudflared/cloudflared /usr/local/bin/
+
+# run as non-privileged user
+USER nonroot
+
+# command / entrypoint of container
+ENTRYPOINT ["cloudflared", "--no-autoupdate"]
+CMD ["version"]

Makefile

@@ -131,6 +131,10 @@ endif
 container:
 	docker build --build-arg=TARGET_ARCH=$(TARGET_ARCH) --build-arg=TARGET_OS=$(TARGET_OS) -t cloudflare/cloudflared-$(TARGET_OS)-$(TARGET_ARCH):"$(VERSION)" .
 
+.PHONY: generate-docker-version
+generate-docker-version:
+	echo latest $(VERSION) > versions
+
 .PHONY: test
 test: vet
 ifndef CI
@@ -145,24 +149,8 @@ endif
 test-ssh-server:
 	docker-compose -f ssh_server_tests/docker-compose.yml up
 
-define publish_package
-	chmod 664 $(BINARY_NAME)*.$(1); \
-	for HOST in $(CF_PKG_HOSTS); do \
-		ssh-keyscan -t ecdsa $$HOST >> ~/.ssh/known_hosts; \
-		scp -p -4 $(BINARY_NAME)*.$(1) cfsync@$$HOST:/state/cf-pkg/staging/$(2)/$(TARGET_PUBLIC_REPO)/$(BINARY_NAME)/; \
-	done
-endef
-
-.PHONY: publish-deb
-publish-deb: cloudflared-deb
-	$(call publish_package,deb,apt)
-
-.PHONY: publish-rpm
-publish-rpm: cloudflared-rpm
-	$(call publish_package,rpm,yum)
-
 cloudflared.1: cloudflared_man_template
-	cat cloudflared_man_template | sed -e 's/\$${VERSION}/$(VERSION)/; s/\$${DATE}/$(DATE)/' > cloudflared.1
+	sed -e 's/\$${VERSION}/$(VERSION)/; s/\$${DATE}/$(DATE)/' cloudflared_man_template > cloudflared.1
 
 install: cloudflared cloudflared.1
 	mkdir -p $(DESTDIR)$(INSTALL_BINDIR) $(DESTDIR)$(INSTALL_MANDIR)
@@ -198,7 +186,7 @@ cloudflared-pkg: cloudflared cloudflared.1
 	$(call build_package,osxpkg)
 
 .PHONY: cloudflared-msi
-cloudflared-msi: cloudflared
+cloudflared-msi:
 	wixl --define Version=$(VERSION) --define Path=$(EXECUTABLE_PATH) --output cloudflared-$(VERSION)-$(TARGET_ARCH).msi cloudflared.wxs
 
 .PHONY: cloudflared-darwin-amd64.tgz
@@ -288,10 +276,17 @@ github-mac-upload:
 	python3 github_release.py --path artifacts/cloudflared-darwin-amd64.tgz --release-version $(VERSION) --name cloudflared-darwin-amd64.tgz
 	python3 github_release.py --path artifacts/cloudflared-amd64.pkg --release-version $(VERSION) --name cloudflared-amd64.pkg
 
+.PHONY: github-windows-upload
+github-windows-upload:
+	python3 github_release.py --path built_artifacts/cloudflared-windows-amd64.exe --release-version $(VERSION) --name cloudflared-windows-amd64.exe
+	python3 github_release.py --path built_artifacts/cloudflared-windows-amd64.msi --release-version $(VERSION) --name cloudflared-windows-amd64.msi
+	python3 github_release.py --path built_artifacts/cloudflared-windows-386.exe --release-version $(VERSION) --name cloudflared-windows-386.exe
+	python3 github_release.py --path built_artifacts/cloudflared-windows-386.msi --release-version $(VERSION) --name cloudflared-windows-386.msi
+
 .PHONY: tunnelrpc-deps
 tunnelrpc-deps:
 	which capnp  # https://capnproto.org/install.html
-	which capnpc-go  # go get zombiezen.com/go/capnproto2/capnpc-go
+	which capnpc-go  # go install zombiezen.com/go/capnproto2/capnpc-go@latest
 	capnp compile -ogo tunnelrpc/tunnelrpc.capnp
 
 .PHONY: quic-deps
@@ -302,7 +297,7 @@ quic-deps:
 
 .PHONY: vet
 vet:
-	go vet -mod=vendor ./...
+	go vet -v -mod=vendor ./...
 
 .PHONY: goimports
 goimports:

README.md

@@ -25,12 +25,13 @@ routing), but for legacy reasons this requirement is still necessary:
 
 ## Installing `cloudflared`
 
-Downloads are available as standalone binaries, a Docker image, and Debian, RPM, and Homebrew packages. You can also find releases here on the `cloudflared` GitHub repository.
+Downloads are available as standalone binaries, a Docker image, and Debian, RPM, and Homebrew packages. You can also find releases [here](https://github.com/cloudflare/cloudflared/releases) on the `cloudflared` GitHub repository.
 
 * You can [install on macOS](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/install-and-setup/installation#macos) via Homebrew or by downloading the [latest Darwin amd64 release](https://github.com/cloudflare/cloudflared/releases)
 * Binaries, Debian, and RPM packages for Linux [can be found here](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/install-and-setup/installation#linux)
 * A Docker image of `cloudflared` is [available on DockerHub](https://hub.docker.com/r/cloudflare/cloudflared)
 * You can install on Windows machines with the [steps here](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/install-and-setup/installation#windows)
+* Build from source with the [instructions here](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/install-and-setup/installation#build-from-source)
 
 User documentation for Cloudflare Tunnel can be found at https://developers.cloudflare.com/cloudflare-one/connections/connect-apps
 

RELEASE_NOTES

@@ -1,3 +1,141 @@
+2022.10.2
+- 2022-10-18 TUN-6869: Fix Makefile complaining about missing GO packages
+- 2022-10-18 TUN-6864: Don't reuse port in quic unit tests
+- 2022-10-18 TUN-6868: Return left padded tracing ID when tracing identity is converted to string
+
+2022.10.1
+- 2022-10-16 TUN-6861: Trace ICMP on Windows
+- 2022-10-15 TUN-6860: Send access configuration keys to the edge
+- 2022-10-14 TUN-6858: Trace ICMP reply
+- 2022-10-13 TUN-6855: Add DatagramV2Type for IP packet with trace and tracing spans
+- 2022-10-13 TUN-6856: Refactor to lay foundation for tracing ICMP
+- 2022-10-13 TUN-6604: Trace icmp echo request on Linux and Darwin
+- 2022-10-12 Fix log message (#591)
+- 2022-10-12 TUN-6853: Reuse source port when connecting to the edge for quic connections
+- 2022-10-11 TUN-6829: Allow user of datagramsession to control logging level of errors
+- 2022-10-10 RTG-2276 Update qtls and go mod tidy
+- 2022-10-05 Add post-quantum flag to quick tunnel
+- 2022-10-05 TUN-6823: Update github release message to pull from KV
+- 2022-10-04 TUN-6825: Fix cloudflared:version images require arch hyphens
+- 2022-10-03 TUN-6806: Add ingress rule number to log when filtering due to middlware handler
+- 2022-08-17 Label correct container
+- 2022-08-16 Fix typo in help text for `cloudflared tunnel route lb`
+- 2022-07-18 drop usage of cat when sed is invoked to generate the manpage
+- 2021-03-15 update-build-readme
+- 2021-03-15 fix link
+
+2022.10.0
+- 2022-09-30 TUN-6755: Remove unused publish functions
+- 2022-09-30 TUN-6813: Only proxy ICMP packets when warp-routing is enabled
+- 2022-09-29 TUN-6811: Ping group range should be parsed as int32
+- 2022-09-29 TUN-6812: Drop IP packets if ICMP proxy is not initialized
+- 2022-09-28 TUN-6716: Document limitation of Windows ICMP proxy
+- 2022-09-28 TUN-6810: Add component test for post-quantum
+- 2022-09-27 TUN-6715: Provide suggestion to add cloudflared to ping_group_range if it failed to open ICMP socket
+- 2022-09-22 TUN-6792: Fix brew core release by not auditing the formula
+- 2022-09-22 TUN-6774: Validate OriginRequest.Access to add Ingress.Middleware
+- 2022-09-22 TUN-6775: Add middleware.Handler verification to ProxyHTTP
+- 2022-09-22 TUN-6791: Calculate ICMPv6 checksum
+- 2022-09-22 TUN-6801: Add punycode alternatives for ingress rules
+- 2022-09-21 TUN-6772: Add a JWT Validator as an ingress verifier
+- 2022-09-21 TUN-6772: Add a JWT Validator as an ingress verifier
+- 2022-09-21 TUN-6774: Validate OriginRequest.Access to add Ingress.Middleware
+- 2022-09-21 TUN-6772: Add a JWT Validator as an ingress verifier
+- 2022-09-20 TUN-6741: ICMP proxy tries to listen on specific IPv4 & IPv6 when possible
+2022.9.1
+- 2022-09-20 TUN-6777: Fix race condition in TestFunnelIdleTimeout
+- 2022-09-20 TUN-6595: Enable datagramv2 and icmp proxy by default
+- 2022-09-20 TUN-6773: Add access based configuration to ingress.OriginRequestConfig
+- 2022-09-19 TUN-6778: Cleanup logs about ICMP
+- 2022-09-19 TUN-6779: cloudflared should also use the root CAs from system pool to validate edge certificate
+- 2022-09-19 TUN-6780: Add support for certReload to also include support for client certificates
+- 2022-09-16 TUN-6767: Build ICMP proxy for Windows only when CGO is enabled
+- 2022-09-15 TUN-6590: Use Windows Teamcity agent to build binary
+- 2022-09-13 TUN-6592: Decrement TTL and return ICMP time exceed if it's 0
+- 2022-09-09 TUN-6749: Fix icmp_generic build
+- 2022-09-09 TUN-6744: On posix platforms, assign unique echo ID per (src, dst, echo ID)
+- 2022-09-08 TUN-6743: Support ICMPv6 echo on Windows
+- 2022-09-08 TUN-6689: Utilize new RegisterUDPSession to begin tracing
+- 2022-09-07 TUN-6688: Update RegisterUdpSession capnproto to include trace context
+- 2022-09-06 TUN-6740: Detect no UDP packets allowed and fallback from QUIC in that case
+- 2022-09-06 TUN-6654: Support ICMPv6 on Linux and Darwin
+- 2022-09-02 TUN-6696: Refactor flow into funnel and close idle funnels
+- 2022-09-02 TUN-6718: Bump go and go-boring 1.18.6
+- 2022-08-29 TUN-6531: Implement ICMP proxy for Windows using IcmpSendEcho
+- 2022-08-24 RTG-1339 Support post-quantum hybrid key exchange
+
+2022.9.0
+- 2022-09-05 TUN-6737: Fix datagramV2Type should be declared in its own block so it starts at 0
+- 2022-09-01 TUN-6725: Fix testProxySSEAllData
+- 2022-09-01 TUN-6726: Fix maxDatagramPayloadSize for Windows QUIC datagrams
+- 2022-09-01 TUN-6729: Fix flaky TestClosePreviousProxies
+- 2022-09-01 TUN-6728: Verify http status code ingress rule
+- 2022-08-25 TUN-6695: Implement ICMP proxy for linux
+
+2022.8.4
+- 2022-08-31 TUN-6717: Update Github action to run with Go 1.19
+- 2022-08-31 TUN-6720: Remove forcibly closing connection during reconnect signal
+- 2022-08-29 Release 2022.8.3
+
+2022.8.3
+- 2022-08-26 TUN-6708: Fix replace flow logic
+- 2022-08-25 TUN-6705: Tunnel should retry connections forever
+- 2022-08-25 TUN-6704: Honor protocol flag when edge discovery is unreachable
+- 2022-08-25 TUN-6699: Add metric for packet too big dropped
+- 2022-08-24 TUN-6691: Properly error check for net.ErrClosed
+- 2022-08-22 TUN-6679: Allow client side of quic request to close body
+- 2022-08-22 TUN-6586: Change ICMP proxy to only build for Darwin and use echo ID to track flows
+- 2022-08-18 TUN-6530: Implement ICMPv4 proxy
+- 2022-08-17 TUN-6666: Define packet package
+- 2022-08-17 TUN-6667: DatagramMuxerV2 provides a method to receive RawPacket
+- 2022-08-16 TUN-6657: Ask for Tunnel ID and Configuration on Bug Report
+- 2022-08-16 TUN-6676: Add suport for trailers in http2 connections
+- 2022-08-11 TUN-6575: Consume cf-trace-id from incoming http2 TCP requests
+
+2022.8.2
+- 2022-08-16 TUN-6656: Docker for arm64 should not be deployed in an amd64 container
+
+2022.8.1
+- 2022-08-15 TUN-6617: Updated CHANGES.md for protocol stickiness
+- 2022-08-12 EDGEPLAT-3918: bump go and go-boring to 1.18.5
+- 2022-08-12 TUN-6652: Publish dockerfile for both amd64 and arm64
+- 2022-08-11 TUN-6617: Dont fallback to http2 if QUIC conn was successful.
+- 2022-08-11 TUN-6617: Dont fallback to http2 if QUIC conn was successful.
+- 2022-08-11 Revert "TUN-6617: Dont fallback to http2 if QUIC conn was successful."
+- 2022-08-11 TUN-6617: Dont fallback to http2 if QUIC conn was successful.
+- 2022-08-01 TUN-6584: Define QUIC datagram v2 format to support proxying IP packets
+
+2022.8.0
+- 2022-08-10 TUN-6637: Upgrade quic-go
+- 2022-08-10 TUN-6646: Add support to SafeStreamCloser to close only write side of stream
+- 2022-08-09 TUN-6642: Fix unexpected close of quic stream triggered by upstream origin close
+- 2022-08-09 TUN-6639: Validate cyclic ingress configuration
+- 2022-08-08 TUN-6637: Upgrade go version and quic-go
+- 2022-08-08 TUN-6639: Validate cyclic ingress configuration
+- 2022-08-04 EDGEPLAT-3918: build cloudflared for Bookworm
+- 2022-08-02 Revert "TUN-6576: Consume cf-trace-id from incoming TCP requests to create root span"
+- 2022-07-27 TUN-6601: Update gopkg.in/yaml.v3 references in modules
+- 2022-07-26 TUN-6576: Consume cf-trace-id from incoming TCP requests to create root span
+- 2022-07-26 TUN-6576: Consume cf-trace-id from incoming TCP requests to create root span
+- 2022-07-25 TUN-6598: Remove auto assignees on github issues
+- 2022-07-20 TUN-6583: Remove legacy --ui flag
+- 2022-07-20 cURL supports stdin and uses os pipes directly without copying
+- 2022-07-07 TUN-6517: Use QUIC stream context while proxying HTTP requests and TCP connections
+2022.7.1
+- 2022-07-06 TUN-6503: Fix transport fallback from QUIC in face of dial error "no network activity"
+
+2022.7.0
+- 2022-07-05 TUN-6499: Remove log that is per datagram
+- 2022-06-24 TUN-6460: Rename metric label location to edge_location
+- 2022-06-24 TUN-6459: Add cloudflared user-agent to access calls
+- 2022-06-17 TUN-6427: Differentiate between upstream request closed/canceled and failed origin requests
+- 2022-06-17 TUN-6388: Fix first tunnel connection not retrying
+- 2022-06-13 TUN-6384: Correct duplicate connection error to fetch new IP first
+- 2022-06-13 TUN-6373: Add edge-ip-version to remotely pushed configuration
+- 2022-06-07 TUN-6010: Add component tests for --edge-ip-version
+- 2022-05-20 TUN-6007: Implement new edge discovery algorithm
+- 2022-02-18 Ensure service install directories are created before writing file
+
 2022.6.3
 - 2022-06-20 TUN-6362: Add armhf support to cloudflare packaging
 

build-packages.sh

@@ -7,15 +7,6 @@ export CGO_ENABLED=0
 # This controls the directory the built artifacts go into
 export ARTIFACT_DIR=built_artifacts/
 mkdir -p $ARTIFACT_DIR
-windowsArchs=("amd64" "386")
-export TARGET_OS=windows
-for arch in ${windowsArchs[@]}; do
-    export TARGET_ARCH=$arch
-    make cloudflared-msi
-    mv ./cloudflared.exe $ARTIFACT_DIR/cloudflared-windows-$arch.exe
-    mv cloudflared-$VERSION-$arch.msi $ARTIFACT_DIR/cloudflared-windows-$arch.msi
-done
-
 
 linuxArchs=("386" "amd64" "arm" "armhf" "arm64")
 export TARGET_OS=linux

cfsetup.yaml

@@ -1,8 +1,8 @@
-pinned_go: &pinned_go go=1.17.10-1
-pinned_go_fips: &pinned_go_fips go-boring=1.17.9-1
+pinned_go: &pinned_go go=1.18.6-1
+pinned_go_fips: &pinned_go_fips go-boring=1.18.6-1
 
 build_dir: &build_dir /cfsetup_build
-default-flavor: buster
+default-flavor: bullseye
 stretch: &stretch
   build:
     build_dir: *build_dir
@@ -78,6 +78,13 @@ stretch: &stretch
       # same logic as above, but for FIPS packages only
       - ./build-packages-fips.sh
       - make github-release-built-pkgs
+  generate-versions-file:
+    build_dir: *build_dir
+    builddeps: 
+      - *pinned_go
+      - build-essential
+    post-cache:
+      - make generate-docker-version
   build-deb:
     build_dir: *build_dir
     builddeps: &build_deb_deps
@@ -119,21 +126,9 @@ stretch: &stretch
       - export GOOS=linux
       - export GOARCH=arm64
       - make cloudflared-deb
-  publish-deb:
-    build_dir: *build_dir
-    builddeps:
-      - *pinned_go
-      - build-essential
-      - fakeroot
-      - rubygem-fpm
-      - openssh-client
-    post-cache:
-      - export GOOS=linux
-      - export GOARCH=amd64
-      - make publish-deb
   github-release-macos-amd64:
     build_dir: *build_dir
-    builddeps:
+    builddeps: &build_pygithub
       - *pinned_go
       - build-essential
       - python3-dev
@@ -145,6 +140,27 @@ stretch: &stretch
       - pip3 install pygithub==1.55
     post-cache:
       - make github-mac-upload
+  github-release-windows:
+    build_dir: *build_dir
+    builddeps:
+      - *pinned_go
+      - build-essential
+      - python3-dev
+      - libffi-dev
+      - python3-setuptools
+      - python3-pip
+      - wget
+      # libmsi and libgcab are libraries the wixl binary depends on.
+      - libmsi-dev
+      - libgcab-dev
+    pre-cache:
+      - wget https://github.com/sudarshan-reddy/msitools/releases/download/v0.101b/wixl -P /usr/local/bin
+      - chmod a+x /usr/local/bin/wixl
+      - pip3 install pynacl==1.4.0
+      - pip3 install pygithub==1.55
+    post-cache:
+      - .teamcity/package-windows.sh
+      - make github-windows-upload
   test:
     build_dir: *build_dir
     builddeps:
@@ -152,7 +168,7 @@ stretch: &stretch
       - build-essential
       - gotest-to-teamcity
     pre-cache: &test_pre_cache
-      - go get golang.org/x/tools/cmd/goimports
+      - go install golang.org/x/tools/cmd/goimports@latest
     post-cache:
       - export GOOS=linux
       - export GOARCH=amd64
@@ -174,6 +190,26 @@ stretch: &stretch
       - ./fmt-check.sh
       - make test | gotest-to-teamcity
   component-test:
+    build_dir: *build_dir
+    builddeps:
+      - *pinned_go
+      - python3.7
+      - python3-pip
+      - python3-setuptools
+      # procps installs the ps command which is needed in test_sysv_service because the init script
+      # uses ps pid to determine if the agent is running
+      - procps
+    pre-cache-copy-paths:
+      - component-tests/requirements.txt
+    pre-cache: &component_test_pre_cache
+      - sudo pip3 install --upgrade -r component-tests/requirements.txt
+    post-cache: &component_test_post_cache
+      # Creates and routes a Named Tunnel for this build. Also constructs config file from env vars.
+      - python3 component-tests/setup.py --type create
+      - pytest component-tests -o log_cli=true --log-cli-level=INFO
+      # The Named Tunnel is deleted and its route unprovisioned here.
+      - python3 component-tests/setup.py --type cleanup
+  component-test-fips:
     build_dir: *build_dir
     builddeps:
       - *pinned_go_fips
@@ -185,32 +221,21 @@ stretch: &stretch
       - procps
     pre-cache-copy-paths:
       - component-tests/requirements.txt
-    pre-cache:
-      - sudo pip3 install --upgrade -r component-tests/requirements.txt
-    post-cache:
-      # Creates and routes a Named Tunnel for this build. Also constructs config file from env vars.
-      - python3 component-tests/setup.py --type create
-      - pytest component-tests -o log_cli=true --log-cli-level=INFO
-      # The Named Tunnel is deleted and its route unprovisioned here.
-      - python3 component-tests/setup.py --type cleanup
+    pre-cache: *component_test_pre_cache
+    post-cache: *component_test_post_cache
   update-homebrew:
     builddeps:
       - openssh-client
       - s3cmd
       - jq
       - build-essential
+      - procps
     post-cache:
       - .teamcity/update-homebrew.sh
       - .teamcity/update-homebrew-core.sh
   github-message-release:
     build_dir: *build_dir
-    builddeps:
-      - *pinned_go
-      - build-essential
-      - python3-dev
-      - libffi-dev
-      - python3-setuptools
-      - python3-pip
+    builddeps: *build_pygithub
     pre-cache: *install_pygithub
     post-cache:
       - make github-message
@@ -221,13 +246,13 @@ stretch: &stretch
       - build-essential
       - python3
       - genisoimage
-      - jetez
     pre-cache:
       - ln -s /usr/bin/genisoimage /usr/bin/mkisofs
     post-cache:
+      - export CGO_ENABLED=0
       - export GOOS=freebsd
       - export GOARCH=amd64
-      - make cloudflared-junos
+      - make cloudflared
   publish-junos:
     build_dir: *build_dir
     builddeps:
@@ -246,18 +271,4 @@ stretch: &stretch
 
 buster: *stretch
 bullseye: *stretch
-centos-7:
-  publish-rpm:
-    build_dir: *build_dir
-    builddeps: &el7_builddeps
-      - https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
-    pre-cache:
-      - yum install -y fakeroot
-      - yum upgrade -y binutils-2.27-44.base.el7.x86_64
-      - wget https://go.dev/dl/go1.17.10.linux-amd64.tar.gz -P /tmp/
-      - tar -C /usr/local -xzf /tmp/go1.17.10.linux-amd64.tar.gz
-    post-cache:
-      - export PATH=$PATH:/usr/local/go/bin
-      - export GOOS=linux
-      - export GOARCH=amd64
-      - make publish-rpm
+bookworm: *stretch

cmd/cloudflared/access/cmd.go

@@ -56,11 +56,13 @@ const sentryDSN = "https://56a9c9fa5c364ab28f34b14f35ea0f1b@sentry.io/189878"
 
 var (
 	shutdownC chan struct{}
+	userAgent = "DEV"
 )
 
 // Init will initialize and store vars from the main program
-func Init(shutdown chan struct{}) {
+func Init(shutdown chan struct{}, version string) {
 	shutdownC = shutdown
+	userAgent = fmt.Sprintf("cloudflared/%s", version)
 }
 
 // Flags return the global flags for Access related commands (hopefully none)
@@ -294,6 +296,7 @@ func curl(c *cli.Context) error {
 // run kicks off a shell task and pipe the results to the respective std pipes
 func run(cmd string, args ...string) error {
 	c := exec.Command(cmd, args...)
+	c.Stdin = os.Stdin
 	stderr, err := c.StderrPipe()
 	if err != nil {
 		return err
@@ -505,7 +508,7 @@ func isTokenValid(options *carrier.StartOptions, log *zerolog.Logger) (bool, err
 	if err != nil {
 		return false, errors.Wrap(err, "Could not create access request")
 	}
-
+	req.Header.Set("User-Agent", userAgent)
 	// Do not follow redirects
 	client := &http.Client{
 		CheckRedirect: func(req *http.Request, via []*http.Request) error {

cmd/cloudflared/main.go

@@ -21,6 +21,7 @@ import (
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/metrics"
 	"github.com/cloudflare/cloudflared/overwatch"
+	"github.com/cloudflare/cloudflared/token"
 	"github.com/cloudflare/cloudflared/tracing"
 	"github.com/cloudflare/cloudflared/watcher"
 )
@@ -85,9 +86,10 @@ func main() {
 	app.Commands = commands(cli.ShowVersion)
 
 	tunnel.Init(bInfo, graceShutdownC) // we need this to support the tunnel sub command...
-	access.Init(graceShutdownC)
+	access.Init(graceShutdownC, Version)
 	updater.Init(Version)
 	tracing.Init(Version)
+	token.Init(Version)
 	runApp(app, graceShutdownC)
 }
 

cmd/cloudflared/service_template.go

@@ -8,6 +8,7 @@ import (
 	"io/ioutil"
 	"os"
 	"os/exec"
+	"path"
 	"text/template"
 
 	homedir "github.com/mitchellh/go-homedir"
@@ -52,10 +53,17 @@ func (st *ServiceTemplate) Generate(args *ServiceTemplateArgs) error {
 	if err != nil {
 		return fmt.Errorf("error generating %s: %v", st.Path, err)
 	}
-	fileMode := os.FileMode(0644)
+	fileMode := os.FileMode(0o644)
 	if st.FileMode != 0 {
 		fileMode = st.FileMode
 	}
+
+	plistFolder := path.Dir(resolvedPath)
+	err = os.MkdirAll(plistFolder, 0o755)
+	if err != nil {
+		return fmt.Errorf("error creating %s: %v", plistFolder, err)
+	}
+
 	err = ioutil.WriteFile(resolvedPath, buffer.Bytes(), fileMode)
 	if err != nil {
 		return fmt.Errorf("error writing %s: %v", resolvedPath, err)

cmd/cloudflared/tunnel/cmd.go

@@ -25,7 +25,6 @@ import (
 	"github.com/cloudflare/cloudflared/cfapi"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/proxydns"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/ui"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/updater"
 	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/connection"
@@ -218,7 +217,7 @@ func runAdhocNamedTunnel(sc *subcommandContext, name, credentialsOutputPath stri
 
 // runClassicTunnel creates a "classic" non-named tunnel
 func runClassicTunnel(sc *subcommandContext) error {
-	return StartServer(sc.c, buildInfo, nil, sc.log, sc.isUIEnabled)
+	return StartServer(sc.c, buildInfo, nil, sc.log)
 }
 
 func routeFromFlag(c *cli.Context) (route cfapi.HostnameRoute, ok bool) {
@@ -236,7 +235,6 @@ func StartServer(
 	info *cliutil.BuildInfo,
 	namedTunnel *connection.NamedTunnelProperties,
 	log *zerolog.Logger,
-	isUIEnabled bool,
 ) error {
 	_ = raven.SetDSN(sentryDSN)
 	var wg sync.WaitGroup
@@ -331,9 +329,9 @@ func StartServer(
 		return fmt.Errorf(errText)
 	}
 
-	logTransport := logger.CreateTransportLoggerFromContext(c, isUIEnabled)
+	logTransport := logger.CreateTransportLoggerFromContext(c, logger.EnableTerminalLog)
 
-	observer := connection.NewObserver(log, logTransport, isUIEnabled)
+	observer := connection.NewObserver(log, logTransport)
 
 	// Send Quick Tunnel URL to UI if applicable
 	var quickTunnelURL string
@@ -377,7 +375,7 @@ func StartServer(
 		errC <- metrics.ServeMetrics(metricsListener, ctx.Done(), readinessServer, quickTunnelURL, orchestrator, log)
 	}()
 
-	reconnectCh := make(chan supervisor.ReconnectSignal, 1)
+	reconnectCh := make(chan supervisor.ReconnectSignal, c.Int("ha-connections"))
 	if c.IsSet("stdin-control") {
 		log.Info().Msg("Enabling control through stdin")
 		go stdinControl(reconnectCh, log)
@@ -392,18 +390,6 @@ func StartServer(
 		errC <- supervisor.StartTunnelDaemon(ctx, tunnelConfig, orchestrator, connectedSignal, reconnectCh, graceShutdownC)
 	}()
 
-	if isUIEnabled {
-		tunnelUI := ui.NewUIModel(
-			info.Version(),
-			hostname,
-			metricsListener.Addr().String(),
-			orchestratorConfig.Ingress,
-			tunnelConfig.HAConnections,
-		)
-		app := tunnelUI.Launch(ctx, log, logTransport)
-		observer.RegisterSink(app)
-	}
-
 	gracePeriod, err := gracePeriod(c)
 	if err != nil {
 		return err
@@ -528,7 +514,7 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			Usage:   "Cloudflare Edge ip address version to connect with. {4, 6, auto}",
 			EnvVars: []string{"TUNNEL_EDGE_IP_VERSION"},
 			Value:   "4",
-			Hidden:  true,
+			Hidden:  false,
 		}),
 		altsrc.NewStringFlag(&cli.StringFlag{
 			Name:    tlsconfig.CaCertFlag,
@@ -663,9 +649,9 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 		}),
 		altsrc.NewBoolFlag(&cli.BoolFlag{
 			Name:   uiFlag,
-			Usage:  "Launch tunnel UI. Tunnel logs are scrollable via 'j', 'k', or arrow keys.",
+			Usage:  "(depreciated) Launch tunnel UI. Tunnel logs are scrollable via 'j', 'k', or arrow keys.",
 			Value:  false,
-			Hidden: shouldHide,
+			Hidden: true,
 		}),
 		altsrc.NewStringFlag(&cli.StringFlag{
 			Name:   "quick-service",
@@ -679,6 +665,13 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			EnvVars: []string{"TUNNEL_MAX_FETCH_SIZE"},
 			Hidden:  true,
 		}),
+		altsrc.NewBoolFlag(&cli.BoolFlag{
+			Name:    "post-quantum",
+			Usage:   "When given creates an experimental post-quantum secure tunnel",
+			Aliases: []string{"pq"},
+			EnvVars: []string{"TUNNEL_POST_QUANTUM"},
+			Hidden:  FipsEnabled,
+		}),
 		selectProtocolFlag,
 		overwriteDNSFlag,
 	}...)

cmd/cloudflared/tunnel/configuration.go

@@ -4,6 +4,9 @@ import (
 	"crypto/tls"
 	"fmt"
 	"io/ioutil"
+	mathRand "math/rand"
+	"net"
+	"net/netip"
 	"os"
 	"path/filepath"
 	"strings"
@@ -43,9 +46,9 @@ var (
 	LogFieldHostname = "hostname"
 
 	secretFlags     = [2]*altsrc.StringFlag{credentialsContentsFlag, tunnelTokenFlag}
-	defaultFeatures = []string{supervisor.FeatureAllowRemoteConfig, supervisor.FeatureSerializedHeaders}
+	defaultFeatures = []string{supervisor.FeatureAllowRemoteConfig, supervisor.FeatureSerializedHeaders, supervisor.FeatureDatagramV2}
 
-	configFlags = []string{"autoupdate-freq", "no-autoupdate", "retries", "protocol", "loglevel", "transport-loglevel", "origincert", "metrics", "metrics-update-freq"}
+	configFlags = []string{"autoupdate-freq", "no-autoupdate", "retries", "protocol", "loglevel", "transport-loglevel", "origincert", "metrics", "metrics-update-freq", "edge-ip-version"}
 )
 
 // returns the first path that contains a cert.pem file. If none of the DefaultConfigSearchDirectories
@@ -220,6 +223,19 @@ func prepareTunnelConfig(
 	)
 
 	transportProtocol := c.String("protocol")
+
+	needPQ := c.Bool("post-quantum")
+	if needPQ {
+		if FipsEnabled {
+			return nil, nil, fmt.Errorf("post-quantum not supported in FIPS mode")
+		}
+		// Error if the user tries to force a non-quic transport protocol
+		if transportProtocol != connection.AutoSelectFlag && transportProtocol != connection.QUIC.String() {
+			return nil, nil, fmt.Errorf("post-quantum is only supported with the quic transport")
+		}
+		transportProtocol = connection.QUIC.String()
+	}
+
 	protocolFetcher := edgediscovery.ProtocolPercentage
 
 	cfg := config.GetConfiguration()
@@ -230,6 +246,9 @@ func prepareTunnelConfig(
 		}
 		log.Info().Msgf("Generated Connector ID: %s", clientUUID)
 		features := append(c.StringSlice("features"), defaultFeatures...)
+		if needPQ {
+			features = append(features, supervisor.FeaturePostQuantum)
+		}
 		if c.IsSet(TunnelTokenFlag) {
 			if transportProtocol == connection.AutoSelectFlag {
 				protocolFetcher = func() (edgediscovery.ProtocolPercents, error) {
@@ -291,7 +310,7 @@ func prepareTunnelConfig(
 	}
 
 	warpRoutingEnabled := isWarpRoutingEnabled(cfg.WarpRouting, isNamedTunnel)
-	protocolSelector, err := connection.NewProtocolSelector(transportProtocol, warpRoutingEnabled, namedTunnel, protocolFetcher, supervisor.ResolveTTL, log)
+	protocolSelector, err := connection.NewProtocolSelector(transportProtocol, warpRoutingEnabled, namedTunnel, protocolFetcher, supervisor.ResolveTTL, log, c.Bool("post-quantum"))
 	if err != nil {
 		return nil, nil, err
 	}
@@ -330,6 +349,15 @@ func prepareTunnelConfig(
 		return nil, nil, err
 	}
 
+	var pqKexIdx int
+	if needPQ {
+		pqKexIdx = mathRand.Intn(len(supervisor.PQKexes))
+		log.Info().Msgf(
+			"Using experimental hybrid post-quantum key agreement %s",
+			supervisor.PQKexNames[supervisor.PQKexes[pqKexIdx]],
+		)
+	}
+
 	tunnelConfig := &supervisor.TunnelConfig{
 		GracePeriod:     gracePeriod,
 		ReplaceExisting: c.Bool("force"),
@@ -355,6 +383,14 @@ func prepareTunnelConfig(
 		MuxerConfig:      muxerConfig,
 		ProtocolSelector: protocolSelector,
 		EdgeTLSConfigs:   edgeTLSConfigs,
+		NeedPQ:           needPQ,
+		PQKexIdx:         pqKexIdx,
+	}
+	packetConfig, err := newPacketConfig(c, log)
+	if err != nil {
+		log.Warn().Err(err).Msg("ICMP proxy feature is disabled")
+	} else {
+		tunnelConfig.PacketConfig = packetConfig
 	}
 	orchestratorConfig := &orchestration.Config{
 		Ingress:            &ingressRules,
@@ -425,3 +461,147 @@ func parseConfigIPVersion(version string) (v allregions.ConfigIPVersion, err err
 	}
 	return
 }
+
+func newPacketConfig(c *cli.Context, logger *zerolog.Logger) (*ingress.GlobalRouterConfig, error) {
+	ipv4Src, err := determineICMPv4Src(c.String("icmpv4-src"), logger)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to determine IPv4 source address for ICMP proxy")
+	}
+	logger.Info().Msgf("ICMP proxy will use %s as source for IPv4", ipv4Src)
+
+	ipv6Src, zone, err := determineICMPv6Src(c.String("icmpv6-src"), logger, ipv4Src)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to determine IPv6 source address for ICMP proxy")
+	}
+	if zone != "" {
+		logger.Info().Msgf("ICMP proxy will use %s in zone %s as source for IPv6", ipv6Src, zone)
+	} else {
+		logger.Info().Msgf("ICMP proxy will use %s as source for IPv6", ipv6Src)
+	}
+
+	icmpRouter, err := ingress.NewICMPRouter(ipv4Src, ipv6Src, zone, logger)
+	if err != nil {
+		return nil, err
+	}
+	return &ingress.GlobalRouterConfig{
+		ICMPRouter: icmpRouter,
+		IPv4Src:    ipv4Src,
+		IPv6Src:    ipv6Src,
+		Zone:       zone,
+	}, nil
+}
+
+func determineICMPv4Src(userDefinedSrc string, logger *zerolog.Logger) (netip.Addr, error) {
+	if userDefinedSrc != "" {
+		addr, err := netip.ParseAddr(userDefinedSrc)
+		if err != nil {
+			return netip.Addr{}, err
+		}
+		if addr.Is4() {
+			return addr, nil
+		}
+		return netip.Addr{}, fmt.Errorf("expect IPv4, but %s is IPv6", userDefinedSrc)
+	}
+
+	addr, err := findLocalAddr(net.ParseIP("192.168.0.1"), 53)
+	if err != nil {
+		addr = netip.IPv4Unspecified()
+		logger.Debug().Err(err).Msgf("Failed to determine the IPv4 for this machine. It will use %s to send/listen for ICMPv4 echo", addr)
+	}
+	return addr, nil
+}
+
+type interfaceIP struct {
+	name string
+	ip   net.IP
+}
+
+func determineICMPv6Src(userDefinedSrc string, logger *zerolog.Logger, ipv4Src netip.Addr) (addr netip.Addr, zone string, err error) {
+	if userDefinedSrc != "" {
+		userDefinedIP, zone, _ := strings.Cut(userDefinedSrc, "%")
+		addr, err := netip.ParseAddr(userDefinedIP)
+		if err != nil {
+			return netip.Addr{}, "", err
+		}
+		if addr.Is6() {
+			return addr, zone, nil
+		}
+		return netip.Addr{}, "", fmt.Errorf("expect IPv6, but %s is IPv4", userDefinedSrc)
+	}
+
+	// Loop through all the interfaces, the preference is
+	// 1. The interface where ipv4Src is in
+	// 2. Interface with IPv6 address
+	// 3. Unspecified interface
+
+	interfaces, err := net.Interfaces()
+	if err != nil {
+		return netip.IPv6Unspecified(), "", nil
+	}
+
+	interfacesWithIPv6 := make([]interfaceIP, 0)
+	for _, interf := range interfaces {
+		interfaceAddrs, err := interf.Addrs()
+		if err != nil {
+			continue
+		}
+
+		foundIPv4SrcInterface := false
+		for _, interfaceAddr := range interfaceAddrs {
+			if ipnet, ok := interfaceAddr.(*net.IPNet); ok {
+				ip := ipnet.IP
+				if ip.Equal(ipv4Src.AsSlice()) {
+					foundIPv4SrcInterface = true
+				}
+				if ip.To4() == nil {
+					interfacesWithIPv6 = append(interfacesWithIPv6, interfaceIP{
+						name: interf.Name,
+						ip:   ip,
+					})
+				}
+			}
+		}
+		// Found the interface of ipv4Src. Loop through the addresses to see if there is an IPv6
+		if foundIPv4SrcInterface {
+			for _, interfaceAddr := range interfaceAddrs {
+				if ipnet, ok := interfaceAddr.(*net.IPNet); ok {
+					ip := ipnet.IP
+					if ip.To4() == nil {
+						addr, err := netip.ParseAddr(ip.String())
+						if err == nil {
+							return addr, interf.Name, nil
+						}
+					}
+				}
+			}
+		}
+	}
+
+	for _, interf := range interfacesWithIPv6 {
+		addr, err := netip.ParseAddr(interf.ip.String())
+		if err == nil {
+			return addr, interf.name, nil
+		}
+	}
+	logger.Debug().Err(err).Msgf("Failed to determine the IPv6 for this machine. It will use %s to send/listen for ICMPv6 echo", netip.IPv6Unspecified())
+
+	return netip.IPv6Unspecified(), "", nil
+}
+
+// FindLocalAddr tries to dial UDP and returns the local address picked by the OS
+func findLocalAddr(dst net.IP, port int) (netip.Addr, error) {
+	udpConn, err := net.DialUDP("udp", nil, &net.UDPAddr{
+		IP:   dst,
+		Port: port,
+	})
+	if err != nil {
+		return netip.Addr{}, err
+	}
+	defer udpConn.Close()
+	localAddrPort, err := netip.ParseAddrPort(udpConn.LocalAddr().String())
+	if err != nil {
+		return netip.Addr{}, err
+	}
+	localAddr := localAddrPort.Addr()
+	return localAddr, nil
+}

cmd/cloudflared/tunnel/fips.go

@@ -0,0 +1,3 @@
+package tunnel
+
+var FipsEnabled bool

cmd/cloudflared/tunnel/quick_tunnel.go

@@ -78,7 +78,6 @@ func RunQuickTunnel(sc *subcommandContext) error {
 		buildInfo,
 		&connection.NamedTunnelProperties{Credentials: credentials, QuickTunnelUrl: data.Result.Hostname},
 		sc.log,
-		sc.isUIEnabled,
 	)
 }
 

cmd/cloudflared/tunnel/subcommand_context.go

@@ -31,10 +31,9 @@ func (e errInvalidJSONCredential) Error() string {
 // subcommandContext carries structs shared between subcommands, to reduce number of arguments needed to
 // pass between subcommands, and make sure they are only initialized once
 type subcommandContext struct {
-	c           *cli.Context
-	log         *zerolog.Logger
-	isUIEnabled bool
-	fs          fileSystem
+	c   *cli.Context
+	log *zerolog.Logger
+	fs  fileSystem
 
 	// These fields should be accessed using their respective Getter
 	tunnelstoreClient cfapi.Client
@@ -42,16 +41,10 @@ type subcommandContext struct {
 }
 
 func newSubcommandContext(c *cli.Context) (*subcommandContext, error) {
-	isUIEnabled := c.IsSet(uiFlag) && c.String("name") != ""
-
-	// If UI is enabled, terminal log output should be disabled -- log should be written into a UI log window instead
-	log := logger.CreateLoggerFromContext(c, isUIEnabled)
-
 	return &subcommandContext{
-		c:           c,
-		log:         log,
-		isUIEnabled: isUIEnabled,
-		fs:          realFileSystem{},
+		c:   c,
+		log: logger.CreateLoggerFromContext(c, logger.EnableTerminalLog),
+		fs:  realFileSystem{},
 	}, nil
 }
 
@@ -312,7 +305,6 @@ func (sc *subcommandContext) runWithCredentials(credentials connection.Credentia
 		buildInfo,
 		&connection.NamedTunnelProperties{Credentials: credentials},
 		sc.log,
-		sc.isUIEnabled,
 	)
 }
 

cmd/cloudflared/tunnel/subcommand_context_test.go

@@ -35,7 +35,6 @@ func Test_subcommandContext_findCredentials(t *testing.T) {
 	type fields struct {
 		c                 *cli.Context
 		log               *zerolog.Logger
-		isUIEnabled       bool
 		fs                fileSystem
 		tunnelstoreClient cfapi.Client
 		userCredential    *userCredential
@@ -168,7 +167,6 @@ func Test_subcommandContext_findCredentials(t *testing.T) {
 			sc := &subcommandContext{
 				c:                 tt.fields.c,
 				log:               tt.fields.log,
-				isUIEnabled:       tt.fields.isUIEnabled,
 				fs:                tt.fields.fs,
 				tunnelstoreClient: tt.fields.tunnelstoreClient,
 				userCredential:    tt.fields.userCredential,
@@ -307,7 +305,6 @@ func Test_subcommandContext_Delete(t *testing.T) {
 			sc := &subcommandContext{
 				c:                 tt.fields.c,
 				log:               tt.fields.log,
-				isUIEnabled:       tt.fields.isUIEnabled,
 				fs:                tt.fields.fs,
 				tunnelstoreClient: tt.fields.tunnelstoreClient,
 				userCredential:    tt.fields.userCredential,

cmd/cloudflared/tunnel/subcommands.go

@@ -140,6 +140,13 @@ var (
 		EnvVars: []string{"TUNNEL_TRANSPORT_PROTOCOL"},
 		Hidden:  true,
 	})
+	postQuantumFlag = altsrc.NewBoolFlag(&cli.BoolFlag{
+		Name:    "post-quantum",
+		Usage:   "When given creates an experimental post-quantum secure tunnel",
+		Aliases: []string{"pq"},
+		EnvVars: []string{"TUNNEL_POST_QUANTUM"},
+		Hidden:  FipsEnabled,
+	})
 	sortInfoByFlag = &cli.StringFlag{
 		Name:    "sort-by",
 		Value:   "createdAt",
@@ -169,6 +176,16 @@ var (
 		Usage:   "Base64 encoded secret to set for the tunnel. The decoded secret must be at least 32 bytes long. If not specified, a random 32-byte secret will be generated.",
 		EnvVars: []string{"TUNNEL_CREATE_SECRET"},
 	}
+	icmpv4SrcFlag = &cli.StringFlag{
+		Name:    "icmpv4-src",
+		Usage:   "Source address to send/receive ICMPv4 messages. If not provided cloudflared will dial a local address to determine the source IP or fallback to 0.0.0.0.",
+		EnvVars: []string{"TUNNEL_ICMPV4_SRC"},
+	}
+	icmpv6SrcFlag = &cli.StringFlag{
+		Name:    "icmpv6-src",
+		Usage:   "Source address and the interface name to send/receive ICMPv6 messages. If not provided cloudflared will dial a local address to determine the source IP or fallback to ::.",
+		EnvVars: []string{"TUNNEL_ICMPV6_SRC"},
+	}
 )
 
 func buildCreateCommand() *cli.Command {
@@ -602,9 +619,12 @@ func buildRunCommand() *cli.Command {
 		forceFlag,
 		credentialsFileFlag,
 		credentialsContentsFlag,
+		postQuantumFlag,
 		selectProtocolFlag,
 		featuresFlag,
 		tunnelTokenFlag,
+		icmpv4SrcFlag,
+		icmpv6SrcFlag,
 	}
 	flags = append(flags, configureProxyFlags(false)...)
 	return &cli.Command{
@@ -801,7 +821,7 @@ Further information about managing Cloudflare WARP traffic to your tunnel is ava
 				Name:        "lb",
 				Action:      cliutil.ConfiguredAction(routeLbCommand),
 				Usage:       "Use this tunnel as a load balancer origin, creating pool and load balancer if necessary",
-				UsageText:   "cloudflared tunnel route dns [TUNNEL] [HOSTNAME] [LB-POOL]",
+				UsageText:   "cloudflared tunnel route lb [TUNNEL] [HOSTNAME] [LB-POOL]",
 				Description: `Creates Load Balancer with an origin pool that points to the tunnel.`,
 			},
 			buildRouteIPSubcommand(),

cmd/cloudflared/ui/launch_ui.go

@@ -1,223 +0,0 @@
-package ui
-
-import (
-	"context"
-	"fmt"
-	"strings"
-
-	"github.com/gdamore/tcell"
-	"github.com/rivo/tview"
-	"github.com/rs/zerolog"
-
-	"github.com/cloudflare/cloudflared/connection"
-	"github.com/cloudflare/cloudflared/ingress"
-)
-
-type connState struct {
-	location string
-}
-
-type uiModel struct {
-	version       string
-	edgeURL       string
-	metricsURL    string
-	localServices []string
-	connections   []connState
-}
-
-type palette struct {
-	url          string
-	connected    string
-	defaultText  string
-	disconnected string
-	reconnecting string
-	unregistered string
-}
-
-func NewUIModel(version, hostname, metricsURL string, ing *ingress.Ingress, haConnections int) *uiModel {
-	localServices := make([]string, len(ing.Rules))
-	for i, rule := range ing.Rules {
-		localServices[i] = rule.Service.String()
-	}
-	return &uiModel{
-		version:       version,
-		edgeURL:       hostname,
-		metricsURL:    metricsURL,
-		localServices: localServices,
-		connections:   make([]connState, haConnections),
-	}
-}
-
-func (data *uiModel) Launch(
-	ctx context.Context,
-	log, transportLog *zerolog.Logger,
-) connection.EventSink {
-	// Configure the logger to stream logs into the textview
-
-	// Add TextView as a group to write output to
-	logTextView := NewDynamicColorTextView()
-	// TODO: Format log for UI
-	//log.Add(logTextView, logger.NewUIFormatter(time.RFC3339), logLevels...)
-	//transportLog.Add(logTextView, logger.NewUIFormatter(time.RFC3339), logLevels...)
-
-	// Construct the UI
-	palette := palette{
-		url:          "lightblue",
-		connected:    "lime",
-		defaultText:  "white",
-		disconnected: "red",
-		reconnecting: "orange",
-		unregistered: "orange",
-	}
-
-	app := tview.NewApplication()
-
-	grid := tview.NewGrid().SetGap(1, 0)
-	frame := tview.NewFrame(grid)
-	header := fmt.Sprintf("cloudflared [::b]%s", data.version)
-
-	frame.AddText(header, true, tview.AlignLeft, tcell.ColorWhite)
-
-	// Create table to store connection info and status
-	connTable := tview.NewTable()
-	// SetColumns takes a value for each column, representing the size of the column
-	// Numbers <= 0 represent proportional widths and positive numbers represent absolute widths
-	grid.SetColumns(20, 0)
-
-	// SetRows takes a value for each row, representing the size of the row
-	grid.SetRows(1, 1, len(data.connections), 1, 0)
-
-	// AddItem takes a primitive tview type, row, column, rowSpan, columnSpan, minGridHeight, minGridWidth, and focus
-	grid.AddItem(tview.NewTextView().SetText("Tunnel:"), 0, 0, 1, 1, 0, 0, false)
-	grid.AddItem(tview.NewTextView().SetText("Status:"), 1, 0, 1, 1, 0, 0, false)
-	grid.AddItem(tview.NewTextView().SetText("Connections:"), 2, 0, 1, 1, 0, 0, false)
-
-	grid.AddItem(tview.NewTextView().SetText("Metrics:"), 3, 0, 1, 1, 0, 0, false)
-
-	tunnelHostText := tview.NewTextView().SetText(data.edgeURL)
-
-	grid.AddItem(tunnelHostText, 0, 1, 1, 1, 0, 0, false)
-	status := fmt.Sprintf("[%s]\u2022[%s] Proxying to [%s::b]%s", palette.connected, palette.defaultText, palette.url, strings.Join(data.localServices, ", "))
-	grid.AddItem(NewDynamicColorTextView().SetText(status), 1, 1, 1, 1, 0, 0, false)
-
-	grid.AddItem(connTable, 2, 1, 1, 1, 0, 0, false)
-
-	grid.AddItem(NewDynamicColorTextView().SetText(fmt.Sprintf("Metrics at [%s::b]http://%s/metrics", palette.url, data.metricsURL)), 3, 1, 1, 1, 0, 0, false)
-
-	// Add TextView to stream logs
-	// Logs are displayed in a new grid so a border can be set around them
-	logGrid := tview.NewGrid().SetBorders(true).AddItem(logTextView.SetChangedFunc(handleNewText(app, logTextView)), 0, 0, 5, 2, 0, 0, false)
-	// LogFrame holds the Logs header as well as the grid with the textView for streamed logs
-	logFrame := tview.NewFrame(logGrid).AddText("[::b]Logs:[::-]", true, tview.AlignLeft, tcell.ColorWhite).SetBorders(0, 0, 0, 0, 0, 0)
-	// Footer for log frame
-	logFrame.AddText("[::d]Use Ctrl+C to exit[::-]", false, tview.AlignRight, tcell.ColorWhite)
-	grid.AddItem(logFrame, 4, 0, 5, 2, 0, 0, false)
-
-	go func() {
-		<-ctx.Done()
-		app.Stop()
-		return
-	}()
-
-	go func() {
-		if err := app.SetRoot(frame, true).Run(); err != nil {
-			log.Error().Msgf("Error launching UI: %s", err)
-		}
-	}()
-
-	return connection.EventSinkFunc(func(event connection.Event) {
-		switch event.EventType {
-		case connection.Connected:
-			data.setConnTableCell(event, connTable, palette)
-		case connection.Disconnected, connection.Reconnecting, connection.Unregistering:
-			data.changeConnStatus(event, connTable, log, palette)
-		case connection.SetURL:
-			tunnelHostText.SetText(event.URL)
-			data.edgeURL = event.URL
-		case connection.RegisteringTunnel:
-			if data.edgeURL == "" {
-				tunnelHostText.SetText(fmt.Sprintf("Registering tunnel connection %d...", event.Index))
-			}
-		}
-		app.Draw()
-	})
-}
-
-func NewDynamicColorTextView() *tview.TextView {
-	return tview.NewTextView().SetDynamicColors(true)
-}
-
-// Re-draws application when new logs are streamed to UI
-func handleNewText(app *tview.Application, logTextView *tview.TextView) func() {
-	return func() {
-		app.Draw()
-		// SetFocus to enable scrolling in textview
-		app.SetFocus(logTextView)
-	}
-}
-
-func (data *uiModel) changeConnStatus(event connection.Event, table *tview.Table, log *zerolog.Logger, palette palette) {
-	index := int(event.Index)
-	// Get connection location and state
-	connState := data.getConnState(index)
-	// Check if connection is already displayed in UI
-	if connState == nil {
-		log.Info().Msg("Connection is not in the UI table")
-		return
-	}
-
-	locationState := event.Location
-
-	if event.EventType == connection.Reconnecting {
-		locationState = "Reconnecting..."
-	}
-
-	connectionNum := index + 1
-	// Get table cell
-	cell := table.GetCell(index, 0)
-	// Change dot color in front of text as well as location state
-	text := newCellText(palette, connectionNum, locationState, event.EventType)
-	cell.SetText(text)
-}
-
-// Return connection location and row in UI table
-func (data *uiModel) getConnState(connID int) *connState {
-	if connID < len(data.connections) {
-		return &data.connections[connID]
-	}
-
-	return nil
-}
-
-func (data *uiModel) setConnTableCell(event connection.Event, table *tview.Table, palette palette) {
-	index := int(event.Index)
-	connectionNum := index + 1
-
-	// Update slice to keep track of connection location and state in UI table
-	data.connections[index].location = event.Location
-
-	// Update text in table cell to show disconnected state
-	text := newCellText(palette, connectionNum, event.Location, event.EventType)
-	cell := tview.NewTableCell(text)
-	table.SetCell(index, 0, cell)
-}
-
-func newCellText(palette palette, connectionNum int, location string, connectedStatus connection.Status) string {
-	// HA connection indicator formatted as: "• #<CONNECTION_INDEX>: <COLO>",
-	//  where the left middle dot's color depends on the status of the connection
-	const connFmtString = "[%s]\u2022[%s] #%d: %s"
-
-	var dotColor string
-	switch connectedStatus {
-	case connection.Connected:
-		dotColor = palette.connected
-	case connection.Disconnected:
-		dotColor = palette.disconnected
-	case connection.Reconnecting:
-		dotColor = palette.reconnecting
-	case connection.Unregistering:
-		dotColor = palette.unregistered
-	}
-
-	return fmt.Sprintf(connFmtString, dotColor, palette.defaultText, connectionNum, location)
-}

component-tests/cli.py

@@ -0,0 +1,95 @@
+import json
+import subprocess
+from time import sleep
+
+from setup import get_config_from_file
+
+SINGLE_CASE_TIMEOUT = 600
+
+class CloudflaredCli:
+    def __init__(self, config, config_path, logger):
+        self.basecmd = [config.cloudflared_binary, "tunnel"]
+        if config_path is not None:
+            self.basecmd += ["--config", str(config_path)]
+        origincert = get_config_from_file()["origincert"]
+        if origincert:
+            self.basecmd += ["--origincert", origincert]
+        self.logger = logger
+
+    def _run_command(self, subcmd, subcmd_name, needs_to_pass=True):
+        cmd = self.basecmd + subcmd
+        # timeout limits the time a subprocess can run. This is useful to guard against running a tunnel when
+        # command/args are in wrong order.
+        result = run_subprocess(cmd, subcmd_name, self.logger, check=needs_to_pass, capture_output=True, timeout=15)
+        return result
+
+    def list_tunnels(self):
+        cmd_args = ["list", "--output", "json"]
+        listed = self._run_command(cmd_args, "list")
+        return json.loads(listed.stdout)
+
+    def get_tunnel_info(self, tunnel_id):
+        info = self._run_command(["info", "--output", "json", tunnel_id], "info")
+        return json.loads(info.stdout)
+
+    def __enter__(self):
+        self.basecmd += ["run"]
+        self.process = subprocess.Popen(self.basecmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        self.logger.info(f"Run cmd {self.basecmd}")
+        return self.process
+
+    def __exit__(self, exc_type, exc_value, exc_traceback):
+        terminate_gracefully(self.process, self.logger, self.basecmd)
+        self.logger.debug(f"{self.basecmd} logs: {self.process.stderr.read()}")
+
+
+def terminate_gracefully(process, logger, cmd):
+    process.terminate()
+    process_terminated = wait_for_terminate(process)
+    if not process_terminated:
+        process.kill()
+        logger.warning(f"{cmd}: cloudflared did not terminate within wait period. Killing process. logs: \
+                stdout: {process.stdout.read()}, stderr: {process.stderr.read()}")
+
+
+def wait_for_terminate(opened_subprocess, attempts=10, poll_interval=1):
+    """
+        wait_for_terminate polls the opened_subprocess every x seconds for a given number of attempts.
+        It returns true if the subprocess was terminated and false if it didn't.
+    """
+    for _ in range(attempts):
+        if _is_process_stopped(opened_subprocess):
+            return True
+        sleep(poll_interval)
+    return False
+
+
+def _is_process_stopped(process):
+    return process.poll() is not None
+
+
+def cert_path():
+    return get_config_from_file()["origincert"]
+
+
+class SubprocessError(Exception):
+    def __init__(self, program, exit_code, cause):
+        self.program = program
+        self.exit_code = exit_code
+        self.cause = cause
+
+
+def run_subprocess(cmd, cmd_name, logger, timeout=SINGLE_CASE_TIMEOUT, **kargs):
+    kargs["timeout"] = timeout
+    try:
+        result = subprocess.run(cmd, **kargs)
+        logger.debug(f"{cmd} log: {result.stdout}", extra={"cmd": cmd_name})
+        return result
+    except subprocess.CalledProcessError as e:
+        err = f"{cmd} return exit code {e.returncode}, stderr" + e.stderr.decode("utf-8")
+        logger.error(err, extra={"cmd": cmd_name, "return_code": e.returncode})
+        raise SubprocessError(cmd[0], e.returncode, e)
+    except subprocess.TimeoutExpired as e:
+        err = f"{cmd} timeout after {e.timeout} seconds, stdout: {e.stdout}, stderr: {e.stderr}"
+        logger.error(err, extra={"cmd": cmd_name, "return_code": "timeout"})
+        raise e

component-tests/constants.py

@@ -1,6 +1,7 @@
 METRICS_PORT = 51000
 MAX_RETRIES = 5
 BACKOFF_SECS = 7
+MAX_LOG_LINES = 50
 
 PROXY_DNS_PORT = 9053
 

component-tests/test_edge_discovery.py

@@ -0,0 +1,165 @@
+import ipaddress
+import socket
+
+import pytest
+
+from constants import protocols
+from cli import CloudflaredCli
+from util import get_tunnel_connector_id, LOGGER, wait_tunnel_ready, write_config
+
+
+class TestEdgeDiscovery:
+    def _extra_config(self, protocol, edge_ip_version):
+        config = {
+            "protocol": protocol,
+        }
+        if edge_ip_version:
+            config["edge-ip-version"] = edge_ip_version
+        return config
+
+    @pytest.mark.parametrize("protocol", protocols())
+    def test_default_only(self, tmp_path, component_tests_config, protocol):
+        """
+        This test runs a tunnel to connect via IPv4-only edge addresses (default is unset "--edge-ip-version 4")
+        """
+        if self.has_ipv6_only():
+            pytest.skip("Host has IPv6 only support and current default is IPv4 only")
+        self.expect_address_connections(
+            tmp_path, component_tests_config, protocol, None, self.expect_ipv4_address)
+
+    @pytest.mark.parametrize("protocol", protocols())
+    def test_ipv4_only(self, tmp_path, component_tests_config, protocol):
+        """
+        This test runs a tunnel to connect via IPv4-only edge addresses
+        """
+        if self.has_ipv6_only():
+            pytest.skip("Host has IPv6 only support")
+        self.expect_address_connections(
+            tmp_path, component_tests_config, protocol, "4", self.expect_ipv4_address)
+
+    @pytest.mark.parametrize("protocol", protocols())
+    def test_ipv6_only(self, tmp_path, component_tests_config, protocol):
+        """
+        This test runs a tunnel to connect via IPv6-only edge addresses
+        """
+        if self.has_ipv4_only():
+            pytest.skip("Host has IPv4 only support")
+        self.expect_address_connections(
+            tmp_path, component_tests_config, protocol, "6", self.expect_ipv6_address)
+
+    @pytest.mark.parametrize("protocol", protocols())
+    def test_auto_ip64(self, tmp_path, component_tests_config, protocol):
+        """
+        This test runs a tunnel to connect via auto with a preference of IPv6 then IPv4 addresses for a dual stack host
+
+        This test also assumes that the host has IPv6 preference.
+        """
+        if not self.has_dual_stack(address_family_preference=socket.AddressFamily.AF_INET6):
+            pytest.skip("Host does not support dual stack with IPv6 preference")
+        self.expect_address_connections(
+            tmp_path, component_tests_config, protocol, "auto", self.expect_ipv6_address)
+
+    @pytest.mark.parametrize("protocol", protocols())
+    def test_auto_ip46(self, tmp_path, component_tests_config, protocol):
+        """
+        This test runs a tunnel to connect via auto with a preference of IPv4 then IPv6 addresses for a dual stack host
+
+        This test also assumes that the host has IPv4 preference.
+        """
+        if not self.has_dual_stack(address_family_preference=socket.AddressFamily.AF_INET):
+            pytest.skip("Host does not support dual stack with IPv4 preference")
+        self.expect_address_connections(
+            tmp_path, component_tests_config, protocol, "auto", self.expect_ipv4_address)
+
+    def expect_address_connections(self, tmp_path, component_tests_config, protocol, edge_ip_version, assert_address_type):
+        config = component_tests_config(
+            self._extra_config(protocol, edge_ip_version))
+        config_path = write_config(tmp_path, config.full_config)
+        LOGGER.debug(config)
+        with CloudflaredCli(config, config_path, LOGGER):
+            wait_tunnel_ready(tunnel_url=config.get_url(),
+                              require_min_connections=4)
+            cfd_cli = CloudflaredCli(config, config_path, LOGGER)
+            tunnel_id = config.get_tunnel_id()
+            info = cfd_cli.get_tunnel_info(tunnel_id)
+            connector_id = get_tunnel_connector_id()
+            connector = next(
+                (c for c in info["conns"] if c["id"] == connector_id), None)
+            assert connector, f"Expected connection info from get tunnel info for the connected instance: {info}"
+            conns = connector["conns"]
+            assert conns == None or len(
+                conns) == 4, f"There should be 4 connections registered: {conns}"
+            for conn in conns:
+                origin_ip = conn["origin_ip"]
+                assert origin_ip, f"No available origin_ip for this connection: {conn}"
+                assert_address_type(origin_ip)
+
+    def expect_ipv4_address(self, address):
+        assert type(ipaddress.ip_address(
+            address)) is ipaddress.IPv4Address, f"Expected connection from origin to be a valid IPv4 address: {address}"
+
+    def expect_ipv6_address(self, address):
+        assert type(ipaddress.ip_address(
+            address)) is ipaddress.IPv6Address, f"Expected connection from origin to be a valid IPv6 address: {address}"
+
+    def get_addresses(self):
+        """
+        Returns a list of addresses for the host.
+        """
+        host_addresses = socket.getaddrinfo(
+            "region1.v2.argotunnel.com", 7844, socket.AF_UNSPEC, socket.SOCK_STREAM)
+        assert len(
+            host_addresses) > 0, "No addresses returned from getaddrinfo"
+        return host_addresses
+
+    def has_dual_stack(self, address_family_preference=None):
+        """
+        Returns true if the host has dual stack support and can optionally check 
+        the provided IP family preference.
+        """
+        dual_stack = not self.has_ipv6_only() and not self.has_ipv4_only()
+        if address_family_preference:
+            address = self.get_addresses()[0]
+            return dual_stack and address[0] == address_family_preference
+
+        return dual_stack
+
+    def has_ipv6_only(self):
+        """
+        Returns True if the host has only IPv6 address support.
+        """
+        return self.attempt_connection(socket.AddressFamily.AF_INET6) and not self.attempt_connection(socket.AddressFamily.AF_INET)
+
+    def has_ipv4_only(self):
+        """
+        Returns True if the host has only IPv4 address support.
+        """
+        return self.attempt_connection(socket.AddressFamily.AF_INET) and not self.attempt_connection(socket.AddressFamily.AF_INET6)
+
+    def attempt_connection(self, address_family):
+        """
+        Returns True if a successful socket connection can be made to the 
+        remote host with the provided address family to validate host support 
+        for the provided address family.
+        """
+        address = None
+        for a in self.get_addresses():
+            if a[0] == address_family:
+                address = a
+                break
+        if address is None:
+            # Couldn't even lookup the address family so we can't connect
+            return False
+        af, socktype, proto, canonname, sockaddr = address
+        s = None
+        try:
+            s = socket.socket(af, socktype, proto)
+        except OSError:
+            return False
+        try:
+            s.connect(sockaddr)
+        except OSError:
+            s.close()
+            return False
+        s.close()
+        return True

component-tests/test_logging.py

@@ -2,6 +2,7 @@
 import json
 import os
 
+from constants import MAX_LOG_LINES
 from util import start_cloudflared, wait_tunnel_ready, send_requests
 
 # Rolling logger rotate log files after 1 MB
@@ -11,14 +12,24 @@ expect_message = "Starting Hello"
 
 
 def assert_log_to_terminal(cloudflared):
-    stderr = cloudflared.stderr.read(1500)
-    assert expect_message.encode() in stderr, f"{stderr} doesn't contain {expect_message}"
+    for _ in range(0, MAX_LOG_LINES):
+        line = cloudflared.stderr.readline()
+        if not line:
+            break
+        if expect_message.encode() in line:
+            return
+    raise Exception(f"terminal log doesn't contain {expect_message}")
 
 
 def assert_log_in_file(file):
     with open(file, "r") as f:
-        log = f.read(2000)
-        assert expect_message in log, f"{log} doesn't contain {expect_message}"
+        for _ in range(0, MAX_LOG_LINES):
+            line = f.readline()
+            if not line:
+                break
+            if expect_message in line:
+                return
+    raise Exception(f"log file doesn't contain {expect_message}")
 
 
 def assert_json_log(file):

component-tests/test_pq.py

@@ -0,0 +1,17 @@
+from util import LOGGER, nofips, start_cloudflared, wait_tunnel_ready
+
+
+@nofips
+class TestPostQuantum:
+    def _extra_config(self):
+        config = {
+            "protocol": "quic",
+        }
+        return config
+
+    def test_post_quantum(self, tmp_path, component_tests_config):
+        config = component_tests_config(self._extra_config())
+        LOGGER.debug(config)
+        with start_cloudflared(tmp_path, config, cfd_args=["run", "--post-quantum"], new_process=True):
+            wait_tunnel_ready(tunnel_url=config.get_url(),
+                              require_min_connections=4)

component-tests/test_reconnect.py

@@ -47,17 +47,12 @@ class TestReconnect:
         cloudflared.stdin.flush()
 
     def assert_reconnect(self, config, cloudflared, repeat):
-        wait_tunnel_ready(tunnel_url=config.get_url(), require_min_connections=self.default_ha_conns)
+        wait_tunnel_ready(tunnel_url=config.get_url(),
+                          require_min_connections=self.default_ha_conns)
         for _ in range(repeat):
-            for i in range(self.default_ha_conns):
+            for _ in range(self.default_ha_conns):
                 self.send_reconnect(cloudflared, self.default_reconnect_secs)
-                expect_connections = self.default_ha_conns-i-1
-                if expect_connections > 0:
-                    # Don't check if tunnel returns 200 here because there is a race condition between wait_tunnel_ready
-                    # retrying to get 200 response and reconnecting
-                    wait_tunnel_ready(require_min_connections=expect_connections)
-                else:
-                    check_tunnel_not_connected()
-
+            check_tunnel_not_connected()
             sleep(self.default_reconnect_secs * 2)
-            wait_tunnel_ready(tunnel_url=config.get_url(), require_min_connections=self.default_ha_conns)
+            wait_tunnel_ready(tunnel_url=config.get_url(),
+                              require_min_connections=self.default_ha_conns)

component-tests/test_service.py

@@ -1,7 +1,6 @@
 #!/usr/bin/env python
 import os
 import pathlib
-import platform
 import subprocess
 from contextlib import contextmanager
 from pathlib import Path
@@ -10,12 +9,7 @@ import pytest
 
 import test_logging
 from conftest import CfdModes
-from util import start_cloudflared, wait_tunnel_ready, write_config
-
-
-def select_platform(plat):
-    return pytest.mark.skipif(
-        platform.system() != plat, reason=f"Only runs on {plat}")
+from util import select_platform, start_cloudflared, wait_tunnel_ready, write_config
 
 
 def default_config_dir():

component-tests/util.py

@@ -1,9 +1,12 @@
 import logging
 import os
+import platform
 import subprocess
 from contextlib import contextmanager
 from time import sleep
 
+import pytest
+
 import requests
 import yaml
 from retrying import retry
@@ -13,6 +16,17 @@ from constants import METRICS_PORT, MAX_RETRIES, BACKOFF_SECS
 LOGGER = logging.getLogger(__name__)
 
 
+def select_platform(plat):
+    return pytest.mark.skipif(
+        platform.system() != plat, reason=f"Only runs on {plat}")
+
+def fips_enabled():
+    env_fips = os.getenv("COMPONENT_TESTS_FIPS")
+    return env_fips is not None and env_fips != "0"
+
+nofips = pytest.mark.skipif(
+        fips_enabled(), reason=f"Only runs without FIPS (COMPONENT_TESTS_FIPS=0)")
+
 def write_config(directory, config):
     config_path = directory / "config.yml"
     with open(config_path, 'w') as outfile:
@@ -54,13 +68,15 @@ def cloudflared_cmd(config, config_path, cfd_args, cfd_pre_args, root):
 def run_cloudflared_background(cmd, allow_input, capture_output):
     output = subprocess.PIPE if capture_output else subprocess.DEVNULL
     stdin = subprocess.PIPE if allow_input else None
+    cfd = None
     try:
         cfd = subprocess.Popen(cmd, stdin=stdin, stdout=output, stderr=output)
         yield cfd
     finally:
-        cfd.terminate()
-        if capture_output:
-            LOGGER.info(f"cloudflared log: {cfd.stderr.read()}")
+        if cfd:
+            cfd.terminate()
+            if capture_output:
+                LOGGER.info(f"cloudflared log: {cfd.stderr.read()}")
 
 
 def wait_tunnel_ready(tunnel_url=None, require_min_connections=1, cfd_logs=None):
@@ -99,18 +115,31 @@ def _log_cloudflared_logs(cfd_logs):
             LOGGER.warning(line)
 
 
-@retry(stop_max_attempt_number=MAX_RETRIES * BACKOFF_SECS, wait_fixed=1000)
+@retry(stop_max_attempt_number=MAX_RETRIES, wait_fixed=BACKOFF_SECS * 1000)
 def check_tunnel_not_connected():
     url = f'http://localhost:{METRICS_PORT}/ready'
 
     try:
-        resp = requests.get(url, timeout=1)
+        resp = requests.get(url, timeout=BACKOFF_SECS)
         assert resp.status_code == 503, f"Expect {url} returns 503, got {resp.status_code}"
+        assert resp.json()[
+            "readyConnections"] == 0, "Expected all connections to be terminated (pending reconnect)"
     # cloudflared might already terminate
     except requests.exceptions.ConnectionError as e:
         LOGGER.warning(f"Failed to connect to {url}, error: {e}")
 
 
+def get_tunnel_connector_id():
+    url = f'http://localhost:{METRICS_PORT}/ready'
+
+    try:
+        resp = requests.get(url, timeout=1)
+        return resp.json()["connectorId"]
+    # cloudflared might already terminated
+    except requests.exceptions.ConnectionError as e:
+        LOGGER.warning(f"Failed to connect to {url}, error: {e}")
+
+
 # In some cases we don't need to check response status, such as when sending batch requests to generate logs
 def send_requests(url, count, require_ok=True):
     errors = 0

config/configuration.go

@@ -229,6 +229,19 @@ type OriginRequestConfig struct {
 	IPRules []IngressIPRule `yaml:"ipRules" json:"ipRules,omitempty"`
 	// Attempt to connect to origin with HTTP/2
 	Http2Origin *bool `yaml:"http2Origin" json:"http2Origin,omitempty"`
+	// Access holds all access related configs
+	Access *AccessConfig `yaml:"access" json:"access,omitempty"`
+}
+
+type AccessConfig struct {
+	// Required when set to true will fail every request that does not arrive through an access authenticated endpoint.
+	Required bool `yaml:"required" json:"required,omitempty"`
+
+	// TeamName is the organization team name to get the public key certificates for.
+	TeamName string `yaml:"teamName" json:"teamName"`
+
+	// AudTag is the AudTag to verify access JWT against.
+	AudTag []string `yaml:"audTag" json:"audTag"`
 }
 
 type IngressIPRule struct {

connection/connection.go

@@ -24,14 +24,22 @@ const (
 	LogFieldConnIndex      = "connIndex"
 	MaxGracePeriod         = time.Minute * 3
 	MaxConcurrentStreams   = math.MaxUint32
+
+	contentTypeHeader = "content-type"
+	sseContentType    = "text/event-stream"
+	grpcContentType   = "application/grpc"
 )
 
-var switchingProtocolText = fmt.Sprintf("%d %s", http.StatusSwitchingProtocols, http.StatusText(http.StatusSwitchingProtocols))
+var (
+	switchingProtocolText = fmt.Sprintf("%d %s", http.StatusSwitchingProtocols, http.StatusText(http.StatusSwitchingProtocols))
+	flushableContentTypes = []string{sseContentType, grpcContentType}
+)
 
 type Orchestrator interface {
 	UpdateConfig(version int32, config []byte) *pogs.UpdateConfigurationResponse
 	GetConfigJSON() ([]byte, error)
 	GetOriginProxy() (OriginProxy, error)
+	WarpRoutingEnabled() (enabled bool)
 }
 
 type NamedTunnelProperties struct {
@@ -123,23 +131,24 @@ func (t Type) String() string {
 
 // OriginProxy is how data flows from cloudflared to the origin services running behind it.
 type OriginProxy interface {
-	ProxyHTTP(w ResponseWriter, tr *tracing.TracedRequest, isWebsocket bool) error
+	ProxyHTTP(w ResponseWriter, tr *tracing.TracedHTTPRequest, isWebsocket bool) error
 	ProxyTCP(ctx context.Context, rwa ReadWriteAcker, req *TCPRequest) error
 }
 
 // TCPRequest defines the input format needed to perform a TCP proxy.
 type TCPRequest struct {
-	Dest    string
-	CFRay   string
-	LBProbe bool
-	FlowID  string
+	Dest      string
+	CFRay     string
+	LBProbe   bool
+	FlowID    string
+	CfTraceID string
 }
 
 // ReadWriteAcker is a readwriter with the ability to Acknowledge to the downstream (edge) that the origin has
 // accepted the connection.
 type ReadWriteAcker interface {
 	io.ReadWriter
-	AckConnection() error
+	AckConnection(tracePropagation string) error
 }
 
 // HTTPResponseReadWriteAcker is an HTTP implementation of ReadWriteAcker.
@@ -168,22 +177,28 @@ func (h *HTTPResponseReadWriteAcker) Write(p []byte) (int, error) {
 
 // AckConnection acks an HTTP connection by sending a switch protocols status code that enables the caller to
 // upgrade to streams.
-func (h *HTTPResponseReadWriteAcker) AckConnection() error {
+func (h *HTTPResponseReadWriteAcker) AckConnection(tracePropagation string) error {
 	resp := &http.Response{
 		Status:        switchingProtocolText,
 		StatusCode:    http.StatusSwitchingProtocols,
 		ContentLength: -1,
+		Header:        http.Header{},
 	}
 
 	if secWebsocketKey := h.req.Header.Get("Sec-WebSocket-Key"); secWebsocketKey != "" {
 		resp.Header = websocket.NewResponseHeader(h.req)
 	}
 
+	if tracePropagation != "" {
+		resp.Header.Add(tracing.CanonicalCloudflaredTracingHeader, tracePropagation)
+	}
+
 	return h.w.WriteRespHeaders(resp.StatusCode, resp.Header)
 }
 
 type ResponseWriter interface {
 	WriteRespHeaders(status int, header http.Header) error
+	AddTrailer(trailerName, trailerValue string)
 	io.Writer
 }
 
@@ -192,10 +207,18 @@ type ConnectedFuse interface {
 	IsConnected() bool
 }
 
-func IsServerSentEvent(headers http.Header) bool {
-	if contentType := headers.Get("content-type"); contentType != "" {
-		return strings.HasPrefix(strings.ToLower(contentType), "text/event-stream")
+// Helper method to let the caller know what content-types should require a flush on every
+// write to a ResponseWriter.
+func shouldFlush(headers http.Header) bool {
+	if contentType := headers.Get(contentTypeHeader); contentType != "" {
+		contentType = strings.ToLower(contentType)
+		for _, c := range flushableContentTypes {
+			if strings.HasPrefix(contentType, c) {
+				return true
+			}
+		}
 	}
+
 	return false
 }
 

connection/connection_test.go

@@ -6,11 +6,9 @@ import (
 	"io"
 	"math/rand"
 	"net/http"
-	"testing"
 	"time"
 
 	"github.com/rs/zerolog"
-	"github.com/stretchr/testify/assert"
 
 	"github.com/cloudflare/cloudflared/tracing"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
@@ -30,6 +28,8 @@ var (
 	testLargeResp = make([]byte, largeFileSize)
 )
 
+var _ ReadWriteAcker = (*HTTPResponseReadWriteAcker)(nil)
+
 type testRequest struct {
 	name           string
 	endpoint       string
@@ -56,11 +56,15 @@ func (mcr *mockOrchestrator) GetOriginProxy() (OriginProxy, error) {
 	return mcr.originProxy, nil
 }
 
+func (mcr *mockOrchestrator) WarpRoutingEnabled() (enabled bool) {
+	return true
+}
+
 type mockOriginProxy struct{}
 
 func (moc *mockOriginProxy) ProxyHTTP(
 	w ResponseWriter,
-	tr *tracing.TracedRequest,
+	tr *tracing.TracedHTTPRequest,
 	isWebsocket bool,
 ) error {
 	req := tr.Request
@@ -195,40 +199,3 @@ func (mcf mockConnectedFuse) Connected() {}
 func (mcf mockConnectedFuse) IsConnected() bool {
 	return true
 }
-
-func TestIsEventStream(t *testing.T) {
-	tests := []struct {
-		headers       http.Header
-		isEventStream bool
-	}{
-		{
-			headers:       newHeader("Content-Type", "text/event-stream"),
-			isEventStream: true,
-		},
-		{
-			headers:       newHeader("content-type", "text/event-stream"),
-			isEventStream: true,
-		},
-		{
-			headers:       newHeader("Content-Type", "text/event-stream; charset=utf-8"),
-			isEventStream: true,
-		},
-		{
-			headers:       newHeader("Content-Type", "application/json"),
-			isEventStream: false,
-		},
-		{
-			headers:       http.Header{},
-			isEventStream: false,
-		},
-	}
-	for _, test := range tests {
-		assert.Equal(t, test.isEventStream, IsServerSentEvent(test.headers))
-	}
-}
-
-func newHeader(key, value string) http.Header {
-	header := http.Header{}
-	header.Add(key, value)
-	return header
-}

connection/control.go

@@ -2,7 +2,9 @@ package connection
 
 import (
 	"context"
+	"fmt"
 	"io"
+	"net"
 	"time"
 
 	"github.com/rs/zerolog"
@@ -19,6 +21,8 @@ type controlStream struct {
 	connectedFuse         ConnectedFuse
 	namedTunnelProperties *NamedTunnelProperties
 	connIndex             uint8
+	edgeAddress           net.IP
+	protocol              Protocol
 
 	newRPCClientFunc RPCClientFunc
 
@@ -45,9 +49,11 @@ func NewControlStream(
 	connectedFuse ConnectedFuse,
 	namedTunnelConfig *NamedTunnelProperties,
 	connIndex uint8,
+	edgeAddress net.IP,
 	newRPCClientFunc RPCClientFunc,
 	gracefulShutdownC <-chan struct{},
 	gracePeriod time.Duration,
+	protocol Protocol,
 ) ControlStreamHandler {
 	if newRPCClientFunc == nil {
 		newRPCClientFunc = newRegistrationRPCClient
@@ -58,8 +64,10 @@ func NewControlStream(
 		namedTunnelProperties: namedTunnelConfig,
 		newRPCClientFunc:      newRPCClientFunc,
 		connIndex:             connIndex,
+		edgeAddress:           edgeAddress,
 		gracefulShutdownC:     gracefulShutdownC,
 		gracePeriod:           gracePeriod,
+		protocol:              protocol,
 	}
 }
 
@@ -71,11 +79,14 @@ func (c *controlStream) ServeControlStream(
 ) error {
 	rpcClient := c.newRPCClientFunc(ctx, rw, c.observer.log)
 
-	registrationDetails, err := rpcClient.RegisterConnection(ctx, c.namedTunnelProperties, connOptions, c.connIndex, c.observer)
+	registrationDetails, err := rpcClient.RegisterConnection(ctx, c.namedTunnelProperties, connOptions, c.connIndex, c.edgeAddress, c.observer)
 	if err != nil {
 		rpcClient.Close()
 		return err
 	}
+
+	c.observer.logServerInfo(c.connIndex, registrationDetails.Location, c.edgeAddress, fmt.Sprintf("Connection %s registered", registrationDetails.UUID))
+	c.observer.sendConnectedEvent(c.connIndex, c.protocol, registrationDetails.Location)
 	c.connectedFuse.Connected()
 
 	// if conn index is 0 and tunnel is not remotely managed, then send local ingress rules configuration

connection/errors.go

@@ -18,6 +18,19 @@ func (e DupConnRegisterTunnelError) Error() string {
 	return "already connected to this server, trying another address"
 }
 
+// Dial to edge server with quic failed
+type EdgeQuicDialError struct {
+	Cause error
+}
+
+func (e *EdgeQuicDialError) Error() string {
+	return "failed to dial to edge with quic: " + e.Cause.Error()
+}
+
+func (e *EdgeQuicDialError) Unwrap() error {
+	return e.Cause
+}
+
 // RegisterTunnel error from server
 type ServerRegisterTunnelError struct {
 	Cause     error

connection/event.go

@@ -5,6 +5,7 @@ type Event struct {
 	Index     uint8
 	EventType Status
 	Location  string
+	Protocol  Protocol
 	URL       string
 }
 

connection/h2mux.go

@@ -69,6 +69,7 @@ func NewH2muxConnection(
 	connIndex uint8,
 	observer *Observer,
 	gracefulShutdownC <-chan struct{},
+	log *zerolog.Logger,
 ) (*h2muxConnection, error, bool) {
 	h := &h2muxConnection{
 		orchestrator:      orchestrator,
@@ -79,6 +80,7 @@ func NewH2muxConnection(
 		observer:          observer,
 		gracefulShutdownC: gracefulShutdownC,
 		newRPCClientFunc:  newRegistrationRPCClient,
+		log:               log,
 	}
 
 	// Establish a muxed connection with the edge
@@ -234,7 +236,7 @@ func (h *h2muxConnection) ServeStream(stream *h2mux.MuxedStream) error {
 		return err
 	}
 
-	err = originProxy.ProxyHTTP(respWriter, tracing.NewTracedRequest(req), sourceConnectionType == TypeWebsocket)
+	err = originProxy.ProxyHTTP(respWriter, tracing.NewTracedHTTPRequest(req, h.log), sourceConnectionType == TypeWebsocket)
 	if err != nil {
 		respWriter.WriteErrorResponse()
 	}
@@ -257,6 +259,10 @@ type h2muxRespWriter struct {
 	*h2mux.MuxedStream
 }
 
+func (rp *h2muxRespWriter) AddTrailer(trailerName, trailerValue string) {
+	// do nothing. we don't support trailers over h2mux
+}
+
 func (rp *h2muxRespWriter) WriteRespHeaders(status int, header http.Header) error {
 	headers := H1ResponseToH2ResponseHeaders(status, header)
 	headers = append(headers, h2mux.Header{Name: ResponseMetaHeader, Value: responseMetaHeaderOrigin})

connection/h2mux_test.go

@@ -47,8 +47,8 @@ func newH2MuxConnection(t require.TestingT) (*h2muxConnection, *h2mux.Muxer) {
 		edgeMuxChan <- edgeMux
 	}()
 	var connIndex = uint8(0)
-	testObserver := NewObserver(&log, &log, false)
-	h2muxConn, err, _ := NewH2muxConnection(testOrchestrator, testGracePeriod, testMuxerConfig, originConn, connIndex, testObserver, nil)
+	testObserver := NewObserver(&log, &log)
+	h2muxConn, err, _ := NewH2muxConnection(testOrchestrator, testGracePeriod, testMuxerConfig, originConn, connIndex, testObserver, nil, &log)
 	require.NoError(t, err)
 	return h2muxConn, <-edgeMuxChan
 }

connection/http2.go

@@ -132,7 +132,7 @@ func (c *HTTP2Connection) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	case TypeWebsocket, TypeHTTP:
 		stripWebsocketUpgradeHeader(r)
 		// Check for tracing on request
-		tr := tracing.NewTracedRequest(r)
+		tr := tracing.NewTracedHTTPRequest(r, c.log)
 		if err := originProxy.ProxyHTTP(respWriter, tr, connType == TypeWebsocket); err != nil {
 			err := fmt.Errorf("Failed to proxy HTTP: %w", err)
 			c.log.Error().Err(err)
@@ -149,9 +149,10 @@ func (c *HTTP2Connection) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
 		rws := NewHTTPResponseReadWriterAcker(respWriter, r)
 		if err := originProxy.ProxyTCP(r.Context(), rws, &TCPRequest{
-			Dest:    host,
-			CFRay:   FindCfRayHeader(r),
-			LBProbe: IsLBProbeRequest(r),
+			Dest:      host,
+			CFRay:     FindCfRayHeader(r),
+			LBProbe:   IsLBProbeRequest(r),
+			CfTraceID: r.Header.Get(tracing.TracerContextName),
 		}); err != nil {
 			respWriter.WriteErrorResponse()
 		}
@@ -190,11 +191,12 @@ func (c *HTTP2Connection) close() {
 }
 
 type http2RespWriter struct {
-	r           io.Reader
-	w           http.ResponseWriter
-	flusher     http.Flusher
-	shouldFlush bool
-	log         *zerolog.Logger
+	r             io.Reader
+	w             http.ResponseWriter
+	flusher       http.Flusher
+	shouldFlush   bool
+	statusWritten bool
+	log           *zerolog.Logger
 }
 
 func NewHTTP2RespWriter(r *http.Request, w http.ResponseWriter, connType Type, log *zerolog.Logger) (*http2RespWriter, error) {
@@ -218,11 +220,20 @@ func NewHTTP2RespWriter(r *http.Request, w http.ResponseWriter, connType Type, l
 	}, nil
 }
 
+func (rp *http2RespWriter) AddTrailer(trailerName, trailerValue string) {
+	if !rp.statusWritten {
+		rp.log.Warn().Msg("Tried to add Trailer to response before status written. Ignoring...")
+		return
+	}
+
+	rp.w.Header().Add(http2.TrailerPrefix+trailerName, trailerValue)
+}
+
 func (rp *http2RespWriter) WriteRespHeaders(status int, header http.Header) error {
 	dest := rp.w.Header()
 	userHeaders := make(http.Header, len(header))
 	for name, values := range header {
-		// Since these are http2 headers, they're required to be lowercase
+		// lowercase headers for simplicity check
 		h2name := strings.ToLower(name)
 
 		if h2name == "content-length" {
@@ -233,7 +244,7 @@ func (rp *http2RespWriter) WriteRespHeaders(status int, header http.Header) erro
 
 		if h2name == tracing.IntCloudflaredTracingHeader {
 			// Add cf-int-cloudflared-tracing header outside of serialized userHeaders
-			rp.w.Header()[tracing.CanonicalCloudflaredTracingHeader] = values
+			dest[tracing.CanonicalCloudflaredTracingHeader] = values
 			continue
 		}
 
@@ -246,18 +257,21 @@ func (rp *http2RespWriter) WriteRespHeaders(status int, header http.Header) erro
 
 	// Perform user header serialization and set them in the single header
 	dest.Set(CanonicalResponseUserHeaders, SerializeHeaders(userHeaders))
+
 	rp.setResponseMetaHeader(responseMetaHeaderOrigin)
 	// HTTP2 removes support for 101 Switching Protocols https://tools.ietf.org/html/rfc7540#section-8.1.1
 	if status == http.StatusSwitchingProtocols {
 		status = http.StatusOK
 	}
 	rp.w.WriteHeader(status)
-	if IsServerSentEvent(header) {
+	if shouldFlush(header) {
 		rp.shouldFlush = true
 	}
 	if rp.shouldFlush {
 		rp.flusher.Flush()
 	}
+
+	rp.statusWritten = true
 	return nil
 }
 

connection/http2_test.go

@@ -33,7 +33,7 @@ func newTestHTTP2Connection() (*HTTP2Connection, net.Conn) {
 	edgeConn, cfdConn := net.Pipe()
 	var connIndex = uint8(0)
 	log := zerolog.Nop()
-	obs := NewObserver(&log, &log, false)
+	obs := NewObserver(&log, &log)
 	controlStream := NewControlStream(
 		obs,
 		mockConnectedFuse{},
@@ -41,7 +41,9 @@ func newTestHTTP2Connection() (*HTTP2Connection, net.Conn) {
 		connIndex,
 		nil,
 		nil,
+		nil,
 		1*time.Second,
+		HTTP2,
 	)
 	return NewHTTP2Connection(
 		cfdConn,
@@ -176,6 +178,7 @@ func (mc mockNamedTunnelRPCClient) RegisterConnection(
 	properties *NamedTunnelProperties,
 	options *tunnelpogs.ConnectionOptions,
 	connIndex uint8,
+	edgeAddress net.IP,
 	observer *Observer,
 ) (*tunnelpogs.ConnectionDetails, error) {
 	if mc.shouldFail != nil {
@@ -354,15 +357,17 @@ func TestServeControlStream(t *testing.T) {
 		unregistered: make(chan struct{}),
 	}
 
-	obs := NewObserver(&log, &log, false)
+	obs := NewObserver(&log, &log)
 	controlStream := NewControlStream(
 		obs,
 		mockConnectedFuse{},
 		&NamedTunnelProperties{},
 		1,
+		nil,
 		rpcClientFactory.newMockRPCClient,
 		nil,
 		1*time.Second,
+		HTTP2,
 	)
 	http2Conn.controlStreamHandler = controlStream
 
@@ -404,15 +409,17 @@ func TestFailRegistration(t *testing.T) {
 		unregistered: make(chan struct{}),
 	}
 
-	obs := NewObserver(&log, &log, false)
+	obs := NewObserver(&log, &log)
 	controlStream := NewControlStream(
 		obs,
 		mockConnectedFuse{},
 		&NamedTunnelProperties{},
 		http2Conn.connIndex,
+		nil,
 		rpcClientFactory.newMockRPCClient,
 		nil,
 		1*time.Second,
+		HTTP2,
 	)
 	http2Conn.controlStreamHandler = controlStream
 
@@ -449,16 +456,18 @@ func TestGracefulShutdownHTTP2(t *testing.T) {
 	events := &eventCollectorSink{}
 
 	shutdownC := make(chan struct{})
-	obs := NewObserver(&log, &log, false)
+	obs := NewObserver(&log, &log)
 	obs.RegisterSink(events)
 	controlStream := NewControlStream(
 		obs,
 		mockConnectedFuse{},
 		&NamedTunnelProperties{},
 		http2Conn.connIndex,
+		nil,
 		rpcClientFactory.newMockRPCClient,
 		shutdownC,
 		1*time.Second,
+		HTTP2,
 	)
 
 	http2Conn.controlStreamHandler = controlStream

connection/metrics.go

@@ -367,7 +367,7 @@ func initTunnelMetrics() *tunnelMetrics {
 			Name:      "server_locations",
 			Help:      "Where each tunnel is connected to. 1 means current location, 0 means previous locations.",
 		},
-		[]string{"connection_id", "location"},
+		[]string{"connection_id", "edge_location"},
 	)
 	prometheus.MustRegister(serverLocations)
 

connection/observer.go

@@ -18,7 +18,6 @@ type Observer struct {
 	logTransport    *zerolog.Logger
 	metrics         *tunnelMetrics
 	tunnelEventChan chan Event
-	uiEnabled       bool
 	addSinkChan     chan EventSink
 }
 
@@ -26,12 +25,11 @@ type EventSink interface {
 	OnTunnelEvent(event Event)
 }
 
-func NewObserver(log, logTransport *zerolog.Logger, uiEnabled bool) *Observer {
+func NewObserver(log, logTransport *zerolog.Logger) *Observer {
 	o := &Observer{
 		log:             log,
 		logTransport:    logTransport,
 		metrics:         newTunnelMetrics(),
-		uiEnabled:       uiEnabled,
 		tunnelEventChan: make(chan Event, observerChannelBufferSize),
 		addSinkChan:     make(chan EventSink, observerChannelBufferSize),
 	}
@@ -57,8 +55,8 @@ func (o *Observer) sendRegisteringEvent(connIndex uint8) {
 	o.sendEvent(Event{Index: connIndex, EventType: RegisteringTunnel})
 }
 
-func (o *Observer) sendConnectedEvent(connIndex uint8, location string) {
-	o.sendEvent(Event{Index: connIndex, EventType: Connected, Location: location})
+func (o *Observer) sendConnectedEvent(connIndex uint8, protocol Protocol, location string) {
+	o.sendEvent(Event{Index: connIndex, EventType: Connected, Protocol: protocol, Location: location})
 }
 
 func (o *Observer) SendURL(url string) {

connection/observer_test.go

@@ -12,7 +12,7 @@ import (
 )
 
 func TestSendUrl(t *testing.T) {
-	observer := NewObserver(&log, &log, false)
+	observer := NewObserver(&log, &log)
 
 	observer.SendURL("my-url.com")
 	assert.Equal(t, 1.0, getCounterValue(t, observer.metrics.userHostnamesCounts, "https://my-url.com"))
@@ -63,7 +63,7 @@ func TestRegisterServerLocation(t *testing.T) {
 }
 
 func TestObserverEventsDontBlock(t *testing.T) {
-	observer := NewObserver(&log, &log, false)
+	observer := NewObserver(&log, &log)
 	var mu sync.Mutex
 	observer.RegisterSink(EventSinkFunc(func(_ Event) {
 		// callback will block if lock is already held

connection/protocol.go

@@ -1,6 +1,7 @@
 package connection
 
 import (
+	"errors"
 	"fmt"
 	"hash/fnv"
 	"sync"
@@ -130,6 +131,7 @@ type autoProtocolSelector struct {
 	refreshAfter    time.Time
 	ttl             time.Duration
 	log             *zerolog.Logger
+	needPQ          bool
 }
 
 func newAutoProtocolSelector(
@@ -139,6 +141,7 @@ func newAutoProtocolSelector(
 	fetchFunc PercentageFetcher,
 	ttl time.Duration,
 	log *zerolog.Logger,
+	needPQ bool,
 ) *autoProtocolSelector {
 	return &autoProtocolSelector{
 		current:         current,
@@ -148,6 +151,7 @@ func newAutoProtocolSelector(
 		refreshAfter:    time.Now().Add(ttl),
 		ttl:             ttl,
 		log:             log,
+		needPQ:          needPQ,
 	}
 }
 
@@ -187,6 +191,9 @@ func getProtocol(protocolPool []Protocol, fetchFunc PercentageFetcher, switchThr
 func (s *autoProtocolSelector) Fallback() (Protocol, bool) {
 	s.lock.RLock()
 	defer s.lock.RUnlock()
+	if s.needPQ {
+		return 0, false
+	}
 	return s.current.fallback()
 }
 
@@ -199,9 +206,14 @@ func NewProtocolSelector(
 	fetchFunc PercentageFetcher,
 	ttl time.Duration,
 	log *zerolog.Logger,
+	needPQ bool,
 ) (ProtocolSelector, error) {
 	// Classic tunnel is only supported with h2mux
 	if namedTunnel == nil {
+		if needPQ {
+			return nil, errors.New("Classic tunnel does not support post-quantum")
+		}
+
 		return &staticProtocolSelector{
 			current: H2mux,
 		}, nil
@@ -209,8 +221,11 @@ func NewProtocolSelector(
 
 	threshold := switchThreshold(namedTunnel.Credentials.AccountTag)
 	fetchedProtocol, err := getProtocol([]Protocol{QUIC, HTTP2}, fetchFunc, threshold)
-	if err != nil {
-		log.Err(err).Msg("Unable to lookup protocol. Defaulting to `http2`. If this fails, you can set `--protocol h2mux` in your cloudflared command.")
+	if err != nil && protocolFlag == "auto" {
+		log.Err(err).Msg("Unable to lookup protocol. Defaulting to `http2`. If this fails, you can attempt `--protocol quic` instead.")
+		if needPQ {
+			return nil, errors.New("http2 does not support post-quantum")
+		}
 		return &staticProtocolSelector{
 			current: HTTP2,
 		}, nil
@@ -221,10 +236,10 @@ func NewProtocolSelector(
 			protocolFlag = HTTP2.String()
 			fetchedProtocol = HTTP2Warp
 		}
-		return selectWarpRoutingProtocols(protocolFlag, fetchFunc, ttl, log, threshold, fetchedProtocol)
+		return selectWarpRoutingProtocols(protocolFlag, fetchFunc, ttl, log, threshold, fetchedProtocol, needPQ)
 	}
 
-	return selectNamedTunnelProtocols(protocolFlag, fetchFunc, ttl, log, threshold, fetchedProtocol)
+	return selectNamedTunnelProtocols(protocolFlag, fetchFunc, ttl, log, threshold, fetchedProtocol, needPQ)
 }
 
 func selectNamedTunnelProtocols(
@@ -234,6 +249,7 @@ func selectNamedTunnelProtocols(
 	log *zerolog.Logger,
 	threshold int32,
 	protocol Protocol,
+	needPQ bool,
 ) (ProtocolSelector, error) {
 	// If the user picks a protocol, then we stick to it no matter what.
 	switch protocolFlag {
@@ -248,7 +264,7 @@ func selectNamedTunnelProtocols(
 	// If the user does not pick (hopefully the majority) then we use the one derived from the TXT DNS record and
 	// fallback on failures.
 	if protocolFlag == AutoSelectFlag {
-		return newAutoProtocolSelector(protocol, []Protocol{QUIC, HTTP2, H2mux}, threshold, fetchFunc, ttl, log), nil
+		return newAutoProtocolSelector(protocol, []Protocol{QUIC, HTTP2, H2mux}, threshold, fetchFunc, ttl, log, needPQ), nil
 	}
 
 	return nil, fmt.Errorf("Unknown protocol %s, %s", protocolFlag, AvailableProtocolFlagMessage)
@@ -261,6 +277,7 @@ func selectWarpRoutingProtocols(
 	log *zerolog.Logger,
 	threshold int32,
 	protocol Protocol,
+	needPQ bool,
 ) (ProtocolSelector, error) {
 	// If the user picks a protocol, then we stick to it no matter what.
 	switch protocolFlag {
@@ -273,7 +290,7 @@ func selectWarpRoutingProtocols(
 	// If the user does not pick (hopefully the majority) then we use the one derived from the TXT DNS record and
 	// fallback on failures.
 	if protocolFlag == AutoSelectFlag {
-		return newAutoProtocolSelector(protocol, []Protocol{QUICWarp, HTTP2Warp}, threshold, fetchFunc, ttl, log), nil
+		return newAutoProtocolSelector(protocol, []Protocol{QUICWarp, HTTP2Warp}, threshold, fetchFunc, ttl, log, needPQ), nil
 	}
 
 	return nil, fmt.Errorf("Unknown protocol %s, %s", protocolFlag, AvailableProtocolFlagMessage)

connection/protocol_test.go

@@ -219,7 +219,7 @@ func TestNewProtocolSelector(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			selector, err := NewProtocolSelector(test.protocol, test.warpRoutingEnabled, test.namedTunnelConfig, test.fetchFunc, testNoTTL, &log)
+			selector, err := NewProtocolSelector(test.protocol, test.warpRoutingEnabled, test.namedTunnelConfig, test.fetchFunc, testNoTTL, &log, false)
 			if test.wantErr {
 				assert.Error(t, err, fmt.Sprintf("test %s failed", test.name))
 			} else {
@@ -237,7 +237,7 @@ func TestNewProtocolSelector(t *testing.T) {
 
 func TestAutoProtocolSelectorRefresh(t *testing.T) {
 	fetcher := dynamicMockFetcher{}
-	selector, err := NewProtocolSelector(AutoSelectFlag, noWarpRoutingEnabled, testNamedTunnelProperties, fetcher.fetch(), testNoTTL, &log)
+	selector, err := NewProtocolSelector(AutoSelectFlag, noWarpRoutingEnabled, testNamedTunnelProperties, fetcher.fetch(), testNoTTL, &log, false)
 	assert.NoError(t, err)
 	assert.Equal(t, H2mux, selector.Current())
 
@@ -267,7 +267,7 @@ func TestAutoProtocolSelectorRefresh(t *testing.T) {
 func TestHTTP2ProtocolSelectorRefresh(t *testing.T) {
 	fetcher := dynamicMockFetcher{}
 	// Since the user chooses http2 on purpose, we always stick to it.
-	selector, err := NewProtocolSelector("http2", noWarpRoutingEnabled, testNamedTunnelProperties, fetcher.fetch(), testNoTTL, &log)
+	selector, err := NewProtocolSelector("http2", noWarpRoutingEnabled, testNamedTunnelProperties, fetcher.fetch(), testNoTTL, &log, false)
 	assert.NoError(t, err)
 	assert.Equal(t, HTTP2, selector.Current())
 
@@ -297,7 +297,7 @@ func TestHTTP2ProtocolSelectorRefresh(t *testing.T) {
 func TestProtocolSelectorRefreshTTL(t *testing.T) {
 	fetcher := dynamicMockFetcher{}
 	fetcher.protocolPercents = edgediscovery.ProtocolPercents{edgediscovery.ProtocolPercent{Protocol: "quic", Percentage: 100}}
-	selector, err := NewProtocolSelector(AutoSelectFlag, noWarpRoutingEnabled, testNamedTunnelProperties, fetcher.fetch(), time.Hour, &log)
+	selector, err := NewProtocolSelector(AutoSelectFlag, noWarpRoutingEnabled, testNamedTunnelProperties, fetcher.fetch(), time.Hour, &log, false)
 	assert.NoError(t, err)
 	assert.Equal(t, QUIC, selector.Current())
 

connection/quic.go

@@ -7,18 +7,24 @@ import (
 	"io"
 	"net"
 	"net/http"
+	"net/netip"
 	"strconv"
 	"strings"
+	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/google/uuid"
 	"github.com/lucas-clemente/quic-go"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
 	"golang.org/x/sync/errgroup"
 
 	"github.com/cloudflare/cloudflared/datagramsession"
 	"github.com/cloudflare/cloudflared/ingress"
+	"github.com/cloudflare/cloudflared/packet"
 	quicpogs "github.com/cloudflare/cloudflared/quic"
 	"github.com/cloudflare/cloudflared/tracing"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
@@ -33,14 +39,25 @@ const (
 	HTTPHostKey = "HttpHost"
 
 	QUICMetadataFlowID = "FlowID"
+	// emperically this capacity has been working well
+	demuxChanCapacity = 16
+)
+
+var (
+	portForConnIndex = make(map[uint8]int, 0)
+	portMapMutex     sync.Mutex
 )
 
 // QUICConnection represents the type that facilitates Proxying via QUIC streams.
 type QUICConnection struct {
-	session              quic.Connection
-	logger               *zerolog.Logger
-	orchestrator         Orchestrator
-	sessionManager       datagramsession.Manager
+	session      quic.Connection
+	logger       *zerolog.Logger
+	orchestrator Orchestrator
+	// sessionManager tracks active sessions. It receives datagrams from quic connection via datagramMuxer
+	sessionManager datagramsession.Manager
+	// datagramMuxer mux/demux datagrams from quic connection
+	datagramMuxer        *quicpogs.DatagramMuxerV2
+	packetRouter         *ingress.PacketRouter
 	controlStreamHandler ControlStreamHandler
 	connOptions          *tunnelpogs.ConnectionOptions
 }
@@ -49,29 +66,42 @@ type QUICConnection struct {
 func NewQUICConnection(
 	quicConfig *quic.Config,
 	edgeAddr net.Addr,
+	connIndex uint8,
 	tlsConfig *tls.Config,
 	orchestrator Orchestrator,
 	connOptions *tunnelpogs.ConnectionOptions,
 	controlStreamHandler ControlStreamHandler,
 	logger *zerolog.Logger,
+	packetRouterConfig *ingress.GlobalRouterConfig,
 ) (*QUICConnection, error) {
-	session, err := quic.DialAddr(edgeAddr.String(), tlsConfig, quicConfig)
-	if err != nil {
-		return nil, fmt.Errorf("failed to dial to edge: %w", err)
-	}
-
-	datagramMuxer, err := quicpogs.NewDatagramMuxer(session, logger)
+	udpConn, err := createUDPConnForConnIndex(connIndex, logger)
 	if err != nil {
 		return nil, err
 	}
 
-	sessionManager := datagramsession.NewManager(datagramMuxer, logger)
+	session, err := quic.Dial(udpConn, edgeAddr, edgeAddr.String(), tlsConfig, quicConfig)
+	if err != nil {
+		return nil, &EdgeQuicDialError{Cause: err}
+	}
+
+	// wrap the session, so that the UDPConn is closed after session is closed.
+	session = &wrapCloseableConnQuicConnection{
+		session,
+		udpConn,
+	}
+
+	sessionDemuxChan := make(chan *packet.Session, demuxChanCapacity)
+	datagramMuxer := quicpogs.NewDatagramMuxerV2(session, logger, sessionDemuxChan)
+	sessionManager := datagramsession.NewManager(logger, datagramMuxer.SendToSession, sessionDemuxChan)
+	packetRouter := ingress.NewPacketRouter(packetRouterConfig, datagramMuxer, logger, orchestrator.WarpRoutingEnabled)
 
 	return &QUICConnection{
 		session:              session,
 		orchestrator:         orchestrator,
 		logger:               logger,
 		sessionManager:       sessionManager,
+		datagramMuxer:        datagramMuxer,
+		packetRouter:         packetRouter,
 		controlStreamHandler: controlStreamHandler,
 		connOptions:          connOptions,
 	}, nil
@@ -107,6 +137,14 @@ func (q *QUICConnection) Serve(ctx context.Context) error {
 		defer cancel()
 		return q.sessionManager.Serve(ctx)
 	})
+	errGroup.Go(func() error {
+		defer cancel()
+		return q.datagramMuxer.ServeReceive(ctx)
+	})
+	errGroup.Go(func() error {
+		defer cancel()
+		return q.packetRouter.Serve(ctx)
+	})
 
 	return errGroup.Wait()
 }
@@ -143,15 +181,21 @@ func (q *QUICConnection) acceptStream(ctx context.Context) error {
 }
 
 func (q *QUICConnection) runStream(quicStream quic.Stream) {
+	ctx := quicStream.Context()
 	stream := quicpogs.NewSafeStreamCloser(quicStream)
 	defer stream.Close()
 
-	if err := q.handleStream(stream); err != nil {
+	// we are going to fuse readers/writers from stream <- cloudflared -> origin, and we want to guarantee that
+	// code executed in the code path of handleStream don't trigger an earlier close to the downstream write stream.
+	// So, we wrap the stream with a no-op write closer and only this method can actually close write side of the stream.
+	// A call to close will simulate a close to the read-side, which will fail subsequent reads.
+	noCloseStream := &nopCloserReadWriter{ReadWriteCloser: stream}
+	if err := q.handleStream(ctx, noCloseStream); err != nil {
 		q.logger.Err(err).Msg("Failed to handle QUIC stream")
 	}
 }
 
-func (q *QUICConnection) handleStream(stream io.ReadWriteCloser) error {
+func (q *QUICConnection) handleStream(ctx context.Context, stream io.ReadWriteCloser) error {
 	signature, err := quicpogs.DetermineProtocol(stream)
 	if err != nil {
 		return err
@@ -162,7 +206,7 @@ func (q *QUICConnection) handleStream(stream io.ReadWriteCloser) error {
 		if err != nil {
 			return err
 		}
-		return q.handleDataStream(reqServerStream)
+		return q.handleDataStream(ctx, reqServerStream)
 	case quicpogs.RPCStreamProtocolSignature:
 		rpcStream, err := quicpogs.NewRPCServerStream(stream, signature)
 		if err != nil {
@@ -174,13 +218,13 @@ func (q *QUICConnection) handleStream(stream io.ReadWriteCloser) error {
 	}
 }
 
-func (q *QUICConnection) handleDataStream(stream *quicpogs.RequestServerStream) error {
+func (q *QUICConnection) handleDataStream(ctx context.Context, stream *quicpogs.RequestServerStream) error {
 	request, err := stream.ReadConnectRequestData()
 	if err != nil {
 		return err
 	}
 
-	if err := q.dispatchRequest(stream, err, request); err != nil {
+	if err := q.dispatchRequest(ctx, stream, err, request); err != nil {
 		_ = stream.WriteConnectResponseData(err)
 		q.logger.Err(err).Str("type", request.Type.String()).Str("dest", request.Dest).Msg("Request failed")
 	}
@@ -188,7 +232,7 @@ func (q *QUICConnection) handleDataStream(stream *quicpogs.RequestServerStream)
 	return nil
 }
 
-func (q *QUICConnection) dispatchRequest(stream *quicpogs.RequestServerStream, err error, request *quicpogs.ConnectRequest) error {
+func (q *QUICConnection) dispatchRequest(ctx context.Context, stream *quicpogs.RequestServerStream, err error, request *quicpogs.ConnectRequest) error {
 	originProxy, err := q.orchestrator.GetOriginProxy()
 	if err != nil {
 		return err
@@ -196,7 +240,7 @@ func (q *QUICConnection) dispatchRequest(stream *quicpogs.RequestServerStream, e
 
 	switch request.Type {
 	case quicpogs.ConnectionTypeHTTP, quicpogs.ConnectionTypeWebsocket:
-		tracedReq, err := buildHTTPRequest(request, stream)
+		tracedReq, err := buildHTTPRequest(ctx, request, stream, q.logger)
 		if err != nil {
 			return err
 		}
@@ -206,9 +250,10 @@ func (q *QUICConnection) dispatchRequest(stream *quicpogs.RequestServerStream, e
 	case quicpogs.ConnectionTypeTCP:
 		rwa := &streamReadWriteAcker{stream}
 		metadata := request.MetadataMap()
-		return originProxy.ProxyTCP(context.Background(), rwa, &TCPRequest{
-			Dest:   request.Dest,
-			FlowID: metadata[QUICMetadataFlowID],
+		return originProxy.ProxyTCP(ctx, rwa, &TCPRequest{
+			Dest:      request.Dest,
+			FlowID:    metadata[QUICMetadataFlowID],
+			CfTraceID: metadata[tracing.TracerContextName],
 		})
 	}
 	return nil
@@ -219,24 +264,42 @@ func (q *QUICConnection) handleRPCStream(rpcStream *quicpogs.RPCServerStream) er
 }
 
 // RegisterUdpSession is the RPC method invoked by edge to register and run a session
-func (q *QUICConnection) RegisterUdpSession(ctx context.Context, sessionID uuid.UUID, dstIP net.IP, dstPort uint16, closeAfterIdleHint time.Duration) error {
+func (q *QUICConnection) RegisterUdpSession(ctx context.Context, sessionID uuid.UUID, dstIP net.IP, dstPort uint16, closeAfterIdleHint time.Duration, traceContext string) (*tunnelpogs.RegisterUdpSessionResponse, error) {
+	traceCtx := tracing.NewTracedContext(ctx, traceContext, q.logger)
+	ctx, registerSpan := traceCtx.Tracer().Start(traceCtx, "register-session", trace.WithAttributes(
+		attribute.String("session-id", sessionID.String()),
+		attribute.String("dst", fmt.Sprintf("%s:%d", dstIP, dstPort)),
+	))
 	// Each session is a series of datagram from an eyeball to a dstIP:dstPort.
 	// (src port, dst IP, dst port) uniquely identifies a session, so it needs a dedicated connected socket.
 	originProxy, err := ingress.DialUDP(dstIP, dstPort)
 	if err != nil {
 		q.logger.Err(err).Msgf("Failed to create udp proxy to %s:%d", dstIP, dstPort)
-		return err
+		tracing.EndWithErrorStatus(registerSpan, err)
+		return nil, err
 	}
+	registerSpan.SetAttributes(
+		attribute.Bool("socket-bind-success", true),
+		attribute.String("src", originProxy.LocalAddr().String()),
+	)
+
 	session, err := q.sessionManager.RegisterSession(ctx, sessionID, originProxy)
 	if err != nil {
 		q.logger.Err(err).Str("sessionID", sessionID.String()).Msgf("Failed to register udp session")
-		return err
+		tracing.EndWithErrorStatus(registerSpan, err)
+		return nil, err
 	}
 
 	go q.serveUDPSession(session, closeAfterIdleHint)
 
 	q.logger.Debug().Str("sessionID", sessionID.String()).Str("src", originProxy.LocalAddr().String()).Str("dst", fmt.Sprintf("%s:%d", dstIP, dstPort)).Msgf("Registered session")
-	return nil
+	tracing.End(registerSpan)
+
+	resp := tunnelpogs.RegisterUdpSessionResponse{
+		Spans: traceCtx.GetProtoSpans(),
+	}
+
+	return &resp, nil
 }
 
 func (q *QUICConnection) serveUDPSession(session *datagramsession.Session, closeAfterIdleHint time.Duration) {
@@ -295,8 +358,12 @@ type streamReadWriteAcker struct {
 }
 
 // AckConnection acks response back to the proxy.
-func (s *streamReadWriteAcker) AckConnection() error {
-	return s.WriteConnectResponseData(nil)
+func (s *streamReadWriteAcker) AckConnection(tracePropagation string) error {
+	metadata := quicpogs.Metadata{
+		Key: tracing.CanonicalCloudflaredTracingHeader,
+		Val: tracePropagation,
+	}
+	return s.WriteConnectResponseData(nil, metadata)
 }
 
 // httpResponseAdapter translates responses written by the HTTP Proxy into ones that can be used in QUIC.
@@ -308,6 +375,10 @@ func newHTTPResponseAdapter(s *quicpogs.RequestServerStream) httpResponseAdapter
 	return httpResponseAdapter{s}
 }
 
+func (hrw httpResponseAdapter) AddTrailer(trailerName, trailerValue string) {
+	// we do not support trailers over QUIC
+}
+
 func (hrw httpResponseAdapter) WriteRespHeaders(status int, header http.Header) error {
 	metadata := make([]quicpogs.Metadata, 0)
 	metadata = append(metadata, quicpogs.Metadata{Key: "HttpStatus", Val: strconv.Itoa(status)})
@@ -324,14 +395,19 @@ func (hrw httpResponseAdapter) WriteErrorResponse(err error) {
 	hrw.WriteConnectResponseData(err, quicpogs.Metadata{Key: "HttpStatus", Val: strconv.Itoa(http.StatusBadGateway)})
 }
 
-func buildHTTPRequest(connectRequest *quicpogs.ConnectRequest, body io.ReadCloser) (*tracing.TracedRequest, error) {
+func buildHTTPRequest(
+	ctx context.Context,
+	connectRequest *quicpogs.ConnectRequest,
+	body io.ReadCloser,
+	log *zerolog.Logger,
+) (*tracing.TracedHTTPRequest, error) {
 	metadata := connectRequest.MetadataMap()
 	dest := connectRequest.Dest
 	method := metadata[HTTPMethodKey]
 	host := metadata[HTTPHostKey]
 	isWebsocket := connectRequest.Type == quicpogs.ConnectionTypeWebsocket
 
-	req, err := http.NewRequest(method, dest, body)
+	req, err := http.NewRequestWithContext(ctx, method, dest, body)
 	if err != nil {
 		return nil, err
 	}
@@ -366,7 +442,7 @@ func buildHTTPRequest(connectRequest *quicpogs.ConnectRequest, body io.ReadClose
 	stripWebsocketUpgradeHeader(req)
 
 	// Check for tracing on request
-	tracedReq := tracing.NewTracedRequest(req)
+	tracedReq := tracing.NewTracedHTTPRequest(req, log)
 	return tracedReq, err
 }
 
@@ -384,3 +460,107 @@ func isTransferEncodingChunked(req *http.Request) bool {
 	// separated value as well.
 	return strings.Contains(strings.ToLower(transferEncodingVal), "chunked")
 }
+
+// A helper struct that guarantees a call to close only affects read side, but not write side.
+type nopCloserReadWriter struct {
+	io.ReadWriteCloser
+
+	// for use by Read only
+	// we don't need a memory barrier here because there is an implicit assumption that
+	// Read calls can't happen concurrently by different go-routines.
+	sawEOF bool
+	// should be updated and read using atomic primitives.
+	// value is read in Read method and written in Close method, which could be done by different
+	// go-routines.
+	closed uint32
+}
+
+func (np *nopCloserReadWriter) Read(p []byte) (n int, err error) {
+	if np.sawEOF {
+		return 0, io.EOF
+	}
+
+	if atomic.LoadUint32(&np.closed) > 0 {
+		return 0, fmt.Errorf("closed by handler")
+	}
+
+	n, err = np.ReadWriteCloser.Read(p)
+	if err == io.EOF {
+		np.sawEOF = true
+	}
+
+	return
+}
+
+func (np *nopCloserReadWriter) Close() error {
+	atomic.StoreUint32(&np.closed, 1)
+
+	return nil
+}
+
+// muxerWrapper wraps DatagramMuxerV2 to satisfy the packet.FunnelUniPipe interface
+type muxerWrapper struct {
+	muxer *quicpogs.DatagramMuxerV2
+}
+
+func (rp *muxerWrapper) SendPacket(dst netip.Addr, pk packet.RawPacket) error {
+	return rp.muxer.SendPacket(quicpogs.RawPacket(pk))
+}
+
+func (rp *muxerWrapper) ReceivePacket(ctx context.Context) (packet.RawPacket, error) {
+	pk, err := rp.muxer.ReceivePacket(ctx)
+	if err != nil {
+		return packet.RawPacket{}, err
+	}
+	rawPacket, ok := pk.(quicpogs.RawPacket)
+	if ok {
+		return packet.RawPacket(rawPacket), nil
+	}
+	return packet.RawPacket{}, fmt.Errorf("unexpected packet type %+v", pk)
+}
+
+func (rp *muxerWrapper) Close() error {
+	return nil
+}
+
+func createUDPConnForConnIndex(connIndex uint8, logger *zerolog.Logger) (*net.UDPConn, error) {
+	portMapMutex.Lock()
+	defer portMapMutex.Unlock()
+
+	// if port was not set yet, it will be zero, so bind will randomly allocate one.
+	if port, ok := portForConnIndex[connIndex]; ok {
+		udpConn, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.IPv4zero, Port: port})
+		// if there wasn't an error, or if port was 0 (independently of error or not, just return)
+		if err == nil {
+			return udpConn, nil
+		} else {
+			logger.Debug().Err(err).Msgf("Unable to reuse port %d for connIndex %d. Falling back to random allocation.", port, connIndex)
+		}
+	}
+
+	// if we reached here, then there was an error or port as not been allocated it.
+	udpConn, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.IPv4zero, Port: 0})
+	if err == nil {
+		udpAddr, ok := (udpConn.LocalAddr()).(*net.UDPAddr)
+		if !ok {
+			return nil, fmt.Errorf("unable to cast to udpConn")
+		}
+		portForConnIndex[connIndex] = udpAddr.Port
+	} else {
+		delete(portForConnIndex, connIndex)
+	}
+
+	return udpConn, err
+}
+
+type wrapCloseableConnQuicConnection struct {
+	quic.Connection
+	udpConn *net.UDPConn
+}
+
+func (w *wrapCloseableConnQuicConnection) CloseWithError(errorCode quic.ApplicationErrorCode, reason string) error {
+	err := w.Connection.CloseWithError(errorCode, reason)
+	w.udpConn.Close()
+
+	return err
+}

connection/quic_test.go

@@ -10,7 +10,7 @@ import (
 	"net/http"
 	"net/url"
 	"os"
-	"sync"
+	"strings"
 	"testing"
 	"time"
 
@@ -25,17 +25,21 @@ import (
 	"github.com/cloudflare/cloudflared/datagramsession"
 	quicpogs "github.com/cloudflare/cloudflared/quic"
 	"github.com/cloudflare/cloudflared/tracing"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
 var (
 	testTLSServerConfig = quicpogs.GenerateTLSConfig()
 	testQUICConfig      = &quic.Config{
-		KeepAlivePeriod: 5 * time.Second,
-		EnableDatagrams: true,
+		ConnectionIDLength: 16,
+		KeepAlivePeriod:    5 * time.Second,
+		EnableDatagrams:    true,
 	}
 )
 
+var _ ReadWriteAcker = (*streamReadWriteAcker)(nil)
+
 // TestQUICServer tests if a quic server accepts and responds to a quic client with the acceptance protocol.
 // It also serves as a demonstration for communication with the QUIC connection started by a cloudflared.
 func TestQUICServer(t *testing.T) {
@@ -137,24 +141,33 @@ func TestQUICServer(t *testing.T) {
 		},
 	}
 
-	for _, test := range tests {
+	for i, test := range tests {
+		test := test // capture range variable
 		t.Run(test.desc, func(t *testing.T) {
 			ctx, cancel := context.WithCancel(context.Background())
-			var wg sync.WaitGroup
-			wg.Add(1)
+
+			quicListener, err := quic.Listen(udpListener, testTLSServerConfig, testQUICConfig)
+			require.NoError(t, err)
+
+			serverDone := make(chan struct{})
 			go func() {
-				defer wg.Done()
 				quicServer(
-					t, udpListener, testTLSServerConfig, testQUICConfig,
-					test.dest, test.connectionType, test.metadata, test.message, test.expectedResponse,
+					ctx, t, quicListener, test.dest, test.connectionType, test.metadata, test.message, test.expectedResponse,
 				)
+				close(serverDone)
 			}()
 
-			qc := testQUICConnection(udpListener.LocalAddr(), t)
-			go qc.Serve(ctx)
+			qc := testQUICConnection(udpListener.LocalAddr(), t, uint8(i))
 
-			wg.Wait()
+			connDone := make(chan struct{})
+			go func() {
+				qc.Serve(ctx)
+				close(connDone)
+			}()
+
+			<-serverDone
 			cancel()
+			<-connDone
 		})
 	}
 }
@@ -172,23 +185,16 @@ func (fakeControlStream) IsStopped() bool {
 }
 
 func quicServer(
+	ctx context.Context,
 	t *testing.T,
-	conn net.PacketConn,
-	tlsConf *tls.Config,
-	config *quic.Config,
+	listener quic.Listener,
 	dest string,
 	connectionType quicpogs.ConnectionType,
 	metadata []quicpogs.Metadata,
 	message []byte,
 	expectedResponse []byte,
 ) {
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	earlyListener, err := quic.Listen(conn, tlsConf, config)
-	require.NoError(t, err)
-
-	session, err := earlyListener.Accept(ctx)
+	session, err := listener.Accept(ctx)
 	require.NoError(t, err)
 
 	quicStream, err := session.OpenStreamSync(context.Background())
@@ -220,7 +226,7 @@ func quicServer(
 
 type mockOriginProxyWithRequest struct{}
 
-func (moc *mockOriginProxyWithRequest) ProxyHTTP(w ResponseWriter, tr *tracing.TracedRequest, isWebsocket bool) error {
+func (moc *mockOriginProxyWithRequest) ProxyHTTP(w ResponseWriter, tr *tracing.TracedHTTPRequest, isWebsocket bool) error {
 	// These are a series of crude tests to ensure the headers and http related data is transferred from
 	// metadata.
 	r := tr.Request
@@ -475,9 +481,11 @@ func TestBuildHTTPRequest(t *testing.T) {
 		},
 	}
 
+	log := zerolog.Nop()
 	for _, test := range tests {
+		test := test // capture range variable
 		t.Run(test.name, func(t *testing.T) {
-			req, err := buildHTTPRequest(test.connectRequest, test.body)
+			req, err := buildHTTPRequest(context.Background(), test.connectRequest, test.body, &log)
 			assert.NoError(t, err)
 			test.req = test.req.WithContext(req.Context())
 			assert.Equal(t, test.req, req.Request)
@@ -486,7 +494,7 @@ func TestBuildHTTPRequest(t *testing.T) {
 }
 
 func (moc *mockOriginProxyWithRequest) ProxyTCP(ctx context.Context, rwa ReadWriteAcker, tcpRequest *TCPRequest) error {
-	rwa.AckConnection()
+	rwa.AckConnection("")
 	io.Copy(rwa, rwa)
 	return nil
 }
@@ -500,6 +508,7 @@ func TestServeUDPSession(t *testing.T) {
 	defer udpListener.Close()
 
 	ctx, cancel := context.WithCancel(context.Background())
+	val := udpListener.LocalAddr()
 
 	// Establish QUIC connection with edge
 	edgeQUICSessionChan := make(chan quic.Connection)
@@ -512,7 +521,8 @@ func TestServeUDPSession(t *testing.T) {
 		edgeQUICSessionChan <- edgeQUICSession
 	}()
 
-	qc := testQUICConnection(udpListener.LocalAddr(), t)
+	// Random index to avoid reusing port
+	qc := testQUICConnection(val, t, 28)
 	go qc.Serve(ctx)
 
 	edgeQUICSession := <-edgeQUICSessionChan
@@ -522,6 +532,75 @@ func TestServeUDPSession(t *testing.T) {
 	cancel()
 }
 
+func TestNopCloserReadWriterCloseBeforeEOF(t *testing.T) {
+	readerWriter := nopCloserReadWriter{ReadWriteCloser: &mockReaderNoopWriter{Reader: strings.NewReader("123456789")}}
+	buffer := make([]byte, 5)
+
+	n, err := readerWriter.Read(buffer)
+	require.NoError(t, err)
+	require.Equal(t, n, 5)
+
+	// close
+	require.NoError(t, readerWriter.Close())
+
+	// read should get error
+	n, err = readerWriter.Read(buffer)
+	require.Equal(t, n, 0)
+	require.Equal(t, err, fmt.Errorf("closed by handler"))
+}
+
+func TestNopCloserReadWriterCloseAfterEOF(t *testing.T) {
+	readerWriter := nopCloserReadWriter{ReadWriteCloser: &mockReaderNoopWriter{Reader: strings.NewReader("123456789")}}
+	buffer := make([]byte, 20)
+
+	n, err := readerWriter.Read(buffer)
+	require.NoError(t, err)
+	require.Equal(t, n, 9)
+
+	// force another read to read eof
+	_, err = readerWriter.Read(buffer)
+	require.Equal(t, err, io.EOF)
+
+	// close
+	require.NoError(t, readerWriter.Close())
+
+	// read should get EOF still
+	n, err = readerWriter.Read(buffer)
+	require.Equal(t, n, 0)
+	require.Equal(t, err, io.EOF)
+}
+
+func TestCreateUDPConnReuseSourcePort(t *testing.T) {
+	logger := zerolog.Nop()
+	conn, err := createUDPConnForConnIndex(0, &logger)
+	require.NoError(t, err)
+
+	getPortFunc := func(conn *net.UDPConn) int {
+		addr := conn.LocalAddr().(*net.UDPAddr)
+		return addr.Port
+	}
+
+	initialPort := getPortFunc(conn)
+
+	// close conn
+	conn.Close()
+
+	// should get the same port as before.
+	conn, err = createUDPConnForConnIndex(0, &logger)
+	require.NoError(t, err)
+	require.Equal(t, initialPort, getPortFunc(conn))
+
+	// new index, should get a different port
+	conn1, err := createUDPConnForConnIndex(1, &logger)
+	require.NoError(t, err)
+	require.NotEqual(t, initialPort, getPortFunc(conn1))
+
+	// not closing the conn and trying to obtain a new conn for same index should give a different random port
+	conn, err = createUDPConnForConnIndex(0, &logger)
+	require.NoError(t, err)
+	require.NotEqual(t, initialPort, getPortFunc(conn))
+}
+
 func serveSession(ctx context.Context, qc *QUICConnection, edgeQUICSession quic.Connection, closeType closeReason, expectedReason string, t *testing.T) {
 	var (
 		payload = []byte(t.Name())
@@ -538,8 +617,12 @@ func serveSession(ctx context.Context, qc *QUICConnection, edgeQUICSession quic.
 		close(sessionDone)
 	}()
 
-	// Send a message to the quic session on edge side, it should be deumx to this datagram session
-	muxedPayload := append(payload, sessionID[:]...)
+	// Send a message to the quic session on edge side, it should be deumx to this datagram v2 session
+	muxedPayload, err := quicpogs.SuffixSessionID(sessionID, payload)
+	require.NoError(t, err)
+	muxedPayload, err = quicpogs.SuffixType(muxedPayload, quicpogs.DatagramTypeUDP)
+	require.NoError(t, err)
+
 	err = edgeQUICSession.SendMessage(muxedPayload)
 	require.NoError(t, err)
 
@@ -608,8 +691,8 @@ type mockSessionRPCServer struct {
 	calledUnregisterChan chan struct{}
 }
 
-func (s mockSessionRPCServer) RegisterUdpSession(ctx context.Context, sessionID uuid.UUID, dstIP net.IP, dstPort uint16, closeIdleAfter time.Duration) error {
-	return fmt.Errorf("mockSessionRPCServer doesn't implement RegisterUdpSession")
+func (s mockSessionRPCServer) RegisterUdpSession(ctx context.Context, sessionID uuid.UUID, dstIP net.IP, dstPort uint16, closeIdleAfter time.Duration, traceContext string) (*pogs.RegisterUdpSessionResponse, error) {
+	return nil, fmt.Errorf("mockSessionRPCServer doesn't implement RegisterUdpSession")
 }
 
 func (s mockSessionRPCServer) UnregisterUdpSession(ctx context.Context, sessionID uuid.UUID, reason string) error {
@@ -623,7 +706,7 @@ func (s mockSessionRPCServer) UnregisterUdpSession(ctx context.Context, sessionI
 	return nil
 }
 
-func testQUICConnection(udpListenerAddr net.Addr, t *testing.T) *QUICConnection {
+func testQUICConnection(udpListenerAddr net.Addr, t *testing.T, index uint8) *QUICConnection {
 	tlsClientConfig := &tls.Config{
 		InsecureSkipVerify: true,
 		NextProtos:         []string{"argotunnel"},
@@ -633,12 +716,26 @@ func testQUICConnection(udpListenerAddr net.Addr, t *testing.T) *QUICConnection
 	qc, err := NewQUICConnection(
 		testQUICConfig,
 		udpListenerAddr,
+		index,
 		tlsClientConfig,
 		&mockOrchestrator{originProxy: &mockOriginProxyWithRequest{}},
 		&tunnelpogs.ConnectionOptions{},
 		fakeControlStream{},
 		&log,
+		nil,
 	)
 	require.NoError(t, err)
 	return qc
 }
+
+type mockReaderNoopWriter struct {
+	io.Reader
+}
+
+func (m *mockReaderNoopWriter) Write(p []byte) (n int, err error) {
+	return len(p), nil
+}
+
+func (m *mockReaderNoopWriter) Close() error {
+	return nil
+}

connection/rpc.go

@@ -58,6 +58,7 @@ type NamedTunnelRPCClient interface {
 		config *NamedTunnelProperties,
 		options *tunnelpogs.ConnectionOptions,
 		connIndex uint8,
+		edgeAddress net.IP,
 		observer *Observer,
 	) (*tunnelpogs.ConnectionDetails, error)
 	SendLocalConfiguration(
@@ -95,6 +96,7 @@ func (rsc *registrationServerClient) RegisterConnection(
 	properties *NamedTunnelProperties,
 	options *tunnelpogs.ConnectionOptions,
 	connIndex uint8,
+	edgeAddress net.IP,
 	observer *Observer,
 ) (*tunnelpogs.ConnectionDetails, error) {
 	conn, err := rsc.client.RegisterConnection(
@@ -115,9 +117,6 @@ func (rsc *registrationServerClient) RegisterConnection(
 
 	observer.metrics.regSuccess.WithLabelValues("registerConnection").Inc()
 
-	observer.logServerInfo(connIndex, conn.Location, options.OriginLocalIP, fmt.Sprintf("Connection %s registered", conn.UUID))
-	observer.sendConnectedEvent(connIndex, conn.Location)
-
 	return conn, nil
 }
 
@@ -291,9 +290,13 @@ func (h *h2muxConnection) registerNamedTunnel(
 	rpcClient := h.newRPCClientFunc(ctx, stream, h.observer.log)
 	defer rpcClient.Close()
 
-	if _, err = rpcClient.RegisterConnection(ctx, namedTunnel, connOptions, h.connIndex, h.observer); err != nil {
+	registrationDetails, err := rpcClient.RegisterConnection(ctx, namedTunnel, connOptions, h.connIndex, nil, h.observer)
+	if err != nil {
 		return err
 	}
+	h.observer.logServerInfo(h.connIndex, registrationDetails.Location, nil, fmt.Sprintf("Connection %s registered", registrationDetails.UUID))
+	h.observer.sendConnectedEvent(h.connIndex, H2mux, registrationDetails.Location)
+
 	return nil
 }
 

datagramsession/event.go

@@ -42,9 +42,3 @@ func (sc *errClosedSession) Error() string {
 		return fmt.Sprintf("session closed by local due to %s", sc.message)
 	}
 }
-
-// newDatagram is an event when transport receives new datagram
-type newDatagram struct {
-	sessionID uuid.UUID
-	payload   []byte
-}

datagramsession/manager.go

@@ -7,9 +7,9 @@ import (
 	"time"
 
 	"github.com/google/uuid"
-	"github.com/lucas-clemente/quic-go"
 	"github.com/rs/zerolog"
-	"golang.org/x/sync/errgroup"
+
+	"github.com/cloudflare/cloudflared/packet"
 )
 
 const (
@@ -36,26 +36,25 @@ type Manager interface {
 type manager struct {
 	registrationChan   chan *registerSessionEvent
 	unregistrationChan chan *unregisterSessionEvent
-	datagramChan       chan *newDatagram
-	closedChan         chan struct{}
-	transport          transport
+	sendFunc           transportSender
+	receiveChan        <-chan *packet.Session
+	closedChan         <-chan struct{}
 	sessions           map[uuid.UUID]*Session
 	log                *zerolog.Logger
 	// timeout waiting for an API to finish. This can be overriden in test
 	timeout time.Duration
 }
 
-func NewManager(transport transport, log *zerolog.Logger) *manager {
+func NewManager(log *zerolog.Logger, sendF transportSender, receiveChan <-chan *packet.Session) *manager {
 	return &manager{
 		registrationChan:   make(chan *registerSessionEvent),
 		unregistrationChan: make(chan *unregisterSessionEvent),
-		// datagramChan is buffered, so it can read more datagrams from transport while the event loop is processing other events
-		datagramChan: make(chan *newDatagram, requestChanCapacity),
-		closedChan:   make(chan struct{}),
-		transport:    transport,
-		sessions:     make(map[uuid.UUID]*Session),
-		log:          log,
-		timeout:      defaultReqTimeout,
+		sendFunc:           sendF,
+		receiveChan:        receiveChan,
+		closedChan:         make(chan struct{}),
+		sessions:           make(map[uuid.UUID]*Session),
+		log:                log,
+		timeout:            defaultReqTimeout,
 	}
 }
 
@@ -65,49 +64,21 @@ func (m *manager) UpdateLogger(log *zerolog.Logger) {
 }
 
 func (m *manager) Serve(ctx context.Context) error {
-	errGroup, ctx := errgroup.WithContext(ctx)
-	errGroup.Go(func() error {
-		for {
-			sessionID, payload, err := m.transport.ReceiveFrom()
-			if err != nil {
-				if aerr, ok := err.(*quic.ApplicationError); ok && uint64(aerr.ErrorCode) == uint64(quic.NoError) {
-					return nil
-				} else {
-					return err
-				}
-			}
-			datagram := &newDatagram{
-				sessionID: sessionID,
-				payload:   payload,
-			}
-			select {
-			case <-ctx.Done():
-				return ctx.Err()
-			// Only the event loop routine can update/lookup the sessions map to avoid concurrent access
-			// Send the datagram to the event loop. It will find the session to send to
-			case m.datagramChan <- datagram:
-			}
+	for {
+		select {
+		case <-ctx.Done():
+			m.shutdownSessions(ctx.Err())
+			return ctx.Err()
+		// receiveChan is buffered, so the transport can read more datagrams from transport while the event loop is
+		// processing other events
+		case datagram := <-m.receiveChan:
+			m.sendToSession(datagram)
+		case registration := <-m.registrationChan:
+			m.registerSession(ctx, registration)
+		case unregistration := <-m.unregistrationChan:
+			m.unregisterSession(unregistration)
 		}
-	})
-	errGroup.Go(func() error {
-		for {
-			select {
-			case <-ctx.Done():
-				return nil
-			case datagram := <-m.datagramChan:
-				m.sendToSession(datagram)
-			case registration := <-m.registrationChan:
-				m.registerSession(ctx, registration)
-			// TODO: TUN-5422: Unregister inactive session upon timeout
-			case unregistration := <-m.unregistrationChan:
-				m.unregisterSession(unregistration)
-			}
-		}
-	})
-	err := errGroup.Wait()
-	close(m.closedChan)
-	m.shutdownSessions(err)
-	return err
+	}
 }
 
 func (m *manager) shutdownSessions(err error) {
@@ -149,16 +120,17 @@ func (m *manager) registerSession(ctx context.Context, registration *registerSes
 }
 
 func (m *manager) newSession(id uuid.UUID, dstConn io.ReadWriteCloser) *Session {
+	logger := m.log.With().Str("sessionID", id.String()).Logger()
 	return &Session{
-		ID:        id,
-		transport: m.transport,
-		dstConn:   dstConn,
+		ID:       id,
+		sendFunc: m.sendFunc,
+		dstConn:  dstConn,
 		// activeAtChan has low capacity. It can be full when there are many concurrent read/write. markActive() will
 		// drop instead of blocking because last active time only needs to be an approximation
 		activeAtChan: make(chan time.Time, 2),
 		// capacity is 2 because close() and dstToTransport routine in Serve() can write to this channel
 		closeChan: make(chan error, 2),
-		log:       m.log,
+		log:       &logger,
 	}
 }
 
@@ -191,16 +163,13 @@ func (m *manager) unregisterSession(unregistration *unregisterSessionEvent) {
 	}
 }
 
-func (m *manager) sendToSession(datagram *newDatagram) {
-	session, ok := m.sessions[datagram.sessionID]
+func (m *manager) sendToSession(datagram *packet.Session) {
+	session, ok := m.sessions[datagram.ID]
 	if !ok {
-		m.log.Error().Str("sessionID", datagram.sessionID.String()).Msg("session not found")
+		m.log.Error().Str("sessionID", datagram.ID.String()).Msg("session not found")
 		return
 	}
 	// session writes to destination over a connected UDP socket, which should not be blocking, so this call doesn't
 	// need to run in another go routine
-	_, err := session.transportToDst(datagram.payload)
-	if err != nil {
-		m.log.Err(err).Str("sessionID", datagram.sessionID.String()).Msg("Failed to write payload to session")
-	}
+	session.transportToDst(datagram.Payload)
 }

datagramsession/manager_test.go

@@ -14,22 +14,30 @@ import (
 	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/require"
 	"golang.org/x/sync/errgroup"
+
+	"github.com/cloudflare/cloudflared/packet"
+)
+
+var (
+	nopLogger = zerolog.Nop()
 )
 
 func TestManagerServe(t *testing.T) {
 	const (
-		sessions            = 20
-		msgs                = 50
+		sessions            = 2
+		msgs                = 5
 		remoteUnregisterMsg = "eyeball closed connection"
 	)
 
-	mg, transport := newTestManager(1)
-
-	eyeballTracker := make(map[uuid.UUID]*datagramChannel)
-	for i := 0; i < sessions; i++ {
-		sessionID := uuid.New()
-		eyeballTracker[sessionID] = newDatagramChannel(1)
+	requestChan := make(chan *packet.Session)
+	transport := mockQUICTransport{
+		sessions: make(map[uuid.UUID]chan []byte),
 	}
+	for i := 0; i < sessions; i++ {
+		transport.sessions[uuid.New()] = make(chan []byte)
+	}
+
+	mg := NewManager(&nopLogger, transport.MuxSession, requestChan)
 
 	ctx, cancel := context.WithCancel(context.Background())
 	serveDone := make(chan struct{})
@@ -38,53 +46,42 @@ func TestManagerServe(t *testing.T) {
 		close(serveDone)
 	}(ctx)
 
-	go func(ctx context.Context) {
-		for {
-			sessionID, payload, err := transport.respChan.Receive(ctx)
-			if err != nil {
-				require.Equal(t, context.Canceled, err)
-				return
-			}
-			respChan := eyeballTracker[sessionID]
-			require.NoError(t, respChan.Send(ctx, sessionID, payload))
-		}
-	}(ctx)
-
 	errGroup, ctx := errgroup.WithContext(ctx)
-	for sID, receiver := range eyeballTracker {
+	for sessionID, eyeballRespChan := range transport.sessions {
+		// Assign loop variables to local variables
+		sID := sessionID
+		payload := testPayload(sID)
+		expectResp := testResponse(payload)
+
+		cfdConn, originConn := net.Pipe()
+
+		origin := mockOrigin{
+			expectMsgCount: msgs,
+			expectedMsg:    payload,
+			expectedResp:   expectResp,
+			conn:           originConn,
+		}
+
+		eyeball := mockEyeballSession{
+			id:               sID,
+			expectedMsgCount: msgs,
+			expectedMsg:      payload,
+			expectedResponse: expectResp,
+			respReceiver:     eyeballRespChan,
+		}
+
 		// Assign loop variables to local variables
-		sessionID := sID
-		eyeballRespReceiver := receiver
 		errGroup.Go(func() error {
-			payload := testPayload(sessionID)
-			expectResp := testResponse(payload)
-
-			cfdConn, originConn := net.Pipe()
-
-			origin := mockOrigin{
-				expectMsgCount: msgs,
-				expectedMsg:    payload,
-				expectedResp:   expectResp,
-				conn:           originConn,
-			}
-			eyeball := mockEyeball{
-				expectMsgCount:  msgs,
-				expectedMsg:     expectResp,
-				expectSessionID: sessionID,
-				respReceiver:    eyeballRespReceiver,
-			}
-
+			session, err := mg.RegisterSession(ctx, sID, cfdConn)
+			require.NoError(t, err)
 			reqErrGroup, reqCtx := errgroup.WithContext(ctx)
 			reqErrGroup.Go(func() error {
 				return origin.serve()
 			})
 			reqErrGroup.Go(func() error {
-				return eyeball.serve(reqCtx)
+				return eyeball.serve(reqCtx, requestChan)
 			})
 
-			session, err := mg.RegisterSession(ctx, sessionID, cfdConn)
-			require.NoError(t, err)
-
 			sessionDone := make(chan struct{})
 			go func() {
 				closedByRemote, err := session.Serve(ctx, time.Minute*2)
@@ -97,23 +94,17 @@ func TestManagerServe(t *testing.T) {
 				close(sessionDone)
 			}()
 
-			for i := 0; i < msgs; i++ {
-				require.NoError(t, transport.newRequest(ctx, sessionID, testPayload(sessionID)))
-			}
-
 			// Make sure eyeball and origin have received all messages before unregistering the session
 			require.NoError(t, reqErrGroup.Wait())
 
-			require.NoError(t, mg.UnregisterSession(ctx, sessionID, remoteUnregisterMsg, true))
+			require.NoError(t, mg.UnregisterSession(ctx, sID, remoteUnregisterMsg, true))
 			<-sessionDone
-
 			return nil
 		})
 	}
 
 	require.NoError(t, errGroup.Wait())
 	cancel()
-	transport.close()
 	<-serveDone
 }
 
@@ -122,7 +113,7 @@ func TestTimeout(t *testing.T) {
 		testTimeout = time.Millisecond * 50
 	)
 
-	mg, _ := newTestManager(1)
+	mg := NewManager(&nopLogger, nil, nil)
 	mg.timeout = testTimeout
 	ctx := context.Background()
 	sessionID := uuid.New()
@@ -135,9 +126,51 @@ func TestTimeout(t *testing.T) {
 	require.ErrorIs(t, err, context.DeadlineExceeded)
 }
 
-func TestCloseTransportCloseSessions(t *testing.T) {
-	mg, transport := newTestManager(1)
-	ctx := context.Background()
+func TestUnregisterSessionCloseSession(t *testing.T) {
+	sessionID := uuid.New()
+	payload := []byte(t.Name())
+	sender := newMockTransportSender(sessionID, payload)
+	mg := NewManager(&nopLogger, sender.muxSession, nil)
+	ctx, cancel := context.WithCancel(context.Background())
+
+	managerDone := make(chan struct{})
+	go func() {
+		err := mg.Serve(ctx)
+		require.Error(t, err)
+		close(managerDone)
+	}()
+
+	cfdConn, originConn := net.Pipe()
+	session, err := mg.RegisterSession(ctx, sessionID, cfdConn)
+	require.NoError(t, err)
+	require.NotNil(t, session)
+
+	unregisteredChan := make(chan struct{})
+	go func() {
+		_, err := originConn.Write(payload)
+		require.NoError(t, err)
+
+		err = mg.UnregisterSession(ctx, sessionID, "eyeball closed session", true)
+		require.NoError(t, err)
+
+		close(unregisteredChan)
+	}()
+
+	closedByRemote, err := session.Serve(ctx, time.Minute)
+	require.True(t, closedByRemote)
+	require.Error(t, err)
+
+	<-unregisteredChan
+	cancel()
+	<-managerDone
+}
+
+func TestManagerCtxDoneCloseSessions(t *testing.T) {
+	sessionID := uuid.New()
+	payload := []byte(t.Name())
+	sender := newMockTransportSender(sessionID, payload)
+	mg := NewManager(&nopLogger, sender.muxSession, nil)
+	ctx, cancel := context.WithCancel(context.Background())
 
 	var wg sync.WaitGroup
 	wg.Add(1)
@@ -147,35 +180,26 @@ func TestCloseTransportCloseSessions(t *testing.T) {
 		require.Error(t, err)
 	}()
 
-	cfdConn, eyeballConn := net.Pipe()
-	session, err := mg.RegisterSession(ctx, uuid.New(), cfdConn)
+	cfdConn, originConn := net.Pipe()
+	session, err := mg.RegisterSession(ctx, sessionID, cfdConn)
 	require.NoError(t, err)
 	require.NotNil(t, session)
 
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
-		_, err := eyeballConn.Write([]byte(t.Name()))
+		_, err := originConn.Write(payload)
 		require.NoError(t, err)
-		transport.close()
+		cancel()
 	}()
 
 	closedByRemote, err := session.Serve(ctx, time.Minute)
-	require.True(t, closedByRemote)
+	require.False(t, closedByRemote)
 	require.Error(t, err)
 
 	wg.Wait()
 }
 
-func newTestManager(capacity uint) (*manager, *mockQUICTransport) {
-	log := zerolog.Nop()
-	transport := &mockQUICTransport{
-		reqChan:  newDatagramChannel(capacity),
-		respChan: newDatagramChannel(capacity),
-	}
-	return NewManager(transport, &log), transport
-}
-
 type mockOrigin struct {
 	expectMsgCount int
 	expectedMsg    []byte
@@ -197,7 +221,6 @@ func (mo *mockOrigin) serve() error {
 		if !bytes.Equal(readBuffer[:n], mo.expectedMsg) {
 			return fmt.Errorf("Expect %v, read %v", mo.expectedMsg, readBuffer[:n])
 		}
-
 		_, err = mo.conn.Write(mo.expectedResp)
 		if err != nil {
 			return err
@@ -214,72 +237,35 @@ func testResponse(msg []byte) []byte {
 	return []byte(fmt.Sprintf("Response to %v", msg))
 }
 
-type mockEyeball struct {
-	expectMsgCount  int
-	expectedMsg     []byte
-	expectSessionID uuid.UUID
-	respReceiver    *datagramChannel
+type mockQUICTransport struct {
+	sessions map[uuid.UUID]chan []byte
 }
 
-func (me *mockEyeball) serve(ctx context.Context) error {
-	for i := 0; i < me.expectMsgCount; i++ {
-		sessionID, msg, err := me.respReceiver.Receive(ctx)
-		if err != nil {
-			return err
-		}
-		if sessionID != me.expectSessionID {
-			return fmt.Errorf("Expect session %s, got %s", me.expectSessionID, sessionID)
-		}
-		if !bytes.Equal(msg, me.expectedMsg) {
-			return fmt.Errorf("Expect %v, read %v", me.expectedMsg, msg)
-		}
-	}
+func (me *mockQUICTransport) MuxSession(session *packet.Session) error {
+	s := me.sessions[session.ID]
+	s <- session.Payload
 	return nil
 }
 
-// datagramChannel is a channel for Datagram with wrapper to send/receive with context
-type datagramChannel struct {
-	datagramChan chan *newDatagram
-	closedChan   chan struct{}
+type mockEyeballSession struct {
+	id               uuid.UUID
+	expectedMsgCount int
+	expectedMsg      []byte
+	expectedResponse []byte
+	respReceiver     <-chan []byte
 }
 
-func newDatagramChannel(capacity uint) *datagramChannel {
-	return &datagramChannel{
-		datagramChan: make(chan *newDatagram, capacity),
-		closedChan:   make(chan struct{}),
-	}
-}
-
-func (rc *datagramChannel) Send(ctx context.Context, sessionID uuid.UUID, payload []byte) error {
-	select {
-	case <-ctx.Done():
-		return ctx.Err()
-	case <-rc.closedChan:
-		return &errClosedSession{
-			message:  fmt.Errorf("datagram channel closed").Error(),
-			byRemote: true,
+func (me *mockEyeballSession) serve(ctx context.Context, requestChan chan *packet.Session) error {
+	for i := 0; i < me.expectedMsgCount; i++ {
+		requestChan <- &packet.Session{
+			ID:      me.id,
+			Payload: me.expectedMsg,
 		}
-	case rc.datagramChan <- &newDatagram{sessionID: sessionID, payload: payload}:
-		return nil
-	}
-}
-
-func (rc *datagramChannel) Receive(ctx context.Context) (uuid.UUID, []byte, error) {
-	select {
-	case <-ctx.Done():
-		return uuid.Nil, nil, ctx.Err()
-	case <-rc.closedChan:
-		err := &errClosedSession{
-			message:  fmt.Errorf("datagram channel closed").Error(),
-			byRemote: true,
+		resp := <-me.respReceiver
+		if !bytes.Equal(resp, me.expectedResponse) {
+			return fmt.Errorf("Expect %v, read %v", me.expectedResponse, resp)
 		}
-		return uuid.Nil, nil, err
-	case msg := <-rc.datagramChan:
-		return msg.sessionID, msg.payload, nil
+		fmt.Println("Resp", resp)
 	}
-}
-
-func (rc *datagramChannel) Close() {
-	// No need to close msgChan, it will be garbage collect once there is no reference to it
-	close(rc.closedChan)
+	return nil
 }

datagramsession/session.go

@@ -2,12 +2,16 @@ package datagramsession
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"io"
+	"net"
 	"time"
 
 	"github.com/google/uuid"
 	"github.com/rs/zerolog"
+
+	"github.com/cloudflare/cloudflared/packet"
 )
 
 const (
@@ -18,21 +22,27 @@ func SessionIdleErr(timeout time.Duration) error {
 	return fmt.Errorf("session idle for %v", timeout)
 }
 
+type transportSender func(session *packet.Session) error
+
+// ErrVithVariableSeverity are errors that have variable severity
+type ErrVithVariableSeverity interface {
+	error
+	// LogLevel return the severity of this error
+	LogLevel() zerolog.Level
+}
+
 // Session is a bidirectional pipe of datagrams between transport and dstConn
-// Currently the only implementation of transport is quic DatagramMuxer
 // Destination can be a connection with origin or with eyeball
 // When the destination is origin:
-// - Datagrams from edge are read by Manager from the transport. Manager finds the corresponding Session and calls the
-//   write method of the Session to send to origin
-// - Datagrams from origin are read from conn and SentTo transport. Transport will return them to eyeball
+// - Manager receives datagrams from receiveChan and calls the transportToDst method of the Session to send to origin
+// - Datagrams from origin are read from conn and Send to transport using the transportSender callback. Transport will return them to eyeball
 // When the destination is eyeball:
-// - Datagrams from eyeball are read from conn and SentTo transport. Transport will send them to cloudflared
-// - Datagrams from cloudflared are read by Manager from the transport. Manager finds the corresponding Session and calls the
-//   write method of the Session to send to eyeball
+// - Datagrams from eyeball are read from conn and Send to transport. Transport will send them to cloudflared using the transportSender callback.
+// - Manager receives datagrams from receiveChan and calls the transportToDst method of the Session to send to the eyeball
 type Session struct {
-	ID        uuid.UUID
-	transport transport
-	dstConn   io.ReadWriteCloser
+	ID       uuid.UUID
+	sendFunc transportSender
+	dstConn  io.ReadWriteCloser
 	// activeAtChan is used to communicate the last read/write time
 	activeAtChan chan time.Time
 	closeChan    chan error
@@ -46,9 +56,20 @@ func (s *Session) Serve(ctx context.Context, closeAfterIdle time.Duration) (clos
 		const maxPacketSize = 1500
 		readBuffer := make([]byte, maxPacketSize)
 		for {
-			if err := s.dstToTransport(readBuffer); err != nil {
-				s.closeChan <- err
-				return
+			if closeSession, err := s.dstToTransport(readBuffer); err != nil {
+				if errors.Is(err, net.ErrClosed) {
+					s.log.Debug().Msg("Destination connection closed")
+				} else {
+					level := zerolog.ErrorLevel
+					if variableErr, ok := err.(ErrVithVariableSeverity); ok {
+						level = variableErr.LogLevel()
+					}
+					s.log.WithLevel(level).Err(err).Msg("Failed to send session payload from destination to transport")
+				}
+				if closeSession {
+					s.closeChan <- err
+					return
+				}
 			}
 		}
 	}()
@@ -89,32 +110,29 @@ func (s *Session) waitForCloseCondition(ctx context.Context, closeAfterIdle time
 	}
 }
 
-func (s *Session) dstToTransport(buffer []byte) error {
+func (s *Session) dstToTransport(buffer []byte) (closeSession bool, err error) {
 	n, err := s.dstConn.Read(buffer)
 	s.markActive()
 	// https://pkg.go.dev/io#Reader suggests caller should always process n > 0 bytes
-	if n > 0 {
-		if n <= int(s.transport.MTU()) {
-			err = s.transport.SendTo(s.ID, buffer[:n])
-		} else {
-			// drop packet for now, eventually reply with ICMP for PMTUD
-			s.log.Debug().
-				Str("session", s.ID.String()).
-				Int("len", n).
-				Int("mtu", s.transport.MTU()).
-				Msg("dropped packet exceeding MTU")
+	if n > 0 || err == nil {
+		session := packet.Session{
+			ID:      s.ID,
+			Payload: buffer[:n],
+		}
+		if sendErr := s.sendFunc(&session); sendErr != nil {
+			return false, sendErr
 		}
 	}
-	// Some UDP application might send 0-size payload.
-	if err == nil && n == 0 {
-		err = s.transport.SendTo(s.ID, []byte{})
-	}
-	return err
+	return err != nil, err
 }
 
 func (s *Session) transportToDst(payload []byte) (int, error) {
 	s.markActive()
-	return s.dstConn.Write(payload)
+	n, err := s.dstConn.Write(payload)
+	if err != nil {
+		s.log.Err(err).Msg("Failed to write payload to session")
+	}
+	return n, err
 }
 
 // Sends the last active time to the idle checker loop without blocking. activeAtChan will only be full when there

datagramsession/session_test.go

@@ -11,8 +11,11 @@ import (
 	"time"
 
 	"github.com/google/uuid"
+	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/require"
 	"golang.org/x/sync/errgroup"
+
+	"github.com/cloudflare/cloudflared/packet"
 )
 
 // TestCloseSession makes sure a session will stop after context is done
@@ -41,7 +44,8 @@ func testSessionReturns(t *testing.T, closeBy closeMethod, closeAfterIdle time.D
 	cfdConn, originConn := net.Pipe()
 	payload := testPayload(sessionID)
 
-	mg, _ := newTestManager(1)
+	log := zerolog.Nop()
+	mg := NewManager(&log, nil, nil)
 	session := mg.newSession(sessionID, cfdConn)
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -114,7 +118,9 @@ func testActiveSessionNotClosed(t *testing.T, readFromDst bool, writeToDst bool)
 	cfdConn, originConn := net.Pipe()
 	payload := testPayload(sessionID)
 
-	mg, _ := newTestManager(100)
+	respChan := make(chan *packet.Session)
+	sender := newMockTransportSender(sessionID, payload)
+	mg := NewManager(&nopLogger, sender.muxSession, respChan)
 	session := mg.newSession(sessionID, cfdConn)
 
 	startTime := time.Now()
@@ -177,7 +183,7 @@ func testActiveSessionNotClosed(t *testing.T, readFromDst bool, writeToDst bool)
 
 func TestMarkActiveNotBlocking(t *testing.T) {
 	const concurrentCalls = 50
-	mg, _ := newTestManager(1)
+	mg := NewManager(&nopLogger, nil, nil)
 	session := mg.newSession(uuid.New(), nil)
 	var wg sync.WaitGroup
 	wg.Add(concurrentCalls)
@@ -190,11 +196,16 @@ func TestMarkActiveNotBlocking(t *testing.T) {
 	wg.Wait()
 }
 
+// Some UDP application might send 0-size payload.
 func TestZeroBytePayload(t *testing.T) {
 	sessionID := uuid.New()
 	cfdConn, originConn := net.Pipe()
 
-	mg, transport := newTestManager(1)
+	sender := sendOnceTransportSender{
+		baseSender: newMockTransportSender(sessionID, make([]byte, 0)),
+		sentChan:   make(chan struct{}),
+	}
+	mg := NewManager(&nopLogger, sender.muxSession, nil)
 	session := mg.newSession(sessionID, cfdConn)
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -215,11 +226,39 @@ func TestZeroBytePayload(t *testing.T) {
 		return nil
 	})
 
-	receivedSessionID, payload, err := transport.respChan.Receive(ctx)
-	require.NoError(t, err)
-	require.Len(t, payload, 0)
-	require.Equal(t, sessionID, receivedSessionID)
-
+	<-sender.sentChan
 	cancel()
 	require.NoError(t, errGroup.Wait())
 }
+
+type mockTransportSender struct {
+	expectedSessionID uuid.UUID
+	expectedPayload   []byte
+}
+
+func newMockTransportSender(expectedSessionID uuid.UUID, expectedPayload []byte) *mockTransportSender {
+	return &mockTransportSender{
+		expectedSessionID: expectedSessionID,
+		expectedPayload:   expectedPayload,
+	}
+}
+
+func (mts *mockTransportSender) muxSession(session *packet.Session) error {
+	if session.ID != mts.expectedSessionID {
+		return fmt.Errorf("Expect session %s, got %s", mts.expectedSessionID, session.ID)
+	}
+	if !bytes.Equal(session.Payload, mts.expectedPayload) {
+		return fmt.Errorf("Expect %v, read %v", mts.expectedPayload, session.Payload)
+	}
+	return nil
+}
+
+type sendOnceTransportSender struct {
+	baseSender *mockTransportSender
+	sentChan   chan struct{}
+}
+
+func (sots *sendOnceTransportSender) muxSession(session *packet.Session) error {
+	defer close(sots.sentChan)
+	return sots.baseSender.muxSession(session)
+}

datagramsession/transport.go

@@ -1,13 +0,0 @@
-package datagramsession
-
-import "github.com/google/uuid"
-
-// Transport is a connection between cloudflared and edge that can multiplex datagrams from multiple sessions
-type transport interface {
-	// SendTo writes payload for a session to the transport
-	SendTo(sessionID uuid.UUID, payload []byte) error
-	// ReceiveFrom reads the next datagram from the transport
-	ReceiveFrom() (uuid.UUID, []byte, error)
-	// Max transmission unit to receive from the transport
-	MTU() int
-}

datagramsession/transport_test.go

@@ -1,36 +0,0 @@
-package datagramsession
-
-import (
-	"context"
-
-	"github.com/google/uuid"
-)
-
-type mockQUICTransport struct {
-	reqChan  *datagramChannel
-	respChan *datagramChannel
-}
-
-func (mt *mockQUICTransport) SendTo(sessionID uuid.UUID, payload []byte) error {
-	buf := make([]byte, len(payload))
-	// The QUIC implementation copies data to another buffer before returning https://github.com/lucas-clemente/quic-go/blob/v0.24.0/session.go#L1967-L1975
-	copy(buf, payload)
-	return mt.respChan.Send(context.Background(), sessionID, buf)
-}
-
-func (mt *mockQUICTransport) ReceiveFrom() (uuid.UUID, []byte, error) {
-	return mt.reqChan.Receive(context.Background())
-}
-
-func (mt *mockQUICTransport) MTU() int {
-	return 1280
-}
-
-func (mt *mockQUICTransport) newRequest(ctx context.Context, sessionID uuid.UUID, payload []byte) error {
-	return mt.reqChan.Send(ctx, sessionID, payload)
-}
-
-func (mt *mockQUICTransport) close() {
-	mt.reqChan.Close()
-	mt.respChan.Close()
-}

dev.Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.17.10 as builder
+FROM golang:1.18 as builder
 ENV GO111MODULE=on \
     CGO_ENABLED=0
 WORKDIR /go/src/github.com/cloudflare/cloudflared/

edgediscovery/allregions/address.go

@@ -0,0 +1,64 @@
+package allregions
+
+// Region contains cloudflared edge addresses. The edge is partitioned into several regions for
+// redundancy purposes.
+type AddrSet map[*EdgeAddr]UsedBy
+
+// AddrUsedBy finds the address used by the given connection in this region.
+// Returns nil if the connection isn't using any IP.
+func (a AddrSet) AddrUsedBy(connID int) *EdgeAddr {
+	for addr, used := range a {
+		if used.Used && used.ConnID == connID {
+			return addr
+		}
+	}
+	return nil
+}
+
+// AvailableAddrs counts how many unused addresses this region contains.
+func (a AddrSet) AvailableAddrs() int {
+	n := 0
+	for _, usedby := range a {
+		if !usedby.Used {
+			n++
+		}
+	}
+	return n
+}
+
+// GetUnusedIP returns a random unused address in this region.
+// Returns nil if all addresses are in use.
+func (a AddrSet) GetUnusedIP(excluding *EdgeAddr) *EdgeAddr {
+	for addr, usedby := range a {
+		if !usedby.Used && addr != excluding {
+			return addr
+		}
+	}
+	return nil
+}
+
+// Use the address, assigning it to a proxy connection.
+func (a AddrSet) Use(addr *EdgeAddr, connID int) {
+	if addr == nil {
+		return
+	}
+	a[addr] = InUse(connID)
+}
+
+// GetAnyAddress returns an arbitrary address from the region.
+func (a AddrSet) GetAnyAddress() *EdgeAddr {
+	for addr := range a {
+		return addr
+	}
+	return nil
+}
+
+// GiveBack the address, ensuring it is no longer assigned to an IP.
+// Returns true if the address is in this region.
+func (a AddrSet) GiveBack(addr *EdgeAddr) (ok bool) {
+	if _, ok := a[addr]; !ok {
+		return false
+	}
+	a[addr] = Unused()
+	return true
+}

edgediscovery/allregions/address_test.go

@@ -0,0 +1,247 @@
+package allregions
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestAddrSet_AddrUsedBy(t *testing.T) {
+	type args struct {
+		connID int
+	}
+	tests := []struct {
+		name    string
+		addrSet AddrSet
+		args    args
+		want    *EdgeAddr
+	}{
+		{
+			name: "happy trivial test",
+			addrSet: AddrSet{
+				&addr0: InUse(0),
+			},
+			args: args{connID: 0},
+			want: &addr0,
+		},
+		{
+			name: "sad trivial test",
+			addrSet: AddrSet{
+				&addr0: InUse(0),
+			},
+			args: args{connID: 1},
+			want: nil,
+		},
+		{
+			name: "sad test",
+			addrSet: AddrSet{
+				&addr0: InUse(0),
+				&addr1: InUse(1),
+				&addr2: InUse(2),
+			},
+			args: args{connID: 3},
+			want: nil,
+		},
+		{
+			name: "happy test",
+			addrSet: AddrSet{
+				&addr0: InUse(0),
+				&addr1: InUse(1),
+				&addr2: InUse(2),
+			},
+			args: args{connID: 1},
+			want: &addr1,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := tt.addrSet.AddrUsedBy(tt.args.connID); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("Region.AddrUsedBy() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestAddrSet_AvailableAddrs(t *testing.T) {
+	tests := []struct {
+		name    string
+		addrSet AddrSet
+		want    int
+	}{
+		{
+			name: "contains addresses",
+			addrSet: AddrSet{
+				&addr0: InUse(0),
+				&addr1: Unused(),
+				&addr2: InUse(2),
+			},
+			want: 1,
+		},
+		{
+			name: "all free",
+			addrSet: AddrSet{
+				&addr0: Unused(),
+				&addr1: Unused(),
+				&addr2: Unused(),
+			},
+			want: 3,
+		},
+		{
+			name: "all used",
+			addrSet: AddrSet{
+				&addr0: InUse(0),
+				&addr1: InUse(1),
+				&addr2: InUse(2),
+			},
+			want: 0,
+		},
+		{
+			name:    "empty",
+			addrSet: AddrSet{},
+			want:    0,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := tt.addrSet.AvailableAddrs(); got != tt.want {
+				t.Errorf("Region.AvailableAddrs() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestAddrSet_GetUnusedIP(t *testing.T) {
+	type args struct {
+		excluding *EdgeAddr
+	}
+	tests := []struct {
+		name    string
+		addrSet AddrSet
+		args    args
+		want    *EdgeAddr
+	}{
+		{
+			name: "happy test with excluding set",
+			addrSet: AddrSet{
+				&addr0: Unused(),
+				&addr1: Unused(),
+				&addr2: InUse(2),
+			},
+			args: args{excluding: &addr0},
+			want: &addr1,
+		},
+		{
+			name: "happy test with no excluding",
+			addrSet: AddrSet{
+				&addr0: InUse(0),
+				&addr1: Unused(),
+				&addr2: InUse(2),
+			},
+			args: args{excluding: nil},
+			want: &addr1,
+		},
+		{
+			name: "sad test with no excluding",
+			addrSet: AddrSet{
+				&addr0: InUse(0),
+				&addr1: InUse(1),
+				&addr2: InUse(2),
+			},
+			args: args{excluding: nil},
+			want: nil,
+		},
+		{
+			name: "sad test with excluding",
+			addrSet: AddrSet{
+				&addr0: Unused(),
+				&addr1: InUse(1),
+				&addr2: InUse(2),
+			},
+			args: args{excluding: &addr0},
+			want: nil,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := tt.addrSet.GetUnusedIP(tt.args.excluding); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("Region.GetUnusedIP() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestAddrSet_GiveBack(t *testing.T) {
+	type args struct {
+		addr *EdgeAddr
+	}
+	tests := []struct {
+		name           string
+		addrSet        AddrSet
+		args           args
+		wantOk         bool
+		availableAfter int
+	}{
+		{
+			name: "sad test with excluding",
+			addrSet: AddrSet{
+				&addr1: InUse(1),
+			},
+			args:           args{addr: &addr1},
+			wantOk:         true,
+			availableAfter: 1,
+		},
+		{
+			name: "sad test with excluding",
+			addrSet: AddrSet{
+				&addr1: InUse(1),
+			},
+			args:           args{addr: &addr2},
+			wantOk:         false,
+			availableAfter: 0,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if gotOk := tt.addrSet.GiveBack(tt.args.addr); gotOk != tt.wantOk {
+				t.Errorf("Region.GiveBack() = %v, want %v", gotOk, tt.wantOk)
+			}
+			if tt.availableAfter != tt.addrSet.AvailableAddrs() {
+				t.Errorf("Region.AvailableAddrs() = %v, want %v", tt.addrSet.AvailableAddrs(), tt.availableAfter)
+			}
+		})
+	}
+}
+
+func TestAddrSet_GetAnyAddress(t *testing.T) {
+	tests := []struct {
+		name    string
+		addrSet AddrSet
+		wantNil bool
+	}{
+		{
+			name:    "Sad test -- GetAnyAddress should only fail if the region is empty",
+			addrSet: AddrSet{},
+			wantNil: true,
+		},
+		{
+			name: "Happy test (all addresses unused)",
+			addrSet: AddrSet{
+				&addr0: Unused(),
+			},
+			wantNil: false,
+		},
+		{
+			name: "Happy test (GetAnyAddress can still return addresses used by proxy conns)",
+			addrSet: AddrSet{
+				&addr0: InUse(2),
+			},
+			wantNil: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := tt.addrSet.GetAnyAddress(); tt.wantNil != (got == nil) {
+				t.Errorf("Region.GetAnyAddress() = %v, but should it return nil? %v", got, tt.wantNil)
+			}
+		})
+	}
+}

edgediscovery/allregions/discovery.go

@@ -13,7 +13,7 @@ import (
 
 const (
 	// Used to discover HA origintunneld servers
-	srvService = "origintunneld"
+	srvService = "v2-origintunneld"
 	srvProto   = "tcp"
 	srvName    = "argotunnel.com"
 
@@ -115,6 +115,9 @@ func edgeDiscovery(log *zerolog.Logger, srvService string) ([][]*EdgeAddr, error
 		if err != nil {
 			return nil, err
 		}
+		for _, e := range edgeAddrs {
+			log.Debug().Msgf("Edge Address: %+v", *e)
+		}
 		resolvedAddrPerCNAME = append(resolvedAddrPerCNAME, edgeAddrs)
 	}
 
@@ -187,7 +190,6 @@ func ResolveAddrs(addrs []string, log *zerolog.Logger) (resolved []*EdgeAddr) {
 			UDP:       udpAddr,
 			IPVersion: version,
 		})
-
 	}
 	return
 }

edgediscovery/allregions/mocks_for_test.go

@@ -9,6 +9,115 @@ import (
 	"testing/quick"
 )
 
+var (
+	v4Addrs = []*EdgeAddr{&addr0, &addr1, &addr2, &addr3}
+	v6Addrs = []*EdgeAddr{&addr4, &addr5, &addr6, &addr7}
+	addr0   = EdgeAddr{
+		TCP: &net.TCPAddr{
+			IP:   net.ParseIP("123.4.5.0"),
+			Port: 8000,
+			Zone: "",
+		},
+		UDP: &net.UDPAddr{
+			IP:   net.ParseIP("123.4.5.0"),
+			Port: 8000,
+			Zone: "",
+		},
+		IPVersion: V4,
+	}
+	addr1 = EdgeAddr{
+		TCP: &net.TCPAddr{
+			IP:   net.ParseIP("123.4.5.1"),
+			Port: 8000,
+			Zone: "",
+		},
+		UDP: &net.UDPAddr{
+			IP:   net.ParseIP("123.4.5.1"),
+			Port: 8000,
+			Zone: "",
+		},
+		IPVersion: V4,
+	}
+	addr2 = EdgeAddr{
+		TCP: &net.TCPAddr{
+			IP:   net.ParseIP("123.4.5.2"),
+			Port: 8000,
+			Zone: "",
+		},
+		UDP: &net.UDPAddr{
+			IP:   net.ParseIP("123.4.5.2"),
+			Port: 8000,
+			Zone: "",
+		},
+		IPVersion: V4,
+	}
+	addr3 = EdgeAddr{
+		TCP: &net.TCPAddr{
+			IP:   net.ParseIP("123.4.5.3"),
+			Port: 8000,
+			Zone: "",
+		},
+		UDP: &net.UDPAddr{
+			IP:   net.ParseIP("123.4.5.3"),
+			Port: 8000,
+			Zone: "",
+		},
+		IPVersion: V4,
+	}
+	addr4 = EdgeAddr{
+		TCP: &net.TCPAddr{
+			IP:   net.ParseIP("2606:4700:a0::1"),
+			Port: 8000,
+			Zone: "",
+		},
+		UDP: &net.UDPAddr{
+			IP:   net.ParseIP("2606:4700:a0::1"),
+			Port: 8000,
+			Zone: "",
+		},
+		IPVersion: V6,
+	}
+	addr5 = EdgeAddr{
+		TCP: &net.TCPAddr{
+			IP:   net.ParseIP("2606:4700:a0::2"),
+			Port: 8000,
+			Zone: "",
+		},
+		UDP: &net.UDPAddr{
+			IP:   net.ParseIP("2606:4700:a0::2"),
+			Port: 8000,
+			Zone: "",
+		},
+		IPVersion: V6,
+	}
+	addr6 = EdgeAddr{
+		TCP: &net.TCPAddr{
+			IP:   net.ParseIP("2606:4700:a0::3"),
+			Port: 8000,
+			Zone: "",
+		},
+		UDP: &net.UDPAddr{
+			IP:   net.ParseIP("2606:4700:a0::3"),
+			Port: 8000,
+			Zone: "",
+		},
+		IPVersion: V6,
+	}
+	addr7 = EdgeAddr{
+		TCP: &net.TCPAddr{
+			IP:   net.ParseIP("2606:4700:a0::4"),
+			Port: 8000,
+			Zone: "",
+		},
+		UDP: &net.UDPAddr{
+			IP:   net.ParseIP("2606:4700:a0::4"),
+			Port: 8000,
+			Zone: "",
+		},
+		IPVersion: V6,
+	}
+)
+
 type mockAddrs struct {
 	// a set of synthetic SRV records
 	addrMap map[net.SRV][]*EdgeAddr

edgediscovery/allregions/region.go

@@ -1,79 +1,155 @@
 package allregions
 
+import "time"
+
+const (
+	timeoutDuration = 10 * time.Minute
+)
+
 // Region contains cloudflared edge addresses. The edge is partitioned into several regions for
 // redundancy purposes.
 type Region struct {
-	connFor map[*EdgeAddr]UsedBy
+	primaryIsActive bool
+	active          AddrSet
+	primary         AddrSet
+	secondary       AddrSet
+	primaryTimeout  time.Time
+	timeoutDuration time.Duration
 }
 
 // NewRegion creates a region with the given addresses, which are all unused.
-func NewRegion(addrs []*EdgeAddr) Region {
+func NewRegion(addrs []*EdgeAddr, overrideIPVersion ConfigIPVersion) Region {
 	// The zero value of UsedBy is Unused(), so we can just initialize the map's values with their
 	// zero values.
-	connFor := make(map[*EdgeAddr]UsedBy)
-	for _, addr := range addrs {
-		connFor[addr] = Unused()
+	connForv4 := make(AddrSet)
+	connForv6 := make(AddrSet)
+	systemPreference := V6
+	for i, addr := range addrs {
+		if i == 0 {
+			// First family of IPs returned is system preference of IP
+			systemPreference = addr.IPVersion
+		}
+		switch addr.IPVersion {
+		case V4:
+			connForv4[addr] = Unused()
+		case V6:
+			connForv6[addr] = Unused()
+		}
 	}
+
+	// Process as system preference
+	var primary AddrSet
+	var secondary AddrSet
+	switch systemPreference {
+	case V4:
+		primary = connForv4
+		secondary = connForv6
+	case V6:
+		primary = connForv6
+		secondary = connForv4
+	}
+
+	// Override with provided preference
+	switch overrideIPVersion {
+	case IPv4Only:
+		primary = connForv4
+		secondary = make(AddrSet) // empty
+	case IPv6Only:
+		primary = connForv6
+		secondary = make(AddrSet) // empty
+	case Auto:
+		// no change
+	default:
+		// no change
+	}
+
 	return Region{
-		connFor: connFor,
+		primaryIsActive: true,
+		active:          primary,
+		primary:         primary,
+		secondary:       secondary,
+		timeoutDuration: timeoutDuration,
 	}
 }
 
 // AddrUsedBy finds the address used by the given connection in this region.
 // Returns nil if the connection isn't using any IP.
 func (r *Region) AddrUsedBy(connID int) *EdgeAddr {
-	for addr, used := range r.connFor {
-		if used.Used && used.ConnID == connID {
-			return addr
-		}
+	edgeAddr := r.primary.AddrUsedBy(connID)
+	if edgeAddr == nil {
+		edgeAddr = r.secondary.AddrUsedBy(connID)
 	}
-	return nil
+	return edgeAddr
 }
 
 // AvailableAddrs counts how many unused addresses this region contains.
 func (r Region) AvailableAddrs() int {
-	n := 0
-	for _, usedby := range r.connFor {
-		if !usedby.Used {
-			n++
-		}
-	}
-	return n
+	return r.active.AvailableAddrs()
 }
 
-// GetUnusedIP returns a random unused address in this region.
-// Returns nil if all addresses are in use.
-func (r Region) GetUnusedIP(excluding *EdgeAddr) *EdgeAddr {
-	for addr, usedby := range r.connFor {
-		if !usedby.Used && addr != excluding {
-			return addr
-		}
-	}
-	return nil
-}
-
-// Use the address, assigning it to a proxy connection.
-func (r Region) Use(addr *EdgeAddr, connID int) {
-	if addr == nil {
-		return
-	}
-	r.connFor[addr] = InUse(connID)
-}
-
-// GetAnyAddress returns an arbitrary address from the region.
-func (r Region) GetAnyAddress() *EdgeAddr {
-	for addr := range r.connFor {
+// AssignAnyAddress returns a random unused address in this region now
+// assigned to the connID excluding the provided EdgeAddr.
+// Returns nil if all addresses are in use for the region.
+func (r Region) AssignAnyAddress(connID int, excluding *EdgeAddr) *EdgeAddr {
+	if addr := r.active.GetUnusedIP(excluding); addr != nil {
+		r.active.Use(addr, connID)
 		return addr
 	}
 	return nil
 }
 
+// GetAnyAddress returns an arbitrary address from the region.
+func (r Region) GetAnyAddress() *EdgeAddr {
+	return r.active.GetAnyAddress()
+}
+
 // GiveBack the address, ensuring it is no longer assigned to an IP.
 // Returns true if the address is in this region.
-func (r Region) GiveBack(addr *EdgeAddr) (ok bool) {
-	if _, ok := r.connFor[addr]; !ok {
-		return false
+func (r *Region) GiveBack(addr *EdgeAddr, hasConnectivityError bool) (ok bool) {
+	if ok = r.primary.GiveBack(addr); !ok {
+		// Attempt to give back the address in the secondary set
+		if ok = r.secondary.GiveBack(addr); !ok {
+			// Address is not in this region
+			return
+		}
 	}
-	r.connFor[addr] = Unused()
-	return true
+
+	// No connectivity error: no worry
+	if !hasConnectivityError {
+		return
+	}
+
+	// If using primary and returned address is IPv6 and secondary is available
+	if r.primaryIsActive && addr.IPVersion == V6 && len(r.secondary) > 0 {
+		r.active = r.secondary
+		r.primaryIsActive = false
+		r.primaryTimeout = time.Now().Add(r.timeoutDuration)
+		return
+	}
+
+	// Do nothing for IPv4 or if secondary is empty
+	if r.primaryIsActive {
+		return
+	}
+
+	// Immediately return to primary pool, regardless of current primary timeout
+	if addr.IPVersion == V4 {
+		activatePrimary(r)
+		return
+	}
+
+	// Timeout exceeded and can be reset to primary pool
+	if r.primaryTimeout.Before(time.Now()) {
+		activatePrimary(r)
+		return
+	}
+
+	return
+}
+
+// activatePrimary sets the primary set to the active set and resets the timeout.
+func activatePrimary(r *Region) {
+	r.active = r.primary
+	r.primaryIsActive = true
+	r.primaryTimeout = time.Now() // reset timeout
 }

edgediscovery/allregions/region_test.go

@@ -1,284 +1,357 @@
 package allregions
 
 import (
-	"reflect"
+	"net"
 	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
 )
 
+func makeAddrSet(addrs []*EdgeAddr) AddrSet {
+	addrSet := make(AddrSet, len(addrs))
+	for _, addr := range addrs {
+		addrSet[addr] = Unused()
+	}
+	return addrSet
+}
+
 func TestRegion_New(t *testing.T) {
-	r := NewRegion([]*EdgeAddr{&addr0, &addr1, &addr2})
-	if r.AvailableAddrs() != 3 {
-		t.Errorf("r.AvailableAddrs() == %v but want 3", r.AvailableAddrs())
-	}
-}
-
-func TestRegion_AddrUsedBy(t *testing.T) {
-	type fields struct {
-		connFor map[*EdgeAddr]UsedBy
-	}
-	type args struct {
-		connID int
-	}
 	tests := []struct {
-		name   string
-		fields fields
-		args   args
-		want   *EdgeAddr
+		name          string
+		addrs         []*EdgeAddr
+		mode          ConfigIPVersion
+		expectedAddrs int
+		primary       AddrSet
+		secondary     AddrSet
 	}{
 		{
-			name: "happy trivial test",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{
-				&addr0: InUse(0),
-			}},
-			args: args{connID: 0},
-			want: &addr0,
+			name:          "IPv4 addresses with IPv4Only",
+			addrs:         v4Addrs,
+			mode:          IPv4Only,
+			expectedAddrs: len(v4Addrs),
+			primary:       makeAddrSet(v4Addrs),
+			secondary:     AddrSet{},
 		},
 		{
-			name: "sad trivial test",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{
-				&addr0: InUse(0),
-			}},
-			args: args{connID: 1},
-			want: nil,
+			name:          "IPv6 addresses with IPv4Only",
+			addrs:         v6Addrs,
+			mode:          IPv4Only,
+			expectedAddrs: 0,
+			primary:       AddrSet{},
+			secondary:     AddrSet{},
 		},
 		{
-			name: "sad test",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{
-				&addr0: InUse(0),
-				&addr1: InUse(1),
-				&addr2: InUse(2),
-			}},
-			args: args{connID: 3},
-			want: nil,
+			name:          "IPv6 addresses with IPv6Only",
+			addrs:         v6Addrs,
+			mode:          IPv6Only,
+			expectedAddrs: len(v6Addrs),
+			primary:       makeAddrSet(v6Addrs),
+			secondary:     AddrSet{},
 		},
 		{
-			name: "happy test",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{
-				&addr0: InUse(0),
-				&addr1: InUse(1),
-				&addr2: InUse(2),
-			}},
-			args: args{connID: 1},
-			want: &addr1,
+			name:          "IPv6 addresses with IPv4Only",
+			addrs:         v6Addrs,
+			mode:          IPv4Only,
+			expectedAddrs: 0,
+			primary:       AddrSet{},
+			secondary:     AddrSet{},
+		},
+		{
+			name:          "IPv4 (first) and IPv6 addresses with Auto",
+			addrs:         append(v4Addrs, v6Addrs...),
+			mode:          Auto,
+			expectedAddrs: len(v4Addrs),
+			primary:       makeAddrSet(v4Addrs),
+			secondary:     makeAddrSet(v6Addrs),
+		},
+		{
+			name:          "IPv6 (first) and IPv4 addresses with Auto",
+			addrs:         append(v6Addrs, v4Addrs...),
+			mode:          Auto,
+			expectedAddrs: len(v6Addrs),
+			primary:       makeAddrSet(v6Addrs),
+			secondary:     makeAddrSet(v4Addrs),
+		},
+		{
+			name:          "IPv4 addresses with Auto",
+			addrs:         v4Addrs,
+			mode:          Auto,
+			expectedAddrs: len(v4Addrs),
+			primary:       makeAddrSet(v4Addrs),
+			secondary:     AddrSet{},
+		},
+		{
+			name:          "IPv6 addresses with Auto",
+			addrs:         v6Addrs,
+			mode:          Auto,
+			expectedAddrs: len(v6Addrs),
+			primary:       makeAddrSet(v6Addrs),
+			secondary:     AddrSet{},
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			r := &Region{
-				connFor: tt.fields.connFor,
+			r := NewRegion(tt.addrs, tt.mode)
+			assert.Equal(t, tt.expectedAddrs, r.AvailableAddrs())
+			assert.Equal(t, tt.primary, r.primary)
+			assert.Equal(t, tt.secondary, r.secondary)
+		})
+	}
+}
+
+func TestRegion_AnyAddress_EmptyActiveSet(t *testing.T) {
+	tests := []struct {
+		name  string
+		addrs []*EdgeAddr
+		mode  ConfigIPVersion
+	}{
+		{
+			name:  "IPv6 addresses with IPv4Only",
+			addrs: v6Addrs,
+			mode:  IPv4Only,
+		},
+		{
+			name:  "IPv4 addresses with IPv6Only",
+			addrs: v4Addrs,
+			mode:  IPv6Only,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			r := NewRegion(tt.addrs, tt.mode)
+			addr := r.GetAnyAddress()
+			assert.Nil(t, addr)
+			addr = r.AssignAnyAddress(0, nil)
+			assert.Nil(t, addr)
+		})
+	}
+}
+
+func TestRegion_AssignAnyAddress_FullyUsedActiveSet(t *testing.T) {
+	tests := []struct {
+		name  string
+		addrs []*EdgeAddr
+		mode  ConfigIPVersion
+	}{
+		{
+			name:  "IPv6 addresses with IPv6Only",
+			addrs: v6Addrs,
+			mode:  IPv6Only,
+		},
+		{
+			name:  "IPv4 addresses with IPv4Only",
+			addrs: v4Addrs,
+			mode:  IPv4Only,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			r := NewRegion(tt.addrs, tt.mode)
+			total := r.active.AvailableAddrs()
+			for i := 0; i < total; i++ {
+				addr := r.AssignAnyAddress(i, nil)
+				assert.NotNil(t, addr)
 			}
-			if got := r.AddrUsedBy(tt.args.connID); !reflect.DeepEqual(got, tt.want) {
-				t.Errorf("Region.AddrUsedBy() = %v, want %v", got, tt.want)
+			addr := r.AssignAnyAddress(9, nil)
+			assert.Nil(t, addr)
+		})
+	}
+}
+
+var giveBackTests = []struct {
+	name          string
+	addrs         []*EdgeAddr
+	mode          ConfigIPVersion
+	expectedAddrs int
+	primary       AddrSet
+	secondary     AddrSet
+	primarySwap   bool
+}{
+	{
+		name:          "IPv4 addresses with IPv4Only",
+		addrs:         v4Addrs,
+		mode:          IPv4Only,
+		expectedAddrs: len(v4Addrs),
+		primary:       makeAddrSet(v4Addrs),
+		secondary:     AddrSet{},
+		primarySwap:   false,
+	},
+	{
+		name:          "IPv6 addresses with IPv6Only",
+		addrs:         v6Addrs,
+		mode:          IPv6Only,
+		expectedAddrs: len(v6Addrs),
+		primary:       makeAddrSet(v6Addrs),
+		secondary:     AddrSet{},
+		primarySwap:   false,
+	},
+	{
+		name:          "IPv4 (first) and IPv6 addresses with Auto",
+		addrs:         append(v4Addrs, v6Addrs...),
+		mode:          Auto,
+		expectedAddrs: len(v4Addrs),
+		primary:       makeAddrSet(v4Addrs),
+		secondary:     makeAddrSet(v6Addrs),
+		primarySwap:   false,
+	},
+	{
+		name:          "IPv6 (first) and IPv4 addresses with Auto",
+		addrs:         append(v6Addrs, v4Addrs...),
+		mode:          Auto,
+		expectedAddrs: len(v6Addrs),
+		primary:       makeAddrSet(v6Addrs),
+		secondary:     makeAddrSet(v4Addrs),
+		primarySwap:   true,
+	},
+	{
+		name:          "IPv4 addresses with Auto",
+		addrs:         v4Addrs,
+		mode:          Auto,
+		expectedAddrs: len(v4Addrs),
+		primary:       makeAddrSet(v4Addrs),
+		secondary:     AddrSet{},
+		primarySwap:   false,
+	},
+	{
+		name:          "IPv6 addresses with Auto",
+		addrs:         v6Addrs,
+		mode:          Auto,
+		expectedAddrs: len(v6Addrs),
+		primary:       makeAddrSet(v6Addrs),
+		secondary:     AddrSet{},
+		primarySwap:   false,
+	},
+}
+
+func TestRegion_GiveBack_NoConnectivityError(t *testing.T) {
+	for _, tt := range giveBackTests {
+		t.Run(tt.name, func(t *testing.T) {
+			r := NewRegion(tt.addrs, tt.mode)
+			addr := r.AssignAnyAddress(0, nil)
+			assert.NotNil(t, addr)
+			assert.True(t, r.GiveBack(addr, false))
+		})
+	}
+}
+
+func TestRegion_GiveBack_ForeignAddr(t *testing.T) {
+	invalid := EdgeAddr{
+		TCP: &net.TCPAddr{
+			IP:   net.ParseIP("123.4.5.0"),
+			Port: 8000,
+			Zone: "",
+		},
+		UDP: &net.UDPAddr{
+			IP:   net.ParseIP("123.4.5.0"),
+			Port: 8000,
+			Zone: "",
+		},
+		IPVersion: V4,
+	}
+	for _, tt := range giveBackTests {
+		t.Run(tt.name, func(t *testing.T) {
+			r := NewRegion(tt.addrs, tt.mode)
+			assert.False(t, r.GiveBack(&invalid, false))
+			assert.False(t, r.GiveBack(&invalid, true))
+		})
+	}
+}
+
+func TestRegion_GiveBack_SwapPrimary(t *testing.T) {
+	for _, tt := range giveBackTests {
+		t.Run(tt.name, func(t *testing.T) {
+			r := NewRegion(tt.addrs, tt.mode)
+			addr := r.AssignAnyAddress(0, nil)
+			assert.NotNil(t, addr)
+			assert.True(t, r.GiveBack(addr, true))
+			assert.Equal(t, tt.primarySwap, !r.primaryIsActive)
+			if tt.primarySwap {
+				assert.Equal(t, r.secondary, r.active)
+				assert.False(t, r.primaryTimeout.IsZero())
+			} else {
+				assert.Equal(t, r.primary, r.active)
+				assert.True(t, r.primaryTimeout.IsZero())
 			}
 		})
 	}
 }
 
-func TestRegion_AvailableAddrs(t *testing.T) {
-	type fields struct {
-		connFor map[*EdgeAddr]UsedBy
-	}
-	tests := []struct {
-		name   string
-		fields fields
-		want   int
-	}{
-		{
-			name: "contains addresses",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{
-				&addr0: InUse(0),
-				&addr1: Unused(),
-				&addr2: InUse(2),
-			}},
-			want: 1,
-		},
-		{
-			name: "all free",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{
-				&addr0: Unused(),
-				&addr1: Unused(),
-				&addr2: Unused(),
-			}},
-			want: 3,
-		},
-		{
-			name: "all used",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{
-				&addr0: InUse(0),
-				&addr1: InUse(1),
-				&addr2: InUse(2),
-			}},
-			want: 0,
-		},
-		{
-			name:   "empty",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{}},
-			want:   0,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			r := Region{
-				connFor: tt.fields.connFor,
-			}
-			if got := r.AvailableAddrs(); got != tt.want {
-				t.Errorf("Region.AvailableAddrs() = %v, want %v", got, tt.want)
-			}
-		})
-	}
+func TestRegion_GiveBack_IPv4_ResetPrimary(t *testing.T) {
+	r := NewRegion(append(v6Addrs, v4Addrs...), Auto)
+	// Exhaust all IPv6 addresses
+	a0 := r.AssignAnyAddress(0, nil)
+	a1 := r.AssignAnyAddress(1, nil)
+	a2 := r.AssignAnyAddress(2, nil)
+	a3 := r.AssignAnyAddress(3, nil)
+	assert.NotNil(t, a0)
+	assert.NotNil(t, a1)
+	assert.NotNil(t, a2)
+	assert.NotNil(t, a3)
+	// Give back the first IPv6 address to fallback to secondary IPv4 address set
+	assert.True(t, r.GiveBack(a0, true))
+	assert.False(t, r.primaryIsActive)
+	// Give back another IPv6 address
+	assert.True(t, r.GiveBack(a1, true))
+	// Primary shouldn't change
+	assert.False(t, r.primaryIsActive)
+	// Request an address (should be IPv4 from secondary)
+	a4_v4 := r.AssignAnyAddress(4, nil)
+	assert.NotNil(t, a4_v4)
+	assert.Equal(t, V4, a4_v4.IPVersion)
+	a5_v4 := r.AssignAnyAddress(5, nil)
+	assert.NotNil(t, a5_v4)
+	assert.Equal(t, V4, a5_v4.IPVersion)
+	a6_v4 := r.AssignAnyAddress(6, nil)
+	assert.NotNil(t, a6_v4)
+	assert.Equal(t, V4, a6_v4.IPVersion)
+	// Return IPv4 address (without failure)
+	// Primary shouldn't change because it is not a connectivity failure
+	assert.True(t, r.GiveBack(a4_v4, false))
+	assert.False(t, r.primaryIsActive)
+	// Return IPv4 address (with failure)
+	// Primary should change because it is a connectivity failure
+	assert.True(t, r.GiveBack(a5_v4, true))
+	assert.True(t, r.primaryIsActive)
+	// Return IPv4 address (with failure)
+	// Primary shouldn't change because the address is returned to the inactive
+	// secondary address set
+	assert.True(t, r.GiveBack(a6_v4, true))
+	assert.True(t, r.primaryIsActive)
+	// Return IPv6 address (without failure)
+	// Primary shoudn't change because it is not a connectivity failure
+	assert.True(t, r.GiveBack(a2, false))
+	assert.True(t, r.primaryIsActive)
 }
 
-func TestRegion_GetUnusedIP(t *testing.T) {
-	type fields struct {
-		connFor map[*EdgeAddr]UsedBy
-	}
-	type args struct {
-		excluding *EdgeAddr
-	}
-	tests := []struct {
-		name   string
-		fields fields
-		args   args
-		want   *EdgeAddr
-	}{
-		{
-			name: "happy test with excluding set",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{
-				&addr0: Unused(),
-				&addr1: Unused(),
-				&addr2: InUse(2),
-			}},
-			args: args{excluding: &addr0},
-			want: &addr1,
-		},
-		{
-			name: "happy test with no excluding",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{
-				&addr0: InUse(0),
-				&addr1: Unused(),
-				&addr2: InUse(2),
-			}},
-			args: args{excluding: nil},
-			want: &addr1,
-		},
-		{
-			name: "sad test with no excluding",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{
-				&addr0: InUse(0),
-				&addr1: InUse(1),
-				&addr2: InUse(2),
-			}},
-			args: args{excluding: nil},
-			want: nil,
-		},
-		{
-			name: "sad test with excluding",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{
-				&addr0: Unused(),
-				&addr1: InUse(1),
-				&addr2: InUse(2),
-			}},
-			args: args{excluding: &addr0},
-			want: nil,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			r := Region{
-				connFor: tt.fields.connFor,
-			}
-			if got := r.GetUnusedIP(tt.args.excluding); !reflect.DeepEqual(got, tt.want) {
-				t.Errorf("Region.GetUnusedIP() = %v, want %v", got, tt.want)
-			}
-		})
-	}
-}
-
-func TestRegion_GiveBack(t *testing.T) {
-	type fields struct {
-		connFor map[*EdgeAddr]UsedBy
-	}
-	type args struct {
-		addr *EdgeAddr
-	}
-	tests := []struct {
-		name           string
-		fields         fields
-		args           args
-		wantOk         bool
-		availableAfter int
-	}{
-		{
-			name: "sad test with excluding",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{
-				&addr1: InUse(1),
-			}},
-			args:           args{addr: &addr1},
-			wantOk:         true,
-			availableAfter: 1,
-		},
-		{
-			name: "sad test with excluding",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{
-				&addr1: InUse(1),
-			}},
-			args:           args{addr: &addr2},
-			wantOk:         false,
-			availableAfter: 0,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			r := Region{
-				connFor: tt.fields.connFor,
-			}
-			if gotOk := r.GiveBack(tt.args.addr); gotOk != tt.wantOk {
-				t.Errorf("Region.GiveBack() = %v, want %v", gotOk, tt.wantOk)
-			}
-			if tt.availableAfter != r.AvailableAddrs() {
-				t.Errorf("Region.AvailableAddrs() = %v, want %v", r.AvailableAddrs(), tt.availableAfter)
-			}
-		})
-	}
-}
-
-func TestRegion_GetAnyAddress(t *testing.T) {
-	type fields struct {
-		connFor map[*EdgeAddr]UsedBy
-	}
-	tests := []struct {
-		name    string
-		fields  fields
-		wantNil bool
-	}{
-		{
-			name:    "Sad test -- GetAnyAddress should only fail if the region is empty",
-			fields:  fields{connFor: map[*EdgeAddr]UsedBy{}},
-			wantNil: true,
-		},
-		{
-			name: "Happy test (all addresses unused)",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{
-				&addr0: Unused(),
-			}},
-			wantNil: false,
-		},
-		{
-			name: "Happy test (GetAnyAddress can still return addresses used by proxy conns)",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{
-				&addr0: InUse(2),
-			}},
-			wantNil: false,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			r := Region{
-				connFor: tt.fields.connFor,
-			}
-			if got := r.GetAnyAddress(); tt.wantNil != (got == nil) {
-				t.Errorf("Region.GetAnyAddress() = %v, but should it return nil? %v", got, tt.wantNil)
-			}
-		})
-	}
+func TestRegion_GiveBack_Timeout(t *testing.T) {
+	r := NewRegion(append(v6Addrs, v4Addrs...), Auto)
+	a0 := r.AssignAnyAddress(0, nil)
+	a1 := r.AssignAnyAddress(1, nil)
+	a2 := r.AssignAnyAddress(2, nil)
+	assert.NotNil(t, a0)
+	assert.NotNil(t, a1)
+	assert.NotNil(t, a2)
+	// Give back IPv6 address to set timeout
+	assert.True(t, r.GiveBack(a0, true))
+	assert.False(t, r.primaryIsActive)
+	assert.False(t, r.primaryTimeout.IsZero())
+	// Request an address (should be IPv4 from secondary)
+	a3_v4 := r.AssignAnyAddress(3, nil)
+	assert.NotNil(t, a3_v4)
+	assert.Equal(t, V4, a3_v4.IPVersion)
+	assert.False(t, r.primaryIsActive)
+	// Give back IPv6 address inside timeout (no change)
+	assert.True(t, r.GiveBack(a2, true))
+	assert.False(t, r.primaryIsActive)
+	assert.False(t, r.primaryTimeout.IsZero())
+	// Accelerate timeout
+	r.primaryTimeout = time.Now().Add(-time.Minute)
+	// Return IPv6 address
+	assert.True(t, r.GiveBack(a1, true))
+	assert.True(t, r.primaryIsActive)
+	// Returning an IPv4 address after primary is active shouldn't change primary
+	// even with a connectivity error
+	assert.True(t, r.GiveBack(a3_v4, true))
+	assert.True(t, r.primaryIsActive)
 }

edgediscovery/allregions/regions.go

@@ -18,7 +18,7 @@ type Regions struct {
 // ------------------------------------
 
 // ResolveEdge resolves the Cloudflare edge, returning all regions discovered.
-func ResolveEdge(log *zerolog.Logger, region string) (*Regions, error) {
+func ResolveEdge(log *zerolog.Logger, region string, overrideIPVersion ConfigIPVersion) (*Regions, error) {
 	edgeAddrs, err := edgeDiscovery(log, getRegionalServiceName(region))
 	if err != nil {
 		return nil, err
@@ -27,8 +27,8 @@ func ResolveEdge(log *zerolog.Logger, region string) (*Regions, error) {
 		return nil, fmt.Errorf("expected at least 2 Cloudflare Regions regions, but SRV only returned %v", len(edgeAddrs))
 	}
 	return &Regions{
-		region1: NewRegion(edgeAddrs[0]),
-		region2: NewRegion(edgeAddrs[1]),
+		region1: NewRegion(edgeAddrs[0], overrideIPVersion),
+		region2: NewRegion(edgeAddrs[1], overrideIPVersion),
 	}, nil
 }
 
@@ -56,8 +56,8 @@ func NewNoResolve(addrs []*EdgeAddr) *Regions {
 	}
 
 	return &Regions{
-		region1: NewRegion(region1),
-		region2: NewRegion(region2),
+		region1: NewRegion(region1, Auto),
+		region2: NewRegion(region2, Auto),
 	}
 }
 
@@ -95,14 +95,12 @@ func (rs *Regions) GetUnusedAddr(excluding *EdgeAddr, connID int) *EdgeAddr {
 // getAddrs tries to grab address form `first` region, then `second` region
 // this is an unrolled loop over 2 element array
 func getAddrs(excluding *EdgeAddr, connID int, first *Region, second *Region) *EdgeAddr {
-	addr := first.GetUnusedIP(excluding)
+	addr := first.AssignAnyAddress(connID, excluding)
 	if addr != nil {
-		first.Use(addr, connID)
 		return addr
 	}
-	addr = second.GetUnusedIP(excluding)
+	addr = second.AssignAnyAddress(connID, excluding)
 	if addr != nil {
-		second.Use(addr, connID)
 		return addr
 	}
 
@@ -116,18 +114,18 @@ func (rs *Regions) AvailableAddrs() int {
 
 // GiveBack the address so that other connections can use it.
 // Returns true if the address is in this edge.
-func (rs *Regions) GiveBack(addr *EdgeAddr) bool {
-	if found := rs.region1.GiveBack(addr); found {
+func (rs *Regions) GiveBack(addr *EdgeAddr, hasConnectivityError bool) bool {
+	if found := rs.region1.GiveBack(addr, hasConnectivityError); found {
 		return found
 	}
-	return rs.region2.GiveBack(addr)
+	return rs.region2.GiveBack(addr, hasConnectivityError)
 }
 
 // Return regionalized service name if `region` isn't empty, otherwise return the global service name for origintunneld
 func getRegionalServiceName(region string) string {
 	if region != "" {
-		return region + "-" + srvService // Example: `us-origintunneld`
+		return region + "-" + srvService // Example: `us-v2-origintunneld`
 	}
 
-	return srvService // Global service is just `origintunneld`
+	return srvService // Global service is just `v2-origintunneld`
 }

edgediscovery/allregions/regions_test.go

@@ -1,134 +1,215 @@
 package allregions
 
 import (
-	"net"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 )
 
-var (
-	addr0 = EdgeAddr{
-		TCP: &net.TCPAddr{
-			IP:   net.ParseIP("123.4.5.0"),
-			Port: 8000,
-			Zone: "",
-		},
-		UDP: &net.UDPAddr{
-			IP:   net.ParseIP("123.4.5.0"),
-			Port: 8000,
-			Zone: "",
-		},
+func makeRegions(addrs []*EdgeAddr, mode ConfigIPVersion) Regions {
+	r1addrs := make([]*EdgeAddr, 0)
+	r2addrs := make([]*EdgeAddr, 0)
+	for i, addr := range addrs {
+		if i%2 == 0 {
+			r1addrs = append(r1addrs, addr)
+		} else {
+			r2addrs = append(r2addrs, addr)
+		}
 	}
-	addr1 = EdgeAddr{
-		TCP: &net.TCPAddr{
-			IP:   net.ParseIP("123.4.5.1"),
-			Port: 8000,
-			Zone: "",
-		},
-		UDP: &net.UDPAddr{
-			IP:   net.ParseIP("123.4.5.1"),
-			Port: 8000,
-			Zone: "",
-		},
-	}
-	addr2 = EdgeAddr{
-		TCP: &net.TCPAddr{
-			IP:   net.ParseIP("123.4.5.2"),
-			Port: 8000,
-			Zone: "",
-		},
-		UDP: &net.UDPAddr{
-			IP:   net.ParseIP("123.4.5.2"),
-			Port: 8000,
-			Zone: "",
-		},
-	}
-	addr3 = EdgeAddr{
-		TCP: &net.TCPAddr{
-			IP:   net.ParseIP("123.4.5.3"),
-			Port: 8000,
-			Zone: "",
-		},
-		UDP: &net.UDPAddr{
-			IP:   net.ParseIP("123.4.5.3"),
-			Port: 8000,
-			Zone: "",
-		},
-	}
-)
-
-func makeRegions() Regions {
-	r1 := NewRegion([]*EdgeAddr{&addr0, &addr1})
-	r2 := NewRegion([]*EdgeAddr{&addr2, &addr3})
+	r1 := NewRegion(r1addrs, mode)
+	r2 := NewRegion(r2addrs, mode)
 	return Regions{region1: r1, region2: r2}
 }
 
 func TestRegions_AddrUsedBy(t *testing.T) {
-	rs := makeRegions()
-	addr1 := rs.GetUnusedAddr(nil, 1)
-	assert.Equal(t, addr1, rs.AddrUsedBy(1))
-	addr2 := rs.GetUnusedAddr(nil, 2)
-	assert.Equal(t, addr2, rs.AddrUsedBy(2))
-	addr3 := rs.GetUnusedAddr(nil, 3)
-	assert.Equal(t, addr3, rs.AddrUsedBy(3))
+	tests := []struct {
+		name  string
+		addrs []*EdgeAddr
+		mode  ConfigIPVersion
+	}{
+		{
+			name:  "IPv4 addresses with IPv4Only",
+			addrs: v4Addrs,
+			mode:  IPv4Only,
+		},
+		{
+			name:  "IPv6 addresses with IPv6Only",
+			addrs: v6Addrs,
+			mode:  IPv6Only,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			rs := makeRegions(tt.addrs, tt.mode)
+			addr1 := rs.GetUnusedAddr(nil, 1)
+			assert.Equal(t, addr1, rs.AddrUsedBy(1))
+			addr2 := rs.GetUnusedAddr(nil, 2)
+			assert.Equal(t, addr2, rs.AddrUsedBy(2))
+			addr3 := rs.GetUnusedAddr(nil, 3)
+			assert.Equal(t, addr3, rs.AddrUsedBy(3))
+		})
+	}
 }
 
 func TestRegions_Giveback_Region1(t *testing.T) {
-	rs := makeRegions()
-	rs.region1.Use(&addr0, 0)
-	rs.region1.Use(&addr1, 1)
-	rs.region2.Use(&addr2, 2)
-	rs.region2.Use(&addr3, 3)
+	tests := []struct {
+		name  string
+		addrs []*EdgeAddr
+		mode  ConfigIPVersion
+	}{
+		{
+			name:  "IPv4 addresses with IPv4Only",
+			addrs: v4Addrs,
+			mode:  IPv4Only,
+		},
+		{
+			name:  "IPv6 addresses with IPv6Only",
+			addrs: v6Addrs,
+			mode:  IPv6Only,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			rs := makeRegions(tt.addrs, tt.mode)
+			addr := rs.region1.AssignAnyAddress(0, nil)
+			rs.region1.AssignAnyAddress(1, nil)
+			rs.region2.AssignAnyAddress(2, nil)
+			rs.region2.AssignAnyAddress(3, nil)
 
-	assert.Equal(t, 0, rs.AvailableAddrs())
+			assert.Equal(t, 0, rs.AvailableAddrs())
 
-	rs.GiveBack(&addr0)
-	assert.Equal(t, &addr0, rs.GetUnusedAddr(nil, 3))
+			rs.GiveBack(addr, false)
+			assert.Equal(t, addr, rs.GetUnusedAddr(nil, 0))
+		})
+	}
 }
 
 func TestRegions_Giveback_Region2(t *testing.T) {
-	rs := makeRegions()
-	rs.region1.Use(&addr0, 0)
-	rs.region1.Use(&addr1, 1)
-	rs.region2.Use(&addr2, 2)
-	rs.region2.Use(&addr3, 3)
+	tests := []struct {
+		name  string
+		addrs []*EdgeAddr
+		mode  ConfigIPVersion
+	}{
+		{
+			name:  "IPv4 addresses with IPv4Only",
+			addrs: v4Addrs,
+			mode:  IPv4Only,
+		},
+		{
+			name:  "IPv6 addresses with IPv6Only",
+			addrs: v6Addrs,
+			mode:  IPv6Only,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			rs := makeRegions(tt.addrs, tt.mode)
+			rs.region1.AssignAnyAddress(0, nil)
+			rs.region1.AssignAnyAddress(1, nil)
+			addr := rs.region2.AssignAnyAddress(2, nil)
+			rs.region2.AssignAnyAddress(3, nil)
 
-	assert.Equal(t, 0, rs.AvailableAddrs())
+			assert.Equal(t, 0, rs.AvailableAddrs())
 
-	rs.GiveBack(&addr2)
-	assert.Equal(t, &addr2, rs.GetUnusedAddr(nil, 2))
+			rs.GiveBack(addr, false)
+			assert.Equal(t, addr, rs.GetUnusedAddr(nil, 2))
+		})
+	}
 }
 
 func TestRegions_GetUnusedAddr_OneAddrLeft(t *testing.T) {
-	rs := makeRegions()
+	tests := []struct {
+		name  string
+		addrs []*EdgeAddr
+		mode  ConfigIPVersion
+	}{
+		{
+			name:  "IPv4 addresses with IPv4Only",
+			addrs: v4Addrs,
+			mode:  IPv4Only,
+		},
+		{
+			name:  "IPv6 addresses with IPv6Only",
+			addrs: v6Addrs,
+			mode:  IPv6Only,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			rs := makeRegions(tt.addrs, tt.mode)
+			rs.region1.AssignAnyAddress(0, nil)
+			rs.region1.AssignAnyAddress(1, nil)
+			rs.region2.AssignAnyAddress(2, nil)
+			addr := rs.region2.active.GetUnusedIP(nil)
 
-	rs.region1.Use(&addr0, 0)
-	rs.region1.Use(&addr1, 1)
-	rs.region2.Use(&addr2, 2)
-
-	assert.Equal(t, 1, rs.AvailableAddrs())
-	assert.Equal(t, &addr3, rs.GetUnusedAddr(nil, 3))
+			assert.Equal(t, 1, rs.AvailableAddrs())
+			assert.Equal(t, addr, rs.GetUnusedAddr(nil, 3))
+		})
+	}
 }
 
 func TestRegions_GetUnusedAddr_Excluding_Region1(t *testing.T) {
-	rs := makeRegions()
+	tests := []struct {
+		name  string
+		addrs []*EdgeAddr
+		mode  ConfigIPVersion
+	}{
+		{
+			name:  "IPv4 addresses with IPv4Only",
+			addrs: v4Addrs,
+			mode:  IPv4Only,
+		},
+		{
+			name:  "IPv6 addresses with IPv6Only",
+			addrs: v6Addrs,
+			mode:  IPv6Only,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			rs := makeRegions(tt.addrs, tt.mode)
 
-	rs.region1.Use(&addr0, 0)
-	rs.region1.Use(&addr1, 1)
+			rs.region1.AssignAnyAddress(0, nil)
+			rs.region1.AssignAnyAddress(1, nil)
+			addr := rs.region2.active.GetUnusedIP(nil)
+			a2 := rs.region2.active.GetUnusedIP(addr)
 
-	assert.Equal(t, 2, rs.AvailableAddrs())
-	assert.Equal(t, &addr3, rs.GetUnusedAddr(&addr2, 3))
+			assert.Equal(t, 2, rs.AvailableAddrs())
+			assert.Equal(t, addr, rs.GetUnusedAddr(a2, 3))
+		})
+	}
 }
 
 func TestRegions_GetUnusedAddr_Excluding_Region2(t *testing.T) {
-	rs := makeRegions()
+	tests := []struct {
+		name  string
+		addrs []*EdgeAddr
+		mode  ConfigIPVersion
+	}{
+		{
+			name:  "IPv4 addresses with IPv4Only",
+			addrs: v4Addrs,
+			mode:  IPv4Only,
+		},
+		{
+			name:  "IPv6 addresses with IPv6Only",
+			addrs: v6Addrs,
+			mode:  IPv6Only,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			rs := makeRegions(tt.addrs, tt.mode)
 
-	rs.region2.Use(&addr2, 0)
-	rs.region2.Use(&addr3, 1)
+			rs.region2.AssignAnyAddress(0, nil)
+			rs.region2.AssignAnyAddress(1, nil)
+			addr := rs.region1.active.GetUnusedIP(nil)
+			a2 := rs.region1.active.GetUnusedIP(addr)
 
-	assert.Equal(t, 2, rs.AvailableAddrs())
-	assert.Equal(t, &addr1, rs.GetUnusedAddr(&addr0, 1))
+			assert.Equal(t, 2, rs.AvailableAddrs())
+			assert.Equal(t, addr, rs.GetUnusedAddr(a2, 1))
+		})
+	}
 }
 
 func TestNewNoResolveBalancesRegions(t *testing.T) {

edgediscovery/edgediscovery.go

@@ -1,7 +1,6 @@
 package edgediscovery
 
 import (
-	"fmt"
 	"sync"
 
 	"github.com/rs/zerolog"
@@ -11,9 +10,16 @@ import (
 
 const (
 	LogFieldConnIndex = "connIndex"
+	LogFieldIPAddress = "ip"
 )
 
-var errNoAddressesLeft = fmt.Errorf("there are no free edge addresses left")
+var errNoAddressesLeft = ErrNoAddressesLeft{}
+
+type ErrNoAddressesLeft struct{}
+
+func (e ErrNoAddressesLeft) Error() string {
+	return "there are no free edge addresses left to resolve to"
+}
 
 // Edge finds addresses on the Cloudflare edge and hands them out to connections.
 type Edge struct {
@@ -28,8 +34,8 @@ type Edge struct {
 
 // ResolveEdge runs the initial discovery of the Cloudflare edge, finding Addrs that can be allocated
 // to connections.
-func ResolveEdge(log *zerolog.Logger, region string) (*Edge, error) {
-	regions, err := allregions.ResolveEdge(log, region)
+func ResolveEdge(log *zerolog.Logger, region string, edgeIpVersion allregions.ConfigIPVersion) (*Edge, error) {
+	regions, err := allregions.ResolveEdge(log, region, edgeIpVersion)
 	if err != nil {
 		return new(Edge), err
 	}
@@ -51,15 +57,6 @@ func StaticEdge(log *zerolog.Logger, hostnames []string) (*Edge, error) {
 	}, nil
 }
 
-// MockEdge creates a Cloudflare Edge from arbitrary TCP addresses. Used for testing.
-func MockEdge(log *zerolog.Logger, addrs []*allregions.EdgeAddr) *Edge {
-	regions := allregions.NewNoResolve(addrs)
-	return &Edge{
-		log:     log,
-		regions: regions,
-	}
-}
-
 // ------------------------------------
 // Methods
 // ------------------------------------
@@ -93,12 +90,15 @@ func (ed *Edge) GetAddr(connIndex int) (*allregions.EdgeAddr, error) {
 		log.Debug().Msg("edgediscovery - GetAddr: No addresses left to give proxy connection")
 		return nil, errNoAddressesLeft
 	}
-	log.Debug().Msg("edgediscovery - GetAddr: Giving connection its new address")
+	log = ed.log.With().
+		Int(LogFieldConnIndex, connIndex).
+		IPAddr(LogFieldIPAddress, addr.UDP.IP).Logger()
+	log.Debug().Msgf("edgediscovery - GetAddr: Giving connection its new address")
 	return addr, nil
 }
 
 // GetDifferentAddr gives back the proxy connection's edge Addr and uses a new one.
-func (ed *Edge) GetDifferentAddr(connIndex int) (*allregions.EdgeAddr, error) {
+func (ed *Edge) GetDifferentAddr(connIndex int, hasConnectivityError bool) (*allregions.EdgeAddr, error) {
 	log := ed.log.With().Int(LogFieldConnIndex, connIndex).Logger()
 
 	ed.Lock()
@@ -106,7 +106,7 @@ func (ed *Edge) GetDifferentAddr(connIndex int) (*allregions.EdgeAddr, error) {
 
 	oldAddr := ed.regions.AddrUsedBy(connIndex)
 	if oldAddr != nil {
-		ed.regions.GiveBack(oldAddr)
+		ed.regions.GiveBack(oldAddr, hasConnectivityError)
 	}
 	addr := ed.regions.GetUnusedAddr(oldAddr, connIndex)
 	if addr == nil {
@@ -114,8 +114,10 @@ func (ed *Edge) GetDifferentAddr(connIndex int) (*allregions.EdgeAddr, error) {
 		// note: if oldAddr were not nil, it will become available on the next iteration
 		return nil, errNoAddressesLeft
 	}
-	log.Debug().Msgf("edgediscovery - GetDifferentAddr: Giving connection its new address: %v from the address list: %v",
-		addr, ed.regions.AvailableAddrs())
+	log = ed.log.With().
+		Int(LogFieldConnIndex, connIndex).
+		IPAddr(LogFieldIPAddress, addr.UDP.IP).Logger()
+	log.Debug().Msgf("edgediscovery - GetDifferentAddr: Giving connection its new address from the address list: %v", ed.regions.AvailableAddrs())
 	return addr, nil
 }
 
@@ -128,9 +130,11 @@ func (ed *Edge) AvailableAddrs() int {
 
 // GiveBack the address so that other connections can use it.
 // Returns true if the address is in this edge.
-func (ed *Edge) GiveBack(addr *allregions.EdgeAddr) bool {
+func (ed *Edge) GiveBack(addr *allregions.EdgeAddr, hasConnectivityError bool) bool {
 	ed.Lock()
 	defer ed.Unlock()
-	ed.log.Debug().Msg("edgediscovery - GiveBack: Address now unused")
-	return ed.regions.GiveBack(addr)
+	log := ed.log.With().
+		IPAddr(LogFieldIPAddress, addr.UDP.IP).Logger()
+	log.Debug().Msgf("edgediscovery - GiveBack: Address now unused")
+	return ed.regions.GiveBack(addr, hasConnectivityError)
 }

edgediscovery/edgediscovery_test.go

@@ -11,56 +11,113 @@ import (
 )
 
 var (
-	addr0 = allregions.EdgeAddr{
+	testLogger = zerolog.Nop()
+	v4Addrs    = []*allregions.EdgeAddr{&addr0, &addr1, &addr2, &addr3}
+	v6Addrs    = []*allregions.EdgeAddr{&addr4, &addr5, &addr6, &addr7}
+	addr0      = allregions.EdgeAddr{
 		TCP: &net.TCPAddr{
-			IP:   net.ParseIP("123.0.0.0"),
+			IP:   net.ParseIP("123.4.5.0"),
 			Port: 8000,
 			Zone: "",
 		},
 		UDP: &net.UDPAddr{
-			IP:   net.ParseIP("123.0.0.0"),
+			IP:   net.ParseIP("123.4.5.0"),
 			Port: 8000,
 			Zone: "",
 		},
+		IPVersion: allregions.V4,
 	}
 	addr1 = allregions.EdgeAddr{
 		TCP: &net.TCPAddr{
-			IP:   net.ParseIP("123.0.0.1"),
+			IP:   net.ParseIP("123.4.5.1"),
 			Port: 8000,
 			Zone: "",
 		},
 		UDP: &net.UDPAddr{
-			IP:   net.ParseIP("123.0.0.1"),
+			IP:   net.ParseIP("123.4.5.1"),
 			Port: 8000,
 			Zone: "",
 		},
+		IPVersion: allregions.V4,
 	}
 	addr2 = allregions.EdgeAddr{
 		TCP: &net.TCPAddr{
-			IP:   net.ParseIP("123.0.0.2"),
+			IP:   net.ParseIP("123.4.5.2"),
 			Port: 8000,
 			Zone: "",
 		},
 		UDP: &net.UDPAddr{
-			IP:   net.ParseIP("123.0.0.2"),
+			IP:   net.ParseIP("123.4.5.2"),
 			Port: 8000,
 			Zone: "",
 		},
+		IPVersion: allregions.V4,
 	}
 	addr3 = allregions.EdgeAddr{
 		TCP: &net.TCPAddr{
-			IP:   net.ParseIP("123.0.0.3"),
+			IP:   net.ParseIP("123.4.5.3"),
 			Port: 8000,
 			Zone: "",
 		},
 		UDP: &net.UDPAddr{
-			IP:   net.ParseIP("123.0.0.3"),
+			IP:   net.ParseIP("123.4.5.3"),
 			Port: 8000,
 			Zone: "",
 		},
+		IPVersion: allregions.V4,
+	}
+	addr4 = allregions.EdgeAddr{
+		TCP: &net.TCPAddr{
+			IP:   net.ParseIP("2606:4700:a0::1"),
+			Port: 8000,
+			Zone: "",
+		},
+		UDP: &net.UDPAddr{
+			IP:   net.ParseIP("2606:4700:a0::1"),
+			Port: 8000,
+			Zone: "",
+		},
+		IPVersion: allregions.V6,
+	}
+	addr5 = allregions.EdgeAddr{
+		TCP: &net.TCPAddr{
+			IP:   net.ParseIP("2606:4700:a0::2"),
+			Port: 8000,
+			Zone: "",
+		},
+		UDP: &net.UDPAddr{
+			IP:   net.ParseIP("2606:4700:a0::2"),
+			Port: 8000,
+			Zone: "",
+		},
+		IPVersion: allregions.V6,
+	}
+	addr6 = allregions.EdgeAddr{
+		TCP: &net.TCPAddr{
+			IP:   net.ParseIP("2606:4700:a0::3"),
+			Port: 8000,
+			Zone: "",
+		},
+		UDP: &net.UDPAddr{
+			IP:   net.ParseIP("2606:4700:a0::3"),
+			Port: 8000,
+			Zone: "",
+		},
+		IPVersion: allregions.V6,
+	}
+	addr7 = allregions.EdgeAddr{
+		TCP: &net.TCPAddr{
+			IP:   net.ParseIP("2606:4700:a0::4"),
+			Port: 8000,
+			Zone: "",
+		},
+		UDP: &net.UDPAddr{
+			IP:   net.ParseIP("2606:4700:a0::4"),
+			Port: 8000,
+			Zone: "",
+		},
+		IPVersion: allregions.V6,
 	}
-
-	testLogger = zerolog.Nop()
 )
 
 func TestGiveBack(t *testing.T) {
@@ -75,7 +132,7 @@ func TestGiveBack(t *testing.T) {
 	assert.Equal(t, 3, edge.AvailableAddrs())
 
 	// Get it back
-	edge.GiveBack(addr)
+	edge.GiveBack(addr, false)
 	assert.Equal(t, 4, edge.AvailableAddrs())
 }
 
@@ -107,7 +164,7 @@ func TestGetAddrForRPC(t *testing.T) {
 	assert.Equal(t, 4, edge.AvailableAddrs())
 
 	// Get it back
-	edge.GiveBack(addr)
+	edge.GiveBack(addr, false)
 	assert.Equal(t, 4, edge.AvailableAddrs())
 }
 
@@ -122,13 +179,13 @@ func TestOnePerRegion(t *testing.T) {
 	assert.NotNil(t, a1)
 
 	// if the first address is bad, get the second one
-	a2, err := edge.GetDifferentAddr(connID)
+	a2, err := edge.GetDifferentAddr(connID, false)
 	assert.NoError(t, err)
 	assert.NotNil(t, a2)
 	assert.NotEqual(t, a1, a2)
 
 	// now that second one is bad, get the first one again
-	a3, err := edge.GetDifferentAddr(connID)
+	a3, err := edge.GetDifferentAddr(connID, false)
 	assert.NoError(t, err)
 	assert.Equal(t, a1, a3)
 }
@@ -144,11 +201,11 @@ func TestOnlyOneAddrLeft(t *testing.T) {
 	assert.NotNil(t, addr)
 
 	// If that edge address is "bad", there's no alternative address.
-	_, err = edge.GetDifferentAddr(connID)
+	_, err = edge.GetDifferentAddr(connID, false)
 	assert.Error(t, err)
 
 	// previously bad address should become available again on next iteration.
-	addr, err = edge.GetDifferentAddr(connID)
+	addr, err = edge.GetDifferentAddr(connID, false)
 	assert.NoError(t, err)
 	assert.NotNil(t, addr)
 }
@@ -190,8 +247,17 @@ func TestGetDifferentAddr(t *testing.T) {
 	assert.Equal(t, 3, edge.AvailableAddrs())
 
 	// If the same connection requests another address, it should get the same one.
-	addr2, err := edge.GetDifferentAddr(connID)
+	addr2, err := edge.GetDifferentAddr(connID, false)
 	assert.NoError(t, err)
 	assert.NotEqual(t, addr, addr2)
 	assert.Equal(t, 3, edge.AvailableAddrs())
 }
+
+// MockEdge creates a Cloudflare Edge from arbitrary TCP addresses. Used for testing.
+func MockEdge(log *zerolog.Logger, addrs []*allregions.EdgeAddr) *Edge {
+	regions := allregions.NewNoResolve(addrs)
+	return &Edge{
+		log:     log,
+		regions: regions,
+	}
+}

fips/fips.go.linux-amd64

@@ -2,4 +2,11 @@
 
 package main
 
-import _ "crypto/tls/fipsonly"
+import (
+    _ "crypto/tls/fipsonly"
+    "github.com/cloudflare/cloudflared/cmd/cloudflared/tunnel"
+)
+
+func init () {
+    tunnel.FipsEnabled = true
+}

github_message.py

@@ -1,29 +1,56 @@
 #!/usr/bin/python3
 """
-Creates Github Releases Notes with content hashes
+Create Github Releases Notes with binary checksums from Workers KV
 """
 
 import argparse
 import logging
 import os
-import hashlib
-import glob
+import requests
 
-from github import Github, GithubException, UnknownObjectException
+from github import Github, UnknownObjectException
 
 FORMAT = "%(levelname)s - %(asctime)s: %(message)s"
-logging.basicConfig(format=FORMAT)
+logging.basicConfig(format=FORMAT, level=logging.INFO)
 
 CLOUDFLARED_REPO = os.environ.get("GITHUB_REPO", "cloudflare/cloudflared")
 GITHUB_CONFLICT_CODE = "already_exists"
+BASE_KV_URL = 'https://api.cloudflare.com/client/v4/accounts/'
 
-def get_sha256(filename):
-    """ get the sha256 of a file """
-    sha256_hash = hashlib.sha256()
-    with open(filename,"rb") as f:
-        for byte_block in iter(lambda: f.read(4096),b""):
-            sha256_hash.update(byte_block)
-        return sha256_hash.hexdigest()
+
+def kv_get_keys(prefix, account, namespace, api_token):
+    """ get the KV keys for a given prefix """
+    response = requests.get(
+        BASE_KV_URL + account + "/storage/kv/namespaces/" +
+        namespace + "/keys" + "?prefix=" + prefix,
+        headers={
+            "Content-Type": "application/json",
+            "Authorization": "Bearer " + api_token,
+        },
+    )
+    if response.status_code != 200:
+        jsonResponse = response.json()
+        errors = jsonResponse["errors"]
+        if len(errors) > 0:
+            raise Exception("failed to get checksums: {0}", errors[0])
+    return response.json()["result"]
+
+
+def kv_get_value(key, account, namespace, api_token):
+    """ get the KV value for a provided key """
+    response = requests.get(
+        BASE_KV_URL + account + "/storage/kv/namespaces/" + namespace + "/values/" + key,
+        headers={
+            "Content-Type": "application/json",
+            "Authorization": "Bearer " + api_token,
+        },
+    )
+    if response.status_code != 200:
+        jsonResponse = response.json()
+        errors = jsonResponse["errors"]
+        if len(errors) > 0:
+            raise Exception("failed to get checksums: {0}", errors[0])
+    return response.text
 
 
 def update_or_add_message(msg, name, sha):
@@ -42,6 +69,7 @@ def update_or_add_message(msg, name, sha):
         return '{0}{1}```'.format(msg[:back], new_text)
     return '{0} \n### SHA256 Checksums:\n```\n{1}```'.format(msg, new_text)
 
+
 def get_release(repo, version):
     """ Get a Github Release matching the version tag. """
     try:
@@ -55,11 +83,20 @@ def get_release(repo, version):
 def parse_args():
     """ Parse and validate args """
     parser = argparse.ArgumentParser(
-        description="Creates Github Releases and uploads assets."
+        description="Updates a Github Release with checksums from KV"
     )
     parser.add_argument(
         "--api-key", default=os.environ.get("API_KEY"), help="Github API key"
     )
+    parser.add_argument(
+        "--kv-namespace-id", default=os.environ.get("KV_NAMESPACE"), help="workers KV namespace id"
+    )
+    parser.add_argument(
+        "--kv-account-id", default=os.environ.get("KV_ACCOUNT"), help="workers KV account id"
+    )
+    parser.add_argument(
+        "--kv-api-token", default=os.environ.get("KV_API_TOKEN"), help="workers KV API Token"
+    )
     parser.add_argument(
         "--release-version",
         metavar="version",
@@ -80,6 +117,18 @@ def parse_args():
         logging.error("Missing API key")
         is_valid = False
 
+    if not args.kv_namespace_id:
+        logging.error("Missing KV namespace id")
+        is_valid = False
+
+    if not args.kv_account_id:
+        logging.error("Missing KV account id")
+        is_valid = False
+
+    if not args.kv_api_token:
+        logging.error("Missing KV API token")
+        is_valid = False
+
     if is_valid:
         return args
 
@@ -95,16 +144,20 @@ def main():
         repo = client.get_repo(CLOUDFLARED_REPO)
         release = get_release(repo, args.release_version)
 
-        msg = release.body
+        msg = ""
 
-        for filepath in glob.glob("artifacts/*"):
-            pkg_hash = get_sha256(filepath)
-            # add the sha256 of the new artifact to the release message body
-            name = os.path.basename(filepath)
-            msg = update_or_add_message(msg, name, pkg_hash)
+        prefix = f"update_{args.release_version}_"
+        keys = kv_get_keys(prefix, args.kv_account_id,
+                           args.kv_namespace_id, args.kv_api_token)
+        for key in [k["name"] for k in keys]:
+            checksum = kv_get_value(
+                key, args.kv_account_id, args.kv_namespace_id, args.kv_api_token)
+            binary_name = key[len(prefix):]
+            msg = update_or_add_message(msg, binary_name, checksum)
 
         if args.dry_run:
-            logging.info("Skipping asset upload because of dry-run")
+            logging.info("Skipping release message update because of dry-run")
+            logging.info(f"Github message:\n{msg}")
             return
 
         # update the release body text
@@ -115,4 +168,4 @@ def main():
         exit(1)
 
 
-main()
+main()

go.mod

@@ -1,30 +1,30 @@
 module github.com/cloudflare/cloudflared
 
-go 1.17
+go 1.18
 
 require (
 	github.com/cloudflare/brotli-go v0.0.0-20191101163834-d34379f7ff93
 	github.com/cloudflare/golibs v0.0.0-20170913112048-333127dbecfc
 	github.com/coredns/coredns v1.8.7
+	github.com/coreos/go-oidc/v3 v3.4.0
 	github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf
 	github.com/facebookgo/grace v0.0.0-20180706040059-75cf19382434
 	github.com/fsnotify/fsnotify v1.4.9
-	github.com/gdamore/tcell v1.3.0
 	github.com/getsentry/raven-go v0.0.0-20180517221441-ed7bcb39ff10
 	github.com/gobwas/ws v1.0.4
 	github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3
+	github.com/google/gopacket v1.1.19
 	github.com/google/uuid v1.3.0
 	github.com/gorilla/mux v1.8.0
 	github.com/gorilla/websocket v1.4.2
 	github.com/json-iterator/go v1.1.12
-	github.com/lucas-clemente/quic-go v0.27.1
+	github.com/lucas-clemente/quic-go v0.28.1
 	github.com/mattn/go-colorable v0.1.8
 	github.com/miekg/dns v1.1.45
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.12.1
 	github.com/prometheus/client_model v0.2.0
-	github.com/rivo/tview v0.0.0-20200712113419-c65badfc3d92
 	github.com/rs/zerolog v1.20.0
 	github.com/stretchr/testify v1.7.1
 	github.com/urfave/cli/v2 v2.3.0
@@ -35,11 +35,11 @@ require (
 	go.opentelemetry.io/otel/trace v1.6.3
 	go.opentelemetry.io/proto/otlp v0.15.0
 	go.uber.org/automaxprocs v1.4.0
-	golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f
-	golang.org/x/net v0.0.0-20220114011407-0dd24b26b47d
-	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
-	golang.org/x/sys v0.0.0-20220114195835-da31bd327af9
-	golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b
+	golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa
+	golang.org/x/net v0.0.0-20220909164309-bea034e7d591
+	golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f
+	golang.org/x/sys v0.0.0-20220808155132-1c4a2a72c664
+	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211
 	google.golang.org/protobuf v1.28.0
 	gopkg.in/coreos/go-oidc.v2 v2.2.1
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
@@ -55,6 +55,7 @@ require (
 	github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894 // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/cheekybits/genny v1.0.0 // indirect
+	github.com/cloudflare/circl v1.2.1-0.20220809205628-0a9554f37a47 // indirect
 	github.com/coredns/caddy v1.1.1 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
@@ -63,7 +64,6 @@ require (
 	github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
 	github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870 // indirect
 	github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 // indirect
-	github.com/gdamore/encoding v1.0.0 // indirect
 	github.com/go-logr/logr v1.2.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
@@ -73,12 +73,11 @@ require (
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect
 	github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
-	github.com/lucasb-eyer/go-colorful v1.0.3 // indirect
 	github.com/marten-seemann/qtls-go1-16 v0.1.5 // indirect
-	github.com/marten-seemann/qtls-go1-17 v0.1.1 // indirect
-	github.com/marten-seemann/qtls-go1-18 v0.1.1 // indirect
+	github.com/marten-seemann/qtls-go1-17 v0.1.2 // indirect
+	github.com/marten-seemann/qtls-go1-18 v0.1.2 // indirect
+	github.com/marten-seemann/qtls-go1-19 v0.1.0-beta.1 // indirect
 	github.com/mattn/go-isatty v0.0.12 // indirect
-	github.com/mattn/go-runewidth v0.0.8 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
@@ -89,23 +88,33 @@ require (
 	github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 // indirect
 	github.com/prometheus/common v0.32.1 // indirect
 	github.com/prometheus/procfs v0.7.3 // indirect
-	github.com/rivo/uniseg v0.1.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	golang.org/x/mod v0.4.2 // indirect
-	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
+	golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094 // indirect
 	golang.org/x/text v0.3.7 // indirect
 	golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2 // indirect
-	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
+	golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20211223182754-3ac035c7e7cb // indirect
-	google.golang.org/grpc v1.45.0 // indirect
+	google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90 // indirect
+	google.golang.org/grpc v1.47.0 // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 )
 
 replace github.com/urfave/cli/v2 => github.com/ipostelnik/cli/v2 v2.3.1-0.20210324024421-b6ea8234fe3d
 
-replace github.com/lucas-clemente/quic-go => github.com/chungthuang/quic-go v0.27.1-0.20220607112311-13144fbde8da
+replace github.com/lucas-clemente/quic-go => github.com/chungthuang/quic-go v0.27.1-0.20220809135021-ca330f1dec9f
 
 // Avoid 'CVE-2022-21698'
 replace github.com/prometheus/golang_client => github.com/prometheus/golang_client v1.12.1
+
+replace gopkg.in/yaml.v3 => gopkg.in/yaml.v3 v3.0.1
+
+// Post-quantum tunnel RTG-1339
+replace (
+	// branch go1.18
+	github.com/marten-seemann/qtls-go1-18 => github.com/cloudflare/qtls-pq v0.0.0-20221010110824-0053225e48b2
+
+	// branch go1.19
+	github.com/marten-seemann/qtls-go1-19 => github.com/cloudflare/qtls-pq v0.0.0-20221010110800-4f3769902fe0
+)

go.sum

@@ -28,14 +28,23 @@ cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+Y
 cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4=
 cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc=
 cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA=
+cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A=
+cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+nc=
 cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
 cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
 cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
 cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
 cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
 cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
+cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow=
+cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM=
+cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M=
+cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s=
+cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU=
+cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U=
 cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
 cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
+cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY=
 cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
 cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
 cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
@@ -45,6 +54,7 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo
 cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
 cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
+cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y=
 dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBrvjyP0v+ecvNYvCpyZgu5/xkfAUhi6wJj28eUfSU=
@@ -70,7 +80,6 @@ github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBp
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
-github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
 github.com/DataDog/datadog-go v4.4.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
 github.com/DataDog/gostackparse v0.5.0/go.mod h1:lTfqcJKqS9KnXQGnyQMCugq3u1FP6UZMfWR0aitKFMM=
 github.com/DataDog/sketches-go v1.0.0/go.mod h1:O+XkJHWk9w4hDwY2ZUDU31ZC9sNYlYo8DiFsxjYeo1k=
@@ -100,27 +109,33 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g=
 github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=
+github.com/bwesterb/go-ristretto v1.2.2/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
 github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894 h1:JLaf/iINcLyjwbtTsCJjc6rtlASgHeIJPrB6QmwURnA=
 github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA=
-github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
 github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE=
 github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ=
-github.com/chungthuang/quic-go v0.27.1-0.20220607112311-13144fbde8da h1:FmuwbQ8RU/ftTKnfz5diawqvQFH1KDB9wN2Q8S2wqds=
-github.com/chungthuang/quic-go v0.27.1-0.20220607112311-13144fbde8da/go.mod h1:AzgQoPda7N+3IqMMMkywBKggIFo2KT6pfnlrQ2QieeI=
+github.com/chungthuang/quic-go v0.27.1-0.20220809135021-ca330f1dec9f h1:UWC3XjwZzocdNCzzXxq9j/1SdHMZXhcTOsh/+gNRBUQ=
+github.com/chungthuang/quic-go v0.27.1-0.20220809135021-ca330f1dec9f/go.mod h1:oGz5DKK41cJt5+773+BSO9BXDsREY4HLf7+0odGAPO0=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cloudflare/brotli-go v0.0.0-20191101163834-d34379f7ff93 h1:QrGfkZDnMxcWHaYDdB7CmqS9i26OAnUj/xcus/abYkY=
 github.com/cloudflare/brotli-go v0.0.0-20191101163834-d34379f7ff93/go.mod h1:QiTe66jFdP7cUKMCCf/WrvDyYdtdmdZfVcdoLbzaKVY=
+github.com/cloudflare/circl v1.2.1-0.20220809205628-0a9554f37a47 h1:YzpECHxZ9TzO7LpnKmPxItSd79lLgrR5heIlnqU4dTU=
+github.com/cloudflare/circl v1.2.1-0.20220809205628-0a9554f37a47/go.mod h1:qhx8gBILsYlbam7h09SvHDSkjpe3TfLA7b/z4rxJvkE=
 github.com/cloudflare/golibs v0.0.0-20170913112048-333127dbecfc h1:Dvk3ySBsOm5EviLx6VCyILnafPcQinXGP5jbTdHUJgE=
 github.com/cloudflare/golibs v0.0.0-20170913112048-333127dbecfc/go.mod h1:HlgKKR8V5a1wroIDDIz3/A+T+9Janfq+7n1P5sEFdi0=
+github.com/cloudflare/qtls-pq v0.0.0-20221010110800-4f3769902fe0 h1:LEsjEHfKnIJEJU9QEIPVRuslxpBu+2kG2DXhxpkGT+o=
+github.com/cloudflare/qtls-pq v0.0.0-20221010110800-4f3769902fe0/go.mod h1:aIsWqC0WXyUiUxBl/RfxAjDyWE9CCLqvSMnCMTd/+bc=
+github.com/cloudflare/qtls-pq v0.0.0-20221010110824-0053225e48b2 h1:ErNoeVNqFXV+emlf4gY7Ms7/0DbQ8PT2UFxNyWBc51Q=
+github.com/cloudflare/qtls-pq v0.0.0-20221010110824-0053225e48b2/go.mod h1:mW0BgKFFDAiSmOdUwoORtjo0V2vqw5QzVYRtKQqw/Jg=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
@@ -128,11 +143,14 @@ github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XP
 github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
+github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/coredns/caddy v1.1.1 h1:2eYKZT7i6yxIfGP3qLJoJ7HAsDJqYB+X68g4NYjSrE0=
 github.com/coredns/caddy v1.1.1/go.mod h1:A6ntJQlAWuQfFlsd9hvigKbo2WS0VUs2l1e2F+BawD4=
 github.com/coredns/coredns v1.8.7 h1:wVMjAnyFnY7Mc18AFO+9qbGD6ODPtdVUIlzoWrHr3hk=
 github.com/coredns/coredns v1.8.7/go.mod h1:bFmbgEfeRz5aizL2VsQ5LRlsvJuXWkgG/MWG9zxqjVM=
+github.com/coreos/go-oidc/v3 v3.4.0 h1:xz7elHb/LDwm/ERpwHd+5nb7wFHL32rsr6bBOgaeu6g=
+github.com/coreos/go-oidc/v3 v3.4.0/go.mod h1:eHUXhZtXPQLgEaDrOVTgwbgmz1xGOkJNye6h3zkD2Pw=
 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
@@ -166,6 +184,7 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m
 github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
 github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ=
 github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
+github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 h1:0JZ+dUmQeA8IIVUMzysrX4/AKuQwWhV2dYQuPZdvdSQ=
@@ -189,10 +208,6 @@ github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
-github.com/gdamore/encoding v1.0.0 h1:+7OoQ1Bc6eTm5niUzBa0Ctsh6JbMW6Ra+YNuAtDBdko=
-github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo5dl+VrEg=
-github.com/gdamore/tcell v1.3.0 h1:r35w0JBADPZCVQijYebl6YMWWtHRqVEGt7kL2eBADRM=
-github.com/gdamore/tcell v1.3.0/go.mod h1:Hjvr+Ofd+gLglo7RYKxxnzCBmev3BzsS67MebKS4zMM=
 github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg=
 github.com/getsentry/raven-go v0.0.0-20180517221441-ed7bcb39ff10 h1:YO10pIIBftO/kkTFdWhctH96grJ7qiy7bMdiZcIvPKs=
 github.com/getsentry/raven-go v0.0.0-20180517221441-ed7bcb39ff10/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ=
@@ -291,13 +306,16 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
 github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
+github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
+github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8=
+github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
 github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
@@ -322,14 +340,19 @@ github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8=
 github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY=
 github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0=
 github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM=
+github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM=
+github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM=
+github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c=
 github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU=
 github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA=
+github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
 github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
@@ -343,7 +366,6 @@ github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
-github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo=
 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 h1:BZHcxBETFHIdVyhyEfOvn/RdU/QGdLI4y34qQGjGWO0=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks=
@@ -396,9 +418,6 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
-github.com/lucasb-eyer/go-colorful v1.0.2/go.mod h1:0MS4r+7BZKSJ5mw4/S5MPN+qHFF1fYclkSPilDOKW0s=
-github.com/lucasb-eyer/go-colorful v1.0.3 h1:QIbQXiugsb+q10B+MI+7DI1oQLdmnep86tWFlaaUAac=
-github.com/lucasb-eyer/go-colorful v1.0.3/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
 github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI=
 github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
 github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
@@ -407,17 +426,12 @@ github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN
 github.com/marten-seemann/qpack v0.2.1/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc=
 github.com/marten-seemann/qtls-go1-16 v0.1.5 h1:o9JrYPPco/Nukd/HpOHMHZoBDXQqoNtUCmny98/1uqQ=
 github.com/marten-seemann/qtls-go1-16 v0.1.5/go.mod h1:gNpI2Ol+lRS3WwSOtIUUtRwZEQMXjYK+dQSBFbethAk=
-github.com/marten-seemann/qtls-go1-17 v0.1.1 h1:DQjHPq+aOzUeh9/lixAGunn6rIOQyWChPSI4+hgW7jc=
-github.com/marten-seemann/qtls-go1-17 v0.1.1/go.mod h1:C2ekUKcDdz9SDWxec1N/MvcXBpaX9l3Nx67XaR84L5s=
-github.com/marten-seemann/qtls-go1-18 v0.1.1 h1:qp7p7XXUFL7fpBvSS1sWD+uSqPvzNQK43DH+/qEkj0Y=
-github.com/marten-seemann/qtls-go1-18 v0.1.1/go.mod h1:mJttiymBAByA49mhlNZZGrH5u1uXYZJ+RW28Py7f4m4=
+github.com/marten-seemann/qtls-go1-17 v0.1.2 h1:JADBlm0LYiVbuSySCHeY863dNkcpMmDR7s0bLKJeYlQ=
+github.com/marten-seemann/qtls-go1-17 v0.1.2/go.mod h1:C2ekUKcDdz9SDWxec1N/MvcXBpaX9l3Nx67XaR84L5s=
 github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8=
 github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
 github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
-github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
-github.com/mattn/go-runewidth v0.0.8 h1:3tS41NlGYSmhhe/8fhGRzc+z3AYCw1Fe1WAyLuujKs0=
-github.com/mattn/go-runewidth v0.0.8/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4=
@@ -513,16 +527,11 @@ github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1
 github.com/rabbitmq/amqp091-go v1.1.0/go.mod h1:ogQDLSOACsLPsIq0NpbtiifNZi2YOz0VTJ0kHRghqbM=
 github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
-github.com/rivo/tview v0.0.0-20200712113419-c65badfc3d92 h1:rqaqSUdaW+OBbjnsrOoiaJv43mSRARuvsAuirmdxu7E=
-github.com/rivo/tview v0.0.0-20200712113419-c65badfc3d92/go.mod h1:6lkG1x+13OShEf0EaOCaTQYyB7d5nSbb181KtjlS+84=
-github.com/rivo/uniseg v0.1.0 h1:+2KBaVoUmb9XzDsrx/Ct0W/EYOSFf/nWTauy++DprtY=
-github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
 github.com/rs/zerolog v1.20.0 h1:38k9hgtUBdxFwE34yS8rTHmHBa4eN16E4DJlv177LNs=
 github.com/rs/zerolog v1.20.0/go.mod h1:IzD0RJ65iWH0w97OQQebJEvTZYvsCUm9WVLWBQrJRjo=
-github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
@@ -635,8 +644,8 @@ golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20210920023735-84f357641f63/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f h1:OeJjE6G4dgCY4PIXvIRQbE8+RX+uXZyGhUy/ksMGJoc=
-golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa h1:zuSxTR4o9y82ebqCUJYNGJbGPo6sKVl54f/TVDObg1c=
+golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -725,8 +734,16 @@ golang.org/x/net v0.0.0-20210917221730-978cfadd31cf/go.mod h1:9nx3DQGgdP8bBQD5qx
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20220114011407-0dd24b26b47d h1:1n1fc535VhN8SYtD4cDUyNlfpAF2ROMM9+11equK3hs=
-golang.org/x/net v0.0.0-20220114011407-0dd24b26b47d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
+golang.org/x/net v0.0.0-20220909164309-bea034e7d591 h1:D0B/7al0LLrVC8aWF4+oxpv/m8bc7ViFfVS8/gXGdqI=
+golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -744,8 +761,13 @@ golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ
 golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 h1:RerP+noqYHUQ8CMRcPlC2nvTa4dcBIjegkuWdcUDuqg=
 golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
+golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
+golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
+golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE=
+golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094 h1:2o1E+E8TpNLklK9nHiPiK1uzIYrIHt+cQx3ynCwq9V8=
+golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg=
 golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -757,8 +779,9 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f h1:Ax0t5p6N38Ga0dThY21weqDEyz2oklo4IvDkpigvkD8=
+golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -773,7 +796,6 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190626150813-e07cf5db2756/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -830,12 +852,25 @@ golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 h1:XfKQ4OlFl8okEOr5UvAqFRVj8pY/4yfcXrddB8qAbU0=
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220808155132-1c4a2a72c664 h1:v1W7bwXHsnLLloWYTVEdvGvA7BHMeBYsPcF0GLDxIRs=
+golang.org/x/sys v0.0.0-20220808155132-1c4a2a72c664/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b h1:9zKuko04nR4gjZ4+DNjHqRlAJqbJETHwiNKDqTfOjfE=
 golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -915,8 +950,11 @@ golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyj
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
+golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f h1:uF6paiQQebLeSXkrTqHqz0MXhXXS1KgF41eUdBNvxK0=
+golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
 google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
 google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
 google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y=
@@ -950,7 +988,16 @@ google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqiv
 google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE=
 google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI=
 google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I=
+google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo=
 google.golang.org/api v0.64.0/go.mod h1:931CdxA8Rm4t6zqTFGSsgwbAEZ2+GMYurbndwSimebM=
+google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g=
+google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA=
+google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8=
+google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs=
+google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA=
+google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw=
+google.golang.org/api v0.80.0/go.mod h1:xY3nI94gbvBrE0J6NHXhxOmW97HG7Khjkku6AFB3Hyg=
+google.golang.org/api v0.84.0/go.mod h1:NTsGnUFJMYROtiquksZHBWtHfeMC7iYthki7Eq3pa8o=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@@ -1005,6 +1052,7 @@ google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6D
 google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20210329143202-679c6ae281ee/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A=
 google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A=
 google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A=
 google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
@@ -1024,8 +1072,27 @@ google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEc
 google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
 google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
 google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20211223182754-3ac035c7e7cb h1:ZrsicilzPCS/Xr8qtBZZLpy4P9TYXAfl49ctG1/5tgw=
+google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
+google.golang.org/genproto v0.0.0-20211221195035-429b39de9b1c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
 google.golang.org/genproto v0.0.0-20211223182754-3ac035c7e7cb/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
+google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
+google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
+google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI=
+google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI=
+google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI=
+google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI=
+google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E=
+google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
+google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
+google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
+google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
+google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
+google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4=
+google.golang.org/genproto v0.0.0-20220518221133-4f43b3371335/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4=
+google.golang.org/genproto v0.0.0-20220523171625-347a074981d8/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4=
+google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA=
+google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90 h1:4SPz2GL2CXJt28MTF8V6Ap/9ZiVbQlJeGSd9qtA7DLs=
+google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA=
 google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
 google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio=
 google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
@@ -1060,8 +1127,12 @@ google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9K
 google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k=
 google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
 google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
-google.golang.org/grpc v1.45.0 h1:NEpgUqV3Z+ZjkqMsxMg11IaDrXY4RY6CQukSGK0uI1M=
+google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
 google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ=
+google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
+google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
+google.golang.org/grpc v1.47.0 h1:9n77onPX5F3qfFCqjy9dhn8PbNQsIKeVU04J9G7umt8=
+google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
@@ -1106,9 +1177,6 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o=

ingress/config.go

@@ -265,6 +265,9 @@ func originRequestFromConfig(c config.OriginRequestConfig) OriginRequestConfig {
 	if c.Http2Origin != nil {
 		out.Http2Origin = *c.Http2Origin
 	}
+	if c.Access != nil {
+		out.Access = *c.Access
+	}
 	return out
 }
 
@@ -310,6 +313,9 @@ type OriginRequestConfig struct {
 	IPRules []ipaccess.Rule `yaml:"ipRules" json:"ipRules"`
 	// Attempt to connect to origin with HTTP/2
 	Http2Origin bool `yaml:"http2Origin" json:"http2Origin"`
+
+	// Access holds all access related configs
+	Access config.AccessConfig `yaml:"access" json:"access,omitempty"`
 }
 
 func (defaults *OriginRequestConfig) setConnectTimeout(overrides config.OriginRequestConfig) {
@@ -421,6 +427,12 @@ func (defaults *OriginRequestConfig) setHttp2Origin(overrides config.OriginReque
 	}
 }
 
+func (defaults *OriginRequestConfig) setAccess(overrides config.OriginRequestConfig) {
+	if val := overrides.Access; val != nil {
+		defaults.Access = *val
+	}
+}
+
 // SetConfig gets config for the requests that cloudflared sends to origins.
 // Each field has a setter method which sets a value for the field by trying to find:
 //   1. The user config for this rule
@@ -447,6 +459,8 @@ func setConfig(defaults OriginRequestConfig, overrides config.OriginRequestConfi
 	cfg.setProxyType(overrides)
 	cfg.setIPRules(overrides)
 	cfg.setHttp2Origin(overrides)
+	cfg.setAccess(overrides)
+
 	return cfg
 }
 
@@ -457,6 +471,7 @@ func ConvertToRawOriginConfig(c OriginRequestConfig) config.OriginRequestConfig
 	var keepAliveConnections *int
 	var keepAliveTimeout *config.CustomDuration
 	var proxyAddress *string
+	var access *config.AccessConfig
 
 	if c.ConnectTimeout != defaultHTTPConnectTimeout {
 		connectTimeout = &c.ConnectTimeout
@@ -476,6 +491,9 @@ func ConvertToRawOriginConfig(c OriginRequestConfig) config.OriginRequestConfig
 	if c.ProxyAddress != defaultProxyAddress {
 		proxyAddress = &c.ProxyAddress
 	}
+	if c.Access.Required {
+		access = &c.Access
+	}
 
 	return config.OriginRequestConfig{
 		ConnectTimeout:         connectTimeout,
@@ -495,6 +513,7 @@ func ConvertToRawOriginConfig(c OriginRequestConfig) config.OriginRequestConfig
 		ProxyType:              emptyStringToNil(c.ProxyType),
 		IPRules:                convertToRawIPRules(c.IPRules),
 		Http2Origin:            defaultBoolToNil(c.Http2Origin),
+		Access:                 access,
 	}
 }
 

ingress/icmp_darwin.go

@@ -0,0 +1,283 @@
+//go:build darwin
+
+package ingress
+
+// This file implements ICMPProxy for Darwin. It uses a non-privileged ICMP socket to send echo requests and listen for
+// echo replies. The source IP of the requests are rewritten to the bind IP of the socket and the socket reads all
+// messages, so we use echo ID to distinguish the replies. Each (source IP, destination IP, echo ID) is assigned a
+// unique echo ID.
+
+import (
+	"context"
+	"fmt"
+	"math"
+	"net/netip"
+	"strconv"
+	"sync"
+	"time"
+
+	"github.com/rs/zerolog"
+	"go.opentelemetry.io/otel/attribute"
+	"golang.org/x/net/icmp"
+
+	"github.com/cloudflare/cloudflared/packet"
+	"github.com/cloudflare/cloudflared/tracing"
+)
+
+type icmpProxy struct {
+	srcFunnelTracker *packet.FunnelTracker
+	echoIDTracker    *echoIDTracker
+	conn             *icmp.PacketConn
+	// Response is handled in one-by-one, so encoder can be shared between funnels
+	encoder     *packet.Encoder
+	logger      *zerolog.Logger
+	idleTimeout time.Duration
+}
+
+// echoIDTracker tracks which ID has been assigned. It first loops through assignment from lastAssignment to then end,
+// then from the beginning to lastAssignment.
+// ICMP echo are short lived. By the time an ID is revisited, it should have been released.
+type echoIDTracker struct {
+	lock sync.Mutex
+	// maps the source IP, destination IP and original echo ID to a unique echo ID obtained from assignment
+	mapping map[flow3Tuple]uint16
+	// assignment tracks if an ID is assigned using index as the ID
+	// The size of the array is math.MaxUint16 because echo ID is 2 bytes
+	assignment [math.MaxUint16]bool
+	// nextAssignment is the next number to check for assigment
+	nextAssignment uint16
+}
+
+func newEchoIDTracker() *echoIDTracker {
+	return &echoIDTracker{
+		mapping: make(map[flow3Tuple]uint16),
+	}
+}
+
+// Get assignment or assign a new ID.
+func (eit *echoIDTracker) getOrAssign(key flow3Tuple) (id uint16, success bool) {
+	eit.lock.Lock()
+	defer eit.lock.Unlock()
+	id, exists := eit.mapping[key]
+	if exists {
+		return id, true
+	}
+
+	if eit.nextAssignment == math.MaxUint16 {
+		eit.nextAssignment = 0
+	}
+
+	for i, assigned := range eit.assignment[eit.nextAssignment:] {
+		if !assigned {
+			echoID := uint16(i) + eit.nextAssignment
+			eit.set(key, echoID)
+			return echoID, true
+		}
+	}
+	for i, assigned := range eit.assignment[0:eit.nextAssignment] {
+		if !assigned {
+			echoID := uint16(i)
+			eit.set(key, echoID)
+			return echoID, true
+		}
+	}
+	return 0, false
+}
+
+// Caller should hold the lock
+func (eit *echoIDTracker) set(key flow3Tuple, assignedEchoID uint16) {
+	eit.assignment[assignedEchoID] = true
+	eit.mapping[key] = assignedEchoID
+	eit.nextAssignment = assignedEchoID + 1
+}
+
+func (eit *echoIDTracker) release(key flow3Tuple, assigned uint16) bool {
+	eit.lock.Lock()
+	defer eit.lock.Unlock()
+
+	currentEchoID, exists := eit.mapping[key]
+	if exists && assigned == currentEchoID {
+		delete(eit.mapping, key)
+		eit.assignment[assigned] = false
+		return true
+	}
+	return false
+}
+
+type echoFunnelID uint16
+
+func (snf echoFunnelID) Type() string {
+	return "echoID"
+}
+
+func (snf echoFunnelID) String() string {
+	return strconv.FormatUint(uint64(snf), 10)
+}
+
+func newICMPProxy(listenIP netip.Addr, zone string, logger *zerolog.Logger, idleTimeout time.Duration) (*icmpProxy, error) {
+	conn, err := newICMPConn(listenIP, zone)
+	if err != nil {
+		return nil, err
+	}
+	logger.Info().Msgf("Created ICMP proxy listening on %s", conn.LocalAddr())
+	return &icmpProxy{
+		srcFunnelTracker: packet.NewFunnelTracker(),
+		echoIDTracker:    newEchoIDTracker(),
+		encoder:          packet.NewEncoder(),
+		conn:             conn,
+		logger:           logger,
+		idleTimeout:      idleTimeout,
+	}, nil
+}
+
+func (ip *icmpProxy) Request(ctx context.Context, pk *packet.ICMP, responder *packetResponder) error {
+	ctx, span := responder.requestSpan(ctx, pk)
+	defer responder.exportSpan()
+
+	originalEcho, err := getICMPEcho(pk.Message)
+	if err != nil {
+		tracing.EndWithErrorStatus(span, err)
+		return err
+	}
+	span.SetAttributes(
+		attribute.Int("originalEchoID", originalEcho.ID),
+		attribute.Int("seq", originalEcho.Seq),
+	)
+	echoIDTrackerKey := flow3Tuple{
+		srcIP:          pk.Src,
+		dstIP:          pk.Dst,
+		originalEchoID: originalEcho.ID,
+	}
+	// TODO: TUN-6744 assign unique flow per (src, echo ID)
+	assignedEchoID, success := ip.echoIDTracker.getOrAssign(echoIDTrackerKey)
+	if !success {
+		err := fmt.Errorf("failed to assign unique echo ID")
+		tracing.EndWithErrorStatus(span, err)
+		return err
+	}
+	span.SetAttributes(attribute.Int("assignedEchoID", int(assignedEchoID)))
+
+	newFunnelFunc := func() (packet.Funnel, error) {
+		originalEcho, err := getICMPEcho(pk.Message)
+		if err != nil {
+			return nil, err
+		}
+		closeCallback := func() error {
+			ip.echoIDTracker.release(echoIDTrackerKey, assignedEchoID)
+			return nil
+		}
+		icmpFlow := newICMPEchoFlow(pk.Src, closeCallback, ip.conn, responder, int(assignedEchoID), originalEcho.ID, ip.encoder)
+		return icmpFlow, nil
+	}
+	funnelID := echoFunnelID(assignedEchoID)
+	funnel, isNew, err := ip.srcFunnelTracker.GetOrRegister(funnelID, newFunnelFunc)
+	if err != nil {
+		tracing.EndWithErrorStatus(span, err)
+		return err
+	}
+	if isNew {
+		span.SetAttributes(attribute.Bool("newFlow", true))
+		ip.logger.Debug().
+			Str("src", pk.Src.String()).
+			Str("dst", pk.Dst.String()).
+			Int("originalEchoID", originalEcho.ID).
+			Int("assignedEchoID", int(assignedEchoID)).
+			Msg("New flow")
+	}
+	icmpFlow, err := toICMPEchoFlow(funnel)
+	if err != nil {
+		tracing.EndWithErrorStatus(span, err)
+		return err
+	}
+	err = icmpFlow.sendToDst(pk.Dst, pk.Message)
+	if err != nil {
+		tracing.EndWithErrorStatus(span, err)
+		return err
+	}
+	tracing.End(span)
+	return nil
+}
+
+// Serve listens for responses to the requests until context is done
+func (ip *icmpProxy) Serve(ctx context.Context) error {
+	go func() {
+		<-ctx.Done()
+		ip.conn.Close()
+	}()
+	go func() {
+		ip.srcFunnelTracker.ScheduleCleanup(ctx, ip.idleTimeout)
+	}()
+	buf := make([]byte, mtu)
+	icmpDecoder := packet.NewICMPDecoder()
+	for {
+		n, from, err := ip.conn.ReadFrom(buf)
+		if err != nil {
+			return err
+		}
+		reply, err := parseReply(from, buf[:n])
+		if err != nil {
+			ip.logger.Debug().Err(err).Str("dst", from.String()).Msg("Failed to parse ICMP reply, continue to parse as full packet")
+			// In unit test, we found out when the listener listens on 0.0.0.0, the socket reads the full packet after
+			// the second reply
+			if err := ip.handleFullPacket(ctx, icmpDecoder, buf[:n]); err != nil {
+				ip.logger.Debug().Err(err).Str("dst", from.String()).Msg("Failed to parse ICMP reply as full packet")
+			}
+			continue
+		}
+		if !isEchoReply(reply.msg) {
+			ip.logger.Debug().Str("dst", from.String()).Msgf("Drop ICMP %s from reply", reply.msg.Type)
+			continue
+		}
+		if err := ip.sendReply(ctx, reply); err != nil {
+			ip.logger.Error().Err(err).Str("dst", from.String()).Msg("Failed to send ICMP reply")
+			continue
+		}
+	}
+}
+
+func (ip *icmpProxy) handleFullPacket(ctx context.Context, decoder *packet.ICMPDecoder, rawPacket []byte) error {
+	icmpPacket, err := decoder.Decode(packet.RawPacket{Data: rawPacket})
+	if err != nil {
+		return err
+	}
+	echo, err := getICMPEcho(icmpPacket.Message)
+	if err != nil {
+		return err
+	}
+	reply := echoReply{
+		from: icmpPacket.Src,
+		msg:  icmpPacket.Message,
+		echo: echo,
+	}
+	if ip.sendReply(ctx, &reply); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (ip *icmpProxy) sendReply(ctx context.Context, reply *echoReply) error {
+	funnelID := echoFunnelID(reply.echo.ID)
+	funnel, ok := ip.srcFunnelTracker.Get(funnelID)
+	if !ok {
+		return packet.ErrFunnelNotFound
+	}
+	icmpFlow, err := toICMPEchoFlow(funnel)
+	if err != nil {
+		return err
+	}
+
+	_, span := icmpFlow.responder.replySpan(ctx, ip.logger)
+	defer icmpFlow.responder.exportSpan()
+
+	span.SetAttributes(
+		attribute.String("dst", reply.from.String()),
+		attribute.Int("echoID", reply.echo.ID),
+		attribute.Int("seq", reply.echo.Seq),
+		attribute.Int("originalEchoID", icmpFlow.originalEchoID),
+	)
+	if err := icmpFlow.returnToSrc(reply); err != nil {
+		tracing.EndWithErrorStatus(span, err)
+	}
+	tracing.End(span)
+	return nil
+}

ingress/icmp_darwin_test.go

@@ -0,0 +1,121 @@
+//go:build darwin
+
+package ingress
+
+import (
+	"math"
+	"net/netip"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestSingleEchoIDTracker(t *testing.T) {
+	tracker := newEchoIDTracker()
+	key := flow3Tuple{
+		srcIP:          netip.MustParseAddr("172.16.0.1"),
+		dstIP:          netip.MustParseAddr("172.16.0.2"),
+		originalEchoID: 5182,
+	}
+
+	// not assigned yet, so nothing to release
+	require.False(t, tracker.release(key, 0))
+
+	echoID, ok := tracker.getOrAssign(key)
+	require.True(t, ok)
+	require.Equal(t, uint16(0), echoID)
+
+	//  Second time should return the same echo ID
+	echoID, ok = tracker.getOrAssign(key)
+	require.True(t, ok)
+	require.Equal(t, uint16(0), echoID)
+
+	// releasing a different ID returns false
+	require.False(t, tracker.release(key, 1999))
+	require.True(t, tracker.release(key, echoID))
+	// releasing the second time returns false
+	require.False(t, tracker.release(key, echoID))
+
+	// Move to the next IP
+	echoID, ok = tracker.getOrAssign(key)
+	require.True(t, ok)
+	require.Equal(t, uint16(1), echoID)
+}
+
+func TestFullEchoIDTracker(t *testing.T) {
+	var (
+		dstIP          = netip.MustParseAddr("192.168.0.1")
+		originalEchoID = 41820
+	)
+	tracker := newEchoIDTracker()
+	firstSrcIP := netip.MustParseAddr("172.16.0.1")
+	srcIP := firstSrcIP
+
+	for i := uint16(0); i < math.MaxUint16; i++ {
+		key := flow3Tuple{
+			srcIP:          srcIP,
+			dstIP:          dstIP,
+			originalEchoID: originalEchoID,
+		}
+		echoID, ok := tracker.getOrAssign(key)
+		require.True(t, ok)
+		require.Equal(t, i, echoID)
+
+		echoID, ok = tracker.get(key)
+		require.True(t, ok)
+		require.Equal(t, i, echoID)
+
+		srcIP = srcIP.Next()
+	}
+
+	key := flow3Tuple{
+		srcIP:          srcIP.Next(),
+		dstIP:          dstIP,
+		originalEchoID: originalEchoID,
+	}
+	// All echo IDs are assigned
+	echoID, ok := tracker.getOrAssign(key)
+	require.False(t, ok)
+	require.Equal(t, uint16(0), echoID)
+
+	srcIP = firstSrcIP
+	for i := uint16(0); i < math.MaxUint16; i++ {
+		key := flow3Tuple{
+			srcIP:          srcIP,
+			dstIP:          dstIP,
+			originalEchoID: originalEchoID,
+		}
+		ok := tracker.release(key, i)
+		require.True(t, ok)
+
+		echoID, ok = tracker.get(key)
+		require.False(t, ok)
+		require.Equal(t, uint16(0), echoID)
+		srcIP = srcIP.Next()
+	}
+
+	// The IDs are assignable again
+	srcIP = firstSrcIP
+	for i := uint16(0); i < math.MaxUint16; i++ {
+		key := flow3Tuple{
+			srcIP:          srcIP,
+			dstIP:          dstIP,
+			originalEchoID: originalEchoID,
+		}
+		echoID, ok := tracker.getOrAssign(key)
+		require.True(t, ok)
+		require.Equal(t, i, echoID)
+
+		echoID, ok = tracker.get(key)
+		require.True(t, ok)
+		require.Equal(t, i, echoID)
+		srcIP = srcIP.Next()
+	}
+}
+
+func (eit *echoIDTracker) get(key flow3Tuple) (id uint16, exist bool) {
+	eit.lock.Lock()
+	defer eit.lock.Unlock()
+	id, exists := eit.mapping[key]
+	return id, exists
+}

ingress/icmp_generic.go

@@ -0,0 +1,31 @@
+//go:build !darwin && !linux && (!windows || !cgo)
+
+package ingress
+
+import (
+	"context"
+	"fmt"
+	"net/netip"
+	"runtime"
+	"time"
+
+	"github.com/rs/zerolog"
+
+	"github.com/cloudflare/cloudflared/packet"
+)
+
+var errICMPProxyNotImplemented = fmt.Errorf("ICMP proxy is not implemented on %s %s", runtime.GOOS, runtime.GOARCH)
+
+type icmpProxy struct{}
+
+func (ip icmpProxy) Request(ctx context.Context, pk *packet.ICMP, responder *packetResponder) error {
+	return errICMPProxyNotImplemented
+}
+
+func (ip *icmpProxy) Serve(ctx context.Context) error {
+	return errICMPProxyNotImplemented
+}
+
+func newICMPProxy(listenIP netip.Addr, zone string, logger *zerolog.Logger, idleTimeout time.Duration) (*icmpProxy, error) {
+	return nil, errICMPProxyNotImplemented
+}

ingress/icmp_linux.go

@@ -0,0 +1,237 @@
+//go:build linux
+
+package ingress
+
+// This file implements ICMPProxy for Linux. Each (source IP, destination IP, echo ID) opens a non-privileged ICMP socket.
+// The source IP of the requests are rewritten to the bind IP of the socket and echo ID rewritten to the port number of
+// the socket. The kernel ensures the socket only reads replies whose echo ID matches the port number.
+// For more information about the socket, see https://man7.org/linux/man-pages/man7/icmp.7.html and https://lwn.net/Articles/422330/
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"net/netip"
+	"os"
+	"regexp"
+	"strconv"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog"
+	"go.opentelemetry.io/otel/attribute"
+
+	"github.com/cloudflare/cloudflared/packet"
+	"github.com/cloudflare/cloudflared/tracing"
+)
+
+const (
+	// https://lwn.net/Articles/550551/ IPv4 and IPv6 share the same path
+	pingGroupPath = "/proc/sys/net/ipv4/ping_group_range"
+)
+
+var (
+	findGroupIDRegex = regexp.MustCompile(`\d+`)
+)
+
+type icmpProxy struct {
+	srcFunnelTracker *packet.FunnelTracker
+	listenIP         netip.Addr
+	ipv6Zone         string
+	logger           *zerolog.Logger
+	idleTimeout      time.Duration
+}
+
+func newICMPProxy(listenIP netip.Addr, zone string, logger *zerolog.Logger, idleTimeout time.Duration) (*icmpProxy, error) {
+	if err := testPermission(listenIP, zone, logger); err != nil {
+		return nil, err
+	}
+	return &icmpProxy{
+		srcFunnelTracker: packet.NewFunnelTracker(),
+		listenIP:         listenIP,
+		ipv6Zone:         zone,
+		logger:           logger,
+		idleTimeout:      idleTimeout,
+	}, nil
+}
+
+func testPermission(listenIP netip.Addr, zone string, logger *zerolog.Logger) error {
+	// Opens a non-privileged ICMP socket. On Linux the group ID of the process needs to be in ping_group_range
+	// Only check ping_group_range once for IPv4
+	if listenIP.Is4() {
+		if err := checkInPingGroup(); err != nil {
+			logger.Warn().Err(err).Msgf("The user running cloudflared process has a GID (group ID) that is not within ping_group_range. You might need to add that user to a group within that range, or instead update the range to encompass a group the user is already in by modifying %s. Otherwise cloudflared will not be able to ping this network", pingGroupPath)
+			return err
+		}
+	}
+	conn, err := newICMPConn(listenIP, zone)
+	if err != nil {
+		return err
+	}
+	// This conn is only to test if cloudflared has permission to open this type of socket
+	conn.Close()
+	return nil
+}
+
+func checkInPingGroup() error {
+	file, err := os.ReadFile(pingGroupPath)
+	if err != nil {
+		return err
+	}
+	groupID := os.Getgid()
+	// Example content: 999	   59999
+	found := findGroupIDRegex.FindAll(file, 2)
+	if len(found) == 2 {
+		groupMin, err := strconv.ParseInt(string(found[0]), 10, 32)
+		if err != nil {
+			return errors.Wrapf(err, "failed to determine minimum ping group ID")
+		}
+		groupMax, err := strconv.ParseInt(string(found[1]), 10, 32)
+		if err != nil {
+			return errors.Wrapf(err, "failed to determine minimum ping group ID")
+		}
+		if groupID < int(groupMin) || groupID > int(groupMax) {
+			return fmt.Errorf("Group ID %d is not between ping group %d to %d", groupID, groupMin, groupMax)
+		}
+		return nil
+	}
+	return fmt.Errorf("did not find group range in %s", pingGroupPath)
+}
+
+func (ip *icmpProxy) Request(ctx context.Context, pk *packet.ICMP, responder *packetResponder) error {
+	ctx, span := responder.requestSpan(ctx, pk)
+	defer responder.exportSpan()
+
+	originalEcho, err := getICMPEcho(pk.Message)
+	if err != nil {
+		tracing.EndWithErrorStatus(span, err)
+		return err
+	}
+	span.SetAttributes(
+		attribute.Int("originalEchoID", originalEcho.ID),
+		attribute.Int("seq", originalEcho.Seq),
+	)
+
+	newFunnelFunc := func() (packet.Funnel, error) {
+		conn, err := newICMPConn(ip.listenIP, ip.ipv6Zone)
+		if err != nil {
+			tracing.EndWithErrorStatus(span, err)
+			return nil, errors.Wrap(err, "failed to open ICMP socket")
+		}
+		ip.logger.Debug().Msgf("Opened ICMP socket listen on %s", conn.LocalAddr())
+		closeCallback := func() error {
+			return conn.Close()
+		}
+		localUDPAddr, ok := conn.LocalAddr().(*net.UDPAddr)
+		if !ok {
+			return nil, fmt.Errorf("ICMP listener address %s is not net.UDPAddr", conn.LocalAddr())
+		}
+		span.SetAttributes(attribute.Int("port", localUDPAddr.Port))
+
+		echoID := localUDPAddr.Port
+		icmpFlow := newICMPEchoFlow(pk.Src, closeCallback, conn, responder, echoID, originalEcho.ID, packet.NewEncoder())
+		return icmpFlow, nil
+	}
+	funnelID := flow3Tuple{
+		srcIP:          pk.Src,
+		dstIP:          pk.Dst,
+		originalEchoID: originalEcho.ID,
+	}
+	funnel, isNew, err := ip.srcFunnelTracker.GetOrRegister(funnelID, newFunnelFunc)
+	if err != nil {
+		tracing.EndWithErrorStatus(span, err)
+		return err
+	}
+	icmpFlow, err := toICMPEchoFlow(funnel)
+	if err != nil {
+		tracing.EndWithErrorStatus(span, err)
+		return err
+	}
+	if isNew {
+		span.SetAttributes(attribute.Bool("newFlow", true))
+		ip.logger.Debug().
+			Str("src", pk.Src.String()).
+			Str("dst", pk.Dst.String()).
+			Int("originalEchoID", originalEcho.ID).
+			Msg("New flow")
+		go func() {
+			defer ip.srcFunnelTracker.Unregister(funnelID, icmpFlow)
+			if err := ip.listenResponse(ctx, icmpFlow); err != nil {
+				ip.logger.Debug().Err(err).
+					Str("src", pk.Src.String()).
+					Str("dst", pk.Dst.String()).
+					Int("originalEchoID", originalEcho.ID).
+					Msg("Failed to listen for ICMP echo response")
+			}
+		}()
+	}
+	if err := icmpFlow.sendToDst(pk.Dst, pk.Message); err != nil {
+		tracing.EndWithErrorStatus(span, err)
+		return errors.Wrap(err, "failed to send ICMP echo request")
+	}
+	tracing.End(span)
+	return nil
+}
+
+func (ip *icmpProxy) Serve(ctx context.Context) error {
+	ip.srcFunnelTracker.ScheduleCleanup(ctx, ip.idleTimeout)
+	return ctx.Err()
+}
+
+func (ip *icmpProxy) listenResponse(ctx context.Context, flow *icmpEchoFlow) error {
+	buf := make([]byte, mtu)
+	for {
+		retryable, err := ip.handleResponse(ctx, flow, buf)
+		if err != nil && !retryable {
+			return err
+		}
+	}
+}
+
+func (ip *icmpProxy) handleResponse(ctx context.Context, flow *icmpEchoFlow, buf []byte) (retryableErr bool, err error) {
+	_, span := flow.responder.replySpan(ctx, ip.logger)
+	defer flow.responder.exportSpan()
+
+	span.SetAttributes(
+		attribute.Int("originalEchoID", flow.originalEchoID),
+	)
+
+	n, from, err := flow.originConn.ReadFrom(buf)
+	if err != nil {
+		tracing.EndWithErrorStatus(span, err)
+		return false, err
+	}
+	reply, err := parseReply(from, buf[:n])
+	if err != nil {
+		ip.logger.Error().Err(err).Str("dst", from.String()).Msg("Failed to parse ICMP reply")
+		tracing.EndWithErrorStatus(span, err)
+		return true, err
+	}
+	if !isEchoReply(reply.msg) {
+		err := fmt.Errorf("Expect ICMP echo reply, got %s", reply.msg.Type)
+		ip.logger.Debug().Str("dst", from.String()).Msgf("Drop ICMP %s from reply", reply.msg.Type)
+		tracing.EndWithErrorStatus(span, err)
+		return true, err
+	}
+	span.SetAttributes(
+		attribute.String("dst", reply.from.String()),
+		attribute.Int("echoID", reply.echo.ID),
+		attribute.Int("seq", reply.echo.Seq),
+	)
+	if err := flow.returnToSrc(reply); err != nil {
+		ip.logger.Err(err).Str("dst", from.String()).Msg("Failed to send ICMP reply")
+		tracing.EndWithErrorStatus(span, err)
+		return true, err
+	}
+	tracing.End(span)
+	return true, nil
+}
+
+// Only linux uses flow3Tuple as FunnelID
+func (ft flow3Tuple) Type() string {
+	return "srcIP_dstIP_echoID"
+}
+
+func (ft flow3Tuple) String() string {
+	return fmt.Sprintf("%s:%s:%d", ft.srcIP, ft.dstIP, ft.originalEchoID)
+}

ingress/icmp_posix.go

@@ -0,0 +1,176 @@
+//go:build darwin || linux
+
+package ingress
+
+// This file extracts logic shared by Linux and Darwin implementation if ICMPProxy.
+
+import (
+	"fmt"
+	"net"
+	"net/netip"
+
+	"github.com/google/gopacket/layers"
+	"golang.org/x/net/icmp"
+
+	"github.com/cloudflare/cloudflared/packet"
+)
+
+// Opens a non-privileged ICMP socket on Linux and Darwin
+func newICMPConn(listenIP netip.Addr, zone string) (*icmp.PacketConn, error) {
+	if listenIP.Is4() {
+		return icmp.ListenPacket("udp4", listenIP.String())
+	}
+	listenAddr := listenIP.String()
+	if zone != "" {
+		listenAddr = listenAddr + "%" + zone
+	}
+	return icmp.ListenPacket("udp6", listenAddr)
+}
+
+func netipAddr(addr net.Addr) (netip.Addr, bool) {
+	udpAddr, ok := addr.(*net.UDPAddr)
+	if !ok {
+		return netip.Addr{}, false
+	}
+	return netip.AddrFromSlice(udpAddr.IP)
+}
+
+type flow3Tuple struct {
+	srcIP          netip.Addr
+	dstIP          netip.Addr
+	originalEchoID int
+}
+
+// icmpEchoFlow implements the packet.Funnel interface.
+type icmpEchoFlow struct {
+	*packet.ActivityTracker
+	closeCallback  func() error
+	src            netip.Addr
+	originConn     *icmp.PacketConn
+	responder      *packetResponder
+	assignedEchoID int
+	originalEchoID int
+	// it's up to the user to ensure respEncoder is not used concurrently
+	respEncoder *packet.Encoder
+}
+
+func newICMPEchoFlow(src netip.Addr, closeCallback func() error, originConn *icmp.PacketConn, responder *packetResponder, assignedEchoID, originalEchoID int, respEncoder *packet.Encoder) *icmpEchoFlow {
+	return &icmpEchoFlow{
+		ActivityTracker: packet.NewActivityTracker(),
+		closeCallback:   closeCallback,
+		src:             src,
+		originConn:      originConn,
+		responder:       responder,
+		assignedEchoID:  assignedEchoID,
+		originalEchoID:  originalEchoID,
+		respEncoder:     respEncoder,
+	}
+}
+
+func (ief *icmpEchoFlow) Equal(other packet.Funnel) bool {
+	otherICMPFlow, ok := other.(*icmpEchoFlow)
+	if !ok {
+		return false
+	}
+	if otherICMPFlow.src != ief.src {
+		return false
+	}
+	if otherICMPFlow.originalEchoID != ief.originalEchoID {
+		return false
+	}
+	if otherICMPFlow.assignedEchoID != ief.assignedEchoID {
+		return false
+	}
+	return true
+}
+
+func (ief *icmpEchoFlow) Close() error {
+	return ief.closeCallback()
+}
+
+// sendToDst rewrites the echo ID to the one assigned to this flow
+func (ief *icmpEchoFlow) sendToDst(dst netip.Addr, msg *icmp.Message) error {
+	ief.UpdateLastActive()
+	originalEcho, err := getICMPEcho(msg)
+	if err != nil {
+		return err
+	}
+	sendMsg := icmp.Message{
+		Type: msg.Type,
+		Code: msg.Code,
+		Body: &icmp.Echo{
+			ID:   ief.assignedEchoID,
+			Seq:  originalEcho.Seq,
+			Data: originalEcho.Data,
+		},
+	}
+	// For IPv4, the pseudoHeader is not used because the checksum is always calculated
+	var pseudoHeader []byte = nil
+	serializedPacket, err := sendMsg.Marshal(pseudoHeader)
+	if err != nil {
+		return err
+	}
+	_, err = ief.originConn.WriteTo(serializedPacket, &net.UDPAddr{
+		IP: dst.AsSlice(),
+	})
+	return err
+}
+
+// returnToSrc rewrites the echo ID to the original echo ID from the eyeball
+func (ief *icmpEchoFlow) returnToSrc(reply *echoReply) error {
+	ief.UpdateLastActive()
+	reply.echo.ID = ief.originalEchoID
+	reply.msg.Body = reply.echo
+	pk := packet.ICMP{
+		IP: &packet.IP{
+			Src:      reply.from,
+			Dst:      ief.src,
+			Protocol: layers.IPProtocol(reply.msg.Type.Protocol()),
+			TTL:      packet.DefaultTTL,
+		},
+		Message: reply.msg,
+	}
+	serializedPacket, err := ief.respEncoder.Encode(&pk)
+	if err != nil {
+		return err
+	}
+	return ief.responder.returnPacket(serializedPacket)
+}
+
+type echoReply struct {
+	from netip.Addr
+	msg  *icmp.Message
+	echo *icmp.Echo
+}
+
+func parseReply(from net.Addr, rawMsg []byte) (*echoReply, error) {
+	fromAddr, ok := netipAddr(from)
+	if !ok {
+		return nil, fmt.Errorf("cannot convert %s to netip.Addr", from)
+	}
+	proto := layers.IPProtocolICMPv4
+	if fromAddr.Is6() {
+		proto = layers.IPProtocolICMPv6
+	}
+	msg, err := icmp.ParseMessage(int(proto), rawMsg)
+	if err != nil {
+		return nil, err
+	}
+	echo, err := getICMPEcho(msg)
+	if err != nil {
+		return nil, err
+	}
+	return &echoReply{
+		from: fromAddr,
+		msg:  msg,
+		echo: echo,
+	}, nil
+}
+
+func toICMPEchoFlow(funnel packet.Funnel) (*icmpEchoFlow, error) {
+	icmpFlow, ok := funnel.(*icmpEchoFlow)
+	if !ok {
+		return nil, fmt.Errorf("%v is not *ICMPEchoFunnel", funnel)
+	}
+	return icmpFlow, nil
+}

ingress/icmp_posix_test.go

@@ -0,0 +1,78 @@
+//go:build darwin || linux
+
+package ingress
+
+import (
+	"context"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/google/gopacket/layers"
+	"github.com/rs/zerolog"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/net/icmp"
+	"golang.org/x/net/ipv4"
+
+	"github.com/cloudflare/cloudflared/packet"
+)
+
+func TestFunnelIdleTimeout(t *testing.T) {
+	const (
+		idleTimeout = time.Second
+		echoID      = 42573
+		startSeq    = 8129
+	)
+	logger := zerolog.New(os.Stderr)
+	proxy, err := newICMPProxy(localhostIP, "", &logger, idleTimeout)
+	require.NoError(t, err)
+
+	ctx, cancel := context.WithCancel(context.Background())
+
+	proxyDone := make(chan struct{})
+	go func() {
+		proxy.Serve(ctx)
+		close(proxyDone)
+	}()
+
+	// Send a packet to register the flow
+	pk := packet.ICMP{
+		IP: &packet.IP{
+			Src:      localhostIP,
+			Dst:      localhostIP,
+			Protocol: layers.IPProtocolICMPv4,
+		},
+		Message: &icmp.Message{
+			Type: ipv4.ICMPTypeEcho,
+			Code: 0,
+			Body: &icmp.Echo{
+				ID:   echoID,
+				Seq:  startSeq,
+				Data: []byte(t.Name()),
+			},
+		},
+	}
+	muxer := newMockMuxer(0)
+	responder := packetResponder{
+		datagramMuxer: muxer,
+	}
+	require.NoError(t, proxy.Request(ctx, &pk, &responder))
+	validateEchoFlow(t, muxer, &pk)
+
+	// Send second request, should reuse the funnel
+	require.NoError(t, proxy.Request(ctx, &pk, &packetResponder{
+		datagramMuxer: nil,
+	}))
+	validateEchoFlow(t, muxer, &pk)
+
+	time.Sleep(idleTimeout * 2)
+	newMuxer := newMockMuxer(0)
+	newResponder := packetResponder{
+		datagramMuxer: newMuxer,
+	}
+	require.NoError(t, proxy.Request(ctx, &pk, &newResponder))
+	validateEchoFlow(t, newMuxer, &pk)
+
+	cancel()
+	<-proxyDone
+}

ingress/icmp_windows.go

@@ -0,0 +1,577 @@
+//go:build windows && cgo
+
+package ingress
+
+/*
+#include <iphlpapi.h>
+#include <icmpapi.h>
+*/
+import "C"
+import (
+	"context"
+	"encoding/binary"
+	"fmt"
+	"net/netip"
+	"runtime/debug"
+	"sync"
+	"syscall"
+	"time"
+	"unsafe"
+
+	"github.com/google/gopacket/layers"
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog"
+	"go.opentelemetry.io/otel/attribute"
+	"golang.org/x/net/icmp"
+	"golang.org/x/net/ipv4"
+	"golang.org/x/net/ipv6"
+
+	"github.com/cloudflare/cloudflared/packet"
+	"github.com/cloudflare/cloudflared/tracing"
+)
+
+const (
+	// Value defined in https://docs.microsoft.com/en-us/windows/win32/api/winsock2/nf-winsock2-wsasocketw
+	AF_INET6          = 23
+	icmpEchoReplyCode = 0
+	nullParameter     = uintptr(0)
+)
+
+var (
+	Iphlpapi             = syscall.NewLazyDLL("Iphlpapi.dll")
+	IcmpCreateFile_proc  = Iphlpapi.NewProc("IcmpCreateFile")
+	Icmp6CreateFile_proc = Iphlpapi.NewProc("Icmp6CreateFile")
+	IcmpSendEcho_proc    = Iphlpapi.NewProc("IcmpSendEcho")
+	Icmp6SendEcho_proc   = Iphlpapi.NewProc("Icmp6SendEcho2")
+	echoReplySize        = unsafe.Sizeof(echoReply{})
+	echoV6ReplySize      = unsafe.Sizeof(echoV6Reply{})
+	icmpv6ErrMessageSize = 8
+	ioStatusBlockSize    = unsafe.Sizeof(ioStatusBlock{})
+	endian               = binary.LittleEndian
+)
+
+// IP_STATUS code, see https://docs.microsoft.com/en-us/windows/win32/api/ipexport/ns-ipexport-icmp_echo_reply32#members
+// for possible values
+type ipStatus uint32
+
+const (
+	success     ipStatus = 0
+	bufTooSmall          = iota + 11000
+	destNetUnreachable
+	destHostUnreachable
+	destProtocolUnreachable
+	destPortUnreachable
+	noResources
+	badOption
+	hwError
+	packetTooBig
+	reqTimedOut
+	badReq
+	badRoute
+	ttlExpiredTransit
+	ttlExpiredReassembly
+	paramProblem
+	sourceQuench
+	optionTooBig
+	badDestination
+	// Can be returned for malformed ICMP packets
+	generalFailure = 11050
+)
+
+// Additional IP_STATUS codes for ICMPv6  https://docs.microsoft.com/en-us/windows/win32/api/ipexport/ns-ipexport-icmpv6_echo_reply_lh#members
+const (
+	ipv6DestUnreachable ipStatus = iota + 11040
+	ipv6TimeExceeded
+	ipv6BadHeader
+	ipv6UnrecognizedNextHeader
+	ipv6ICMPError
+	ipv6DestScopeMismatch
+)
+
+func (is ipStatus) String() string {
+	switch is {
+	case success:
+		return "Success"
+	case bufTooSmall:
+		return "The reply buffer too small"
+	case destNetUnreachable:
+		return "The destination network was unreachable"
+	case destHostUnreachable:
+		return "The destination host was unreachable"
+	case destProtocolUnreachable:
+		return "The destination protocol was unreachable"
+	case destPortUnreachable:
+		return "The destination port was unreachable"
+	case noResources:
+		return "Insufficient IP resources were available"
+	case badOption:
+		return "A bad IP option was specified"
+	case hwError:
+		return "A hardware error occurred"
+	case packetTooBig:
+		return "The packet was too big"
+	case reqTimedOut:
+		return "The request timed out"
+	case badReq:
+		return "Bad request"
+	case badRoute:
+		return "Bad route"
+	case ttlExpiredTransit:
+		return "The TTL expired in transit"
+	case ttlExpiredReassembly:
+		return "The TTL expired during fragment reassembly"
+	case paramProblem:
+		return "A parameter problem"
+	case sourceQuench:
+		return "Datagrams are arriving too fast to be processed and datagrams may have been discarded"
+	case optionTooBig:
+		return "The IP option was too big"
+	case badDestination:
+		return "Bad destination"
+	case ipv6DestUnreachable:
+		return "IPv6 destination unreachable"
+	case ipv6TimeExceeded:
+		return "IPv6 time exceeded"
+	case ipv6BadHeader:
+		return "IPv6 bad IP header"
+	case ipv6UnrecognizedNextHeader:
+		return "IPv6 unrecognized next header"
+	case ipv6ICMPError:
+		return "IPv6 ICMP error"
+	case ipv6DestScopeMismatch:
+		return "IPv6 destination scope ID mismatch"
+	case generalFailure:
+		return "The ICMP packet might be malformed"
+	default:
+		return fmt.Sprintf("Unknown ip status %d", is)
+	}
+}
+
+// https://docs.microsoft.com/en-us/windows/win32/api/ipexport/ns-ipexport-ip_option_information
+type ipOption struct {
+	TTL         uint8
+	Tos         uint8
+	Flags       uint8
+	OptionsSize uint8
+	OptionsData uintptr
+}
+
+// https://docs.microsoft.com/en-us/windows/win32/api/ipexport/ns-ipexport-icmp_echo_reply
+type echoReply struct {
+	Address       uint32
+	Status        ipStatus
+	RoundTripTime uint32
+	DataSize      uint16
+	Reserved      uint16
+	// The pointer size defers between 32-bit and 64-bit platforms
+	DataPointer *byte
+	Options     ipOption
+}
+
+type echoV6Reply struct {
+	Address       ipv6AddrEx
+	Status        ipStatus
+	RoundTripTime uint32
+}
+
+// https://docs.microsoft.com/en-us/windows/win32/api/ipexport/ns-ipexport-ipv6_address_ex
+// All the fields are in network byte order. The memory alignment is 4 bytes
+type ipv6AddrEx struct {
+	port uint16
+	// flowInfo is uint32. Because of field alignment, when we cast reply buffer to ipv6AddrEx, it starts at the 5th byte
+	// But looking at the raw bytes, flowInfo starts at the 3rd byte. We device flowInfo into 2 uint16 so it's aligned
+	flowInfoUpper uint16
+	flowInfoLower uint16
+	addr          [8]uint16
+	scopeID       uint32
+}
+
+// https://docs.microsoft.com/en-us/windows/win32/winsock/sockaddr-2
+type sockAddrIn6 struct {
+	family int16
+	// Can't embed ipv6AddrEx, that changes the memory alignment
+	port     uint16
+	flowInfo uint32
+	addr     [16]byte
+	scopeID  uint32
+}
+
+func newSockAddrIn6(addr netip.Addr) (*sockAddrIn6, error) {
+	if !addr.Is6() {
+		return nil, fmt.Errorf("%s is not IPv6", addr)
+	}
+	return &sockAddrIn6{
+		family: AF_INET6,
+		port:   10,
+		addr:   addr.As16(),
+	}, nil
+}
+
+// https://docs.microsoft.com/en-us/windows-hardware/drivers/ddi/wdm/ns-wdm-_io_status_block#syntax
+type ioStatusBlock struct {
+	// The first field is an union of NTSTATUS and PVOID. NTSTATUS is int32 while PVOID depends on the platform.
+	// We model it as uintptr whose size depends on if the platform is 32-bit or 64-bit
+	// https://docs.microsoft.com/en-us/openspecs/windows_protocols/ms-erref/596a1078-e883-4972-9bbc-49e60bebca55
+	statusOrPointer uintptr
+	information     uintptr
+}
+
+type icmpProxy struct {
+	// An open handle that can send ICMP requests https://docs.microsoft.com/en-us/windows/win32/api/icmpapi/nf-icmpapi-icmpcreatefile
+	handle uintptr
+	// This is a ICMPv6 if srcSocketAddr is not nil
+	srcSocketAddr *sockAddrIn6
+	logger        *zerolog.Logger
+	// A pool of reusable *packet.Encoder
+	encoderPool sync.Pool
+}
+
+func newICMPProxy(listenIP netip.Addr, zone string, logger *zerolog.Logger, idleTimeout time.Duration) (*icmpProxy, error) {
+	var (
+		srcSocketAddr *sockAddrIn6
+		handle        uintptr
+		err           error
+	)
+	if listenIP.Is4() {
+		handle, _, err = IcmpCreateFile_proc.Call()
+	} else {
+		srcSocketAddr, err = newSockAddrIn6(listenIP)
+		if err != nil {
+			return nil, err
+		}
+		handle, _, err = Icmp6CreateFile_proc.Call()
+	}
+	// Windows procedure calls always return non-nil error constructed from the result of GetLastError.
+	// Caller need to inspect the primary returned value
+	if syscall.Handle(handle) == syscall.InvalidHandle {
+		return nil, errors.Wrap(err, "invalid ICMP handle")
+	}
+	return &icmpProxy{
+		handle:        handle,
+		srcSocketAddr: srcSocketAddr,
+		logger:        logger,
+		encoderPool: sync.Pool{
+			New: func() any {
+				return packet.NewEncoder()
+			},
+		},
+	}, nil
+}
+
+func (ip *icmpProxy) Serve(ctx context.Context) error {
+	<-ctx.Done()
+	syscall.CloseHandle(syscall.Handle(ip.handle))
+	return ctx.Err()
+}
+
+// Request sends an ICMP echo request and wait for a reply or timeout.
+// The async version of Win32 APIs take a callback whose memory is not garbage collected, so we use the synchronous version.
+// It's possible that a slow request will block other requests, so we set the timeout to only 1s.
+func (ip *icmpProxy) Request(ctx context.Context, pk *packet.ICMP, responder *packetResponder) error {
+	defer func() {
+		if r := recover(); r != nil {
+			ip.logger.Error().Interface("error", r).Msgf("Recover panic from sending icmp request/response, error %s", debug.Stack())
+		}
+	}()
+
+	_, requestSpan := responder.requestSpan(ctx, pk)
+	defer responder.exportSpan()
+
+	echo, err := getICMPEcho(pk.Message)
+	if err != nil {
+		return err
+	}
+	requestSpan.SetAttributes(
+		attribute.Int("originalEchoID", echo.ID),
+		attribute.Int("seq", echo.Seq),
+	)
+
+	resp, err := ip.icmpEchoRoundtrip(pk.Dst, echo)
+	if err != nil {
+		ip.logger.Err(err).Msg("ICMP echo roundtrip failed")
+		tracing.EndWithErrorStatus(requestSpan, err)
+		return err
+	}
+	tracing.End(requestSpan)
+	responder.exportSpan()
+
+	_, replySpan := responder.replySpan(ctx, ip.logger)
+	replySpan.SetAttributes(
+		attribute.Int("originalEchoID", echo.ID),
+		attribute.Int("seq", echo.Seq),
+		attribute.Int64("rtt", int64(resp.rtt())),
+		attribute.String("status", resp.status().String()),
+	)
+	err = ip.handleEchoReply(pk, echo, resp, responder)
+	if err != nil {
+		tracing.EndWithErrorStatus(replySpan, err)
+		return errors.Wrap(err, "failed to handle ICMP echo reply")
+	}
+	tracing.End(replySpan)
+	return nil
+}
+
+func (ip *icmpProxy) handleEchoReply(request *packet.ICMP, echoReq *icmp.Echo, resp echoResp, responder *packetResponder) error {
+	var replyType icmp.Type
+	if request.Dst.Is4() {
+		replyType = ipv4.ICMPTypeEchoReply
+	} else {
+		replyType = ipv6.ICMPTypeEchoReply
+	}
+
+	pk := packet.ICMP{
+		IP: &packet.IP{
+			Src:      request.Dst,
+			Dst:      request.Src,
+			Protocol: layers.IPProtocol(request.Type.Protocol()),
+			TTL:      packet.DefaultTTL,
+		},
+		Message: &icmp.Message{
+			Type: replyType,
+			Code: icmpEchoReplyCode,
+			Body: &icmp.Echo{
+				ID:   echoReq.ID,
+				Seq:  echoReq.Seq,
+				Data: resp.payload(),
+			},
+		},
+	}
+
+	cachedEncoder := ip.encoderPool.Get()
+	// The encoded packet is a slice to of the encoder, so we shouldn't return the encoder back to the pool until
+	// the encoded packet is sent.
+	defer ip.encoderPool.Put(cachedEncoder)
+	encoder, ok := cachedEncoder.(*packet.Encoder)
+	if !ok {
+		return fmt.Errorf("encoderPool returned %T, expect *packet.Encoder", cachedEncoder)
+	}
+
+	serializedPacket, err := encoder.Encode(&pk)
+	if err != nil {
+		return err
+	}
+	return responder.returnPacket(serializedPacket)
+}
+
+func (ip *icmpProxy) icmpEchoRoundtrip(dst netip.Addr, echo *icmp.Echo) (echoResp, error) {
+	if dst.Is6() {
+		if ip.srcSocketAddr == nil {
+			return nil, fmt.Errorf("cannot send ICMPv6 using ICMPv4 proxy")
+		}
+		resp, err := ip.icmp6SendEcho(dst, echo)
+		if err != nil {
+
+			return nil, errors.Wrap(err, "failed to send/receive ICMPv6 echo")
+		}
+		return resp, nil
+	}
+	if ip.srcSocketAddr != nil {
+		return nil, fmt.Errorf("cannot send ICMPv4 using ICMPv6 proxy")
+	}
+	resp, err := ip.icmpSendEcho(dst, echo)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to send/receive ICMPv4 echo")
+	}
+	return resp, nil
+}
+
+/*
+	Wrapper to call https://docs.microsoft.com/en-us/windows/win32/api/icmpapi/nf-icmpapi-icmpsendecho
+	Parameters:
+	- IcmpHandle: Handle created by IcmpCreateFile
+	- DestinationAddress: IPv4 in the form of https://docs.microsoft.com/en-us/windows/win32/api/inaddr/ns-inaddr-in_addr#syntax
+	- RequestData: A pointer to echo data
+	- RequestSize: Number of bytes in buffer pointed by echo data
+	- RequestOptions: IP header options
+	- ReplyBuffer: A pointer to the buffer for echoReply, options and data
+	- ReplySize: Number of bytes allocated for ReplyBuffer
+	- Timeout: Timeout in milliseconds to wait for a reply
+	Returns:
+	- the number of replies in uint32 https://docs.microsoft.com/en-us/windows/win32/api/icmpapi/nf-icmpapi-icmpsendecho#return-value
+	To retain the reference allocated objects, conversion from pointer to uintptr must happen as arguments to the
+	syscall function
+*/
+func (ip *icmpProxy) icmpSendEcho(dst netip.Addr, echo *icmp.Echo) (*echoV4Resp, error) {
+	dataSize := len(echo.Data)
+	replySize := echoReplySize + uintptr(dataSize)
+	replyBuf := make([]byte, replySize)
+	noIPHeaderOption := nullParameter
+	inAddr, err := inAddrV4(dst)
+	if err != nil {
+		return nil, err
+	}
+	replyCount, _, err := IcmpSendEcho_proc.Call(
+		ip.handle,
+		uintptr(inAddr),
+		uintptr(unsafe.Pointer(&echo.Data[0])),
+		uintptr(dataSize),
+		noIPHeaderOption,
+		uintptr(unsafe.Pointer(&replyBuf[0])),
+		replySize,
+		icmpRequestTimeoutMs,
+	)
+	if replyCount == 0 {
+		// status is returned in 5th to 8th byte of reply buffer
+		if status, parseErr := unmarshalIPStatus(replyBuf[4:8]); parseErr == nil && status != success {
+			return nil, errors.Wrapf(err, "received ip status: %s", status)
+		}
+		return nil, errors.Wrap(err, "did not receive ICMP echo reply")
+	} else if replyCount > 1 {
+		ip.logger.Warn().Msgf("Received %d ICMP echo replies, only sending 1 back", replyCount)
+	}
+	return newEchoV4Resp(replyBuf)
+}
+
+// Third definition of https://docs.microsoft.com/en-us/windows/win32/api/inaddr/ns-inaddr-in_addr#syntax is address in uint32
+func inAddrV4(ip netip.Addr) (uint32, error) {
+	if !ip.Is4() {
+		return 0, fmt.Errorf("%s is not IPv4", ip)
+	}
+	v4 := ip.As4()
+	return endian.Uint32(v4[:]), nil
+}
+
+type echoResp interface {
+	status() ipStatus
+	rtt() uint32
+	payload() []byte
+}
+
+type echoV4Resp struct {
+	reply *echoReply
+	data  []byte
+}
+
+func (r *echoV4Resp) status() ipStatus {
+	return r.reply.Status
+}
+
+func (r *echoV4Resp) rtt() uint32 {
+	return r.reply.RoundTripTime
+}
+
+func (r *echoV4Resp) payload() []byte {
+	return r.data
+}
+
+func newEchoV4Resp(replyBuf []byte) (*echoV4Resp, error) {
+	if len(replyBuf) == 0 {
+		return nil, fmt.Errorf("reply buffer is empty")
+	}
+	// This is pattern 1 of https://pkg.go.dev/unsafe@master#Pointer, conversion of *replyBuf to *echoReply
+	// replyBuf size is larger than echoReply
+	reply := *(*echoReply)(unsafe.Pointer(&replyBuf[0]))
+	if reply.Status != success {
+		return nil, fmt.Errorf("status %d", reply.Status)
+	}
+	dataBufStart := len(replyBuf) - int(reply.DataSize)
+	if dataBufStart < int(echoReplySize) {
+		return nil, fmt.Errorf("reply buffer size %d is too small to hold data of size %d", len(replyBuf), int(reply.DataSize))
+	}
+	return &echoV4Resp{
+		reply: &reply,
+		data:  replyBuf[dataBufStart:],
+	}, nil
+}
+
+/*
+	Wrapper to call https://docs.microsoft.com/en-us/windows/win32/api/icmpapi/nf-icmpapi-icmp6sendecho2
+	Parameters:
+	- IcmpHandle: Handle created by Icmp6CreateFile
+	- Event (optional): Event object to be signaled when a reply arrives
+	- ApcRoutine (optional): Routine to call when the calling thread is in an alertable thread and a reply arrives
+	- ApcContext (optional): Optional parameter to ApcRoutine
+	- SourceAddress: Source address of the request
+	- DestinationAddress: Destination address of the request
+	- RequestData: A pointer to echo data
+	- RequestSize: Number of bytes in buffer pointed by echo data
+	- RequestOptions (optional): A pointer to the IPv6 header options
+	- ReplyBuffer: A pointer to the buffer for echoReply, options and data
+	- ReplySize: Number of bytes allocated for ReplyBuffer
+	- Timeout: Timeout in milliseconds to wait for a reply
+	Returns:
+	- the number of replies in uint32
+	To retain the reference allocated objects, conversion from pointer to uintptr must happen as arguments to the
+	syscall function
+*/
+
+func (ip *icmpProxy) icmp6SendEcho(dst netip.Addr, echo *icmp.Echo) (*echoV6Resp, error) {
+	dstAddr, err := newSockAddrIn6(dst)
+	if err != nil {
+		return nil, err
+	}
+	dataSize := len(echo.Data)
+	// Reply buffer needs to be big enough to hold an echoV6Reply, echo data, 8 bytes for ICMP error message
+	// and ioStatusBlock
+	replySize := echoV6ReplySize + uintptr(dataSize) + uintptr(icmpv6ErrMessageSize) + ioStatusBlockSize
+	replyBuf := make([]byte, replySize)
+	noEvent := nullParameter
+	noApcRoutine := nullParameter
+	noAppCtx := nullParameter
+	noIPHeaderOption := nullParameter
+	replyCount, _, err := Icmp6SendEcho_proc.Call(
+		ip.handle,
+		noEvent,
+		noApcRoutine,
+		noAppCtx,
+		uintptr(unsafe.Pointer(ip.srcSocketAddr)),
+		uintptr(unsafe.Pointer(dstAddr)),
+		uintptr(unsafe.Pointer(&echo.Data[0])),
+		uintptr(dataSize),
+		noIPHeaderOption,
+		uintptr(unsafe.Pointer(&replyBuf[0])),
+		replySize,
+		icmpRequestTimeoutMs,
+	)
+	if replyCount == 0 {
+		// status is in the 4 bytes after ipv6AddrEx. The reply buffer size is at least size of ipv6AddrEx + 4
+		if status, parseErr := unmarshalIPStatus(replyBuf[unsafe.Sizeof(ipv6AddrEx{}) : unsafe.Sizeof(ipv6AddrEx{})+4]); parseErr == nil && status != success {
+			return nil, fmt.Errorf("received ip status: %s", status)
+		}
+		return nil, errors.Wrap(err, "did not receive ICMP echo reply")
+	} else if replyCount > 1 {
+		ip.logger.Warn().Msgf("Received %d ICMP echo replies, only sending 1 back", replyCount)
+	}
+	return newEchoV6Resp(replyBuf, dataSize)
+}
+
+type echoV6Resp struct {
+	reply *echoV6Reply
+	data  []byte
+}
+
+func (r *echoV6Resp) status() ipStatus {
+	return r.reply.Status
+}
+
+func (r *echoV6Resp) rtt() uint32 {
+	return r.reply.RoundTripTime
+}
+
+func (r *echoV6Resp) payload() []byte {
+	return r.data
+}
+
+func newEchoV6Resp(replyBuf []byte, dataSize int) (*echoV6Resp, error) {
+	if len(replyBuf) == 0 {
+		return nil, fmt.Errorf("reply buffer is empty")
+	}
+	reply := *(*echoV6Reply)(unsafe.Pointer(&replyBuf[0]))
+	if reply.Status != success {
+		return nil, fmt.Errorf("status %d", reply.Status)
+	}
+	if uintptr(len(replyBuf)) < unsafe.Sizeof(reply)+uintptr(dataSize) {
+		return nil, fmt.Errorf("reply buffer size %d is too small to hold reply size %d + data size %d", len(replyBuf), echoV6ReplySize, dataSize)
+	}
+	return &echoV6Resp{
+		reply: &reply,
+		data:  replyBuf[echoV6ReplySize : echoV6ReplySize+uintptr(dataSize)],
+	}, nil
+}
+
+func unmarshalIPStatus(replyBuf []byte) (ipStatus, error) {
+	if len(replyBuf) != 4 {
+		return 0, fmt.Errorf("ipStatus needs to be 4 bytes, got %d", len(replyBuf))
+	}
+	return ipStatus(endian.Uint32(replyBuf)), nil
+}

ingress/icmp_windows_test.go

@@ -0,0 +1,234 @@
+//go:build windows
+
+package ingress
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"io"
+	"net/netip"
+	"testing"
+	"time"
+	"unsafe"
+
+	"golang.org/x/net/icmp"
+
+	"github.com/stretchr/testify/require"
+)
+
+// TestParseEchoReply tests parsing raw bytes from icmpSendEcho into echoResp
+func TestParseEchoReply(t *testing.T) {
+	dst, err := inAddrV4(netip.MustParseAddr("192.168.10.20"))
+	require.NoError(t, err)
+
+	validReplyData := []byte(t.Name())
+	validReply := echoReply{
+		Address:       dst,
+		Status:        success,
+		RoundTripTime: uint32(20),
+		DataSize:      uint16(len(validReplyData)),
+		DataPointer:   &validReplyData[0],
+		Options: ipOption{
+			TTL: 59,
+		},
+	}
+
+	destHostUnreachableReply := validReply
+	destHostUnreachableReply.Status = destHostUnreachable
+
+	tests := []struct {
+		testCase      string
+		replyBuf      []byte
+		expectedReply *echoReply
+		expectedData  []byte
+	}{
+		{
+			testCase: "empty buffer",
+		},
+		{
+			testCase: "status not success",
+			replyBuf: destHostUnreachableReply.marshal(t, []byte{}),
+		},
+		{
+			testCase:      "valid reply",
+			replyBuf:      validReply.marshal(t, validReplyData),
+			expectedReply: &validReply,
+			expectedData:  validReplyData,
+		},
+	}
+
+	for _, test := range tests {
+		resp, err := newEchoV4Resp(test.replyBuf)
+		if test.expectedReply == nil {
+			require.Error(t, err)
+			require.Nil(t, resp)
+		} else {
+			require.NoError(t, err)
+			require.Equal(t, resp.reply, test.expectedReply)
+			require.True(t, bytes.Equal(resp.data, test.expectedData))
+		}
+	}
+}
+
+// TestParseEchoV6Reply tests parsing raw bytes from icmp6SendEcho into echoV6Resp
+func TestParseEchoV6Reply(t *testing.T) {
+	dst := netip.MustParseAddr("2606:3600:4500::3333").As16()
+	var addr [8]uint16
+	for i := 0; i < 8; i++ {
+		addr[i] = binary.BigEndian.Uint16(dst[i*2 : i*2+2])
+	}
+
+	validReplyData := []byte(t.Name())
+	validReply := echoV6Reply{
+		Address: ipv6AddrEx{
+			addr: addr,
+		},
+		Status:        success,
+		RoundTripTime: 25,
+	}
+
+	destHostUnreachableReply := validReply
+	destHostUnreachableReply.Status = ipv6DestUnreachable
+
+	tests := []struct {
+		testCase      string
+		replyBuf      []byte
+		expectedReply *echoV6Reply
+		expectedData  []byte
+	}{
+		{
+			testCase: "empty buffer",
+		},
+		{
+			testCase: "status not success",
+			replyBuf: destHostUnreachableReply.marshal(t, []byte{}),
+		},
+		{
+			testCase:      "valid reply",
+			replyBuf:      validReply.marshal(t, validReplyData),
+			expectedReply: &validReply,
+			expectedData:  validReplyData,
+		},
+	}
+
+	for _, test := range tests {
+		resp, err := newEchoV6Resp(test.replyBuf, len(test.expectedData))
+		if test.expectedReply == nil {
+			require.Error(t, err)
+			require.Nil(t, resp)
+		} else {
+			require.NoError(t, err)
+			require.Equal(t, resp.reply, test.expectedReply)
+			require.True(t, bytes.Equal(resp.data, test.expectedData))
+		}
+	}
+}
+
+//  TestSendEchoErrors makes sure icmpSendEcho handles error cases
+func TestSendEchoErrors(t *testing.T) {
+	testSendEchoErrors(t, netip.IPv4Unspecified())
+	testSendEchoErrors(t, netip.IPv6Unspecified())
+}
+
+func testSendEchoErrors(t *testing.T, listenIP netip.Addr) {
+	proxy, err := newICMPProxy(listenIP, "", &noopLogger, time.Second)
+	require.NoError(t, err)
+
+	echo := icmp.Echo{
+		ID:   6193,
+		Seq:  25712,
+		Data: []byte(t.Name()),
+	}
+	documentIP := netip.MustParseAddr("192.0.2.200")
+	if listenIP.Is6() {
+		documentIP = netip.MustParseAddr("2001:db8::1")
+	}
+	resp, err := proxy.icmpEchoRoundtrip(documentIP, &echo)
+	require.Error(t, err)
+	require.Nil(t, resp)
+}
+
+func (er *echoReply) marshal(t *testing.T, data []byte) []byte {
+	buf := new(bytes.Buffer)
+
+	for _, field := range []any{
+		er.Address,
+		er.Status,
+		er.RoundTripTime,
+		er.DataSize,
+		er.Reserved,
+	} {
+		require.NoError(t, binary.Write(buf, endian, field))
+	}
+
+	require.NoError(t, marshalPointer(buf, uintptr(unsafe.Pointer(er.DataPointer))))
+
+	for _, field := range []any{
+		er.Options.TTL,
+		er.Options.Tos,
+		er.Options.Flags,
+		er.Options.OptionsSize,
+	} {
+		require.NoError(t, binary.Write(buf, endian, field))
+	}
+
+	require.NoError(t, marshalPointer(buf, er.Options.OptionsData))
+
+	padSize := buf.Len() % int(unsafe.Alignof(er))
+	padding := make([]byte, padSize)
+	n, err := buf.Write(padding)
+	require.NoError(t, err)
+	require.Equal(t, padSize, n)
+
+	n, err = buf.Write(data)
+	require.NoError(t, err)
+	require.Equal(t, len(data), n)
+
+	return buf.Bytes()
+}
+
+func marshalPointer(buf io.Writer, ptr uintptr) error {
+	size := unsafe.Sizeof(ptr)
+	switch size {
+	case 4:
+		return binary.Write(buf, endian, uint32(ptr))
+	case 8:
+		return binary.Write(buf, endian, uint64(ptr))
+	default:
+		return fmt.Errorf("unexpected pointer size %d", size)
+	}
+}
+
+func (er *echoV6Reply) marshal(t *testing.T, data []byte) []byte {
+	buf := new(bytes.Buffer)
+
+	for _, field := range []any{
+		er.Address.port,
+		er.Address.flowInfoUpper,
+		er.Address.flowInfoLower,
+		er.Address.addr,
+		er.Address.scopeID,
+	} {
+		require.NoError(t, binary.Write(buf, endian, field))
+	}
+
+	padSize := buf.Len() % int(unsafe.Alignof(er))
+	padding := make([]byte, padSize)
+	n, err := buf.Write(padding)
+	require.NoError(t, err)
+	require.Equal(t, padSize, n)
+
+	for _, field := range []any{
+		er.Status,
+		er.RoundTripTime,
+	} {
+		require.NoError(t, binary.Write(buf, endian, field))
+	}
+
+	n, err = buf.Write(data)
+	require.NoError(t, err)
+	require.Equal(t, len(data), n)
+
+	return buf.Bytes()
+}

ingress/ingress.go

@@ -11,8 +11,10 @@ import (
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
 	"github.com/urfave/cli/v2"
+	"golang.org/x/net/idna"
 
 	"github.com/cloudflare/cloudflared/config"
+	"github.com/cloudflare/cloudflared/ingress/middleware"
 	"github.com/cloudflare/cloudflared/ipaccess"
 )
 
@@ -168,6 +170,28 @@ func (ing Ingress) CatchAll() *Rule {
 	return &ing.Rules[len(ing.Rules)-1]
 }
 
+func validateAccessConfiguration(cfg *config.AccessConfig) error {
+	if !cfg.Required {
+		return nil
+	}
+
+	// It is possible to set `required:true` and not have these two configured yet.
+	// But if one of them is configured, we'd validate for correctness.
+	if len(cfg.AudTag) == 0 && cfg.TeamName == "" {
+		return nil
+	}
+
+	if len(cfg.AudTag) == 0 {
+		return errors.New("access audtag cannot be empty")
+	}
+
+	if cfg.TeamName == "" {
+		return errors.New("access.TeamName cannot be blank")
+	}
+
+	return nil
+}
+
 func validateIngress(ingress []config.UnvalidatedIngressRule, defaults OriginRequestConfig) (Ingress, error) {
 	rules := make([]Rule, len(ingress))
 	for i, r := range ingress {
@@ -182,11 +206,14 @@ func validateIngress(ingress []config.UnvalidatedIngressRule, defaults OriginReq
 			path := strings.TrimPrefix(r.Service, prefix)
 			service = &unixSocketPath{path: path, scheme: "https"}
 		} else if prefix := "http_status:"; strings.HasPrefix(r.Service, prefix) {
-			status, err := strconv.Atoi(strings.TrimPrefix(r.Service, prefix))
+			statusCode, err := strconv.Atoi(strings.TrimPrefix(r.Service, prefix))
 			if err != nil {
-				return Ingress{}, errors.Wrap(err, "invalid HTTP status")
+				return Ingress{}, errors.Wrap(err, "invalid HTTP status code")
 			}
-			srv := newStatusCode(status)
+			if statusCode < 100 || statusCode > 999 {
+				return Ingress{}, fmt.Errorf("invalid HTTP status code: %d", statusCode)
+			}
+			srv := newStatusCode(statusCode)
 			service = &srv
 		} else if r.Service == HelloWorldService || r.Service == "hello-world" || r.Service == "helloworld" {
 			service = new(helloWorld)
@@ -234,10 +261,31 @@ func validateIngress(ingress []config.UnvalidatedIngressRule, defaults OriginReq
 			}
 		}
 
+		var handlers []middleware.Handler
+		if access := r.OriginRequest.Access; access != nil {
+			if err := validateAccessConfiguration(access); err != nil {
+				return Ingress{}, err
+			}
+			if access.Required {
+				verifier := middleware.NewJWTValidator(access.TeamName, "", access.AudTag)
+				handlers = append(handlers, verifier)
+			}
+		}
+
 		if err := validateHostname(r, i, len(ingress)); err != nil {
 			return Ingress{}, err
 		}
 
+		isCatchAllRule := (r.Hostname == "" || r.Hostname == "*") && r.Path == ""
+		punycodeHostname := ""
+		if !isCatchAllRule {
+			punycode, err := idna.Lookup.ToASCII(r.Hostname)
+			// Don't provide the punycode hostname if it is the same as the original hostname
+			if err == nil && punycode != r.Hostname {
+				punycodeHostname = punycode
+			}
+		}
+
 		var pathRegexp *Regexp
 		if r.Path != "" {
 			var err error
@@ -249,10 +297,12 @@ func validateIngress(ingress []config.UnvalidatedIngressRule, defaults OriginReq
 		}
 
 		rules[i] = Rule{
-			Hostname: r.Hostname,
-			Service:  service,
-			Path:     pathRegexp,
-			Config:   cfg,
+			Hostname:         r.Hostname,
+			punycodeHostname: punycodeHostname,
+			Service:          service,
+			Path:             pathRegexp,
+			Handlers:         handlers,
+			Config:           cfg,
 		}
 	}
 	return Ingress{Rules: rules, Defaults: defaults}, nil

ingress/ingress_test.go

@@ -130,6 +130,36 @@ ingress:
 				},
 			},
 		},
+		{
+			name: "Unicode domain",
+			args: args{rawYAML: `
+ingress:
+ - hostname: môô.cloudflare.com
+   service: https://localhost:8000
+ - service: https://localhost:8001
+`},
+			want: []Rule{
+				{
+					Hostname:         "môô.cloudflare.com",
+					punycodeHostname: "xn--m-xgaa.cloudflare.com",
+					Service:          &httpService{url: localhost8000},
+					Config:           defaultConfig,
+				},
+				{
+					Service: &httpService{url: localhost8001},
+					Config:  defaultConfig,
+				},
+			},
+		},
+		{
+			name: "Invalid unicode domain",
+			args: args{rawYAML: fmt.Sprintf(`
+ingress:
+ - hostname: %s
+   service: https://localhost:8000
+`, string(rune(0xd8f3))+".cloudflare.com")},
+			wantErr: true,
+		},
 		{
 			name: "Invalid service",
 			args: args{rawYAML: `
@@ -208,6 +238,14 @@ ingress:
 			args: args{rawYAML: `
 ingress:
  - service: http_status:asdf
+`},
+			wantErr: true,
+		},
+		{
+			name: "Invalid HTTP status code",
+			args: args{rawYAML: `
+ingress:
+ - service: http_status:8080
 `},
 			wantErr: true,
 		},

ingress/middleware/jwtvalidator.go

@@ -0,0 +1,79 @@
+package middleware
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+
+	"github.com/coreos/go-oidc/v3/oidc"
+)
+
+const (
+	headerKeyAccessJWTAssertion = "Cf-Access-Jwt-Assertion"
+)
+
+var (
+	cloudflareAccessCertsURL = "https://%s.cloudflareaccess.com"
+)
+
+// JWTValidator is an implementation of Verifier that validates access based JWT tokens.
+type JWTValidator struct {
+	*oidc.IDTokenVerifier
+	audTags []string
+}
+
+func NewJWTValidator(teamName string, certsURL string, audTags []string) *JWTValidator {
+	if certsURL == "" {
+		certsURL = fmt.Sprintf(cloudflareAccessCertsURL, teamName)
+	}
+	certsEndpoint := fmt.Sprintf("%s/cdn-cgi/access/certs", certsURL)
+
+	config := &oidc.Config{
+		SkipClientIDCheck: true,
+	}
+
+	ctx := context.Background()
+	keySet := oidc.NewRemoteKeySet(ctx, certsEndpoint)
+	verifier := oidc.NewVerifier(certsURL, keySet, config)
+	return &JWTValidator{
+		IDTokenVerifier: verifier,
+		audTags:         audTags,
+	}
+}
+
+func (v *JWTValidator) Name() string {
+	return "AccessJWTValidator"
+}
+
+func (v *JWTValidator) Handle(ctx context.Context, r *http.Request) (*HandleResult, error) {
+	accessJWT := r.Header.Get(headerKeyAccessJWTAssertion)
+	if accessJWT == "" {
+		// log the exact error message here. the message is specific to the handler implementation logic, we don't gain anything
+		// in passing it upstream. and each handler impl know what logging level to use for each.
+		return &HandleResult{
+			ShouldFilterRequest: true,
+			StatusCode:          http.StatusForbidden,
+			Reason:              "no access token in request",
+		}, nil
+	}
+
+	token, err := v.IDTokenVerifier.Verify(ctx, accessJWT)
+	if err != nil {
+		return nil, err
+	}
+
+	// We want at least one audTag to match
+	for _, jwtAudTag := range token.Audience {
+		for _, acceptedAudTag := range v.audTags {
+			if acceptedAudTag == jwtAudTag {
+				return &HandleResult{ShouldFilterRequest: false}, nil
+			}
+		}
+	}
+
+	return &HandleResult{
+		ShouldFilterRequest: true,
+		StatusCode:          http.StatusForbidden,
+		Reason:              fmt.Sprintf("Invalid token in jwt: %v", token.Audience),
+	}, nil
+}

ingress/middleware/middleware.go

@@ -0,0 +1,19 @@
+package middleware
+
+import (
+	"context"
+	"net/http"
+)
+
+type HandleResult struct {
+	// Tells that the request didn't passed the handler and should be filtered
+	ShouldFilterRequest bool
+	// The status code to return in case ShouldFilterRequest is true.
+	StatusCode int
+	Reason     string
+}
+
+type Handler interface {
+	Name() string
+	Handle(ctx context.Context, r *http.Request) (result *HandleResult, err error)
+}

ingress/origin_icmp_proxy.go

@@ -0,0 +1,104 @@
+package ingress
+
+import (
+	"context"
+	"fmt"
+	"net/netip"
+	"time"
+
+	"github.com/rs/zerolog"
+	"golang.org/x/net/icmp"
+	"golang.org/x/net/ipv4"
+	"golang.org/x/net/ipv6"
+
+	"github.com/cloudflare/cloudflared/packet"
+)
+
+const (
+	// funnelIdleTimeout controls how long to wait to close a funnel without send/return
+	funnelIdleTimeout = time.Second * 10
+	mtu               = 1500
+	// icmpRequestTimeoutMs controls how long to wait for a reply
+	icmpRequestTimeoutMs = 1000
+)
+
+var (
+	errPacketNil = fmt.Errorf("packet is nil")
+)
+
+type icmpRouter struct {
+	ipv4Proxy *icmpProxy
+	ipv6Proxy *icmpProxy
+}
+
+// NewICMPRouter doesn't return an error if either ipv4 proxy or ipv6 proxy can be created. The machine might only
+// support one of them
+func NewICMPRouter(ipv4Addr, ipv6Addr netip.Addr, ipv6Zone string, logger *zerolog.Logger) (*icmpRouter, error) {
+	ipv4Proxy, ipv4Err := newICMPProxy(ipv4Addr, "", logger, funnelIdleTimeout)
+	ipv6Proxy, ipv6Err := newICMPProxy(ipv6Addr, ipv6Zone, logger, funnelIdleTimeout)
+	if ipv4Err != nil && ipv6Err != nil {
+		err := fmt.Errorf("cannot create ICMPv4 proxy: %v nor ICMPv6 proxy: %v", ipv4Err, ipv6Err)
+		logger.Debug().Err(err).Msg("ICMP proxy feature is disabled")
+		return nil, err
+	}
+	if ipv4Err != nil {
+		logger.Debug().Err(ipv4Err).Msg("failed to create ICMPv4 proxy, only ICMPv6 proxy is created")
+		ipv4Proxy = nil
+	}
+	if ipv6Err != nil {
+		logger.Debug().Err(ipv6Err).Msg("failed to create ICMPv6 proxy, only ICMPv4 proxy is created")
+		ipv6Proxy = nil
+	}
+	return &icmpRouter{
+		ipv4Proxy: ipv4Proxy,
+		ipv6Proxy: ipv6Proxy,
+	}, nil
+}
+
+func (ir *icmpRouter) Serve(ctx context.Context) error {
+	if ir.ipv4Proxy != nil && ir.ipv6Proxy != nil {
+		errC := make(chan error, 2)
+		go func() {
+			errC <- ir.ipv4Proxy.Serve(ctx)
+		}()
+		go func() {
+			errC <- ir.ipv6Proxy.Serve(ctx)
+		}()
+		return <-errC
+	}
+	if ir.ipv4Proxy != nil {
+		return ir.ipv4Proxy.Serve(ctx)
+	}
+	if ir.ipv6Proxy != nil {
+		return ir.ipv6Proxy.Serve(ctx)
+	}
+	return fmt.Errorf("ICMPv4 proxy and ICMPv6 proxy are both nil")
+}
+
+func (ir *icmpRouter) Request(ctx context.Context, pk *packet.ICMP, responder *packetResponder) error {
+	if pk == nil {
+		return errPacketNil
+	}
+	if pk.Dst.Is4() {
+		if ir.ipv4Proxy != nil {
+			return ir.ipv4Proxy.Request(ctx, pk, responder)
+		}
+		return fmt.Errorf("ICMPv4 proxy was not instantiated")
+	}
+	if ir.ipv6Proxy != nil {
+		return ir.ipv6Proxy.Request(ctx, pk, responder)
+	}
+	return fmt.Errorf("ICMPv6 proxy was not instantiated")
+}
+
+func getICMPEcho(msg *icmp.Message) (*icmp.Echo, error) {
+	echo, ok := msg.Body.(*icmp.Echo)
+	if !ok {
+		return nil, fmt.Errorf("expect ICMP echo, got %s", msg.Type)
+	}
+	return echo, nil
+}
+
+func isEchoReply(msg *icmp.Message) bool {
+	return msg.Type == ipv4.ICMPTypeEchoReply || msg.Type == ipv6.ICMPTypeEchoReply
+}

ingress/origin_icmp_proxy_test.go

@@ -0,0 +1,422 @@
+package ingress
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"net"
+	"net/netip"
+	"runtime"
+	"strings"
+	"sync"
+	"testing"
+
+	"github.com/google/gopacket/layers"
+	"github.com/rs/zerolog"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/net/icmp"
+	"golang.org/x/net/ipv4"
+	"golang.org/x/net/ipv6"
+
+	"github.com/cloudflare/cloudflared/packet"
+	quicpogs "github.com/cloudflare/cloudflared/quic"
+	"github.com/cloudflare/cloudflared/tracing"
+)
+
+var (
+	noopLogger    = zerolog.Nop()
+	localhostIP   = netip.MustParseAddr("127.0.0.1")
+	localhostIPv6 = netip.MustParseAddr("::1")
+)
+
+// TestICMPProxyEcho makes sure we can send ICMP echo via the Request method and receives response via the
+// ListenResponse method
+//
+// Note: if this test fails on your device under Linux, then most likely you need to make sure that your user
+// is allowed in ping_group_range. See the following gist for how to do that:
+// https://github.com/ValentinBELYN/icmplib/blob/main/docs/6-use-icmplib-without-privileges.md
+func TestICMPRouterEcho(t *testing.T) {
+	testICMPRouterEcho(t, true)
+	testICMPRouterEcho(t, false)
+}
+
+func testICMPRouterEcho(t *testing.T, sendIPv4 bool) {
+	const (
+		echoID = 36571
+		endSeq = 20
+	)
+
+	router, err := NewICMPRouter(localhostIP, localhostIPv6, "", &noopLogger)
+	require.NoError(t, err)
+
+	proxyDone := make(chan struct{})
+	ctx, cancel := context.WithCancel(context.Background())
+	go func() {
+		router.Serve(ctx)
+		close(proxyDone)
+	}()
+
+	muxer := newMockMuxer(1)
+	responder := packetResponder{
+		datagramMuxer: muxer,
+	}
+
+	protocol := layers.IPProtocolICMPv6
+	if sendIPv4 {
+		protocol = layers.IPProtocolICMPv4
+	}
+	localIPs := getLocalIPs(t, sendIPv4)
+	ips := make([]*packet.IP, len(localIPs))
+	for i, localIP := range localIPs {
+		ips[i] = &packet.IP{
+			Src:      localIP,
+			Dst:      localIP,
+			Protocol: protocol,
+			TTL:      packet.DefaultTTL,
+		}
+	}
+
+	var icmpType icmp.Type = ipv6.ICMPTypeEchoRequest
+	if sendIPv4 {
+		icmpType = ipv4.ICMPTypeEcho
+	}
+	for seq := 0; seq < endSeq; seq++ {
+		for i, ip := range ips {
+			pk := packet.ICMP{
+				IP: ip,
+				Message: &icmp.Message{
+					Type: icmpType,
+					Code: 0,
+					Body: &icmp.Echo{
+						ID:   echoID + i,
+						Seq:  seq,
+						Data: []byte(fmt.Sprintf("icmp echo seq %d", seq)),
+					},
+				},
+			}
+			require.NoError(t, router.Request(ctx, &pk, &responder))
+			validateEchoFlow(t, muxer, &pk)
+		}
+	}
+	cancel()
+	<-proxyDone
+}
+
+func TestTraceICMPRouterEcho(t *testing.T) {
+	tracingCtx := "ec31ad8a01fde11fdcabe2efdce36873:52726f6cabc144f5:0:1"
+
+	router, err := NewICMPRouter(localhostIP, localhostIPv6, "", &noopLogger)
+	require.NoError(t, err)
+
+	proxyDone := make(chan struct{})
+	ctx, cancel := context.WithCancel(context.Background())
+	go func() {
+		router.Serve(ctx)
+		close(proxyDone)
+	}()
+
+	// Buffer 3 packets, request span, reply span and reply
+	muxer := newMockMuxer(3)
+	tracingIdentity, err := tracing.NewIdentity(tracingCtx)
+	require.NoError(t, err)
+	serializedIdentity, err := tracingIdentity.MarshalBinary()
+	require.NoError(t, err)
+
+	responder := packetResponder{
+		datagramMuxer:      muxer,
+		tracedCtx:          tracing.NewTracedContext(ctx, tracingIdentity.String(), &noopLogger),
+		serializedIdentity: serializedIdentity,
+	}
+
+	echo := &icmp.Echo{
+		ID:   12910,
+		Seq:  182,
+		Data: []byte(t.Name()),
+	}
+	pk := packet.ICMP{
+		IP: &packet.IP{
+			Src:      localhostIP,
+			Dst:      localhostIP,
+			Protocol: layers.IPProtocolICMPv4,
+			TTL:      packet.DefaultTTL,
+		},
+		Message: &icmp.Message{
+			Type: ipv4.ICMPTypeEcho,
+			Code: 0,
+			Body: echo,
+		},
+	}
+
+	require.NoError(t, router.Request(ctx, &pk, &responder))
+	// On Windows, request span is returned before reply
+	if runtime.GOOS != "windows" {
+		validateEchoFlow(t, muxer, &pk)
+	}
+
+	// Request span
+	receivedPacket := <-muxer.cfdToEdge
+	requestSpan, ok := receivedPacket.(*quicpogs.TracingSpanPacket)
+	require.True(t, ok)
+	require.NotEmpty(t, requestSpan.Spans)
+	require.True(t, bytes.Equal(serializedIdentity, requestSpan.TracingIdentity))
+
+	if runtime.GOOS == "windows" {
+		validateEchoFlow(t, muxer, &pk)
+	}
+
+	// Reply span
+	receivedPacket = <-muxer.cfdToEdge
+	replySpan, ok := receivedPacket.(*quicpogs.TracingSpanPacket)
+	require.True(t, ok)
+	require.NotEmpty(t, replySpan.Spans)
+	require.True(t, bytes.Equal(serializedIdentity, replySpan.TracingIdentity))
+	require.False(t, bytes.Equal(requestSpan.Spans, replySpan.Spans))
+
+	echo.Seq++
+	pk.Body = echo
+	// Only first request for a flow is traced. The edge will not send tracing context for the second request
+	newResponder := packetResponder{
+		datagramMuxer: muxer,
+	}
+	require.NoError(t, router.Request(ctx, &pk, &newResponder))
+	validateEchoFlow(t, muxer, &pk)
+
+	select {
+	case receivedPacket = <-muxer.cfdToEdge:
+		panic(fmt.Sprintf("Receive unexpected packet %+v", receivedPacket))
+	default:
+	}
+
+	cancel()
+	<-proxyDone
+}
+
+// TestConcurrentRequests makes sure icmpRouter can send concurrent requests to the same destination with different
+// echo ID. This simulates concurrent ping to the same destination.
+func TestConcurrentRequestsToSameDst(t *testing.T) {
+	const (
+		concurrentPings = 5
+		endSeq          = 5
+	)
+
+	router, err := NewICMPRouter(localhostIP, localhostIPv6, "", &noopLogger)
+	require.NoError(t, err)
+
+	proxyDone := make(chan struct{})
+	ctx, cancel := context.WithCancel(context.Background())
+	go func() {
+		router.Serve(ctx)
+		close(proxyDone)
+	}()
+
+	var wg sync.WaitGroup
+	// icmpv4 and icmpv6 each has concurrentPings
+	wg.Add(concurrentPings * 2)
+	for i := 0; i < concurrentPings; i++ {
+		echoID := 38451 + i
+		go func() {
+			defer wg.Done()
+
+			muxer := newMockMuxer(1)
+			responder := packetResponder{
+				datagramMuxer: muxer,
+			}
+			for seq := 0; seq < endSeq; seq++ {
+				pk := &packet.ICMP{
+					IP: &packet.IP{
+						Src:      localhostIP,
+						Dst:      localhostIP,
+						Protocol: layers.IPProtocolICMPv4,
+						TTL:      packet.DefaultTTL,
+					},
+					Message: &icmp.Message{
+						Type: ipv4.ICMPTypeEcho,
+						Code: 0,
+						Body: &icmp.Echo{
+							ID:   echoID,
+							Seq:  seq,
+							Data: []byte(fmt.Sprintf("icmpv4 echo id %d, seq %d", echoID, seq)),
+						},
+					},
+				}
+				require.NoError(t, router.Request(ctx, pk, &responder))
+				validateEchoFlow(t, muxer, pk)
+			}
+		}()
+		go func() {
+			defer wg.Done()
+			muxer := newMockMuxer(1)
+			responder := packetResponder{
+				datagramMuxer: muxer,
+			}
+			for seq := 0; seq < endSeq; seq++ {
+				pk := &packet.ICMP{
+					IP: &packet.IP{
+						Src:      localhostIPv6,
+						Dst:      localhostIPv6,
+						Protocol: layers.IPProtocolICMPv6,
+						TTL:      packet.DefaultTTL,
+					},
+					Message: &icmp.Message{
+						Type: ipv6.ICMPTypeEchoRequest,
+						Code: 0,
+						Body: &icmp.Echo{
+							ID:   echoID,
+							Seq:  seq,
+							Data: []byte(fmt.Sprintf("icmpv6 echo id %d, seq %d", echoID, seq)),
+						},
+					},
+				}
+				require.NoError(t, router.Request(ctx, pk, &responder))
+				validateEchoFlow(t, muxer, pk)
+			}
+		}()
+	}
+	wg.Wait()
+	cancel()
+	<-proxyDone
+}
+
+// TestICMPProxyRejectNotEcho makes sure it rejects messages other than echo
+func TestICMPRouterRejectNotEcho(t *testing.T) {
+	msgs := []icmp.Message{
+		{
+			Type: ipv4.ICMPTypeDestinationUnreachable,
+			Code: 1,
+			Body: &icmp.DstUnreach{
+				Data: []byte("original packet"),
+			},
+		},
+		{
+			Type: ipv4.ICMPTypeTimeExceeded,
+			Code: 1,
+			Body: &icmp.TimeExceeded{
+				Data: []byte("original packet"),
+			},
+		},
+		{
+			Type: ipv4.ICMPType(2),
+			Code: 0,
+			Body: &icmp.PacketTooBig{
+				MTU:  1280,
+				Data: []byte("original packet"),
+			},
+		},
+	}
+	testICMPRouterRejectNotEcho(t, localhostIP, msgs)
+	msgsV6 := []icmp.Message{
+		{
+			Type: ipv6.ICMPTypeDestinationUnreachable,
+			Code: 3,
+			Body: &icmp.DstUnreach{
+				Data: []byte("original packet"),
+			},
+		},
+		{
+			Type: ipv6.ICMPTypeTimeExceeded,
+			Code: 0,
+			Body: &icmp.TimeExceeded{
+				Data: []byte("original packet"),
+			},
+		},
+		{
+			Type: ipv6.ICMPTypePacketTooBig,
+			Code: 0,
+			Body: &icmp.PacketTooBig{
+				MTU:  1280,
+				Data: []byte("original packet"),
+			},
+		},
+	}
+	testICMPRouterRejectNotEcho(t, localhostIPv6, msgsV6)
+}
+
+func testICMPRouterRejectNotEcho(t *testing.T, srcDstIP netip.Addr, msgs []icmp.Message) {
+	router, err := NewICMPRouter(localhostIP, localhostIPv6, "", &noopLogger)
+	require.NoError(t, err)
+
+	muxer := newMockMuxer(1)
+	responder := packetResponder{
+		datagramMuxer: muxer,
+	}
+	protocol := layers.IPProtocolICMPv4
+	if srcDstIP.Is6() {
+		protocol = layers.IPProtocolICMPv6
+	}
+	for _, m := range msgs {
+		pk := packet.ICMP{
+			IP: &packet.IP{
+				Src:      srcDstIP,
+				Dst:      srcDstIP,
+				Protocol: protocol,
+				TTL:      packet.DefaultTTL,
+			},
+			Message: &m,
+		}
+		require.Error(t, router.Request(context.Background(), &pk, &responder))
+	}
+}
+
+type echoFlowResponder struct {
+	lock     sync.Mutex
+	decoder  *packet.ICMPDecoder
+	respChan chan []byte
+}
+
+func (efr *echoFlowResponder) SendPacket(dst netip.Addr, pk packet.RawPacket) error {
+	efr.lock.Lock()
+	defer efr.lock.Unlock()
+	copiedPacket := make([]byte, len(pk.Data))
+	copy(copiedPacket, pk.Data)
+	efr.respChan <- copiedPacket
+	return nil
+}
+
+func (efr *echoFlowResponder) Close() error {
+	efr.lock.Lock()
+	defer efr.lock.Unlock()
+	close(efr.respChan)
+	return nil
+}
+
+func validateEchoFlow(t *testing.T, muxer *mockMuxer, echoReq *packet.ICMP) {
+	pk := <-muxer.cfdToEdge
+	decoder := packet.NewICMPDecoder()
+	decoded, err := decoder.Decode(packet.RawPacket{Data: pk.Payload()})
+	require.NoError(t, err)
+	require.Equal(t, decoded.Src, echoReq.Dst)
+	require.Equal(t, decoded.Dst, echoReq.Src)
+	require.Equal(t, echoReq.Protocol, decoded.Protocol)
+
+	if echoReq.Type == ipv4.ICMPTypeEcho {
+		require.Equal(t, ipv4.ICMPTypeEchoReply, decoded.Type)
+	} else {
+		require.Equal(t, ipv6.ICMPTypeEchoReply, decoded.Type)
+	}
+	require.Equal(t, 0, decoded.Code)
+	require.NotZero(t, decoded.Checksum)
+
+	require.Equal(t, echoReq.Body, decoded.Body)
+}
+
+func getLocalIPs(t *testing.T, ipv4 bool) []netip.Addr {
+	interfaces, err := net.Interfaces()
+	require.NoError(t, err)
+	localIPs := []netip.Addr{}
+	for _, i := range interfaces {
+		// Skip TUN devices
+		if strings.Contains(i.Name, "tun") {
+			continue
+		}
+		addrs, err := i.Addrs()
+		require.NoError(t, err)
+		for _, addr := range addrs {
+			if ipnet, ok := addr.(*net.IPNet); ok && (ipnet.IP.IsPrivate() || ipnet.IP.IsLoopback()) {
+				if (ipv4 && ipnet.IP.To4() != nil) || (!ipv4 && ipnet.IP.To4() == nil) {
+					localIPs = append(localIPs, netip.MustParseAddr(ipnet.IP.String()))
+				}
+			}
+		}
+	}
+	return localIPs
+}

ingress/packet_router.go

@@ -0,0 +1,186 @@
+package ingress
+
+import (
+	"context"
+	"fmt"
+	"net/netip"
+
+	"github.com/rs/zerolog"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
+
+	"github.com/cloudflare/cloudflared/packet"
+	quicpogs "github.com/cloudflare/cloudflared/quic"
+	"github.com/cloudflare/cloudflared/tracing"
+)
+
+// Upstream of raw packets
+type muxer interface {
+	SendPacket(pk quicpogs.Packet) error
+	// ReceivePacket waits for the next raw packet from upstream
+	ReceivePacket(ctx context.Context) (quicpogs.Packet, error)
+}
+
+// PacketRouter routes packets between Upstream and ICMPRouter. Currently it rejects all other type of ICMP packets
+type PacketRouter struct {
+	globalConfig           *GlobalRouterConfig
+	muxer                  muxer
+	logger                 *zerolog.Logger
+	checkRouterEnabledFunc func() bool
+	icmpDecoder            *packet.ICMPDecoder
+	encoder                *packet.Encoder
+}
+
+// GlobalRouterConfig is the configuration shared by all instance of Router.
+type GlobalRouterConfig struct {
+	ICMPRouter *icmpRouter
+	IPv4Src    netip.Addr
+	IPv6Src    netip.Addr
+	Zone       string
+}
+
+// NewPacketRouter creates a PacketRouter that handles ICMP packets. Packets are read from muxer but dropped if globalConfig is nil.
+func NewPacketRouter(globalConfig *GlobalRouterConfig, muxer muxer, logger *zerolog.Logger, checkRouterEnabledFunc func() bool) *PacketRouter {
+	return &PacketRouter{
+		globalConfig:           globalConfig,
+		muxer:                  muxer,
+		logger:                 logger,
+		checkRouterEnabledFunc: checkRouterEnabledFunc,
+		icmpDecoder:            packet.NewICMPDecoder(),
+		encoder:                packet.NewEncoder(),
+	}
+}
+
+func (r *PacketRouter) Serve(ctx context.Context) error {
+	for {
+		rawPacket, responder, err := r.nextPacket(ctx)
+		if err != nil {
+			return err
+		}
+		r.handlePacket(ctx, rawPacket, responder)
+	}
+}
+
+func (r *PacketRouter) nextPacket(ctx context.Context) (packet.RawPacket, *packetResponder, error) {
+	pk, err := r.muxer.ReceivePacket(ctx)
+	if err != nil {
+		return packet.RawPacket{}, nil, err
+	}
+	responder := &packetResponder{
+		datagramMuxer: r.muxer,
+	}
+	switch pk.Type() {
+	case quicpogs.DatagramTypeIP:
+		return packet.RawPacket{Data: pk.Payload()}, responder, nil
+	case quicpogs.DatagramTypeIPWithTrace:
+		var identity tracing.Identity
+		if err := identity.UnmarshalBinary(pk.Metadata()); err != nil {
+			r.logger.Err(err).Bytes("tracingIdentity", pk.Metadata()).Msg("Failed to unmarshal tracing identity")
+		} else {
+			responder.tracedCtx = tracing.NewTracedContext(ctx, identity.String(), r.logger)
+			responder.serializedIdentity = pk.Metadata()
+		}
+		return packet.RawPacket{Data: pk.Payload()}, responder, nil
+	default:
+		return packet.RawPacket{}, nil, fmt.Errorf("unexpected datagram type %d", pk.Type())
+	}
+}
+
+func (r *PacketRouter) handlePacket(ctx context.Context, rawPacket packet.RawPacket, responder *packetResponder) {
+	// ICMP Proxy feature is disabled, drop packets
+	if r.globalConfig == nil {
+		return
+	}
+
+	if enabled := r.checkRouterEnabledFunc(); !enabled {
+		return
+	}
+
+	icmpPacket, err := r.icmpDecoder.Decode(rawPacket)
+	if err != nil {
+		r.logger.Err(err).Msg("Failed to decode ICMP packet from quic datagram")
+		return
+	}
+
+	if icmpPacket.TTL <= 1 {
+		if err := r.sendTTLExceedMsg(ctx, icmpPacket, rawPacket, r.encoder); err != nil {
+			r.logger.Err(err).Msg("Failed to return ICMP TTL exceed error")
+		}
+		return
+	}
+	icmpPacket.TTL--
+
+	if err := r.globalConfig.ICMPRouter.Request(ctx, icmpPacket, responder); err != nil {
+		r.logger.Err(err).
+			Str("src", icmpPacket.Src.String()).
+			Str("dst", icmpPacket.Dst.String()).
+			Interface("type", icmpPacket.Type).
+			Msg("Failed to send ICMP packet")
+	}
+}
+
+func (r *PacketRouter) sendTTLExceedMsg(ctx context.Context, pk *packet.ICMP, rawPacket packet.RawPacket, encoder *packet.Encoder) error {
+	var srcIP netip.Addr
+	if pk.Dst.Is4() {
+		srcIP = r.globalConfig.IPv4Src
+	} else {
+		srcIP = r.globalConfig.IPv6Src
+	}
+	ttlExceedPacket := packet.NewICMPTTLExceedPacket(pk.IP, rawPacket, srcIP)
+
+	encodedTTLExceed, err := encoder.Encode(ttlExceedPacket)
+	if err != nil {
+		return err
+	}
+	return r.muxer.SendPacket(quicpogs.RawPacket(encodedTTLExceed))
+}
+
+// packetResponder should not be used concurrently. This assumption is upheld because reply packets are ready one-by-one
+type packetResponder struct {
+	datagramMuxer      muxer
+	tracedCtx          *tracing.TracedContext
+	serializedIdentity []byte
+	// hadReply tracks if there has been any reply for this flow
+	hadReply bool
+}
+
+func (pr *packetResponder) tracingEnabled() bool {
+	return pr.tracedCtx != nil
+}
+
+func (pr *packetResponder) returnPacket(rawPacket packet.RawPacket) error {
+	pr.hadReply = true
+	return pr.datagramMuxer.SendPacket(quicpogs.RawPacket(rawPacket))
+}
+
+func (pr *packetResponder) requestSpan(ctx context.Context, pk *packet.ICMP) (context.Context, trace.Span) {
+	if !pr.tracingEnabled() {
+		return ctx, tracing.NewNoopSpan()
+	}
+	return pr.tracedCtx.Tracer().Start(pr.tracedCtx, "icmp-echo-request", trace.WithAttributes(
+		attribute.String("src", pk.Src.String()),
+		attribute.String("dst", pk.Dst.String()),
+	))
+}
+
+func (pr *packetResponder) replySpan(ctx context.Context, logger *zerolog.Logger) (context.Context, trace.Span) {
+	if !pr.tracingEnabled() || pr.hadReply {
+		return ctx, tracing.NewNoopSpan()
+	}
+	return pr.tracedCtx.Tracer().Start(pr.tracedCtx, "icmp-echo-reply")
+}
+
+func (pr *packetResponder) exportSpan() {
+	if !pr.tracingEnabled() {
+		return
+	}
+	spans := pr.tracedCtx.GetProtoSpans()
+	if len(spans) > 0 {
+		// Make sure spans are cleared after they are sent
+		defer pr.tracedCtx.ClearSpans()
+		pr.datagramMuxer.SendPacket(&quicpogs.TracingSpanPacket{
+			Spans:           spans,
+			TracingIdentity: pr.serializedIdentity,
+		})
+	}
+}

ingress/packet_router_test.go

@@ -0,0 +1,241 @@
+package ingress
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"net/netip"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/google/gopacket/layers"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/net/icmp"
+	"golang.org/x/net/ipv4"
+	"golang.org/x/net/ipv6"
+
+	"github.com/cloudflare/cloudflared/packet"
+	quicpogs "github.com/cloudflare/cloudflared/quic"
+)
+
+var (
+	packetConfig = &GlobalRouterConfig{
+		ICMPRouter: nil,
+		IPv4Src:    netip.MustParseAddr("172.16.0.1"),
+		IPv6Src:    netip.MustParseAddr("fd51:2391:523:f4ee::1"),
+	}
+)
+
+func TestRouterReturnTTLExceed(t *testing.T) {
+	muxer := newMockMuxer(0)
+	routerEnabled := &routerEnabledChecker{}
+	routerEnabled.set(true)
+	router := NewPacketRouter(packetConfig, muxer, &noopLogger, routerEnabled.isEnabled)
+	ctx, cancel := context.WithCancel(context.Background())
+	routerStopped := make(chan struct{})
+	go func() {
+		router.Serve(ctx)
+		close(routerStopped)
+	}()
+
+	pk := packet.ICMP{
+		IP: &packet.IP{
+			Src:      netip.MustParseAddr("192.168.1.1"),
+			Dst:      netip.MustParseAddr("10.0.0.1"),
+			Protocol: layers.IPProtocolICMPv4,
+			TTL:      1,
+		},
+		Message: &icmp.Message{
+			Type: ipv4.ICMPTypeEcho,
+			Code: 0,
+			Body: &icmp.Echo{
+				ID:   12481,
+				Seq:  8036,
+				Data: []byte("TTL exceed"),
+			},
+		},
+	}
+	assertTTLExceed(t, &pk, router.globalConfig.IPv4Src, muxer)
+	pk = packet.ICMP{
+		IP: &packet.IP{
+			Src:      netip.MustParseAddr("fd51:2391:523:f4ee::1"),
+			Dst:      netip.MustParseAddr("fd51:2391:697:f4ee::2"),
+			Protocol: layers.IPProtocolICMPv6,
+			TTL:      1,
+		},
+		Message: &icmp.Message{
+			Type: ipv6.ICMPTypeEchoRequest,
+			Code: 0,
+			Body: &icmp.Echo{
+				ID:   42583,
+				Seq:  7039,
+				Data: []byte("TTL exceed"),
+			},
+		},
+	}
+	assertTTLExceed(t, &pk, router.globalConfig.IPv6Src, muxer)
+
+	cancel()
+	<-routerStopped
+}
+
+func TestRouterCheckEnabled(t *testing.T) {
+	muxer := newMockMuxer(0)
+	routerEnabled := &routerEnabledChecker{}
+	router := NewPacketRouter(packetConfig, muxer, &noopLogger, routerEnabled.isEnabled)
+	ctx, cancel := context.WithCancel(context.Background())
+	routerStopped := make(chan struct{})
+	go func() {
+		router.Serve(ctx)
+		close(routerStopped)
+	}()
+
+	pk := packet.ICMP{
+		IP: &packet.IP{
+			Src:      netip.MustParseAddr("192.168.1.1"),
+			Dst:      netip.MustParseAddr("10.0.0.1"),
+			Protocol: layers.IPProtocolICMPv4,
+			TTL:      1,
+		},
+		Message: &icmp.Message{
+			Type: ipv4.ICMPTypeEcho,
+			Code: 0,
+			Body: &icmp.Echo{
+				ID:   12481,
+				Seq:  8036,
+				Data: []byte(t.Name()),
+			},
+		},
+	}
+
+	// router is disabled
+	encoder := packet.NewEncoder()
+	encodedPacket, err := encoder.Encode(&pk)
+	require.NoError(t, err)
+	sendPacket := quicpogs.RawPacket(encodedPacket)
+
+	muxer.edgeToCfd <- sendPacket
+	select {
+	case <-time.After(time.Millisecond * 10):
+	case <-muxer.cfdToEdge:
+		t.Error("Unexpected reply when router is disabled")
+	}
+	routerEnabled.set(true)
+	// router is enabled, expects reply
+	muxer.edgeToCfd <- sendPacket
+	<-muxer.cfdToEdge
+
+	routerEnabled.set(false)
+	// router is disabled
+	muxer.edgeToCfd <- sendPacket
+	select {
+	case <-time.After(time.Millisecond * 10):
+	case <-muxer.cfdToEdge:
+		t.Error("Unexpected reply when router is disabled")
+	}
+
+	cancel()
+	<-routerStopped
+}
+
+func assertTTLExceed(t *testing.T, originalPacket *packet.ICMP, expectedSrc netip.Addr, muxer *mockMuxer) {
+	encoder := packet.NewEncoder()
+	rawPacket, err := encoder.Encode(originalPacket)
+	require.NoError(t, err)
+	muxer.edgeToCfd <- quicpogs.RawPacket(rawPacket)
+
+	resp := <-muxer.cfdToEdge
+	decoder := packet.NewICMPDecoder()
+	decoded, err := decoder.Decode(packet.RawPacket(resp.(quicpogs.RawPacket)))
+	require.NoError(t, err)
+
+	require.Equal(t, expectedSrc, decoded.Src)
+	require.Equal(t, originalPacket.Src, decoded.Dst)
+	require.Equal(t, originalPacket.Protocol, decoded.Protocol)
+	require.Equal(t, packet.DefaultTTL, decoded.TTL)
+	if originalPacket.Dst.Is4() {
+		require.Equal(t, ipv4.ICMPTypeTimeExceeded, decoded.Type)
+	} else {
+		require.Equal(t, ipv6.ICMPTypeTimeExceeded, decoded.Type)
+	}
+	require.Equal(t, 0, decoded.Code)
+	timeExceed, ok := decoded.Body.(*icmp.TimeExceeded)
+	require.True(t, ok)
+	require.True(t, bytes.Equal(rawPacket.Data, timeExceed.Data))
+}
+
+type mockMuxer struct {
+	cfdToEdge chan quicpogs.Packet
+	edgeToCfd chan quicpogs.Packet
+}
+
+func newMockMuxer(capacity int) *mockMuxer {
+	return &mockMuxer{
+		cfdToEdge: make(chan quicpogs.Packet, capacity),
+		edgeToCfd: make(chan quicpogs.Packet, capacity),
+	}
+}
+
+// Copy packet, because icmpProxy expects the encoder buffer to be reusable after the packet is sent
+func (mm *mockMuxer) SendPacket(pk quicpogs.Packet) error {
+	payload := pk.Payload()
+	copiedPayload := make([]byte, len(payload))
+	copy(copiedPayload, payload)
+
+	metadata := pk.Metadata()
+	copiedMetadata := make([]byte, len(metadata))
+	copy(copiedMetadata, metadata)
+
+	var copiedPacket quicpogs.Packet
+	switch pk.Type() {
+	case quicpogs.DatagramTypeIP:
+		copiedPacket = quicpogs.RawPacket(packet.RawPacket{
+			Data: copiedPayload,
+		})
+	case quicpogs.DatagramTypeIPWithTrace:
+		copiedPacket = &quicpogs.TracedPacket{
+			Packet: packet.RawPacket{
+				Data: copiedPayload,
+			},
+			TracingIdentity: copiedMetadata,
+		}
+	case quicpogs.DatagramTypeTracingSpan:
+		copiedPacket = &quicpogs.TracingSpanPacket{
+			Spans:           copiedPayload,
+			TracingIdentity: copiedMetadata,
+		}
+	default:
+		return fmt.Errorf("unexpected metadata type %d", pk.Type())
+	}
+	mm.cfdToEdge <- copiedPacket
+	return nil
+}
+
+func (mm *mockMuxer) ReceivePacket(ctx context.Context) (quicpogs.Packet, error) {
+	select {
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	case pk := <-mm.edgeToCfd:
+		return pk, nil
+	}
+}
+
+type routerEnabledChecker struct {
+	enabled uint32
+}
+
+func (rec *routerEnabledChecker) isEnabled() bool {
+	if atomic.LoadUint32(&rec.enabled) == 0 {
+		return false
+	}
+	return true
+}
+
+func (rec *routerEnabledChecker) set(enabled bool) {
+	if enabled {
+		atomic.StoreUint32(&rec.enabled, 1)
+	} else {
+		atomic.StoreUint32(&rec.enabled, 0)
+	}
+}

ingress/rule.go

@@ -4,6 +4,8 @@ import (
 	"encoding/json"
 	"regexp"
 	"strings"
+
+	"github.com/cloudflare/cloudflared/ingress/middleware"
 )
 
 // Rule routes traffic from a hostname/path on the public internet to the
@@ -12,6 +14,9 @@ type Rule struct {
 	// Requests for this hostname will be proxied to this rule's service.
 	Hostname string `json:"hostname"`
 
+	// punycodeHostname is an additional optional hostname converted to punycode.
+	punycodeHostname string
+
 	// Path is an optional regex that can specify path-driven ingress rules.
 	Path *Regexp `json:"path"`
 
@@ -20,6 +25,9 @@ type Rule struct {
 	// address.
 	Service OriginService `json:"service"`
 
+	// Handlers is a list of functions that acts as a middleware during ProxyHTTP
+	Handlers []middleware.Handler
+
 	// Configure the request cloudflared sends to this specific origin.
 	Config OriginRequestConfig `json:"originRequest"`
 }
@@ -45,9 +53,18 @@ func (r Rule) MultiLineString() string {
 
 // Matches checks if the rule matches a given hostname/path combination.
 func (r *Rule) Matches(hostname, path string) bool {
-	hostMatch := r.Hostname == "" || r.Hostname == "*" || matchHost(r.Hostname, hostname)
+	hostMatch := false
+	if r.Hostname == "" || r.Hostname == "*" {
+		hostMatch = true
+	} else {
+		hostMatch = matchHost(r.Hostname, hostname)
+	}
+	punycodeHostMatch := false
+	if r.punycodeHostname != "" {
+		punycodeHostMatch = matchHost(r.punycodeHostname, hostname)
+	}
 	pathMatch := r.Path == nil || r.Path.Regexp == nil || r.Path.Regexp.MatchString(path)
-	return hostMatch && pathMatch
+	return (hostMatch || punycodeHostMatch) && pathMatch
 }
 
 // Regexp adds unmarshalling from json for regexp.Regexp

ingress/rule_test.go

@@ -14,23 +14,18 @@ import (
 )
 
 func Test_rule_matches(t *testing.T) {
-	type fields struct {
-		Hostname string
-		Path     *Regexp
-		Service  OriginService
-	}
 	type args struct {
 		requestURL *url.URL
 	}
 	tests := []struct {
-		name   string
-		fields fields
-		args   args
-		want   bool
+		name string
+		rule Rule
+		args args
+		want bool
 	}{
 		{
 			name: "Just hostname, pass",
-			fields: fields{
+			rule: Rule{
 				Hostname: "example.com",
 			},
 			args: args{
@@ -38,9 +33,31 @@ func Test_rule_matches(t *testing.T) {
 			},
 			want: true,
 		},
+		{
+			name: "Unicode hostname with unicode request, pass",
+			rule: Rule{
+				Hostname:         "môô.cloudflare.com",
+				punycodeHostname: "xn--m-xgaa.cloudflare.com",
+			},
+			args: args{
+				requestURL: MustParseURL(t, "https://môô.cloudflare.com"),
+			},
+			want: true,
+		},
+		{
+			name: "Unicode hostname with punycode request, pass",
+			rule: Rule{
+				Hostname:         "môô.cloudflare.com",
+				punycodeHostname: "xn--m-xgaa.cloudflare.com",
+			},
+			args: args{
+				requestURL: MustParseURL(t, "https://xn--m-xgaa.cloudflare.com"),
+			},
+			want: true,
+		},
 		{
 			name: "Entire hostname is wildcard, should match everything",
-			fields: fields{
+			rule: Rule{
 				Hostname: "*",
 			},
 			args: args{
@@ -50,7 +67,7 @@ func Test_rule_matches(t *testing.T) {
 		},
 		{
 			name: "Just hostname, fail",
-			fields: fields{
+			rule: Rule{
 				Hostname: "example.com",
 			},
 			args: args{
@@ -60,7 +77,7 @@ func Test_rule_matches(t *testing.T) {
 		},
 		{
 			name: "Just wildcard hostname, pass",
-			fields: fields{
+			rule: Rule{
 				Hostname: "*.example.com",
 			},
 			args: args{
@@ -70,7 +87,7 @@ func Test_rule_matches(t *testing.T) {
 		},
 		{
 			name: "Just wildcard hostname, fail",
-			fields: fields{
+			rule: Rule{
 				Hostname: "*.example.com",
 			},
 			args: args{
@@ -80,7 +97,7 @@ func Test_rule_matches(t *testing.T) {
 		},
 		{
 			name: "Just wildcard outside of subdomain in hostname, fail",
-			fields: fields{
+			rule: Rule{
 				Hostname: "*example.com",
 			},
 			args: args{
@@ -90,7 +107,7 @@ func Test_rule_matches(t *testing.T) {
 		},
 		{
 			name: "Wildcard over multiple subdomains",
-			fields: fields{
+			rule: Rule{
 				Hostname: "*.example.com",
 			},
 			args: args{
@@ -100,7 +117,7 @@ func Test_rule_matches(t *testing.T) {
 		},
 		{
 			name: "Hostname and path",
-			fields: fields{
+			rule: Rule{
 				Hostname: "*.example.com",
 				Path:     &Regexp{Regexp: regexp.MustCompile("/static/.*\\.html")},
 			},
@@ -111,7 +128,7 @@ func Test_rule_matches(t *testing.T) {
 		},
 		{
 			name: "Hostname and empty Regex",
-			fields: fields{
+			rule: Rule{
 				Hostname: "example.com",
 				Path:     &Regexp{},
 			},
@@ -122,7 +139,7 @@ func Test_rule_matches(t *testing.T) {
 		},
 		{
 			name: "Hostname and nil path",
-			fields: fields{
+			rule: Rule{
 				Hostname: "example.com",
 				Path:     nil,
 			},
@@ -134,13 +151,8 @@ func Test_rule_matches(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			r := Rule{
-				Hostname: tt.fields.Hostname,
-				Path:     tt.fields.Path,
-				Service:  tt.fields.Service,
-			}
 			u := tt.args.requestURL
-			if got := r.Matches(u.Hostname(), u.Path); got != tt.want {
+			if got := tt.rule.Matches(u.Hostname(), u.Path); got != tt.want {
 				t.Errorf("rule.matches() = %v, want %v", got, tt.want)
 			}
 		})
@@ -182,25 +194,25 @@ func TestMarshalJSON(t *testing.T) {
 		{
 			name:     "Nil",
 			path:     nil,
-			expected: `{"hostname":"example.com","path":null,"service":"https://localhost:8000","originRequest":{"connectTimeout":30,"tlsTimeout":10,"tcpKeepAlive":30,"noHappyEyeballs":false,"keepAliveTimeout":90,"keepAliveConnections":100,"httpHostHeader":"","originServerName":"","caPool":"","noTLSVerify":false,"disableChunkedEncoding":false,"bastionMode":false,"proxyAddress":"127.0.0.1","proxyPort":0,"proxyType":"","ipRules":null,"http2Origin":false}}`,
+			expected: `{"hostname":"example.com","path":null,"service":"https://localhost:8000","Handlers":null,"originRequest":{"connectTimeout":30,"tlsTimeout":10,"tcpKeepAlive":30,"noHappyEyeballs":false,"keepAliveTimeout":90,"keepAliveConnections":100,"httpHostHeader":"","originServerName":"","caPool":"","noTLSVerify":false,"disableChunkedEncoding":false,"bastionMode":false,"proxyAddress":"127.0.0.1","proxyPort":0,"proxyType":"","ipRules":null,"http2Origin":false,"access":{"teamName":"","audTag":null}}}`,
 			want:     true,
 		},
 		{
 			name:     "Nil regex",
 			path:     &Regexp{Regexp: nil},
-			expected: `{"hostname":"example.com","path":null,"service":"https://localhost:8000","originRequest":{"connectTimeout":30,"tlsTimeout":10,"tcpKeepAlive":30,"noHappyEyeballs":false,"keepAliveTimeout":90,"keepAliveConnections":100,"httpHostHeader":"","originServerName":"","caPool":"","noTLSVerify":false,"disableChunkedEncoding":false,"bastionMode":false,"proxyAddress":"127.0.0.1","proxyPort":0,"proxyType":"","ipRules":null,"http2Origin":false}}`,
+			expected: `{"hostname":"example.com","path":null,"service":"https://localhost:8000","Handlers":null,"originRequest":{"connectTimeout":30,"tlsTimeout":10,"tcpKeepAlive":30,"noHappyEyeballs":false,"keepAliveTimeout":90,"keepAliveConnections":100,"httpHostHeader":"","originServerName":"","caPool":"","noTLSVerify":false,"disableChunkedEncoding":false,"bastionMode":false,"proxyAddress":"127.0.0.1","proxyPort":0,"proxyType":"","ipRules":null,"http2Origin":false,"access":{"teamName":"","audTag":null}}}`,
 			want:     true,
 		},
 		{
 			name:     "Empty",
 			path:     &Regexp{Regexp: regexp.MustCompile("")},
-			expected: `{"hostname":"example.com","path":"","service":"https://localhost:8000","originRequest":{"connectTimeout":30,"tlsTimeout":10,"tcpKeepAlive":30,"noHappyEyeballs":false,"keepAliveTimeout":90,"keepAliveConnections":100,"httpHostHeader":"","originServerName":"","caPool":"","noTLSVerify":false,"disableChunkedEncoding":false,"bastionMode":false,"proxyAddress":"127.0.0.1","proxyPort":0,"proxyType":"","ipRules":null,"http2Origin":false}}`,
+			expected: `{"hostname":"example.com","path":"","service":"https://localhost:8000","Handlers":null,"originRequest":{"connectTimeout":30,"tlsTimeout":10,"tcpKeepAlive":30,"noHappyEyeballs":false,"keepAliveTimeout":90,"keepAliveConnections":100,"httpHostHeader":"","originServerName":"","caPool":"","noTLSVerify":false,"disableChunkedEncoding":false,"bastionMode":false,"proxyAddress":"127.0.0.1","proxyPort":0,"proxyType":"","ipRules":null,"http2Origin":false,"access":{"teamName":"","audTag":null}}}`,
 			want:     true,
 		},
 		{
 			name:     "Basic",
 			path:     &Regexp{Regexp: regexp.MustCompile("/echo")},
-			expected: `{"hostname":"example.com","path":"/echo","service":"https://localhost:8000","originRequest":{"connectTimeout":30,"tlsTimeout":10,"tcpKeepAlive":30,"noHappyEyeballs":false,"keepAliveTimeout":90,"keepAliveConnections":100,"httpHostHeader":"","originServerName":"","caPool":"","noTLSVerify":false,"disableChunkedEncoding":false,"bastionMode":false,"proxyAddress":"127.0.0.1","proxyPort":0,"proxyType":"","ipRules":null,"http2Origin":false}}`,
+			expected: `{"hostname":"example.com","path":"/echo","service":"https://localhost:8000","Handlers":null,"originRequest":{"connectTimeout":30,"tlsTimeout":10,"tcpKeepAlive":30,"noHappyEyeballs":false,"keepAliveTimeout":90,"keepAliveConnections":100,"httpHostHeader":"","originServerName":"","caPool":"","noTLSVerify":false,"disableChunkedEncoding":false,"bastionMode":false,"proxyAddress":"127.0.0.1","proxyPort":0,"proxyType":"","ipRules":null,"http2Origin":false,"access":{"teamName":"","audTag":null}}}`,
 			want:     true,
 		},
 	}

metrics/readiness_test.go

@@ -14,7 +14,7 @@ import (
 
 func TestReadyServer_makeResponse(t *testing.T) {
 	type fields struct {
-		isConnected map[int]bool
+		isConnected map[uint8]tunnelstate.ConnectionInfo
 	}
 	tests := []struct {
 		name                 string
@@ -25,11 +25,11 @@ func TestReadyServer_makeResponse(t *testing.T) {
 		{
 			name: "One connection online => HTTP 200",
 			fields: fields{
-				isConnected: map[int]bool{
-					0: false,
-					1: false,
-					2: true,
-					3: false,
+				isConnected: map[uint8]tunnelstate.ConnectionInfo{
+					0: {IsConnected: false},
+					1: {IsConnected: false},
+					2: {IsConnected: true},
+					3: {IsConnected: false},
 				},
 			},
 			wantOK:               true,
@@ -38,11 +38,11 @@ func TestReadyServer_makeResponse(t *testing.T) {
 		{
 			name: "No connections online => no HTTP 200",
 			fields: fields{
-				isConnected: map[int]bool{
-					0: false,
-					1: false,
-					2: false,
-					3: false,
+				isConnected: map[uint8]tunnelstate.ConnectionInfo{
+					0: {IsConnected: false},
+					1: {IsConnected: false},
+					2: {IsConnected: false},
+					3: {IsConnected: false},
 				},
 			},
 			wantReadyConnections: 0,

orchestration/orchestrator.go

@@ -27,10 +27,12 @@ type Orchestrator struct {
 	// Used by UpdateConfig to make sure one update at a time
 	lock sync.RWMutex
 	// Underlying value is proxy.Proxy, can be read without the lock, but still needs the lock to update
-	proxy  atomic.Value
-	config *Config
-	tags   []tunnelpogs.Tag
-	log    *zerolog.Logger
+	proxy atomic.Value
+	// TODO: TUN-6815 Use atomic.Bool once we upgrade to go 1.19. 1 Means enabled and 0 means disabled
+	warpRoutingEnabled uint32
+	config             *Config
+	tags               []tunnelpogs.Tag
+	log                *zerolog.Logger
 
 	// orchestrator must not handle any more updates after shutdownC is closed
 	shutdownC <-chan struct{}
@@ -122,6 +124,11 @@ func (o *Orchestrator) updateIngress(ingressRules ingress.Ingress, warpRouting i
 	o.proxy.Store(newProxy)
 	o.config.Ingress = &ingressRules
 	o.config.WarpRouting = warpRouting
+	if warpRouting.Enabled {
+		atomic.StoreUint32(&o.warpRoutingEnabled, 1)
+	} else {
+		atomic.StoreUint32(&o.warpRoutingEnabled, 0)
+	}
 
 	// If proxyShutdownC is nil, there is no previous running proxy
 	if o.proxyShutdownC != nil {
@@ -190,6 +197,14 @@ func (o *Orchestrator) GetOriginProxy() (connection.OriginProxy, error) {
 	return proxy, nil
 }
 
+// TODO: TUN-6815 consider storing WarpRouting.Enabled as atomic.Bool once we upgrade to go 1.19
+func (o *Orchestrator) WarpRoutingEnabled() (enabled bool) {
+	if atomic.LoadUint32(&o.warpRoutingEnabled) == 0 {
+		return false
+	}
+	return true
+}
+
 func (o *Orchestrator) waitToCloseLastProxy() {
 	<-o.shutdownC
 	o.lock.Lock()

orchestration/orchestrator_test.go

@@ -55,6 +55,7 @@ func TestUpdateConfiguration(t *testing.T) {
 	initOriginProxy, err := orchestrator.GetOriginProxy()
 	require.NoError(t, err)
 	require.IsType(t, &proxy.Proxy{}, initOriginProxy)
+	require.False(t, orchestrator.WarpRoutingEnabled())
 
 	configJSONV2 := []byte(`
 {
@@ -122,6 +123,7 @@ func TestUpdateConfiguration(t *testing.T) {
 	require.Equal(t, false, configV2.Ingress.Rules[2].Config.NoTLSVerify)
 	require.Equal(t, true, configV2.Ingress.Rules[2].Config.NoHappyEyeballs)
 	require.True(t, configV2.WarpRouting.Enabled)
+	require.Equal(t, configV2.WarpRouting.Enabled, orchestrator.WarpRoutingEnabled())
 	require.Equal(t, configV2.WarpRouting.ConnectTimeout.Duration, 10*time.Second)
 
 	originProxyV2, err := orchestrator.GetOriginProxy()
@@ -166,6 +168,7 @@ func TestUpdateConfiguration(t *testing.T) {
 	require.True(t, configV10.Ingress.Rules[0].Matches("blogs.tunnel.io", "/2022/02/10"))
 	require.Equal(t, ingress.HelloWorldService, configV10.Ingress.Rules[0].Service.String())
 	require.False(t, configV10.WarpRouting.Enabled)
+	require.Equal(t, configV10.WarpRouting.Enabled, orchestrator.WarpRoutingEnabled())
 
 	originProxyV10, err := orchestrator.GetOriginProxy()
 	require.NoError(t, err)
@@ -355,7 +358,7 @@ func proxyHTTP(originProxy connection.OriginProxy, hostname string) (*http.Respo
 		return nil, err
 	}
 
-	err = originProxy.ProxyHTTP(respWriter, tracing.NewTracedRequest(req), false)
+	err = originProxy.ProxyHTTP(respWriter, tracing.NewTracedHTTPRequest(req, &log), false)
 	if err != nil {
 		return nil, err
 	}
@@ -500,7 +503,8 @@ func TestClosePreviousProxies(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, http.StatusTeapot, resp.StatusCode)
 
-	// The hello-world server in config v1 should have been stopped
+	// The hello-world server in config v1 should have been stopped. We wait a bit since it's closed asynchronously.
+	time.Sleep(time.Millisecond * 10)
 	resp, err = proxyHTTP(originProxyV1, hostname)
 	require.Error(t, err)
 	require.Nil(t, resp)
@@ -604,7 +608,7 @@ func TestPersistentConnection(t *testing.T) {
 		respWriter, err := connection.NewHTTP2RespWriter(req, wsRespReadWriter, connection.TypeWebsocket, &log)
 		require.NoError(t, err)
 
-		err = originProxy.ProxyHTTP(respWriter, tracing.NewTracedRequest(req), true)
+		err = originProxy.ProxyHTTP(respWriter, tracing.NewTracedHTTPRequest(req, &log), true)
 		require.NoError(t, err)
 	}()
 

packet/decoder.go

@@ -0,0 +1,184 @@
+package packet
+
+import (
+	"fmt"
+
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/layers"
+	"github.com/pkg/errors"
+	"golang.org/x/net/icmp"
+)
+
+func FindProtocol(p []byte) (layers.IPProtocol, error) {
+	version, err := FindIPVersion(p)
+	if err != nil {
+		return 0, err
+	}
+	switch version {
+	case 4:
+		if len(p) < ipv4MinHeaderLen {
+			return 0, fmt.Errorf("IPv4 packet should have at least %d bytes, got %d bytes", ipv4MinHeaderLen, len(p))
+		}
+		// Protocol is in the 10th byte of IPv4 header
+		return layers.IPProtocol(p[9]), nil
+	case 6:
+		if len(p) < ipv6HeaderLen {
+			return 0, fmt.Errorf("IPv6 packet should have at least %d bytes, got %d bytes", ipv6HeaderLen, len(p))
+		}
+		// Next header is in the 7th byte of IPv6 header
+		return layers.IPProtocol(p[6]), nil
+	default:
+		return 0, fmt.Errorf("unknow ip version %d", version)
+	}
+}
+
+func FindIPVersion(p []byte) (uint8, error) {
+	if len(p) == 0 {
+		return 0, fmt.Errorf("packet length is 0")
+	}
+	return p[0] >> 4, nil
+}
+
+// IPDecoder decodes raw packets into IP. It can process packets sequentially without allocating
+// memory for the layers, so it cannot be called concurrently.
+type IPDecoder struct {
+	ipv4   *layers.IPv4
+	ipv6   *layers.IPv6
+	layers uint8
+
+	v4parser *gopacket.DecodingLayerParser
+	v6parser *gopacket.DecodingLayerParser
+}
+
+func NewIPDecoder() *IPDecoder {
+	var (
+		ipv4 layers.IPv4
+		ipv6 layers.IPv6
+	)
+	dlpv4 := gopacket.NewDecodingLayerParser(layers.LayerTypeIPv4)
+	dlpv4.SetDecodingLayerContainer(gopacket.DecodingLayerSparse(nil))
+	dlpv4.AddDecodingLayer(&ipv4)
+	// Stop parsing when it encounter a layer that it doesn't have a parser
+	dlpv4.IgnoreUnsupported = true
+
+	dlpv6 := gopacket.NewDecodingLayerParser(layers.LayerTypeIPv6)
+	dlpv6.SetDecodingLayerContainer(gopacket.DecodingLayerSparse(nil))
+	dlpv6.AddDecodingLayer(&ipv6)
+	dlpv6.IgnoreUnsupported = true
+
+	return &IPDecoder{
+		ipv4:     &ipv4,
+		ipv6:     &ipv6,
+		layers:   1,
+		v4parser: dlpv4,
+		v6parser: dlpv6,
+	}
+}
+
+func (pd *IPDecoder) Decode(packet RawPacket) (*IP, error) {
+	// Should decode to IP layer
+	decoded, err := pd.decodeByVersion(packet.Data)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, layerType := range decoded {
+		switch layerType {
+		case layers.LayerTypeIPv4:
+			return newIPv4(pd.ipv4)
+		case layers.LayerTypeIPv6:
+			return newIPv6(pd.ipv6)
+		}
+	}
+	return nil, fmt.Errorf("no ip layer is decoded")
+}
+
+func (pd *IPDecoder) decodeByVersion(packet []byte) ([]gopacket.LayerType, error) {
+	version, err := FindIPVersion(packet)
+	if err != nil {
+		return nil, err
+	}
+	decoded := make([]gopacket.LayerType, 0, pd.layers)
+	switch version {
+	case 4:
+		err = pd.v4parser.DecodeLayers(packet, &decoded)
+	case 6:
+		err = pd.v6parser.DecodeLayers(packet, &decoded)
+	default:
+		err = fmt.Errorf("unknow ip version %d", version)
+	}
+	if err != nil {
+		return nil, err
+	}
+	return decoded, nil
+}
+
+// ICMPDecoder decodes raw packets into IP and ICMP. It can process packets sequentially without allocating
+// memory for the layers, so it cannot be called concurrently.
+type ICMPDecoder struct {
+	*IPDecoder
+	icmpv4 *layers.ICMPv4
+	icmpv6 *layers.ICMPv6
+}
+
+func NewICMPDecoder() *ICMPDecoder {
+	ipDecoder := NewIPDecoder()
+
+	var (
+		icmpv4 layers.ICMPv4
+		icmpv6 layers.ICMPv6
+	)
+	ipDecoder.layers++
+	ipDecoder.v4parser.AddDecodingLayer(&icmpv4)
+	ipDecoder.v6parser.AddDecodingLayer(&icmpv6)
+
+	return &ICMPDecoder{
+		IPDecoder: ipDecoder,
+		icmpv4:    &icmpv4,
+		icmpv6:    &icmpv6,
+	}
+}
+
+func (pd *ICMPDecoder) Decode(packet RawPacket) (*ICMP, error) {
+	// Should decode to IP and optionally ICMP layer
+	decoded, err := pd.decodeByVersion(packet.Data)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, layerType := range decoded {
+		switch layerType {
+		case layers.LayerTypeICMPv4:
+			ipv4, err := newIPv4(pd.ipv4)
+			if err != nil {
+				return nil, err
+			}
+			msg, err := icmp.ParseMessage(int(layers.IPProtocolICMPv4), append(pd.icmpv4.Contents, pd.icmpv4.Payload...))
+			if err != nil {
+				return nil, errors.Wrap(err, "failed to parse ICMPv4 message")
+			}
+			return &ICMP{
+				IP:      ipv4,
+				Message: msg,
+			}, nil
+		case layers.LayerTypeICMPv6:
+			ipv6, err := newIPv6(pd.ipv6)
+			if err != nil {
+				return nil, err
+			}
+			msg, err := icmp.ParseMessage(int(layers.IPProtocolICMPv6), append(pd.icmpv6.Contents, pd.icmpv6.Payload...))
+			if err != nil {
+				return nil, errors.Wrap(err, "failed to parse ICMPv6")
+			}
+			return &ICMP{
+				IP:      ipv6,
+				Message: msg,
+			}, nil
+		}
+	}
+	layers := make([]string, len(decoded))
+	for i, l := range decoded {
+		layers[i] = l.String()
+	}
+	return nil, fmt.Errorf("Expect to decode IP and ICMP layers, got %s", layers)
+}

packet/decoder_test.go

@@ -0,0 +1,256 @@
+package packet
+
+import (
+	"net"
+	"net/netip"
+	"testing"
+
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/layers"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/net/icmp"
+	"golang.org/x/net/ipv4"
+	"golang.org/x/net/ipv6"
+)
+
+func TestDecodeIP(t *testing.T) {
+	ipDecoder := NewIPDecoder()
+	icmpDecoder := NewICMPDecoder()
+	udps := []UDP{
+		{
+			IP: IP{
+				Src:      netip.MustParseAddr("172.16.0.1"),
+				Dst:      netip.MustParseAddr("10.0.0.1"),
+				Protocol: layers.IPProtocolUDP,
+			},
+			SrcPort: 31678,
+			DstPort: 53,
+		},
+		{
+
+			IP: IP{
+				Src:      netip.MustParseAddr("fd51:2391:523:f4ee::1"),
+				Dst:      netip.MustParseAddr("fd51:2391:697:f4ee::2"),
+				Protocol: layers.IPProtocolUDP,
+			},
+			SrcPort: 52139,
+			DstPort: 1053,
+		},
+	}
+
+	encoder := NewEncoder()
+	for _, udp := range udps {
+		p, err := encoder.Encode(&udp)
+		require.NoError(t, err)
+
+		ipPacket, err := ipDecoder.Decode(p)
+		require.NoError(t, err)
+		assertIPLayer(t, &udp.IP, ipPacket)
+
+		icmpPacket, err := icmpDecoder.Decode(p)
+		require.Error(t, err)
+		require.Nil(t, icmpPacket)
+	}
+}
+
+func TestDecodeICMP(t *testing.T) {
+	ipDecoder := NewIPDecoder()
+	icmpDecoder := NewICMPDecoder()
+	var (
+		ipv4Packet = IP{
+			Src:      netip.MustParseAddr("172.16.0.1"),
+			Dst:      netip.MustParseAddr("10.0.0.1"),
+			Protocol: layers.IPProtocolICMPv4,
+			TTL:      DefaultTTL,
+		}
+		ipv6Packet = IP{
+			Src:      netip.MustParseAddr("fd51:2391:523:f4ee::1"),
+			Dst:      netip.MustParseAddr("fd51:2391:697:f4ee::2"),
+			Protocol: layers.IPProtocolICMPv6,
+			TTL:      DefaultTTL,
+		}
+		icmpID  = 100
+		icmpSeq = 52819
+	)
+	tests := []struct {
+		testCase string
+		packet   *ICMP
+	}{
+		{
+			testCase: "icmpv4 time exceed",
+			packet: &ICMP{
+				IP: &ipv4Packet,
+				Message: &icmp.Message{
+					Type: ipv4.ICMPTypeTimeExceeded,
+					Code: 0,
+					Body: &icmp.TimeExceeded{
+						Data: []byte("original packet"),
+					},
+				},
+			},
+		},
+		{
+			testCase: "icmpv4 echo",
+			packet: &ICMP{
+				IP: &ipv4Packet,
+				Message: &icmp.Message{
+					Type: ipv4.ICMPTypeEcho,
+					Code: 0,
+					Body: &icmp.Echo{
+						ID:   icmpID,
+						Seq:  icmpSeq,
+						Data: []byte("icmpv4 echo"),
+					},
+				},
+			},
+		},
+		{
+			testCase: "icmpv6 destination unreachable",
+			packet: &ICMP{
+				IP: &ipv6Packet,
+				Message: &icmp.Message{
+					Type: ipv6.ICMPTypeDestinationUnreachable,
+					Code: 4,
+					Body: &icmp.DstUnreach{
+						Data: []byte("original packet"),
+					},
+				},
+			},
+		},
+		{
+			testCase: "icmpv6 echo",
+			packet: &ICMP{
+				IP: &ipv6Packet,
+				Message: &icmp.Message{
+					Type: ipv6.ICMPTypeEchoRequest,
+					Code: 0,
+					Body: &icmp.Echo{
+						ID:   icmpID,
+						Seq:  icmpSeq,
+						Data: []byte("icmpv6 echo"),
+					},
+				},
+			},
+		},
+	}
+
+	encoder := NewEncoder()
+	for _, test := range tests {
+		p, err := encoder.Encode(test.packet)
+		require.NoError(t, err)
+
+		ipPacket, err := ipDecoder.Decode(p)
+		require.NoError(t, err)
+		if ipPacket.Src.Is4() {
+			assertIPLayer(t, &ipv4Packet, ipPacket)
+		} else {
+			assertIPLayer(t, &ipv6Packet, ipPacket)
+		}
+		icmpPacket, err := icmpDecoder.Decode(p)
+		require.NoError(t, err)
+		require.Equal(t, ipPacket, icmpPacket.IP)
+
+		require.Equal(t, test.packet.Type, icmpPacket.Type)
+		require.Equal(t, test.packet.Code, icmpPacket.Code)
+		assertICMPChecksum(t, icmpPacket)
+		require.Equal(t, test.packet.Body, icmpPacket.Body)
+		expectedBody, err := test.packet.Body.Marshal(test.packet.Type.Protocol())
+		require.NoError(t, err)
+		decodedBody, err := icmpPacket.Body.Marshal(test.packet.Type.Protocol())
+		require.NoError(t, err)
+		require.Equal(t, expectedBody, decodedBody)
+	}
+}
+
+// TestDecodeBadPackets makes sure decoders don't decode invalid packets
+func TestDecodeBadPackets(t *testing.T) {
+	var (
+		srcIPv4 = net.ParseIP("172.16.0.1")
+		dstIPv4 = net.ParseIP("10.0.0.1")
+	)
+
+	ipLayer := layers.IPv4{
+		Version:  10,
+		SrcIP:    srcIPv4,
+		DstIP:    dstIPv4,
+		Protocol: layers.IPProtocolICMPv4,
+		TTL:      DefaultTTL,
+	}
+	icmpLayer := layers.ICMPv4{
+		TypeCode: layers.CreateICMPv4TypeCode(uint8(ipv4.ICMPTypeEcho), 0),
+		Id:       100,
+		Seq:      52819,
+	}
+	wrongIPVersion, err := createPacket(&ipLayer, &icmpLayer, nil, nil)
+	require.NoError(t, err)
+
+	tests := []struct {
+		testCase string
+		packet   []byte
+	}{
+		{
+			testCase: "unknown IP version",
+			packet:   wrongIPVersion,
+		},
+		{
+			testCase: "invalid packet",
+			packet:   []byte("not a packet"),
+		},
+		{
+			testCase: "zero length packet",
+			packet:   []byte{},
+		},
+	}
+
+	ipDecoder := NewIPDecoder()
+	icmpDecoder := NewICMPDecoder()
+	for _, test := range tests {
+		ipPacket, err := ipDecoder.Decode(RawPacket{Data: test.packet})
+		require.Error(t, err)
+		require.Nil(t, ipPacket)
+
+		icmpPacket, err := icmpDecoder.Decode(RawPacket{Data: test.packet})
+		require.Error(t, err)
+		require.Nil(t, icmpPacket)
+	}
+}
+
+func createPacket(ipLayer, secondLayer, thirdLayer gopacket.SerializableLayer, body []byte) ([]byte, error) {
+	payload := gopacket.Payload(body)
+	packet := gopacket.NewSerializeBuffer()
+	var err error
+	if thirdLayer != nil {
+		err = gopacket.SerializeLayers(packet, serializeOpts, ipLayer, secondLayer, thirdLayer, payload)
+	} else {
+		err = gopacket.SerializeLayers(packet, serializeOpts, ipLayer, secondLayer, payload)
+	}
+	if err != nil {
+		return nil, err
+	}
+	return packet.Bytes(), nil
+}
+
+func assertIPLayer(t *testing.T, expected, actual *IP) {
+	require.Equal(t, expected.Src, actual.Src)
+	require.Equal(t, expected.Dst, actual.Dst)
+	require.Equal(t, expected.Protocol, actual.Protocol)
+	require.Equal(t, expected.TTL, actual.TTL)
+}
+
+type UDP struct {
+	IP
+	SrcPort, DstPort layers.UDPPort
+}
+
+func (u *UDP) EncodeLayers() ([]gopacket.SerializableLayer, error) {
+	ipLayers, err := u.IP.EncodeLayers()
+	if err != nil {
+		return nil, err
+	}
+	udpLayer := layers.UDP{
+		SrcPort: u.SrcPort,
+		DstPort: u.DstPort,
+	}
+	udpLayer.SetNetworkLayerForChecksum(ipLayers[0].(gopacket.NetworkLayer))
+	return append(ipLayers, &udpLayer), nil
+}

packet/encoder.go

@@ -0,0 +1,39 @@
+package packet
+
+import "github.com/google/gopacket"
+
+var (
+	serializeOpts = gopacket.SerializeOptions{
+		FixLengths:       true,
+		ComputeChecksums: true,
+	}
+)
+
+// RawPacket represents a raw packet or one encoded by Encoder
+type RawPacket struct {
+	Data []byte
+}
+
+type Encoder struct {
+	// buf is reusable because SerializeLayers calls the Clear method before each encoding
+	buf gopacket.SerializeBuffer
+}
+
+func NewEncoder() *Encoder {
+	return &Encoder{
+		buf: gopacket.NewSerializeBuffer(),
+	}
+}
+
+func (e Encoder) Encode(packet Packet) (RawPacket, error) {
+	encodedLayers, err := packet.EncodeLayers()
+	if err != nil {
+		return RawPacket{}, err
+	}
+	if err := gopacket.SerializeLayers(e.buf, serializeOpts, encodedLayers...); err != nil {
+		return RawPacket{}, err
+	}
+	return RawPacket{
+		Data: e.buf.Bytes(),
+	}, nil
+}