Release 2024.9.0

Whenever cloudflared receives a SIGTERM or SIGINT it goes into graceful shutdown mode, which unregisters the connection and closes the control stream. Unregistering makes it so we no longer receive any new requests and makes the edge close the connection, allowing in-flight requests to finish (within a 3 minute period).
 This was working fine for http2 connections, but the quic proxy was cancelling the context as soon as the controls stream ended, forcing the process to stop immediately.

 This commit changes the behavior so that we wait the full grace period before cancelling the request

.teamcity/mac/build.sh

@@ -7,13 +7,17 @@ if [[ "$(uname)" != "Darwin" ]] ; then
     exit 1
 fi
 
+if [[ "amd64" != "${TARGET_ARCH}" && "arm64" != "${TARGET_ARCH}" ]]
+then
+  echo "TARGET_ARCH must be amd64 or arm64"
+  exit 1
+fi
+
 go version
 export GO111MODULE=on
 
 # build 'cloudflared-darwin-amd64.tgz'
 mkdir -p artifacts
-FILENAME="$(pwd)/artifacts/cloudflared-darwin-amd64.tgz"
-PKGNAME="$(pwd)/artifacts/cloudflared-amd64.pkg"
 TARGET_DIRECTORY=".build"
 BINARY_NAME="cloudflared"
 VERSION=$(git describe --tags --always --dirty="-dev")
@@ -25,10 +29,11 @@ INSTALLER_CERT="installer.cer"
 BUNDLE_ID="com.cloudflare.cloudflared"
 SEC_DUP_MSG="security: SecKeychainItemImport: The specified item already exists in the keychain."
 export PATH="$PATH:/usr/local/bin"
+FILENAME="$(pwd)/artifacts/cloudflared-darwin-$TARGET_ARCH.tgz"
+PKGNAME="$(pwd)/artifacts/cloudflared-$TARGET_ARCH.pkg"
 mkdir -p ../src/github.com/cloudflare/    
 cp -r . ../src/github.com/cloudflare/cloudflared
 cd ../src/github.com/cloudflare/cloudflared 
-GOCACHE="$PWD/../../../../" GOPATH="$PWD/../../../../" CGO_ENABLED=1 make cloudflared
 
 # Add code signing private key to the key chain
 if [[ ! -z "$CFD_CODE_SIGN_KEY" ]]; then
@@ -138,22 +143,28 @@ else
   fi
 fi
 
+# cleanup the build directory because the previous execution might have failed without cleaning up.
+rm -rf "${TARGET_DIRECTORY}"
+export TARGET_OS="darwin"
+GOCACHE="$PWD/../../../../" GOPATH="$PWD/../../../../" CGO_ENABLED=1 make cloudflared
+
 # sign the cloudflared binary
 if [[ ! -z "$CODE_SIGN_NAME" ]]; then
   codesign -s "${CODE_SIGN_NAME}" -f -v --timestamp --options runtime ${BINARY_NAME}
-  
+
  # notarize the binary
  # TODO: TUN-5789
 fi
 
+ARCH_TARGET_DIRECTORY="${TARGET_DIRECTORY}/${TARGET_ARCH}-build"
 # creating build directory
-rm -rf $TARGET_DIRECTORY
-mkdir "${TARGET_DIRECTORY}"
-mkdir "${TARGET_DIRECTORY}/contents"
-cp -r ".mac_resources/scripts" "${TARGET_DIRECTORY}/scripts"
+rm -rf $ARCH_TARGET_DIRECTORY
+mkdir -p "${ARCH_TARGET_DIRECTORY}"
+mkdir -p "${ARCH_TARGET_DIRECTORY}/contents"
+cp -r ".mac_resources/scripts" "${ARCH_TARGET_DIRECTORY}/scripts"
 
 # copy cloudflared into the build directory
-cp ${BINARY_NAME} "${TARGET_DIRECTORY}/contents/${PRODUCT}"
+cp ${BINARY_NAME} "${ARCH_TARGET_DIRECTORY}/contents/${PRODUCT}"
 
 # compress cloudflared into a tar and gzipped file
 tar czf "$FILENAME" "${BINARY_NAME}"
@@ -162,8 +173,8 @@ tar czf "$FILENAME" "${BINARY_NAME}"
 if [[ ! -z "$PKG_SIGN_NAME" ]]; then
   pkgbuild --identifier com.cloudflare.${PRODUCT} \
       --version ${VERSION} \
-      --scripts ${TARGET_DIRECTORY}/scripts \
-      --root ${TARGET_DIRECTORY}/contents \
+      --scripts ${ARCH_TARGET_DIRECTORY}/scripts \
+      --root ${ARCH_TARGET_DIRECTORY}/contents \
       --install-location /usr/local/bin \
       --sign "${PKG_SIGN_NAME}" \
       ${PKGNAME}
@@ -173,12 +184,12 @@ if [[ ! -z "$PKG_SIGN_NAME" ]]; then
 else
     pkgbuild --identifier com.cloudflare.${PRODUCT} \
       --version ${VERSION} \
-      --scripts ${TARGET_DIRECTORY}/scripts \
-      --root ${TARGET_DIRECTORY}/contents \
+      --scripts ${ARCH_TARGET_DIRECTORY}/scripts \
+      --root ${ARCH_TARGET_DIRECTORY}/contents \
       --install-location /usr/local/bin \
       ${PKGNAME}
 fi
 
-
-# cleaning up the build directory
-rm -rf $TARGET_DIRECTORY
+# cleanup build directory because this script is not ran within containers,
+# which might lead to future issues in subsequent runs.
+rm -rf "${TARGET_DIRECTORY}"

.teamcity/package-windows.sh

@@ -3,15 +3,17 @@ echo $VERSION
 
 export TARGET_OS=windows
 # This controls the directory the built artifacts go into
-export ARTIFACT_DIR=built_artifacts/
-mkdir -p $ARTIFACT_DIR
+export BUILT_ARTIFACT_DIR=built_artifacts/
+export FINAL_ARTIFACT_DIR=artifacts/
+mkdir -p $BUILT_ARTIFACT_DIR
+mkdir -p $FINAL_ARTIFACT_DIR
 windowsArchs=("amd64" "386")
 for arch in ${windowsArchs[@]}; do
     export TARGET_ARCH=$arch
     # Copy exe into final directory
-    cp ./artifacts/cloudflared-windows-$arch.exe $ARTIFACT_DIR/cloudflared-windows-$arch.exe
-    cp ./artifacts/cloudflared-windows-$arch.exe ./cloudflared.exe
+    cp $BUILT_ARTIFACT_DIR/cloudflared-windows-$arch.exe ./cloudflared.exe
     make cloudflared-msi
     # Copy msi into final directory
-    mv cloudflared-$VERSION-$arch.msi $ARTIFACT_DIR/cloudflared-windows-$arch.msi
-done
+    mv cloudflared-$VERSION-$arch.msi $FINAL_ARTIFACT_DIR/cloudflared-windows-$arch.msi
+    cp $BUILT_ARTIFACT_DIR/cloudflared-windows-$arch.exe $FINAL_ARTIFACT_DIR/cloudflared-windows-$arch.exe 
+done

.teamcity/windows/component-test.ps1

@@ -5,41 +5,6 @@ $ProgressPreference = "SilentlyContinue"
 $WorkingDirectory = Get-Location
 $CloudflaredDirectory = "$WorkingDirectory\go\src\github.com\cloudflare\cloudflared"
 
-Write-Output "Installing python..."
-
-$PythonVersion = "3.10.11"
-$PythonZipFile = "$env:Temp\python-$PythonVersion-embed-amd64.zip"
-$PipInstallFile = "$env:Temp\get-pip.py"
-$PythonZipUrl = "https://www.python.org/ftp/python/$PythonVersion/python-$PythonVersion-embed-amd64.zip"
-$PythonPath = "$WorkingDirectory\Python"
-$PythonBinPath = "$PythonPath\python.exe"
-
-# Download Python zip file
-Invoke-WebRequest -Uri $PythonZipUrl -OutFile $PythonZipFile
-
-# Download Python pip file
-Invoke-WebRequest -Uri "https://bootstrap.pypa.io/get-pip.py" -OutFile $PipInstallFile
-
-# Extract Python files
-Expand-Archive $PythonZipFile -DestinationPath $PythonPath -Force
-
-# Add Python to PATH
-$env:Path = "$PythonPath\Scripts;$PythonPath;$($env:Path)"
-
-Write-Output "Installed to $PythonPath"
-
-# Install pip
-& $PythonBinPath $PipInstallFile
-
-# Add package paths in pythonXX._pth to unblock python -m pip
-$PythonImportPathFile = "$PythonPath\python310._pth"
-$ComponentTestsDir = "$CloudflaredDirectory\component-tests\"
-@($ComponentTestsDir, "Lib\site-packages", $(Get-Content $PythonImportPathFile)) | Set-Content $PythonImportPathFile
-
-# Test Python installation
-& $PythonBinPath --version
-& $PythonBinPath -m pip --version
-
 go env
 go version
 
@@ -48,8 +13,8 @@ $env:CGO_ENABLED = 1
 $env:TARGET_ARCH = "amd64"
 $env:Path = "$Env:Temp\go\bin;$($env:Path)"
 
-& $PythonBinPath --version
-& $PythonBinPath -m pip --version
+python --version
+python -m pip --version
 
 cd $CloudflaredDirectory
 
@@ -72,11 +37,11 @@ if ($LASTEXITCODE -ne 0) { throw "Failed unit tests" }
 
 Write-Output "Running component tests"
 
-& $PythonBinPath -m pip install --upgrade -r component-tests/requirements.txt
-& $PythonBinPath component-tests/setup.py --type create
-& $PythonBinPath -m pytest component-tests -o log_cli=true --log-cli-level=INFO
+python -m pip --disable-pip-version-check install --upgrade -r component-tests/requirements.txt
+python component-tests/setup.py --type create
+python -m pytest component-tests -o log_cli=true --log-cli-level=INFO
 if ($LASTEXITCODE -ne 0) {
-    & $PythonBinPath component-tests/setup.py --type cleanup
+    python component-tests/setup.py --type cleanup
     throw "Failed component tests"
 }
-& $PythonBinPath component-tests/setup.py --type cleanup
+python component-tests/setup.py --type cleanup

CHANGES.md

@@ -1,3 +1,7 @@
+## 2024.9.0
+### Bug Fixes
+- We fixed a bug related to `--grace-period`. Tunnels that use QUIC as transport weren't abiding by this waiting period before forcefully closing the connections to the edge. From now on, both QUIC and HTTP2 tunnels will wait for either the grace period to end (defaults to 30 seconds) or until the last in-flight request is handled. Users that wish to maintain the previous behavior should set `--grace-period` to 0 if `--protocol` is set to `quic`. This will force `cloudflared` to shutdown as soon as either SIGTERM or SIGINT is received.
+
 ## 2024.2.1
 ### Notices
 - Starting from this version, tunnel diagnostics will be enabled by default. This will allow the engineering team to remotely get diagnostics from cloudflared during debug activities. Users still have the capability to opt-out of this feature by defining `--management-diagnostics=false` (or env `TUNNEL_MANAGEMENT_DIAGNOSTICS`).

Makefile

@@ -218,50 +218,24 @@ cloudflared-pkg: cloudflared cloudflared.1
 cloudflared-msi:
 	wixl --define Version=$(VERSION) --define Path=$(EXECUTABLE_PATH) --output cloudflared-$(VERSION)-$(TARGET_ARCH).msi cloudflared.wxs
 
-.PHONY: cloudflared-darwin-amd64.tgz
-cloudflared-darwin-amd64.tgz: cloudflared
-	tar czf cloudflared-darwin-amd64.tgz cloudflared
-	rm cloudflared
+.PHONY: github-release-dryrun
+github-release-dryrun:
+	python3 github_release.py --path $(PWD)/built_artifacts --release-version $(VERSION) --dry-run
 
 .PHONY: github-release
-github-release: cloudflared
-	python3 github_release.py --path $(EXECUTABLE_PATH) --release-version $(VERSION)
-
-.PHONY: github-release-built-pkgs
-github-release-built-pkgs:
+github-release:
 	python3 github_release.py --path $(PWD)/built_artifacts --release-version $(VERSION)
-
-.PHONY: release-pkgs-linux
-release-pkgs-linux:
-	python3 ./release_pkgs.py
-
-.PHONY: github-message
-github-message:
 	python3 github_message.py --release-version $(VERSION)
 
-.PHONY: github-mac-upload
-github-mac-upload:
-	python3 github_release.py --path artifacts/cloudflared-darwin-amd64.tgz --release-version $(VERSION) --name cloudflared-darwin-amd64.tgz
-	python3 github_release.py --path artifacts/cloudflared-amd64.pkg --release-version $(VERSION) --name cloudflared-amd64.pkg
+.PHONY: r2-linux-release
+r2-linux-release:
+	python3 ./release_pkgs.py
 
-.PHONY: github-windows-upload
-github-windows-upload:
-	python3 github_release.py --path built_artifacts/cloudflared-windows-amd64.exe --release-version $(VERSION) --name cloudflared-windows-amd64.exe
-	python3 github_release.py --path built_artifacts/cloudflared-windows-amd64.msi --release-version $(VERSION) --name cloudflared-windows-amd64.msi
-	python3 github_release.py --path built_artifacts/cloudflared-windows-386.exe --release-version $(VERSION) --name cloudflared-windows-386.exe
-	python3 github_release.py --path built_artifacts/cloudflared-windows-386.msi --release-version $(VERSION) --name cloudflared-windows-386.msi
-
-.PHONY: tunnelrpc-deps
-tunnelrpc-deps:
+.PHONY: capnp
+capnp:
 	which capnp  # https://capnproto.org/install.html
 	which capnpc-go  # go install zombiezen.com/go/capnproto2/capnpc-go@latest
-	capnp compile -ogo tunnelrpc/tunnelrpc.capnp
-
-.PHONY: quic-deps
-quic-deps:
-	which capnp
-	which capnpc-go
-	capnp compile -ogo quic/schema/quic_metadata_protocol.capnp
+	capnp compile -ogo tunnelrpc/proto/tunnelrpc.capnp tunnelrpc/proto/quic_metadata_protocol.capnp
 
 .PHONY: vet
 vet:
@@ -269,4 +243,4 @@ vet:
 
 .PHONY: fmt
 fmt:
-	goimports -l -w -local github.com/cloudflare/cloudflared $$(go list -mod=vendor -f '{{.Dir}}' -a ./... | fgrep -v tunnelrpc)
+	goimports -l -w -local github.com/cloudflare/cloudflared $$(go list -mod=vendor -f '{{.Dir}}' -a ./... | fgrep -v tunnelrpc/proto)

RELEASE_NOTES

@@ -1,3 +1,53 @@
+2024.9.0
+- 2024-09-10 TUN-8621: Fix cloudflared version in change notes.
+- 2024-09-06 PPIP-2310: Update quick tunnel disclaimer
+- 2024-08-30 TUN-8621: Prevent QUIC connection from closing before grace period after unregistering
+- 2024-08-09 TUN-8592: Use metadata from the edge to determine if request body is empty for QUIC transport
+- 2024-06-26 TUN-8484: Print response when QuickTunnel can't be unmarshalled
+
+2024.8.3
+- 2024-08-15 TUN-8591 login command without extra text
+- 2024-03-25 remove code that will not be executed
+- 2024-03-25 remove code that will not be executed
+
+2024.8.2
+- 2024-08-05 TUN-8583: change final directory of artifacts
+- 2024-08-05 TUN-8585: Avoid creating GH client when dry-run is true
+
+2024.7.3
+- 2024-07-31 TUN-8546: Fix final artifacts paths
+
+2024.7.2
+- 2024-07-17 TUN-8546: rework MacOS build script
+
+2024.7.1
+- 2024-07-16 TUN-8543: use -p flag to create intermediate directories
+
+2024.7.0
+- 2024-07-05 TUN-8520: add macos arm64 build
+- 2024-07-05 TUN-8523: refactor makefile and cfsetup
+- 2024-07-02 TUN-8504: Use pre-installed python version instead of downloading it on Windows builds
+- 2024-06-26 TUN-8489: Add default noop logger for capnprpc
+- 2024-06-25 TUN-8487: Add user-agent for quick-tunnel requests
+- 2023-12-12 TUN-8057: cloudflared uses new PQ curve ID
+
+2024.6.1
+- 2024-06-12 TUN-8461: Don't log Failed to send session payload if the error is EOF
+- 2024-06-07 TUN-8456: Update quic-go to 0.45 and collect mtu and congestion control metrics
+- 2024-06-06 TUN-8452: Add flag to control QUIC stream-level flow control limit
+- 2024-06-06 TUN-8451: Log QUIC flow control frames and transport parameters received
+- 2024-06-05 TUN-8449: Add flag to control QUIC connection-level flow control limit and increase default to 30MB
+
+2024.6.0
+- 2024-05-30 TUN-8441: Correct UDP total sessions metric to a counter and add new ICMP metrics
+- 2024-05-28 TUN-8422: Add metrics for capnp method calls
+- 2024-05-24 TUN-8424: Refactor capnp registration server
+- 2024-05-23 TUN-8427: Fix BackoffHandler's internally shared clock structure
+- 2024-05-21 TUN-8425: Remove ICMP binding for quick tunnels
+- 2024-05-20 TUN-8423: Deprecate older legacy tunnel capnp interfaces
+- 2024-05-15 TUN-8419: Add capnp safe transport
+- 2024-05-13 TUN-8415: Refactor capnp rpc into a single module
+
 2024.5.0
 - 2024-05-07 TUN-8407: Upgrade go to version 1.22.2
 

build-packages-fips.sh

@@ -3,7 +3,7 @@ VERSION=$(git describe --tags --always --match "[0-9][0-9][0-9][0-9].*.*")
 echo $VERSION
 
 # This controls the directory the built artifacts go into
-export ARTIFACT_DIR=built_artifacts/
+export ARTIFACT_DIR=artifacts/
 mkdir -p $ARTIFACT_DIR
 
 arch=("amd64")
@@ -23,4 +23,4 @@ make cloudflared-rpm
 mv cloudflared-fips-$RPMVERSION-1.$RPMARCH.rpm $ARTIFACT_DIR/cloudflared-fips-linux-$RPMARCH.rpm
 
 # finally move the linux binary as well.
-mv ./cloudflared $ARTIFACT_DIR/cloudflared-fips-linux-$arch
+mv ./cloudflared $ARTIFACT_DIR/cloudflared-fips-linux-$arch

build-packages.sh

@@ -7,7 +7,7 @@ export GOEXPERIMENT=noboringcrypto
 export CGO_ENABLED=0
 
 # This controls the directory the built artifacts go into
-export ARTIFACT_DIR=built_artifacts/
+export ARTIFACT_DIR=artifacts/
 mkdir -p $ARTIFACT_DIR
 
 linuxArchs=("386" "amd64" "arm" "armhf" "arm64")

cfsetup.yaml

@@ -3,34 +3,27 @@ pinned_go: &pinned_go go-boring=1.22.2-1
 build_dir: &build_dir /cfsetup_build
 default-flavor: bullseye
 buster: &buster
-  build:
+  build-linux:
     build_dir: *build_dir
     builddeps: &build_deps
       - *pinned_go
       - build-essential
-      - gotest-to-teamcity
       - fakeroot
       - rubygem-fpm
       - rpm
       - libffi-dev
-      - reprepro
-      - createrepo
     pre-cache: &build_pre_cache
       - export GOCACHE=/cfsetup_build/.cache/go-build
       - go install golang.org/x/tools/cmd/goimports@latest
     post-cache:
-      # TODO: TUN-8126 this is temporary to make sure packages can be built before release
-      - ./build-packages.sh
       # Build binary for component test
       - GOOS=linux GOARCH=amd64 make cloudflared
-  build-fips:
+  build-linux-fips:
     build_dir: *build_dir
     builddeps: *build_deps
     pre-cache: *build_pre_cache
     post-cache:
       - export FIPS=true
-      # TODO: TUN-8126 this is temporary to make sure packages can be built before release
-      - ./build-packages-fips.sh
       # Build binary for component test
       - GOOS=linux GOARCH=amd64 make cloudflared
   cover:
@@ -39,28 +32,21 @@ buster: &buster
     pre-cache: *build_pre_cache
     post-cache:
      - make cover
-  # except FIPS (handled in github-fips-release-pkgs) and macos (handled in github-release-macos-amd64)
-  github-release-pkgs:
+  # except FIPS and macos 
+  build-linux-release:
     build_dir: *build_dir
-    builddeps:
+    builddeps: &build_deps_release
       - *pinned_go
       - build-essential
       - fakeroot
       - rubygem-fpm
       - rpm
-      - wget
-      # libmsi and libgcab are libraries the wixl binary depends on.
-      - libmsi-dev
-      - libgcab-dev
-      - python3-dev
       - libffi-dev
-      - python3-setuptools
+      - python3-dev
       - python3-pip
-      - reprepro
-      - createrepo
-    pre-cache: &github_release_pkgs_pre_cache
-      - wget https://github.com/sudarshan-reddy/msitools/releases/download/v0.101b/wixl -P /usr/local/bin
-      - chmod a+x /usr/local/bin/wixl
+      - python3-setuptools
+      - wget
+    pre-cache: &build_release_pre_cache
       - pip3 install pynacl==1.4.0
       - pip3 install pygithub==1.55
       - pip3 install boto3==1.22.9
@@ -68,32 +54,14 @@ buster: &buster
     post-cache:
       # build all packages (except macos and FIPS) and move them to /cfsetup/built_artifacts
       - ./build-packages.sh
-      # release the packages built and moved to /cfsetup/built_artifacts
-      - make github-release-built-pkgs
-      # publish packages to linux repos
-      - make release-pkgs-linux
   # handle FIPS separately so that we built with gofips compiler
-  github-fips-release-pkgs:
+  build-linux-fips-release:
     build_dir: *build_dir
-    builddeps:
-      - *pinned_go
-      - build-essential
-      - fakeroot
-      - rubygem-fpm
-      - rpm
-      - wget
-      # libmsi and libgcab are libraries the wixl binary depends on.
-      - libmsi-dev
-      - libgcab-dev
-      - python3-dev
-      - libffi-dev
-      - python3-setuptools
-      - python3-pip
-    pre-cache: *github_release_pkgs_pre_cache
+    builddeps: *build_deps_release
+    pre-cache: *build_release_pre_cache
     post-cache:
       # same logic as above, but for FIPS packages only
       - ./build-packages-fips.sh
-      - make github-release-built-pkgs
   generate-versions-file:
     build_dir: *build_dir
     builddeps:
@@ -152,21 +120,7 @@ buster: &buster
       - export GOOS=linux
       - export GOARCH=arm64
       - make cloudflared-deb
-  github-release-macos-amd64:
-    build_dir: *build_dir
-    builddeps: &build_pygithub
-      - *pinned_go
-      - build-essential
-      - python3-dev
-      - libffi-dev
-      - python3-setuptools
-      - python3-pip
-    pre-cache: &install_pygithub
-      - pip3 install pynacl==1.4.0
-      - pip3 install pygithub==1.55
-    post-cache:
-      - make github-mac-upload
-  github-release-windows:
+  package-windows:
     build_dir: *build_dir
     builddeps:
       - *pinned_go
@@ -186,10 +140,16 @@ buster: &buster
       - pip3 install pygithub==1.55
     post-cache:
       - .teamcity/package-windows.sh
-      - make github-windows-upload
   test:
     build_dir: *build_dir
-    builddeps: *build_deps
+    builddeps: &build_deps_tests
+      - *pinned_go
+      - build-essential
+      - fakeroot
+      - rubygem-fpm
+      - rpm
+      - libffi-dev
+      - gotest-to-teamcity
     pre-cache: *build_pre_cache
     post-cache:
       - export GOOS=linux
@@ -199,7 +159,7 @@ buster: &buster
       - make test | gotest-to-teamcity
   test-fips:
     build_dir: *build_dir
-    builddeps: *build_deps
+    builddeps: *build_deps_tests
     pre-cache: *build_pre_cache
     post-cache:
       - export GOOS=linux
@@ -210,7 +170,7 @@ buster: &buster
       - make test | gotest-to-teamcity
   component-test:
     build_dir: *build_dir
-    builddeps:
+    builddeps: &build_deps_component_test
       - *pinned_go
       - python3.7
       - python3-pip
@@ -230,24 +190,61 @@ buster: &buster
       - python3 component-tests/setup.py --type cleanup
   component-test-fips:
     build_dir: *build_dir
-    builddeps:
-      - *pinned_go
-      - python3.7
-      - python3-pip
-      - python3-setuptools
-      # procps installs the ps command which is needed in test_sysv_service because the init script
-      # uses ps pid to determine if the agent is running
-      - procps
+    builddeps: *build_deps_component_test
     pre-cache-copy-paths:
       - component-tests/requirements.txt
     pre-cache: *component_test_pre_cache
     post-cache: *component_test_post_cache
-  github-message-release:
+  github-release-dryrun:
     build_dir: *build_dir
-    builddeps: *build_pygithub
-    pre-cache: *install_pygithub
+    builddeps:
+      - *pinned_go
+      - build-essential
+      - python3-dev
+      - libffi-dev
+      - python3-setuptools
+      - python3-pip
+    pre-cache:
+      - pip3 install pynacl==1.4.0
+      - pip3 install pygithub==1.55
     post-cache:
-      - make github-message
+      - make github-release-dryrun
+  github-release:
+    build_dir: *build_dir
+    builddeps:
+      - *pinned_go
+      - build-essential
+      - python3-dev
+      - libffi-dev
+      - python3-setuptools
+      - python3-pip
+    pre-cache:
+      - pip3 install pynacl==1.4.0
+      - pip3 install pygithub==1.55
+    post-cache:
+      - make github-release
+  r2-linux-release:
+    build_dir: *build_dir
+    builddeps:
+      - *pinned_go
+      - build-essential
+      - fakeroot
+      - rubygem-fpm
+      - rpm
+      - wget
+      - python3-dev
+      - libffi-dev
+      - python3-setuptools
+      - python3-pip
+      - reprepro
+      - createrepo
+    pre-cache:
+      - pip3 install pynacl==1.4.0
+      - pip3 install pygithub==1.55
+      - pip3 install boto3==1.22.9
+      - pip3 install python-gnupg==0.4.9
+    post-cache:
+      - make r2-linux-release
 
 bullseye: *buster
 bookworm: *buster

cmd/cloudflared/access/cmd.go

@@ -26,6 +26,7 @@ import (
 )
 
 const (
+	appURLFlag         = "app"
 	loginQuietFlag     = "quiet"
 	sshHostnameFlag    = "hostname"
 	sshDestinationFlag = "destination"
@@ -83,9 +84,10 @@ func Commands() []*cli.Command {
 			applications from the command line.`,
 			Subcommands: []*cli.Command{
 				{
-					Name:   "login",
-					Action: cliutil.Action(login),
-					Usage:  "login <url of access application>",
+					Name:      "login",
+					Action:    cliutil.Action(login),
+					Usage:     "login <url of access application>",
+					ArgsUsage: "url of Access application",
 					Description: `The login subcommand initiates an authentication flow with your identity provider.
 					The subcommand will launch a browser. For headless systems, a url is provided.
 					Once authenticated with your identity provider, the login command will generate a JSON Web Token (JWT)
@@ -97,6 +99,13 @@ func Commands() []*cli.Command {
 							Aliases: []string{"q"},
 							Usage:   "do not print the jwt to the command line",
 						},
+						&cli.BoolFlag{
+							Name:  "no-verbose",
+							Usage: "print only the jwt to stdout",
+						},
+						&cli.StringFlag{
+							Name: appURLFlag,
+						},
 					},
 				},
 				{
@@ -111,12 +120,12 @@ func Commands() []*cli.Command {
 				{
 					Name:        "token",
 					Action:      cliutil.Action(generateToken),
-					Usage:       "token -app=<url of access application>",
+					Usage:       "token <url of access application>",
 					ArgsUsage:   "url of Access application",
 					Description: `The token subcommand produces a JWT which can be used to authenticate requests.`,
 					Flags: []cli.Flag{
 						&cli.StringFlag{
-							Name: "app",
+							Name: appURLFlag,
 						},
 					},
 				},
@@ -232,9 +241,8 @@ func login(c *cli.Context) error {
 
 	log := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
 
-	args := c.Args()
-	appURL, err := parseURL(args.First())
-	if args.Len() < 1 || err != nil {
+	appURL, err := getAppURLFromArgs(c)
+	if err != nil {
 		log.Error().Msg("Please provide the url of the Access application")
 		return err
 	}
@@ -261,7 +269,14 @@ func login(c *cli.Context) error {
 	if c.Bool(loginQuietFlag) {
 		return nil
 	}
-	fmt.Fprintf(os.Stdout, "Successfully fetched your token:\n\n%s\n\n", cfdToken)
+
+	// Chatty by default for backward compat. The new --app flag
+	// is an implicit opt-out of the backwards-compatible chatty output.
+	if c.Bool("no-verbose") || c.IsSet(appURLFlag) {
+		fmt.Fprint(os.Stdout, cfdToken)
+	} else {
+		fmt.Fprintf(os.Stdout, "Successfully fetched your token:\n\n%s\n\n", cfdToken)
+	}
 
 	return nil
 }
@@ -340,6 +355,17 @@ func run(cmd string, args ...string) error {
 	return c.Run()
 }
 
+func getAppURLFromArgs(c *cli.Context) (*url.URL, error) {
+	var appURLStr string
+	args := c.Args()
+	if args.Len() < 1 {
+		appURLStr = c.String(appURLFlag)
+	} else {
+		appURLStr = args.First()
+	}
+	return parseURL(appURLStr)
+}
+
 // token dumps provided token to stdout
 func generateToken(c *cli.Context) error {
 	err := sentry.Init(sentry.ClientOptions{
@@ -349,8 +375,8 @@ func generateToken(c *cli.Context) error {
 	if err != nil {
 		return err
 	}
-	appURL, err := parseURL(c.String("app"))
-	if err != nil || c.NumFlags() < 1 {
+	appURL, err := getAppURLFromArgs(c)
+	if err != nil {
 		fmt.Fprintln(os.Stderr, "Please provide a url.")
 		return err
 	}

cmd/cloudflared/tunnel/cmd.go

@@ -78,8 +78,8 @@ const (
 	// hostKeyPath is the path of the dir to save SSH host keys too
 	hostKeyPath = "host-key-path"
 
-	// udpUnregisterSessionTimeout is how long we wait before we stop trying to unregister a UDP session from the edge
-	udpUnregisterSessionTimeoutFlag = "udp-unregister-session-timeout"
+	// rpcTimeout is how long to wait for a Capnp RPC request to the edge
+	rpcTimeout = "rpc-timeout"
 
 	// writeStreamTimeout sets if we should have a timeout when writing data to a stream towards the destination (edge/origin).
 	writeStreamTimeout = "write-stream-timeout"
@@ -89,6 +89,14 @@ const (
 	// Note that this may result in packet drops for UDP proxying, since we expect being able to send at least 1280 bytes of inner packets.
 	quicDisablePathMTUDiscovery = "quic-disable-pmtu-discovery"
 
+	// quicConnLevelFlowControlLimit controls the max flow control limit allocated for a QUIC connection. This controls how much data is the
+	// receiver willing to buffer. Once the limit is reached, the sender will send a DATA_BLOCKED frame to indicate it has more data to write,
+	// but it's blocked by flow control
+	quicConnLevelFlowControlLimit = "quic-connection-level-flow-control-limit"
+	// quicStreamLevelFlowControlLimit is similar to quicConnLevelFlowControlLimit but for each QUIC stream. When the sender is blocked,
+	// it will send a STREAM_DATA_BLOCKED frame
+	quicStreamLevelFlowControlLimit = "quic-stream-level-flow-control-limit"
+
 	// uiFlag is to enable launching cloudflared in interactive UI mode
 	uiFlag = "ui"
 
@@ -287,7 +295,7 @@ func routeFromFlag(c *cli.Context) (route cfapi.HostnameRoute, ok bool) {
 func StartServer(
 	c *cli.Context,
 	info *cliutil.BuildInfo,
-	namedTunnel *connection.NamedTunnelProperties,
+	namedTunnel *connection.TunnelProperties,
 	log *zerolog.Logger,
 ) error {
 	err := sentry.Init(sentry.ClientOptions{
@@ -409,6 +417,11 @@ func StartServer(
 		}
 	}
 
+	// Disable ICMP packet routing for quick tunnels
+	if quickTunnelURL != "" {
+		tunnelConfig.PacketConfig = nil
+	}
+
 	internalRules := []ingress.Rule{}
 	if features.Contains(features.FeatureManagementLogs) {
 		serviceIP := c.String("service-op-ip")
@@ -658,9 +671,9 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 		}),
 		altsrc.NewStringSliceFlag(&cli.StringSliceFlag{
 			Name:    "tag",
-			Usage:   "Custom tags used to identify this tunnel, in format `KEY=VALUE`. Multiple tags may be specified",
+			Usage:   "Custom tags used to identify this tunnel via added HTTP request headers to the origin, in format `KEY=VALUE`. Multiple tags may be specified.",
 			EnvVars: []string{"TUNNEL_TAG"},
-			Hidden:  shouldHide,
+			Hidden:  true,
 		}),
 		altsrc.NewDurationFlag(&cli.DurationFlag{
 			Name:   "heartbeat-interval",
@@ -695,7 +708,7 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			Hidden: true,
 		}),
 		altsrc.NewDurationFlag(&cli.DurationFlag{
-			Name:   udpUnregisterSessionTimeoutFlag,
+			Name:   rpcTimeout,
 			Value:  5 * time.Second,
 			Hidden: true,
 		}),
@@ -713,6 +726,20 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			Value:   false,
 			Hidden:  true,
 		}),
+		altsrc.NewIntFlag(&cli.IntFlag{
+			Name:    quicConnLevelFlowControlLimit,
+			EnvVars: []string{"TUNNEL_QUIC_CONN_LEVEL_FLOW_CONTROL_LIMIT"},
+			Usage:   "Use this option to change the connection-level flow control limit for QUIC transport.",
+			Value:   30 * (1 << 20), // 30 MB
+			Hidden:  true,
+		}),
+		altsrc.NewIntFlag(&cli.IntFlag{
+			Name:    quicStreamLevelFlowControlLimit,
+			EnvVars: []string{"TUNNEL_QUIC_STREAM_LEVEL_FLOW_CONTROL_LIMIT"},
+			Usage:   "Use this option to change the connection-level flow control limit for QUIC transport.",
+			Value:   6 * (1 << 20), // 6 MB
+			Hidden:  true,
+		}),
 		altsrc.NewStringFlag(&cli.StringFlag{
 			Name:  connectorLabelFlag,
 			Usage: "Use this option to give a meaningful label to a specific connector. When a tunnel starts up, a connector id unique to the tunnel is generated. This is a uuid. To make it easier to identify a connector, we will use the hostname of the machine the tunnel is running on along with the connector ID. This option exists if one wants to have more control over what their individual connectors are called.",

cmd/cloudflared/tunnel/configuration.go

@@ -27,7 +27,7 @@ import (
 	"github.com/cloudflare/cloudflared/orchestration"
 	"github.com/cloudflare/cloudflared/supervisor"
 	"github.com/cloudflare/cloudflared/tlsconfig"
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
 const (
@@ -108,7 +108,7 @@ func isSecretEnvVar(key string) bool {
 	return false
 }
 
-func dnsProxyStandAlone(c *cli.Context, namedTunnel *connection.NamedTunnelProperties) bool {
+func dnsProxyStandAlone(c *cli.Context, namedTunnel *connection.TunnelProperties) bool {
 	return c.IsSet("proxy-dns") &&
 		!(c.IsSet("name") || // adhoc-named tunnel
 			c.IsSet(ingress.HelloWorldFlag) || // quick or named tunnel
@@ -121,7 +121,7 @@ func prepareTunnelConfig(
 	info *cliutil.BuildInfo,
 	log, logTransport *zerolog.Logger,
 	observer *connection.Observer,
-	namedTunnel *connection.NamedTunnelProperties,
+	namedTunnel *connection.TunnelProperties,
 ) (*supervisor.TunnelConfig, *orchestration.Config, error) {
 	clientID, err := uuid.NewRandom()
 	if err != nil {
@@ -133,7 +133,7 @@ func prepareTunnelConfig(
 		log.Err(err).Msg("Tag parse failure")
 		return nil, nil, errors.Wrap(err, "Tag parse failure")
 	}
-	tags = append(tags, tunnelpogs.Tag{Name: "ID", Value: clientID.String()})
+	tags = append(tags, pogs.Tag{Name: "ID", Value: clientID.String()})
 
 	transportProtocol := c.String("protocol")
 
@@ -166,7 +166,7 @@ func prepareTunnelConfig(
 		)
 	}
 
-	namedTunnel.Client = tunnelpogs.ClientInfo{
+	namedTunnel.Client = pogs.ClientInfo{
 		ClientID: clientID[:],
 		Features: clientFeatures,
 		Version:  info.Version(),
@@ -239,16 +239,18 @@ func prepareTunnelConfig(
 		Observer:        observer,
 		ReportedVersion: info.Version(),
 		// Note TUN-3758 , we use Int because UInt is not supported with altsrc
-		Retries:                     uint(c.Int("retries")),
-		RunFromTerminal:             isRunningFromTerminal(),
-		NamedTunnel:                 namedTunnel,
-		ProtocolSelector:            protocolSelector,
-		EdgeTLSConfigs:              edgeTLSConfigs,
-		FeatureSelector:             featureSelector,
-		MaxEdgeAddrRetries:          uint8(c.Int("max-edge-addr-retries")),
-		UDPUnregisterSessionTimeout: c.Duration(udpUnregisterSessionTimeoutFlag),
-		WriteStreamTimeout:          c.Duration(writeStreamTimeout),
-		DisableQUICPathMTUDiscovery: c.Bool(quicDisablePathMTUDiscovery),
+		Retries:                             uint(c.Int("retries")),
+		RunFromTerminal:                     isRunningFromTerminal(),
+		NamedTunnel:                         namedTunnel,
+		ProtocolSelector:                    protocolSelector,
+		EdgeTLSConfigs:                      edgeTLSConfigs,
+		FeatureSelector:                     featureSelector,
+		MaxEdgeAddrRetries:                  uint8(c.Int("max-edge-addr-retries")),
+		RPCTimeout:                          c.Duration(rpcTimeout),
+		WriteStreamTimeout:                  c.Duration(writeStreamTimeout),
+		DisableQUICPathMTUDiscovery:         c.Bool(quicDisablePathMTUDiscovery),
+		QUICConnectionLevelFlowControlLimit: c.Uint64(quicConnLevelFlowControlLimit),
+		QUICStreamLevelFlowControlLimit:     c.Uint64(quicStreamLevelFlowControlLimit),
 	}
 	packetConfig, err := newPacketConfig(c, log)
 	if err != nil {

cmd/cloudflared/tunnel/quick_tunnel.go

@@ -3,6 +3,7 @@ package tunnel
 import (
 	"encoding/json"
 	"fmt"
+	"io"
 	"net/http"
 	"strings"
 	"time"
@@ -15,10 +16,7 @@ import (
 
 const httpTimeout = 15 * time.Second
 
-const disclaimer = "Thank you for trying Cloudflare Tunnel. Doing so, without a Cloudflare account, is a quick way to" +
-	" experiment and try it out. However, be aware that these account-less Tunnels have no uptime guarantee. If you " +
-	"intend to use Tunnels in production you should use a pre-created named tunnel by following: " +
-	"https://developers.cloudflare.com/cloudflare-one/connections/connect-apps"
+const disclaimer = "Thank you for trying Cloudflare Tunnel. Doing so, without a Cloudflare account, is a quick way to experiment and try it out. However, be aware that these account-less Tunnels have no uptime guarantee, are subject to the Cloudflare Online Services Terms of Use (https://www.cloudflare.com/website-terms/), and Cloudflare reserves the right to investigate your use of Tunnels for violations of such terms. If you intend to use Tunnels in production you should use a pre-created named tunnel by following: https://developers.cloudflare.com/cloudflare-one/connections/connect-apps"
 
 // RunQuickTunnel requests a tunnel from the specified service.
 // We use this to power quick tunnels on trycloudflare.com, but the
@@ -35,14 +33,29 @@ func RunQuickTunnel(sc *subcommandContext) error {
 		Timeout: httpTimeout,
 	}
 
-	resp, err := client.Post(fmt.Sprintf("%s/tunnel", sc.c.String("quick-service")), "application/json", nil)
+	req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/tunnel", sc.c.String("quick-service")), nil)
+	if err != nil {
+		return errors.Wrap(err, "failed to build quick tunnel request")
+	}
+	req.Header.Add("Content-Type", "application/json")
+	req.Header.Add("User-Agent", buildInfo.UserAgent())
+	resp, err := client.Do(req)
 	if err != nil {
 		return errors.Wrap(err, "failed to request quick Tunnel")
 	}
 	defer resp.Body.Close()
 
+	// This will read the entire response into memory so we can print it in case of error
+	rsp_body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return errors.Wrap(err, "failed to read quick-tunnel response")
+	}
+
 	var data QuickTunnelResponse
-	if err := json.NewDecoder(resp.Body).Decode(&data); err != nil {
+	if err := json.Unmarshal(rsp_body, &data); err != nil {
+		rsp_string := string(rsp_body)
+		fields := map[string]interface{}{"status_code": resp.Status}
+		sc.log.Err(err).Fields(fields).Msgf("Error unmarshaling QuickTunnel response: %s", rsp_string)
 		return errors.Wrap(err, "failed to unmarshal quick Tunnel")
 	}
 
@@ -79,7 +92,7 @@ func RunQuickTunnel(sc *subcommandContext) error {
 	return StartServer(
 		sc.c,
 		buildInfo,
-		&connection.NamedTunnelProperties{Credentials: credentials, QuickTunnelUrl: data.Result.Hostname},
+		&connection.TunnelProperties{Credentials: credentials, QuickTunnelUrl: data.Result.Hostname},
 		sc.log,
 	)
 }

cmd/cloudflared/tunnel/subcommand_context.go

@@ -261,7 +261,7 @@ func (sc *subcommandContext) runWithCredentials(credentials connection.Credentia
 	return StartServer(
 		sc.c,
 		buildInfo,
-		&connection.NamedTunnelProperties{Credentials: credentials},
+		&connection.TunnelProperties{Credentials: credentials},
 		sc.log,
 	)
 }

cmd/cloudflared/tunnel/tag.go

@@ -4,23 +4,23 @@ import (
 	"fmt"
 	"regexp"
 
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
 // Restrict key names to characters allowed in an HTTP header name.
 // Restrict key values to printable characters (what is recognised as data in an HTTP header value).
 var tagRegexp = regexp.MustCompile("^([a-zA-Z0-9!#$%&'*+\\-.^_`|~]+)=([[:print:]]+)$")
 
-func NewTagFromCLI(compoundTag string) (tunnelpogs.Tag, bool) {
+func NewTagFromCLI(compoundTag string) (pogs.Tag, bool) {
 	matches := tagRegexp.FindStringSubmatch(compoundTag)
 	if len(matches) == 0 {
-		return tunnelpogs.Tag{}, false
+		return pogs.Tag{}, false
 	}
-	return tunnelpogs.Tag{Name: matches[1], Value: matches[2]}, true
+	return pogs.Tag{Name: matches[1], Value: matches[2]}, true
 }
 
-func NewTagSliceFromCLI(tags []string) ([]tunnelpogs.Tag, error) {
-	var tagSlice []tunnelpogs.Tag
+func NewTagSliceFromCLI(tags []string) ([]pogs.Tag, error) {
+	var tagSlice []pogs.Tag
 	for _, compoundTag := range tags {
 		if tag, ok := NewTagFromCLI(compoundTag); ok {
 			tagSlice = append(tagSlice, tag)

cmd/cloudflared/tunnel/tag_test.go

@@ -3,7 +3,7 @@ package tunnel
 import (
 	"testing"
 
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 
 	"github.com/stretchr/testify/assert"
 )
@@ -11,12 +11,12 @@ import (
 func TestSingleTag(t *testing.T) {
 	testCases := []struct {
 		Input  string
-		Output tunnelpogs.Tag
+		Output pogs.Tag
 		Fail   bool
 	}{
-		{Input: "x=y", Output: tunnelpogs.Tag{Name: "x", Value: "y"}},
-		{Input: "More-Complex=Tag Values", Output: tunnelpogs.Tag{Name: "More-Complex", Value: "Tag Values"}},
-		{Input: "First=Equals=Wins", Output: tunnelpogs.Tag{Name: "First", Value: "Equals=Wins"}},
+		{Input: "x=y", Output: pogs.Tag{Name: "x", Value: "y"}},
+		{Input: "More-Complex=Tag Values", Output: pogs.Tag{Name: "More-Complex", Value: "Tag Values"}},
+		{Input: "First=Equals=Wins", Output: pogs.Tag{Name: "First", Value: "Equals=Wins"}},
 		{Input: "x=", Fail: true},
 		{Input: "=y", Fail: true},
 		{Input: "=", Fail: true},

component-tests/test_tail.py

@@ -47,7 +47,12 @@ class TestTail:
             url = cfd_cli.get_management_wsurl("logs", config, config_path)
             async with connect(url, open_timeout=5, close_timeout=5) as websocket:
                 # send start_streaming
-                await websocket.send('{"type": "start_streaming"}')
+                await websocket.send(json.dumps({
+                    "type": "start_streaming",
+                    "filters": {
+                        "events": ["http"]
+                    }
+                }))
                 # send some http requests to the tunnel to trigger some logs
                 await generate_and_validate_http_events(websocket, config.get_url(), 10)
                 # send stop_streaming
@@ -99,7 +104,8 @@ class TestTail:
                 await websocket.send(json.dumps({
                     "type": "start_streaming",
                     "filters": {
-                        "sampling": 0.5
+                        "sampling": 0.5,
+                        "events": ["http"]
                     }
                 }))
                 # don't expect any http logs

component-tests/test_termination.py

@@ -45,9 +45,10 @@ class TestTermination:
             with connected:
                 connected.wait(self.timeout)
             # Send signal after the SSE connection is established
-            self.terminate_by_signal(cloudflared, signal)
-            self.wait_eyeball_thread(
-                in_flight_req, self.grace_period + self.timeout)
+            with self.within_grace_period():
+                self.terminate_by_signal(cloudflared, signal)
+                self.wait_eyeball_thread(
+                    in_flight_req, self.grace_period + self.timeout)
 
     # test cloudflared terminates before grace period expires when all eyeball
     # connections are drained
@@ -66,7 +67,7 @@ class TestTermination:
 
             with connected:
                 connected.wait(self.timeout)
-            with self.within_grace_period():
+            with self.within_grace_period(has_connection=False):
                 # Send signal after the SSE connection is established
                 self.terminate_by_signal(cloudflared, signal)
                 self.wait_eyeball_thread(in_flight_req, self.grace_period)
@@ -78,7 +79,7 @@ class TestTermination:
         with start_cloudflared(
                 tmp_path, config, cfd_pre_args=["tunnel", "--ha-connections", "1"], new_process=True, capture_output=False) as cloudflared:
             wait_tunnel_ready(tunnel_url=config.get_url())
-            with self.within_grace_period():
+            with self.within_grace_period(has_connection=False):
                 self.terminate_by_signal(cloudflared, signal)
 
     def terminate_by_signal(self, cloudflared, sig):
@@ -92,13 +93,21 @@ class TestTermination:
 
     # Using this context asserts logic within the context is executed within grace period
     @contextmanager
-    def within_grace_period(self):
+    def within_grace_period(self, has_connection=True):
         try:
             start = time.time()
             yield
         finally:
+
+            # If the request takes longer than the grace period then we need to wait at most the grace period.
+            # If the request fell within the grace period cloudflared can close earlier, but to ensure that it doesn't
+            # close immediately we add a minimum boundary. If cloudflared shutdown in less than 1s it's likely that
+            # it shutdown as soon as it received SIGINT. The only way cloudflared can close immediately is if it has no
+            # in-flight requests
+            minimum = 1 if has_connection else 0
             duration = time.time() - start
-            assert duration < self.grace_period
+            # Here we truncate to ensure that we don't fail on minute differences like 10.1 instead of 10
+            assert minimum <= int(duration) <= self.grace_period
 
     def stream_request(self, config, connected, early_terminate):
         expected_terminate_message = "502 Bad Gateway"

connection/connection.go

@@ -42,7 +42,7 @@ type Orchestrator interface {
 	GetOriginProxy() (OriginProxy, error)
 }
 
-type NamedTunnelProperties struct {
+type TunnelProperties struct {
 	Credentials    Credentials
 	Client         pogs.ClientInfo
 	QuickTunnelUrl string

connection/control.go

@@ -6,25 +6,27 @@ import (
 	"net"
 	"time"
 
-	"github.com/rs/zerolog"
+	"github.com/pkg/errors"
 
 	"github.com/cloudflare/cloudflared/management"
+	"github.com/cloudflare/cloudflared/tunnelrpc"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
-// RPCClientFunc derives a named tunnel rpc client that can then be used to register and unregister connections.
-type RPCClientFunc func(context.Context, io.ReadWriteCloser, *zerolog.Logger) NamedTunnelRPCClient
+// registerClient derives a named tunnel rpc client that can then be used to register and unregister connections.
+type registerClientFunc func(context.Context, io.ReadWriteCloser, time.Duration) tunnelrpc.RegistrationClient
 
 type controlStream struct {
 	observer *Observer
 
-	connectedFuse         ConnectedFuse
-	namedTunnelProperties *NamedTunnelProperties
-	connIndex             uint8
-	edgeAddress           net.IP
-	protocol              Protocol
+	connectedFuse    ConnectedFuse
+	tunnelProperties *TunnelProperties
+	connIndex        uint8
+	edgeAddress      net.IP
+	protocol         Protocol
 
-	newRPCClientFunc RPCClientFunc
+	registerClientFunc registerClientFunc
+	registerTimeout    time.Duration
 
 	gracefulShutdownC <-chan struct{}
 	gracePeriod       time.Duration
@@ -47,27 +49,29 @@ type TunnelConfigJSONGetter interface {
 func NewControlStream(
 	observer *Observer,
 	connectedFuse ConnectedFuse,
-	namedTunnelConfig *NamedTunnelProperties,
+	tunnelProperties *TunnelProperties,
 	connIndex uint8,
 	edgeAddress net.IP,
-	newRPCClientFunc RPCClientFunc,
+	registerClientFunc registerClientFunc,
+	registerTimeout time.Duration,
 	gracefulShutdownC <-chan struct{},
 	gracePeriod time.Duration,
 	protocol Protocol,
 ) ControlStreamHandler {
-	if newRPCClientFunc == nil {
-		newRPCClientFunc = newRegistrationRPCClient
+	if registerClientFunc == nil {
+		registerClientFunc = tunnelrpc.NewRegistrationClient
 	}
 	return &controlStream{
-		observer:              observer,
-		connectedFuse:         connectedFuse,
-		namedTunnelProperties: namedTunnelConfig,
-		newRPCClientFunc:      newRPCClientFunc,
-		connIndex:             connIndex,
-		edgeAddress:           edgeAddress,
-		gracefulShutdownC:     gracefulShutdownC,
-		gracePeriod:           gracePeriod,
-		protocol:              protocol,
+		observer:           observer,
+		connectedFuse:      connectedFuse,
+		tunnelProperties:   tunnelProperties,
+		registerClientFunc: registerClientFunc,
+		registerTimeout:    registerTimeout,
+		connIndex:          connIndex,
+		edgeAddress:        edgeAddress,
+		gracefulShutdownC:  gracefulShutdownC,
+		gracePeriod:        gracePeriod,
+		protocol:           protocol,
 	}
 }
 
@@ -77,13 +81,25 @@ func (c *controlStream) ServeControlStream(
 	connOptions *tunnelpogs.ConnectionOptions,
 	tunnelConfigGetter TunnelConfigJSONGetter,
 ) error {
-	rpcClient := c.newRPCClientFunc(ctx, rw, c.observer.log)
+	registrationClient := c.registerClientFunc(ctx, rw, c.registerTimeout)
 
-	registrationDetails, err := rpcClient.RegisterConnection(ctx, c.namedTunnelProperties, connOptions, c.connIndex, c.edgeAddress, c.observer)
+	registrationDetails, err := registrationClient.RegisterConnection(
+		ctx,
+		c.tunnelProperties.Credentials.Auth(),
+		c.tunnelProperties.Credentials.TunnelID,
+		connOptions,
+		c.connIndex,
+		c.edgeAddress)
 	if err != nil {
-		rpcClient.Close()
-		return err
+		defer registrationClient.Close()
+		if err.Error() == DuplicateConnectionError {
+			c.observer.metrics.regFail.WithLabelValues("dup_edge_conn", "registerConnection").Inc()
+			return errDuplicationConnection
+		}
+		c.observer.metrics.regFail.WithLabelValues("server_error", "registerConnection").Inc()
+		return serverRegistrationErrorFromRPC(err)
 	}
+	c.observer.metrics.regSuccess.WithLabelValues("registerConnection").Inc()
 
 	c.observer.logConnected(registrationDetails.UUID, c.connIndex, registrationDetails.Location, c.edgeAddress, c.protocol)
 	c.observer.sendConnectedEvent(c.connIndex, c.protocol, registrationDetails.Location)
@@ -92,35 +108,42 @@ func (c *controlStream) ServeControlStream(
 	// if conn index is 0 and tunnel is not remotely managed, then send local ingress rules configuration
 	if c.connIndex == 0 && !registrationDetails.TunnelIsRemotelyManaged {
 		if tunnelConfig, err := tunnelConfigGetter.GetConfigJSON(); err == nil {
-			if err := rpcClient.SendLocalConfiguration(ctx, tunnelConfig, c.observer); err != nil {
+			if err := registrationClient.SendLocalConfiguration(ctx, tunnelConfig); err != nil {
+				c.observer.metrics.localConfigMetrics.pushesErrors.Inc()
 				c.observer.log.Err(err).Msg("unable to send local configuration")
 			}
+			c.observer.metrics.localConfigMetrics.pushes.Inc()
 		} else {
 			c.observer.log.Err(err).Msg("failed to obtain current configuration")
 		}
 	}
 
-	c.waitForUnregister(ctx, rpcClient)
-	return nil
+	return c.waitForUnregister(ctx, registrationClient)
 }
 
-func (c *controlStream) waitForUnregister(ctx context.Context, rpcClient NamedTunnelRPCClient) {
+func (c *controlStream) waitForUnregister(ctx context.Context, registrationClient tunnelrpc.RegistrationClient) error {
 	// wait for connection termination or start of graceful shutdown
-	defer rpcClient.Close()
+	defer registrationClient.Close()
+	var shutdownError error
 	select {
 	case <-ctx.Done():
+		shutdownError = ctx.Err()
 		break
 	case <-c.gracefulShutdownC:
 		c.stoppedGracefully = true
 	}
 
 	c.observer.sendUnregisteringEvent(c.connIndex)
-	rpcClient.GracefulShutdown(ctx, c.gracePeriod)
+	err := registrationClient.GracefulShutdown(ctx, c.gracePeriod)
+	if err != nil {
+		return errors.Wrap(err, "Error shutting down control stream")
+	}
 	c.observer.log.Info().
 		Int(management.EventTypeKey, int(management.Cloudflared)).
 		Uint8(LogFieldConnIndex, c.connIndex).
 		IPAddr(LogFieldIPAddress, c.edgeAddress).
 		Msg("Unregistered tunnel connection")
+	return shutdownError
 }
 
 func (c *controlStream) IsStopped() bool {

connection/http2.go

@@ -40,8 +40,6 @@ type HTTP2Connection struct {
 	connOptions  *tunnelpogs.ConnectionOptions
 	observer     *Observer
 	connIndex    uint8
-	// newRPCClientFunc allows us to mock RPCs during testing
-	newRPCClientFunc func(context.Context, io.ReadWriteCloser, *zerolog.Logger) NamedTunnelRPCClient
 
 	log                  *zerolog.Logger
 	activeRequestsWG     sync.WaitGroup
@@ -69,7 +67,6 @@ func NewHTTP2Connection(
 		connOptions:          connOptions,
 		observer:             observer,
 		connIndex:            connIndex,
-		newRPCClientFunc:     newRegistrationRPCClient,
 		controlStreamHandler: controlStreamHandler,
 		log:                  log,
 	}

connection/http2_test.go

@@ -20,8 +20,8 @@ import (
 	"github.com/stretchr/testify/require"
 	"golang.org/x/net/http2"
 
+	"github.com/cloudflare/cloudflared/tunnelrpc"
 	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
 var (
@@ -36,10 +36,11 @@ func newTestHTTP2Connection() (*HTTP2Connection, net.Conn) {
 	controlStream := NewControlStream(
 		obs,
 		mockConnectedFuse{},
-		&NamedTunnelProperties{},
+		&TunnelProperties{},
 		connIndex,
 		nil,
 		nil,
+		1*time.Second,
 		nil,
 		1*time.Second,
 		HTTP2,
@@ -168,31 +169,32 @@ type mockNamedTunnelRPCClient struct {
 	unregistered chan struct{}
 }
 
-func (mc mockNamedTunnelRPCClient) SendLocalConfiguration(c context.Context, config []byte, observer *Observer) error {
+func (mc mockNamedTunnelRPCClient) SendLocalConfiguration(c context.Context, config []byte) error {
 	return nil
 }
 
 func (mc mockNamedTunnelRPCClient) RegisterConnection(
-	c context.Context,
-	properties *NamedTunnelProperties,
-	options *tunnelpogs.ConnectionOptions,
+	ctx context.Context,
+	auth pogs.TunnelAuth,
+	tunnelID uuid.UUID,
+	options *pogs.ConnectionOptions,
 	connIndex uint8,
 	edgeAddress net.IP,
-	observer *Observer,
-) (*tunnelpogs.ConnectionDetails, error) {
+) (*pogs.ConnectionDetails, error) {
 	if mc.shouldFail != nil {
 		return nil, mc.shouldFail
 	}
 	close(mc.registered)
-	return &tunnelpogs.ConnectionDetails{
+	return &pogs.ConnectionDetails{
 		Location:                "LIS",
 		UUID:                    uuid.New(),
 		TunnelIsRemotelyManaged: false,
 	}, nil
 }
 
-func (mc mockNamedTunnelRPCClient) GracefulShutdown(ctx context.Context, gracePeriod time.Duration) {
+func (mc mockNamedTunnelRPCClient) GracefulShutdown(ctx context.Context, gracePeriod time.Duration) error {
 	close(mc.unregistered)
+	return nil
 }
 
 func (mockNamedTunnelRPCClient) Close() {}
@@ -203,8 +205,8 @@ type mockRPCClientFactory struct {
 	unregistered chan struct{}
 }
 
-func (mf *mockRPCClientFactory) newMockRPCClient(context.Context, io.ReadWriteCloser, *zerolog.Logger) NamedTunnelRPCClient {
-	return mockNamedTunnelRPCClient{
+func (mf *mockRPCClientFactory) newMockRPCClient(context.Context, io.ReadWriteCloser, time.Duration) tunnelrpc.RegistrationClient {
+	return &mockNamedTunnelRPCClient{
 		shouldFail:   mf.shouldFail,
 		registered:   mf.registered,
 		unregistered: mf.unregistered,
@@ -360,10 +362,11 @@ func TestServeControlStream(t *testing.T) {
 	controlStream := NewControlStream(
 		obs,
 		mockConnectedFuse{},
-		&NamedTunnelProperties{},
+		&TunnelProperties{},
 		1,
 		nil,
 		rpcClientFactory.newMockRPCClient,
+		1*time.Second,
 		nil,
 		1*time.Second,
 		HTTP2,
@@ -412,10 +415,11 @@ func TestFailRegistration(t *testing.T) {
 	controlStream := NewControlStream(
 		obs,
 		mockConnectedFuse{},
-		&NamedTunnelProperties{},
+		&TunnelProperties{},
 		http2Conn.connIndex,
 		nil,
 		rpcClientFactory.newMockRPCClient,
+		1*time.Second,
 		nil,
 		1*time.Second,
 		HTTP2,
@@ -460,10 +464,11 @@ func TestGracefulShutdownHTTP2(t *testing.T) {
 	controlStream := NewControlStream(
 		obs,
 		mockConnectedFuse{},
-		&NamedTunnelProperties{},
+		&TunnelProperties{},
 		http2Conn.connIndex,
 		nil,
 		rpcClientFactory.newMockRPCClient,
+		1*time.Second,
 		shutdownC,
 		1*time.Second,
 		HTTP2,

connection/metrics.go

@@ -43,7 +43,6 @@ type localConfigMetrics struct {
 }
 
 type tunnelMetrics struct {
-	timerRetries    prometheus.Gauge
 	serverLocations *prometheus.GaugeVec
 	// locationLock is a mutex for oldServerLocations
 	locationLock sync.Mutex
@@ -351,15 +350,6 @@ func initTunnelMetrics() *tunnelMetrics {
 	)
 	prometheus.MustRegister(maxConcurrentRequestsPerTunnel)
 
-	timerRetries := prometheus.NewGauge(
-		prometheus.GaugeOpts{
-			Namespace: MetricsNamespace,
-			Subsystem: TunnelSubsystem,
-			Name:      "timer_retries",
-			Help:      "Unacknowledged heart beats count",
-		})
-	prometheus.MustRegister(timerRetries)
-
 	serverLocations := prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Namespace: MetricsNamespace,
@@ -416,7 +406,6 @@ func initTunnelMetrics() *tunnelMetrics {
 	prometheus.MustRegister(registerSuccess)
 
 	return &tunnelMetrics{
-		timerRetries:        timerRetries,
 		serverLocations:     serverLocations,
 		oldServerLocations:  make(map[string]string),
 		muxerMetrics:        newMuxerMetrics(),

connection/quic.go

@@ -28,9 +28,11 @@ import (
 	"github.com/cloudflare/cloudflared/ingress"
 	"github.com/cloudflare/cloudflared/management"
 	"github.com/cloudflare/cloudflared/packet"
-	quicpogs "github.com/cloudflare/cloudflared/quic"
+	cfdquic "github.com/cloudflare/cloudflared/quic"
 	"github.com/cloudflare/cloudflared/tracing"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	rpcquic "github.com/cloudflare/cloudflared/tunnelrpc/quic"
 )
 
 const (
@@ -40,12 +42,26 @@ const (
 	HTTPMethodKey = "HttpMethod"
 	// HTTPHostKey is used to get or set http Method in QUIC ALPN if the underlying proxy connection type is HTTP.
 	HTTPHostKey = "HttpHost"
+	// HTTPRequestBodyHintKey is used in ConnectRequest metadata to indicate if the request has body
+	HTTPRequestBodyHintKey = "HttpReqBodyHint"
 
 	QUICMetadataFlowID = "FlowID"
 	// emperically this capacity has been working well
 	demuxChanCapacity = 16
 )
 
+type RequestBodyHint uint64
+
+const (
+	RequestBodyHintMissing RequestBodyHint = iota
+	RequestBodyHintEmpty
+	RequestBodyHintHasData
+)
+
+func (rbh RequestBodyHint) String() string {
+	return [...]string{"missing", "empty", "data"}[rbh]
+}
+
 var (
 	portForConnIndex = make(map[uint8]int, 0)
 	portMapMutex     sync.Mutex
@@ -59,14 +75,15 @@ type QUICConnection struct {
 	// sessionManager tracks active sessions. It receives datagrams from quic connection via datagramMuxer
 	sessionManager datagramsession.Manager
 	// datagramMuxer mux/demux datagrams from quic connection
-	datagramMuxer        *quicpogs.DatagramMuxerV2
+	datagramMuxer        *cfdquic.DatagramMuxerV2
 	packetRouter         *ingress.PacketRouter
 	controlStreamHandler ControlStreamHandler
 	connOptions          *tunnelpogs.ConnectionOptions
 	connIndex            uint8
 
-	udpUnregisterTimeout time.Duration
-	streamWriteTimeout   time.Duration
+	rpcTimeout         time.Duration
+	streamWriteTimeout time.Duration
+	gracePeriod        time.Duration
 }
 
 // NewQUICConnection returns a new instance of QUICConnection.
@@ -82,8 +99,9 @@ func NewQUICConnection(
 	controlStreamHandler ControlStreamHandler,
 	logger *zerolog.Logger,
 	packetRouterConfig *ingress.GlobalRouterConfig,
-	udpUnregisterTimeout time.Duration,
+	rpcTimeout time.Duration,
 	streamWriteTimeout time.Duration,
+	gracePeriod time.Duration,
 ) (*QUICConnection, error) {
 	udpConn, err := createUDPConnForConnIndex(connIndex, localAddr, logger)
 	if err != nil {
@@ -104,7 +122,7 @@ func NewQUICConnection(
 	}
 
 	sessionDemuxChan := make(chan *packet.Session, demuxChanCapacity)
-	datagramMuxer := quicpogs.NewDatagramMuxerV2(session, logger, sessionDemuxChan)
+	datagramMuxer := cfdquic.NewDatagramMuxerV2(session, logger, sessionDemuxChan)
 	sessionManager := datagramsession.NewManager(logger, datagramMuxer.SendToSession, sessionDemuxChan)
 	packetRouter := ingress.NewPacketRouter(packetRouterConfig, datagramMuxer, logger)
 
@@ -118,8 +136,9 @@ func NewQUICConnection(
 		controlStreamHandler: controlStreamHandler,
 		connOptions:          connOptions,
 		connIndex:            connIndex,
-		udpUnregisterTimeout: udpUnregisterTimeout,
+		rpcTimeout:           rpcTimeout,
 		streamWriteTimeout:   streamWriteTimeout,
+		gracePeriod:          gracePeriod,
 	}, nil
 }
 
@@ -142,8 +161,17 @@ func (q *QUICConnection) Serve(ctx context.Context) error {
 	// In the future, if cloudflared can autonomously push traffic to the edge, we have to make sure the control
 	// stream is already fully registered before the other goroutines can proceed.
 	errGroup.Go(func() error {
-		defer cancel()
-		return q.serveControlStream(ctx, controlStream)
+		// err is equal to nil if we exit due to unregistration. If that happens we want to wait the full
+		// amount of the grace period, allowing requests to finish before we cancel the context, which will
+		// make cloudflared exit.
+		if err := q.serveControlStream(ctx, controlStream); err == nil {
+			select {
+			case <-ctx.Done():
+			case <-time.Tick(q.gracePeriod):
+			}
+		}
+		cancel()
+		return err
 	})
 	errGroup.Go(func() error {
 		defer cancel()
@@ -198,7 +226,7 @@ func (q *QUICConnection) acceptStream(ctx context.Context) error {
 
 func (q *QUICConnection) runStream(quicStream quic.Stream) {
 	ctx := quicStream.Context()
-	stream := quicpogs.NewSafeStreamCloser(quicStream, q.streamWriteTimeout, q.logger)
+	stream := cfdquic.NewSafeStreamCloser(quicStream, q.streamWriteTimeout, q.logger)
 	defer stream.Close()
 
 	// we are going to fuse readers/writers from stream <- cloudflared -> origin, and we want to guarantee that
@@ -206,7 +234,8 @@ func (q *QUICConnection) runStream(quicStream quic.Stream) {
 	// So, we wrap the stream with a no-op write closer and only this method can actually close write side of the stream.
 	// A call to close will simulate a close to the read-side, which will fail subsequent reads.
 	noCloseStream := &nopCloserReadWriter{ReadWriteCloser: stream}
-	if err := q.handleStream(ctx, noCloseStream); err != nil {
+	ss := rpcquic.NewCloudflaredServer(q.handleDataStream, q, q, q.rpcTimeout)
+	if err := ss.Serve(ctx, noCloseStream); err != nil {
 		q.logger.Debug().Err(err).Msg("Failed to handle QUIC stream")
 
 		// if we received an error at this level, then close write side of stream with an error, which will result in
@@ -215,30 +244,7 @@ func (q *QUICConnection) runStream(quicStream quic.Stream) {
 	}
 }
 
-func (q *QUICConnection) handleStream(ctx context.Context, stream io.ReadWriteCloser) error {
-	signature, err := quicpogs.DetermineProtocol(stream)
-	if err != nil {
-		return err
-	}
-	switch signature {
-	case quicpogs.DataStreamProtocolSignature:
-		reqServerStream, err := quicpogs.NewRequestServerStream(stream, signature)
-		if err != nil {
-			return err
-		}
-		return q.handleDataStream(ctx, reqServerStream)
-	case quicpogs.RPCStreamProtocolSignature:
-		rpcStream, err := quicpogs.NewRPCServerStream(stream, signature)
-		if err != nil {
-			return err
-		}
-		return q.handleRPCStream(rpcStream)
-	default:
-		return fmt.Errorf("unknown protocol %v", signature)
-	}
-}
-
-func (q *QUICConnection) handleDataStream(ctx context.Context, stream *quicpogs.RequestServerStream) error {
+func (q *QUICConnection) handleDataStream(ctx context.Context, stream *rpcquic.RequestServerStream) error {
 	request, err := stream.ReadConnectRequestData()
 	if err != nil {
 		return err
@@ -264,22 +270,22 @@ func (q *QUICConnection) handleDataStream(ctx context.Context, stream *quicpogs.
 // dispatchRequest will dispatch the request depending on the type and returns an error if it occurs.
 // More importantly, it also tells if the during processing of the request the ConnectResponse metadata was sent downstream.
 // This is important since it informs
-func (q *QUICConnection) dispatchRequest(ctx context.Context, stream *quicpogs.RequestServerStream, err error, request *quicpogs.ConnectRequest) (error, bool) {
+func (q *QUICConnection) dispatchRequest(ctx context.Context, stream *rpcquic.RequestServerStream, err error, request *pogs.ConnectRequest) (error, bool) {
 	originProxy, err := q.orchestrator.GetOriginProxy()
 	if err != nil {
 		return err, false
 	}
 
 	switch request.Type {
-	case quicpogs.ConnectionTypeHTTP, quicpogs.ConnectionTypeWebsocket:
+	case pogs.ConnectionTypeHTTP, pogs.ConnectionTypeWebsocket:
 		tracedReq, err := buildHTTPRequest(ctx, request, stream, q.connIndex, q.logger)
 		if err != nil {
 			return err, false
 		}
 		w := newHTTPResponseAdapter(stream)
-		return originProxy.ProxyHTTP(&w, tracedReq, request.Type == quicpogs.ConnectionTypeWebsocket), w.connectResponseSent
+		return originProxy.ProxyHTTP(&w, tracedReq, request.Type == pogs.ConnectionTypeWebsocket), w.connectResponseSent
 
-	case quicpogs.ConnectionTypeTCP:
+	case pogs.ConnectionTypeTCP:
 		rwa := &streamReadWriteAcker{RequestServerStream: stream}
 		metadata := request.MetadataMap()
 		return originProxy.ProxyTCP(ctx, rwa, &TCPRequest{
@@ -293,14 +299,6 @@ func (q *QUICConnection) dispatchRequest(ctx context.Context, stream *quicpogs.R
 	}
 }
 
-func (q *QUICConnection) handleRPCStream(rpcStream *quicpogs.RPCServerStream) error {
-	if err := rpcStream.Serve(q, q, q.logger); err != nil {
-		q.logger.Err(err).Msg("failed handling RPC stream")
-	}
-
-	return nil
-}
-
 // RegisterUdpSession is the RPC method invoked by edge to register and run a session
 func (q *QUICConnection) RegisterUdpSession(ctx context.Context, sessionID uuid.UUID, dstIP net.IP, dstPort uint16, closeAfterIdleHint time.Duration, traceContext string) (*tunnelpogs.RegisterUdpSessionResponse, error) {
 	traceCtx := tracing.NewTracedContext(ctx, traceContext, q.logger)
@@ -377,9 +375,9 @@ func (q *QUICConnection) closeUDPSession(ctx context.Context, sessionID uuid.UUI
 		return
 	}
 
-	stream := quicpogs.NewSafeStreamCloser(quicStream, q.streamWriteTimeout, q.logger)
+	stream := cfdquic.NewSafeStreamCloser(quicStream, q.streamWriteTimeout, q.logger)
 	defer stream.Close()
-	rpcClientStream, err := quicpogs.NewRPCClientStream(ctx, stream, q.udpUnregisterTimeout, q.logger)
+	rpcClientStream, err := rpcquic.NewSessionClient(ctx, stream, q.rpcTimeout)
 	if err != nil {
 		// Log this at debug because this is not an error if session was closed due to lost connection
 		// with edge
@@ -408,16 +406,16 @@ func (q *QUICConnection) UpdateConfiguration(ctx context.Context, version int32,
 // streamReadWriteAcker is a light wrapper over QUIC streams with a callback to send response back to
 // the client.
 type streamReadWriteAcker struct {
-	*quicpogs.RequestServerStream
+	*rpcquic.RequestServerStream
 	connectResponseSent bool
 }
 
 // AckConnection acks response back to the proxy.
 func (s *streamReadWriteAcker) AckConnection(tracePropagation string) error {
-	metadata := []quicpogs.Metadata{}
+	metadata := []pogs.Metadata{}
 	// Only add tracing if provided by origintunneld
 	if tracePropagation != "" {
-		metadata = append(metadata, quicpogs.Metadata{
+		metadata = append(metadata, pogs.Metadata{
 			Key: tracing.CanonicalCloudflaredTracingHeader,
 			Val: tracePropagation,
 		})
@@ -428,12 +426,12 @@ func (s *streamReadWriteAcker) AckConnection(tracePropagation string) error {
 
 // httpResponseAdapter translates responses written by the HTTP Proxy into ones that can be used in QUIC.
 type httpResponseAdapter struct {
-	*quicpogs.RequestServerStream
+	*rpcquic.RequestServerStream
 	headers             http.Header
 	connectResponseSent bool
 }
 
-func newHTTPResponseAdapter(s *quicpogs.RequestServerStream) httpResponseAdapter {
+func newHTTPResponseAdapter(s *rpcquic.RequestServerStream) httpResponseAdapter {
 	return httpResponseAdapter{RequestServerStream: s, headers: make(http.Header)}
 }
 
@@ -442,12 +440,12 @@ func (hrw *httpResponseAdapter) AddTrailer(trailerName, trailerValue string) {
 }
 
 func (hrw *httpResponseAdapter) WriteRespHeaders(status int, header http.Header) error {
-	metadata := make([]quicpogs.Metadata, 0)
-	metadata = append(metadata, quicpogs.Metadata{Key: "HttpStatus", Val: strconv.Itoa(status)})
+	metadata := make([]pogs.Metadata, 0)
+	metadata = append(metadata, pogs.Metadata{Key: "HttpStatus", Val: strconv.Itoa(status)})
 	for k, vv := range header {
 		for _, v := range vv {
 			httpHeaderKey := fmt.Sprintf("%s:%s", HTTPHeaderKey, k)
-			metadata = append(metadata, quicpogs.Metadata{Key: httpHeaderKey, Val: v})
+			metadata = append(metadata, pogs.Metadata{Key: httpHeaderKey, Val: v})
 		}
 	}
 
@@ -483,17 +481,17 @@ func (hrw *httpResponseAdapter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
 }
 
 func (hrw *httpResponseAdapter) WriteErrorResponse(err error) {
-	hrw.WriteConnectResponseData(err, quicpogs.Metadata{Key: "HttpStatus", Val: strconv.Itoa(http.StatusBadGateway)})
+	hrw.WriteConnectResponseData(err, pogs.Metadata{Key: "HttpStatus", Val: strconv.Itoa(http.StatusBadGateway)})
 }
 
-func (hrw *httpResponseAdapter) WriteConnectResponseData(respErr error, metadata ...quicpogs.Metadata) error {
+func (hrw *httpResponseAdapter) WriteConnectResponseData(respErr error, metadata ...pogs.Metadata) error {
 	hrw.connectResponseSent = true
 	return hrw.RequestServerStream.WriteConnectResponseData(respErr, metadata...)
 }
 
 func buildHTTPRequest(
 	ctx context.Context,
-	connectRequest *quicpogs.ConnectRequest,
+	connectRequest *pogs.ConnectRequest,
 	body io.ReadCloser,
 	connIndex uint8,
 	log *zerolog.Logger,
@@ -502,7 +500,6 @@ func buildHTTPRequest(
 	dest := connectRequest.Dest
 	method := metadata[HTTPMethodKey]
 	host := metadata[HTTPHostKey]
-	isWebsocket := connectRequest.Type == quicpogs.ConnectionTypeWebsocket
 
 	req, err := http.NewRequestWithContext(ctx, method, dest, body)
 	if err != nil {
@@ -527,13 +524,8 @@ func buildHTTPRequest(
 		return nil, fmt.Errorf("Error setting content-length: %w", err)
 	}
 
-	// Go's client defaults to chunked encoding after a 200ms delay if the following cases are true:
-	//   * the request body blocks
-	//   * the content length is not set (or set to -1)
-	//   * the method doesn't usually have a body (GET, HEAD, DELETE, ...)
-	//   * there is no transfer-encoding=chunked already set.
-	// So, if transfer cannot be chunked and content length is 0, we dont set a request body.
-	if !isWebsocket && !isTransferEncodingChunked(req) && req.ContentLength == 0 {
+	if shouldSetRequestBodyToEmpty(connectRequest, metadata, req) {
+		log.Debug().Str("host", req.Host).Str("method", req.Method).Msg("Set request to have no body")
 		req.Body = http.NoBody
 	}
 	stripWebsocketUpgradeHeader(req)
@@ -558,6 +550,35 @@ func isTransferEncodingChunked(req *http.Request) bool {
 	return strings.Contains(strings.ToLower(transferEncodingVal), "chunked")
 }
 
+// Borrowed from https://github.com/golang/go/blob/go1.22.6/src/net/http/request.go#L1541
+func requestMethodUsuallyLacksBody(req *http.Request) bool {
+	switch strings.ToUpper(req.Method) {
+	case "GET", "HEAD", "DELETE", "OPTIONS", "PROPFIND", "SEARCH":
+		return true
+	}
+	return false
+}
+
+func shouldSetRequestBodyToEmpty(connectRequest *pogs.ConnectRequest, metadata map[string]string, req *http.Request) bool {
+	switch metadata[HTTPRequestBodyHintKey] {
+	case RequestBodyHintEmpty.String():
+		return true
+	case RequestBodyHintHasData.String():
+		return false
+	default:
+	}
+
+	isWebsocket := connectRequest.Type == pogs.ConnectionTypeWebsocket
+	// Go's client defaults to chunked encoding after a 200ms delay if the following cases are true:
+	//   * the request body blocks
+	//   * the content length is not set (or set to -1)
+	//   * the method doesn't usually have a body (GET, HEAD, DELETE, ...)
+	//   * there is no transfer-encoding=chunked already set.
+	// So, if transfer cannot be chunked and content length is 0, we dont set a request body.
+	// Reference: https://github.com/golang/go/blob/go1.22.2/src/net/http/transfer.go#L192-L206
+	return !isWebsocket && requestMethodUsuallyLacksBody(req) && !isTransferEncodingChunked(req) && req.ContentLength == 0
+}
+
 // A helper struct that guarantees a call to close only affects read side, but not write side.
 type nopCloserReadWriter struct {
 	io.ReadWriteCloser
@@ -597,11 +618,11 @@ func (np *nopCloserReadWriter) Close() error {
 
 // muxerWrapper wraps DatagramMuxerV2 to satisfy the packet.FunnelUniPipe interface
 type muxerWrapper struct {
-	muxer *quicpogs.DatagramMuxerV2
+	muxer *cfdquic.DatagramMuxerV2
 }
 
 func (rp *muxerWrapper) SendPacket(dst netip.Addr, pk packet.RawPacket) error {
-	return rp.muxer.SendPacket(quicpogs.RawPacket(pk))
+	return rp.muxer.SendPacket(cfdquic.RawPacket(pk))
 }
 
 func (rp *muxerWrapper) ReceivePacket(ctx context.Context) (packet.RawPacket, error) {
@@ -609,7 +630,7 @@ func (rp *muxerWrapper) ReceivePacket(ctx context.Context) (packet.RawPacket, er
 	if err != nil {
 		return packet.RawPacket{}, err
 	}
-	rawPacket, ok := pk.(quicpogs.RawPacket)
+	rawPacket, ok := pk.(cfdquic.RawPacket)
 	if ok {
 		return packet.RawPacket(rawPacket), nil
 	}

connection/quic_test.go

@@ -3,9 +3,14 @@ package connection
 import (
 	"bytes"
 	"context"
+	"crypto/rand"
+	"crypto/rsa"
 	"crypto/tls"
+	"crypto/x509"
+	"encoding/pem"
 	"fmt"
 	"io"
+	"math/big"
 	"net"
 	"net/http"
 	"net/url"
@@ -23,14 +28,15 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/cloudflare/cloudflared/datagramsession"
-	quicpogs "github.com/cloudflare/cloudflared/quic"
+	cfdquic "github.com/cloudflare/cloudflared/quic"
 	"github.com/cloudflare/cloudflared/tracing"
 	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	rpcquic "github.com/cloudflare/cloudflared/tunnelrpc/quic"
 )
 
 var (
-	testTLSServerConfig = quicpogs.GenerateTLSConfig()
+	testTLSServerConfig = GenerateTLSConfig()
 	testQUICConfig      = &quic.Config{
 		KeepAlivePeriod: 5 * time.Second,
 		EnableDatagrams: true,
@@ -50,16 +56,16 @@ func TestQUICServer(t *testing.T) {
 	var tests = []struct {
 		desc             string
 		dest             string
-		connectionType   quicpogs.ConnectionType
-		metadata         []quicpogs.Metadata
+		connectionType   pogs.ConnectionType
+		metadata         []pogs.Metadata
 		message          []byte
 		expectedResponse []byte
 	}{
 		{
 			desc:           "test http proxy",
 			dest:           "/ok",
-			connectionType: quicpogs.ConnectionTypeHTTP,
-			metadata: []quicpogs.Metadata{
+			connectionType: pogs.ConnectionTypeHTTP,
+			metadata: []pogs.Metadata{
 				{
 					Key: "HttpHeader:Cf-Ray",
 					Val: "123123123",
@@ -78,8 +84,8 @@ func TestQUICServer(t *testing.T) {
 		{
 			desc:           "test http body request streaming",
 			dest:           "/slow_echo_body",
-			connectionType: quicpogs.ConnectionTypeHTTP,
-			metadata: []quicpogs.Metadata{
+			connectionType: pogs.ConnectionTypeHTTP,
+			metadata: []pogs.Metadata{
 				{
 					Key: "HttpHeader:Cf-Ray",
 					Val: "123123123",
@@ -103,8 +109,8 @@ func TestQUICServer(t *testing.T) {
 		{
 			desc:           "test ws proxy",
 			dest:           "/ws/echo",
-			connectionType: quicpogs.ConnectionTypeWebsocket,
-			metadata: []quicpogs.Metadata{
+			connectionType: pogs.ConnectionTypeWebsocket,
+			metadata: []pogs.Metadata{
 				{
 					Key: "HttpHeader:Cf-Cloudflared-Proxy-Connection-Upgrade",
 					Val: "Websocket",
@@ -127,8 +133,8 @@ func TestQUICServer(t *testing.T) {
 		},
 		{
 			desc:             "test tcp proxy",
-			connectionType:   quicpogs.ConnectionTypeTCP,
-			metadata:         []quicpogs.Metadata{},
+			connectionType:   pogs.ConnectionTypeTCP,
+			metadata:         []pogs.Metadata{},
 			message:          []byte("Here is some tcp data"),
 			expectedResponse: []byte("Here is some tcp data"),
 		},
@@ -175,7 +181,7 @@ type fakeControlStream struct {
 	ControlStreamHandler
 }
 
-func (fakeControlStream) ServeControlStream(ctx context.Context, rw io.ReadWriteCloser, connOptions *tunnelpogs.ConnectionOptions, tunnelConfigGetter TunnelConfigJSONGetter) error {
+func (fakeControlStream) ServeControlStream(ctx context.Context, rw io.ReadWriteCloser, connOptions *pogs.ConnectionOptions, tunnelConfigGetter TunnelConfigJSONGetter) error {
 	<-ctx.Done()
 	return nil
 }
@@ -188,8 +194,8 @@ func quicServer(
 	t *testing.T,
 	listener *quic.Listener,
 	dest string,
-	connectionType quicpogs.ConnectionType,
-	metadata []quicpogs.Metadata,
+	connectionType pogs.ConnectionType,
+	metadata []pogs.Metadata,
 	message []byte,
 	expectedResponse []byte,
 ) {
@@ -198,9 +204,9 @@ func quicServer(
 
 	quicStream, err := session.OpenStreamSync(context.Background())
 	require.NoError(t, err)
-	stream := quicpogs.NewSafeStreamCloser(quicStream, defaultQUICTimeout, &log)
+	stream := cfdquic.NewSafeStreamCloser(quicStream, defaultQUICTimeout, &log)
 
-	reqClientStream := quicpogs.RequestClientStream{ReadWriteCloser: stream}
+	reqClientStream := rpcquic.RequestClientStream{ReadWriteCloser: stream}
 	err = reqClientStream.WriteConnectRequestData(dest, connectionType, metadata...)
 	require.NoError(t, err)
 
@@ -265,15 +271,15 @@ func (moc *mockOriginProxyWithRequest) ProxyHTTP(w ResponseWriter, tr *tracing.T
 func TestBuildHTTPRequest(t *testing.T) {
 	var tests = []struct {
 		name           string
-		connectRequest *quicpogs.ConnectRequest
+		connectRequest *pogs.ConnectRequest
 		body           io.ReadCloser
 		req            *http.Request
 	}{
 		{
 			name: "check if http.Request is built correctly with content length",
-			connectRequest: &quicpogs.ConnectRequest{
+			connectRequest: &pogs.ConnectRequest{
 				Dest: "http://test.com",
-				Metadata: []quicpogs.Metadata{
+				Metadata: []pogs.Metadata{
 					{
 						Key: "HttpHeader:Cf-Cloudflared-Proxy-Connection-Upgrade",
 						Val: "Websocket",
@@ -317,9 +323,9 @@ func TestBuildHTTPRequest(t *testing.T) {
 		},
 		{
 			name: "if content length isn't part of request headers, then it's not set",
-			connectRequest: &quicpogs.ConnectRequest{
+			connectRequest: &pogs.ConnectRequest{
 				Dest: "http://test.com",
-				Metadata: []quicpogs.Metadata{
+				Metadata: []pogs.Metadata{
 					{
 						Key: "HttpHeader:Cf-Cloudflared-Proxy-Connection-Upgrade",
 						Val: "Websocket",
@@ -358,9 +364,9 @@ func TestBuildHTTPRequest(t *testing.T) {
 		},
 		{
 			name: "if content length is 0, but transfer-encoding is chunked, body is not nil",
-			connectRequest: &quicpogs.ConnectRequest{
+			connectRequest: &pogs.ConnectRequest{
 				Dest: "http://test.com",
-				Metadata: []quicpogs.Metadata{
+				Metadata: []pogs.Metadata{
 					{
 						Key: "HttpHeader:Another-Header",
 						Val: "Misc",
@@ -400,9 +406,9 @@ func TestBuildHTTPRequest(t *testing.T) {
 		},
 		{
 			name: "if content length is 0, but transfer-encoding is gzip,chunked, body is not nil",
-			connectRequest: &quicpogs.ConnectRequest{
+			connectRequest: &pogs.ConnectRequest{
 				Dest: "http://test.com",
-				Metadata: []quicpogs.Metadata{
+				Metadata: []pogs.Metadata{
 					{
 						Key: "HttpHeader:Another-Header",
 						Val: "Misc",
@@ -442,10 +448,10 @@ func TestBuildHTTPRequest(t *testing.T) {
 		},
 		{
 			name: "if content length is 0, and connect request is a websocket, body is not nil",
-			connectRequest: &quicpogs.ConnectRequest{
-				Type: quicpogs.ConnectionTypeWebsocket,
+			connectRequest: &pogs.ConnectRequest{
+				Type: pogs.ConnectionTypeWebsocket,
 				Dest: "http://test.com",
-				Metadata: []quicpogs.Metadata{
+				Metadata: []pogs.Metadata{
 					{
 						Key: "HttpHeader:Another-Header",
 						Val: "Misc",
@@ -478,6 +484,125 @@ func TestBuildHTTPRequest(t *testing.T) {
 			},
 			body: io.NopCloser(&bytes.Buffer{}),
 		},
+		{
+			name: "if edge sends the body is empty hint, set body to empty",
+			connectRequest: &pogs.ConnectRequest{
+				Dest: "http://test.com",
+				Metadata: []pogs.Metadata{
+					{
+						Key: "HttpHeader:Another-Header",
+						Val: "Misc",
+					},
+					{
+						Key: "HttpHost",
+						Val: "cf.host",
+					},
+					{
+						Key: "HttpMethod",
+						Val: "put",
+					},
+					{
+						Key: HTTPRequestBodyHintKey,
+						Val: RequestBodyHintEmpty.String(),
+					},
+				},
+			},
+			req: &http.Request{
+				Method: "put",
+				URL: &url.URL{
+					Scheme: "http",
+					Host:   "test.com",
+				},
+				Proto:      "HTTP/1.1",
+				ProtoMajor: 1,
+				ProtoMinor: 1,
+				Header: http.Header{
+					"Another-Header": []string{"Misc"},
+				},
+				ContentLength: 0,
+				Host:          "cf.host",
+				Body:          http.NoBody,
+			},
+			body: io.NopCloser(&bytes.Buffer{}),
+		},
+		{
+			name: "if edge sends the body has data hint, don't set body to empty",
+			connectRequest: &pogs.ConnectRequest{
+				Dest: "http://test.com",
+				Metadata: []pogs.Metadata{
+					{
+						Key: "HttpHeader:Another-Header",
+						Val: "Misc",
+					},
+					{
+						Key: "HttpHost",
+						Val: "cf.host",
+					},
+					{
+						Key: "HttpMethod",
+						Val: "put",
+					},
+					{
+						Key: HTTPRequestBodyHintKey,
+						Val: RequestBodyHintHasData.String(),
+					},
+				},
+			},
+			req: &http.Request{
+				Method: "put",
+				URL: &url.URL{
+					Scheme: "http",
+					Host:   "test.com",
+				},
+				Proto:      "HTTP/1.1",
+				ProtoMajor: 1,
+				ProtoMinor: 1,
+				Header: http.Header{
+					"Another-Header": []string{"Misc"},
+				},
+				ContentLength: 0,
+				Host:          "cf.host",
+				Body:          io.NopCloser(&bytes.Buffer{}),
+			},
+			body: io.NopCloser(&bytes.Buffer{}),
+		},
+		{
+			name: "if the http method usually has body, don't set body to empty",
+			connectRequest: &pogs.ConnectRequest{
+				Dest: "http://test.com",
+				Metadata: []pogs.Metadata{
+					{
+						Key: "HttpHeader:Another-Header",
+						Val: "Misc",
+					},
+					{
+						Key: "HttpHost",
+						Val: "cf.host",
+					},
+					{
+						Key: "HttpMethod",
+						Val: "post",
+					},
+				},
+			},
+			req: &http.Request{
+				Method: "post",
+				URL: &url.URL{
+					Scheme: "http",
+					Host:   "test.com",
+				},
+				Proto:      "HTTP/1.1",
+				ProtoMajor: 1,
+				ProtoMinor: 1,
+				Header: http.Header{
+					"Another-Header": []string{"Misc"},
+				},
+				ContentLength: 0,
+				Host:          "cf.host",
+				Body:          io.NopCloser(&bytes.Buffer{}),
+			},
+			body: io.NopCloser(&bytes.Buffer{}),
+		},
 	}
 
 	log := zerolog.Nop()
@@ -617,9 +742,9 @@ func serveSession(ctx context.Context, qc *QUICConnection, edgeQUICSession quic.
 	}()
 
 	// Send a message to the quic session on edge side, it should be deumx to this datagram v2 session
-	muxedPayload, err := quicpogs.SuffixSessionID(sessionID, payload)
+	muxedPayload, err := cfdquic.SuffixSessionID(sessionID, payload)
 	require.NoError(t, err)
-	muxedPayload, err = quicpogs.SuffixType(muxedPayload, quicpogs.DatagramTypeUDP)
+	muxedPayload, err = cfdquic.SuffixType(muxedPayload, cfdquic.DatagramTypeUDP)
 	require.NoError(t, err)
 
 	err = edgeQUICSession.SendDatagram(muxedPayload)
@@ -665,7 +790,7 @@ const (
 	closedByTimeout
 )
 
-func runRPCServer(ctx context.Context, session quic.Connection, sessionRPCServer tunnelpogs.SessionManager, configRPCServer tunnelpogs.ConfigurationManager, t *testing.T) {
+func runRPCServer(ctx context.Context, session quic.Connection, sessionRPCServer pogs.SessionManager, configRPCServer pogs.ConfigurationManager, t *testing.T) {
 	stream, err := session.AcceptStream(ctx)
 	require.NoError(t, err)
 
@@ -674,13 +799,15 @@ func runRPCServer(ctx context.Context, session quic.Connection, sessionRPCServer
 		stream, err = session.AcceptStream(ctx)
 		require.NoError(t, err)
 	}
-	protocol, err := quicpogs.DetermineProtocol(stream)
-	assert.NoError(t, err)
-	rpcServerStream, err := quicpogs.NewRPCServerStream(stream, protocol)
-	assert.NoError(t, err)
-
-	log := zerolog.New(os.Stdout)
-	err = rpcServerStream.Serve(sessionRPCServer, configRPCServer, &log)
+	ss := rpcquic.NewCloudflaredServer(
+		func(_ context.Context, _ *rpcquic.RequestServerStream) error {
+			return nil
+		},
+		sessionRPCServer,
+		configRPCServer,
+		10*time.Second,
+	)
+	err = ss.Serve(ctx, stream)
 	assert.NoError(t, err)
 }
 
@@ -726,7 +853,8 @@ func testQUICConnection(udpListenerAddr net.Addr, t *testing.T, index uint8) *QU
 		fakeControlStream{},
 		&log,
 		nil,
-		5*time.Second,
+		15*time.Second,
+		0*time.Second,
 		0*time.Second,
 	)
 	require.NoError(t, err)
@@ -744,3 +872,27 @@ func (m *mockReaderNoopWriter) Write(p []byte) (n int, err error) {
 func (m *mockReaderNoopWriter) Close() error {
 	return nil
 }
+
+// GenerateTLSConfig sets up a bare-bones TLS config for a QUIC server
+func GenerateTLSConfig() *tls.Config {
+	key, err := rsa.GenerateKey(rand.Reader, 1024)
+	if err != nil {
+		panic(err)
+	}
+	template := x509.Certificate{SerialNumber: big.NewInt(1)}
+	certDER, err := x509.CreateCertificate(rand.Reader, &template, &template, &key.PublicKey, key)
+	if err != nil {
+		panic(err)
+	}
+	keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)})
+	certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER})
+
+	tlsCert, err := tls.X509KeyPair(certPEM, keyPEM)
+	if err != nil {
+		panic(err)
+	}
+	return &tls.Config{
+		Certificates: []tls.Certificate{tlsCert},
+		NextProtos:   []string{"argotunnel"},
+	}
+}

connection/rpc.go

@@ -1,153 +0,0 @@
-package connection
-
-import (
-	"context"
-	"io"
-	"net"
-	"time"
-
-	"github.com/rs/zerolog"
-	"zombiezen.com/go/capnproto2/rpc"
-
-	"github.com/cloudflare/cloudflared/tunnelrpc"
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
-)
-
-type tunnelServerClient struct {
-	client    tunnelpogs.TunnelServer_PogsClient
-	transport rpc.Transport
-}
-
-// NewTunnelRPCClient creates and returns a new RPC client, which will communicate using a stream on the given muxer.
-// This method is exported for supervisor to call Authenticate RPC
-func NewTunnelServerClient(
-	ctx context.Context,
-	stream io.ReadWriteCloser,
-	log *zerolog.Logger,
-) *tunnelServerClient {
-	transport := tunnelrpc.NewTransportLogger(log, rpc.StreamTransport(stream))
-	conn := rpc.NewConn(
-		transport,
-		tunnelrpc.ConnLog(log),
-	)
-	registrationClient := tunnelpogs.RegistrationServer_PogsClient{Client: conn.Bootstrap(ctx), Conn: conn}
-	return &tunnelServerClient{
-		client:    tunnelpogs.TunnelServer_PogsClient{RegistrationServer_PogsClient: registrationClient, Client: conn.Bootstrap(ctx), Conn: conn},
-		transport: transport,
-	}
-}
-
-func (tsc *tunnelServerClient) Authenticate(ctx context.Context, classicTunnel *ClassicTunnelProperties, registrationOptions *tunnelpogs.RegistrationOptions) (tunnelpogs.AuthOutcome, error) {
-	authResp, err := tsc.client.Authenticate(ctx, classicTunnel.OriginCert, classicTunnel.Hostname, registrationOptions)
-	if err != nil {
-		return nil, err
-	}
-	return authResp.Outcome(), nil
-}
-
-func (tsc *tunnelServerClient) Close() {
-	// Closing the client will also close the connection
-	_ = tsc.client.Close()
-	_ = tsc.transport.Close()
-}
-
-type NamedTunnelRPCClient interface {
-	RegisterConnection(
-		c context.Context,
-		config *NamedTunnelProperties,
-		options *tunnelpogs.ConnectionOptions,
-		connIndex uint8,
-		edgeAddress net.IP,
-		observer *Observer,
-	) (*tunnelpogs.ConnectionDetails, error)
-	SendLocalConfiguration(
-		c context.Context,
-		config []byte,
-		observer *Observer,
-	) error
-	GracefulShutdown(ctx context.Context, gracePeriod time.Duration)
-	Close()
-}
-
-type registrationServerClient struct {
-	client    tunnelpogs.RegistrationServer_PogsClient
-	transport rpc.Transport
-}
-
-func newRegistrationRPCClient(
-	ctx context.Context,
-	stream io.ReadWriteCloser,
-	log *zerolog.Logger,
-) NamedTunnelRPCClient {
-	transport := tunnelrpc.NewTransportLogger(log, rpc.StreamTransport(stream))
-	conn := rpc.NewConn(
-		transport,
-		tunnelrpc.ConnLog(log),
-	)
-	return &registrationServerClient{
-		client:    tunnelpogs.RegistrationServer_PogsClient{Client: conn.Bootstrap(ctx), Conn: conn},
-		transport: transport,
-	}
-}
-
-func (rsc *registrationServerClient) RegisterConnection(
-	ctx context.Context,
-	properties *NamedTunnelProperties,
-	options *tunnelpogs.ConnectionOptions,
-	connIndex uint8,
-	edgeAddress net.IP,
-	observer *Observer,
-) (*tunnelpogs.ConnectionDetails, error) {
-	conn, err := rsc.client.RegisterConnection(
-		ctx,
-		properties.Credentials.Auth(),
-		properties.Credentials.TunnelID,
-		connIndex,
-		options,
-	)
-	if err != nil {
-		if err.Error() == DuplicateConnectionError {
-			observer.metrics.regFail.WithLabelValues("dup_edge_conn", "registerConnection").Inc()
-			return nil, errDuplicationConnection
-		}
-		observer.metrics.regFail.WithLabelValues("server_error", "registerConnection").Inc()
-		return nil, serverRegistrationErrorFromRPC(err)
-	}
-
-	observer.metrics.regSuccess.WithLabelValues("registerConnection").Inc()
-
-	return conn, nil
-}
-
-func (rsc *registrationServerClient) SendLocalConfiguration(ctx context.Context, config []byte, observer *Observer) (err error) {
-	observer.metrics.localConfigMetrics.pushes.Inc()
-	defer func() {
-		if err != nil {
-			observer.metrics.localConfigMetrics.pushesErrors.Inc()
-		}
-	}()
-
-	return rsc.client.SendLocalConfiguration(ctx, config)
-}
-
-func (rsc *registrationServerClient) GracefulShutdown(ctx context.Context, gracePeriod time.Duration) {
-	ctx, cancel := context.WithTimeout(ctx, gracePeriod)
-	defer cancel()
-	_ = rsc.client.UnregisterConnection(ctx)
-}
-
-func (rsc *registrationServerClient) Close() {
-	// Closing the client will also close the connection
-	_ = rsc.client.Close()
-	// Closing the transport also closes the stream
-	_ = rsc.transport.Close()
-}
-
-type rpcName string
-
-const (
-	register     rpcName = "register"
-	reconnect    rpcName = "reconnect"
-	unregister   rpcName = "unregister"
-	authenticate rpcName = " authenticate"
-)

datagramsession/metrics.go

@@ -15,7 +15,7 @@ var (
 		Name:      "active_sessions",
 		Help:      "Concurrent count of UDP sessions that are being proxied to any origin",
 	})
-	totalUDPSessions = prometheus.NewGauge(prometheus.GaugeOpts{
+	totalUDPSessions = prometheus.NewCounter(prometheus.CounterOpts{
 		Namespace: namespace,
 		Subsystem: "udp",
 		Name:      "total_sessions",

datagramsession/session.go

@@ -57,7 +57,7 @@ func (s *Session) Serve(ctx context.Context, closeAfterIdle time.Duration) (clos
 		readBuffer := make([]byte, maxPacketSize)
 		for {
 			if closeSession, err := s.dstToTransport(readBuffer); err != nil {
-				if errors.Is(err, net.ErrClosed) {
+				if errors.Is(err, net.ErrClosed) || errors.Is(err, io.EOF) {
 					s.log.Debug().Msg("Destination connection closed")
 				} else {
 					level := zerolog.ErrorLevel

github_release.py

@@ -17,7 +17,7 @@ import re
 from github import Github, GithubException, UnknownObjectException
 
 FORMAT = "%(levelname)s - %(asctime)s: %(message)s"
-logging.basicConfig(format=FORMAT)
+logging.basicConfig(format=FORMAT, level=logging.INFO)
 
 CLOUDFLARED_REPO = os.environ.get("GITHUB_REPO", "cloudflare/cloudflared")
 GITHUB_CONFLICT_CODE = "already_exists"
@@ -214,24 +214,33 @@ def main():
     """ Attempts to upload Asset to Github Release. Creates Release if it doesn't exist """
     try:
         args = parse_args()
-        client = Github(args.api_key)
-        repo = client.get_repo(CLOUDFLARED_REPO)
-        release = get_or_create_release(repo, args.release_version, args.dry_run)
 
         if args.dry_run:
-            logging.info("Skipping asset upload because of dry-run")
+            if os.path.isdir(args.path):
+                onlyfiles = [f for f in listdir(args.path) if isfile(join(args.path, f))]
+                for filename in onlyfiles:
+                    binary_path = os.path.join(args.path, filename)
+                    logging.info("binary: " + binary_path)
+            elif os.path.isfile(args.path):
+                logging.info("binary: " + binary_path)
+            else:
+                logging.error("dryrun failed")
             return
-
-        if os.path.isdir(args.path):
-            onlyfiles = [f for f in listdir(args.path) if isfile(join(args.path, f))]
-            for filename in onlyfiles:
-                binary_path = os.path.join(args.path, filename)
-                upload_asset(release, binary_path, filename, args.release_version, args.kv_account_id, args.namespace_id,
-                args.kv_api_token)
-                move_asset(binary_path, filename)
         else:
-            upload_asset(release, args.path, args.name, args.release_version, args.kv_account_id, args.namespace_id,
-                args.kv_api_token)
+            client = Github(args.api_key)
+            repo = client.get_repo(CLOUDFLARED_REPO)
+            release = get_or_create_release(repo, args.release_version, args.dry_run)
+
+            if os.path.isdir(args.path):
+                onlyfiles = [f for f in listdir(args.path) if isfile(join(args.path, f))]
+                for filename in onlyfiles:
+                    binary_path = os.path.join(args.path, filename)
+                    upload_asset(release, binary_path, filename, args.release_version, args.kv_account_id, args.namespace_id,
+                    args.kv_api_token)
+                    move_asset(binary_path, filename)
+            else:
+                upload_asset(release, args.path, args.name, args.release_version, args.kv_account_id, args.namespace_id,
+                    args.kv_api_token)
 
     except Exception as e:
         logging.exception(e)

go.mod

@@ -23,9 +23,9 @@ require (
 	github.com/miekg/dns v1.1.50
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/pkg/errors v0.9.1
-	github.com/prometheus/client_golang v1.13.0
-	github.com/prometheus/client_model v0.2.0
-	github.com/quic-go/quic-go v0.42.0
+	github.com/prometheus/client_golang v1.19.1
+	github.com/prometheus/client_model v0.5.0
+	github.com/quic-go/quic-go v0.45.0
 	github.com/rs/zerolog v1.20.0
 	github.com/stretchr/testify v1.9.0
 	github.com/urfave/cli/v2 v2.3.0
@@ -38,7 +38,7 @@ require (
 	go.uber.org/automaxprocs v1.4.0
 	golang.org/x/crypto v0.23.0
 	golang.org/x/net v0.25.0
-	golang.org/x/sync v0.6.0
+	golang.org/x/sync v0.7.0
 	golang.org/x/sys v0.20.0
 	golang.org/x/term v0.20.0
 	google.golang.org/protobuf v1.34.1
@@ -80,16 +80,16 @@ require (
 	github.com/onsi/ginkgo/v2 v2.9.5 // indirect
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/common v0.37.0 // indirect
-	github.com/prometheus/procfs v0.8.0 // indirect
+	github.com/prometheus/common v0.48.0 // indirect
+	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	go.opentelemetry.io/otel/metric v1.26.0 // indirect
 	go.uber.org/mock v0.4.0 // indirect
-	golang.org/x/exp v0.0.0-20221205204356-47842c84f3db // indirect
-	golang.org/x/mod v0.11.0 // indirect
+	golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect
+	golang.org/x/mod v0.17.0 // indirect
 	golang.org/x/oauth2 v0.17.0 // indirect
 	golang.org/x/text v0.15.0 // indirect
-	golang.org/x/tools v0.9.1 // indirect
+	golang.org/x/tools v0.21.0 // indirect
 	google.golang.org/appengine v1.6.8 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de // indirect

go.sum

@@ -1,61 +1,15 @@
-cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
-cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
-cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
-cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
-cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
-cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
-cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
-cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
-cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
-cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=
-cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
-cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
-cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
-cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
-cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
-cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
-cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
-cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
-cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
-cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
-cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
-cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
-cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
-cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
-cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
-cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
-cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
-cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
-cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
-cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
-dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/toml v1.2.0 h1:Rt8g24XnyGTyglgET/PRUNlrUeu9F5L+7FilkXfZgs0=
 github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
-github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
-github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
-github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
-github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
-github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
-github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
 github.com/apparentlymart/go-cidr v1.1.0 h1:2mAhrMoF+nhXqxTzSZMUzDHkLjmIHC+Zzn4tdgBZjnU=
 github.com/apparentlymart/go-cidr v1.1.0/go.mod h1:EBcsNrHc3zQeuaeCeCtQruQm+n9/YjEn/vI25Lg7Gwc=
-github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
-github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
-github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
-github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
 github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
-github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
-github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/coredns/caddy v1.1.1 h1:2eYKZT7i6yxIfGP3qLJoJ7HAsDJqYB+X68g4NYjSrE0=
 github.com/coredns/caddy v1.1.1/go.mod h1:A6ntJQlAWuQfFlsd9hvigKbo2WS0VUs2l1e2F+BawD4=
 github.com/coredns/coredns v1.10.0 h1:jCfuWsBjTs0dapkkhISfPCzn5LqvSRtrFtaf/Tjj4DI=
@@ -72,10 +26,6 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
-github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
-github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
-github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 h1:0JZ+dUmQeA8IIVUMzysrX4/AKuQwWhV2dYQuPZdvdSQ=
 github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64=
 github.com/facebookgo/freeport v0.0.0-20150612182905-d4adf43b75b9 h1:wWke/RUCl7VRjQhwPlR/v0glZXNYzBHdNUzf/Am2Nmg=
@@ -105,19 +55,8 @@ github.com/go-chi/cors v1.2.1 h1:xEC8UT3Rlp2QuWNEr4Fs/c2EAGVKBwy/1vHx3bppil4=
 github.com/go-chi/cors v1.2.1/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58=
 github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
 github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
-github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
-github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
-github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-jose/go-jose/v4 v4.0.1 h1:QVEPDE3OluqXBQZDcnNvQrInro2h0e4eqNbnZSWqS6U=
 github.com/go-jose/go-jose/v4 v4.0.1/go.mod h1:WVf9LFMHh/QVrmqrOfqun0C45tMe3RoiKJMPvgWwLfY=
-github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
-github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
-github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
-github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0=
-github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
-github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
-github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
-github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ=
 github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
@@ -133,7 +72,6 @@ github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl
 github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI=
 github.com/go-playground/validator/v10 v10.11.1 h1:prmOlTVv+YjZjmRmNSF3VmspqJIxJWXmqUsHwfTRRkQ=
 github.com/go-playground/validator/v10 v10.11.1/go.mod h1:i+3WkQ1FvaUjjxh1kSvIA4dMGDBiPU55YFDl0WbKdWU=
-github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
 github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
 github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo=
@@ -148,48 +86,15 @@ github.com/gobwas/ws v1.0.4/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/E
 github.com/goccy/go-json v0.9.11 h1:/pAaQDLHEoCq/5FFmSKBswWmK6H0e8g4159Kc/X/nqk=
 github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
-github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3 h1:zN2lZNZRflqFyxVaTIU61KNKQ9C0055u9CAfpmqUvo4=
 github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3/go.mod h1:nPpo7qLxd6XL3hWJG/O60sR8ZKfMCiIoNap5GvD12KU=
-github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
-github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
-github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
-github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
-github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
-github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
-github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
 github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
-github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
-github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
-github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
-github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
-github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
-github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
-github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
-github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
 github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
-github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
-github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
-github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
-github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
@@ -197,23 +102,11 @@ github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8=
 github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
-github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
-github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
-github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
-github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
-github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec=
 github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
-github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
@@ -221,30 +114,15 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 h1:/c3QmbOGMGTOumP2iT/rCwB7b0Q
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1/go.mod h1:5SN9VR2LTsRFsrEC6FHgRbTWrTHu6tqPeKxEQv15giM=
 github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 h1:MJG/KsmcqMwFAkh8mTnAwhyKoB+sTAnY4CACC110tbU=
 github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645/go.mod h1:6iZfnjpejD4L/4DwD7NryNaJyCQdzwWwH2MWhCA90Kw=
-github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/ipostelnik/cli/v2 v2.3.1-0.20210324024421-b6ea8234fe3d h1:PRDnysJ9dF1vUMmEzBu6aHQeUluSQy4eWH3RsSSy/vI=
 github.com/ipostelnik/cli/v2 v2.3.1-0.20210324024421-b6ea8234fe3d/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
-github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
-github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
-github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
-github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
-github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
-github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
-github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
-github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
-github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/klauspost/compress v1.15.11 h1:Lcadnb3RKGin4FYM/orgq0qde+nc15E5Cbqg4B9Sx9c=
 github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM=
-github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
@@ -272,11 +150,8 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
-github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
-github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q=
 github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k=
 github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE=
@@ -289,40 +164,21 @@ github.com/philhofer/fwd v1.1.1 h1:GdGcTjf5RNAxwS4QLsiMzJYj5KEvPJD3Abr261yRQXQ=
 github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
 github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
 github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
-github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
-github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
-github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
-github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
-github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
-github.com/prometheus/client_golang v1.13.0 h1:b71QUfeo5M8gq2+evJdTPfZhYMAU0uKPkyPJ7TPsloU=
-github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ=
-github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
-github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
-github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
-github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
-github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
-github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
-github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE=
-github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA=
-github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
-github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
-github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
-github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
-github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
-github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo=
-github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4=
-github.com/quic-go/quic-go v0.42.0 h1:uSfdap0eveIl8KXnipv9K7nlwZ5IqLlYOpJ58u5utpM=
-github.com/quic-go/quic-go v0.42.0/go.mod h1:132kz4kL3F9vxhW3CtQJLDVwcFe5wdWeJXXijhsO57M=
-github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE=
+github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho=
+github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw=
+github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI=
+github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE=
+github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc=
+github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
+github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
+github.com/quic-go/quic-go v0.45.0 h1:OHmkQGM37luZITyTSu6ff03HP/2IrwDX1ZFiNEhSFUE=
+github.com/quic-go/quic-go v0.45.0/go.mod h1:1dLehS7TIR64+vxGR70GDcatWTOtMX2PUtnKsjbTurI=
 github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
 github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
 github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
@@ -332,12 +188,7 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
-github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
-github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
-github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
@@ -351,16 +202,8 @@ github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVM
 github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
 github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0=
 github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY=
-github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
-go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
-go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
-go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opentelemetry.io/contrib/propagators v0.22.0 h1:KGdv58M2//veiYLIhb31mofaI2LgkIPXXAZVeYVyfd8=
 go.opentelemetry.io/contrib/propagators v0.22.0/go.mod h1:xGOuXr6lLIF9BXipA4pm6UuOSI0M98U6tsI3khbOiwU=
 go.opentelemetry.io/otel v1.0.0-RC2/go.mod h1:w1thVQ7qbAy8MHb0IFj8a5Q2QU0l2ksf8u/CN8m3NOM=
@@ -381,153 +224,45 @@ go.uber.org/automaxprocs v1.4.0 h1:CpDZl6aOlLhReez+8S3eEotD7Jx0Os++lemPlMULQP0=
 go.uber.org/automaxprocs v1.4.0/go.mod h1:/mTEdr7LvHhs0v7mjdxDreTz1OG5zdZGqgOnhWiR/+Q=
 go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU=
 go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc=
-golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=
 golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
-golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
-golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
-golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
-golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
-golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
-golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
-golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
-golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
-golang.org/x/exp v0.0.0-20221205204356-47842c84f3db h1:D/cFflL63o2KSLJIwjlcIt8PR064j/xsmdEJL/YvY/o=
-golang.org/x/exp v0.0.0-20221205204356-47842c84f3db/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
-golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
-golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
-golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
-golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
-golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
-golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
-golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM=
+golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc=
 golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
-golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
-golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
-golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
-golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
-golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
-golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
-golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU=
-golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA=
+golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
-golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
-golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
-golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac=
 golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
-golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
 golang.org/x/oauth2 v0.17.0 h1:6m3ZPmLEFdVxKKWnKq4VqZ60gutO35zm+zrAHVmHyDQ=
 golang.org/x/oauth2 v0.17.0/go.mod h1:OzPDGQiuQMguemayvdylqddI7qcD9lnSDb+1FiwQ5HA=
-golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
-golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
-golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
+golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -537,9 +272,7 @@ golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9sn
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw=
 golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
-golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@@ -547,178 +280,49 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
 golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk=
 golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
-golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
 golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
-golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
-golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
-golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
-golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
-golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
-golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
-golang.org/x/tools v0.9.1 h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo=
-golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc=
+golang.org/x/tools v0.21.0 h1:qc0xYgIbsSDt9EyWz05J5wfa7LOVW0YTLOXrqdLAWIw=
+golang.org/x/tools v0.21.0/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
-google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
-google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
-google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
-google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
-google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
-google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
-google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
-google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
-google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
-google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
-google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
-google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
-google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
-google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM=
 google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds=
-google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
-google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
-google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
-google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U=
-google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
-google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
-google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de h1:F6qOa9AZTYJXOUEr4jDysRDLrm4PHePlge4v4TGAlxY=
 google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:VUhTRKeHn9wwcdrk73nvdC9gF178Tzhmt/qyaFcPLSo=
 google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de h1:jFNzHPIeuzhdRwVhbZdiym9q0ory/xY3sA+v2wPg8I0=
 google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:5iCWqnniDlqZHrd3neWVTOwvh/v6s3232omMecelax8=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de h1:cZGRis4/ot9uVm639a+rHCUaG0JJHEsdyzSQTMX+suY=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:H4O17MA/PE9BsGx3w+a+W2VOLLD1Qf7oJneAoU6WktY=
-google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
-google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
-google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
-google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
-google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
-google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
-google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
-google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
-google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
 google.golang.org/grpc v1.63.0 h1:WjKe+dnvABXyPJMD7KDNLxtoGk5tgk+YFWN6cBWjZE8=
 google.golang.org/grpc v1.63.0/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA=
-google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
-google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
-google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
-google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
-google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
-google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
-google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg=
 google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
-gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
-gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
-gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
-honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 nhooyr.io/websocket v1.8.7 h1:usjR2uOr/zjjkVMy0lW+PPohFok7PCow5sDjLgX4P4g=
 nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0=
-rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
-rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
-rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
 zombiezen.com/go/capnproto2 v2.18.0+incompatible h1:mwfXZniffG5mXokQGHUJWGnqIBggoPfT/CEwon9Yess=
 zombiezen.com/go/capnproto2 v2.18.0+incompatible/go.mod h1:XO5Pr2SbXgqZwn0m0Ru54QBqpOf4K5AYBO+8LAOBQEQ=

ingress/icmp_metrics.go

@@ -0,0 +1,39 @@
+package ingress
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const (
+	namespace = "cloudflared"
+)
+
+var (
+	icmpRequests = prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: namespace,
+		Subsystem: "icmp",
+		Name:      "total_requests",
+		Help:      "Total count of ICMP requests that have been proxied to any origin",
+	})
+	icmpReplies = prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: namespace,
+		Subsystem: "icmp",
+		Name:      "total_replies",
+		Help:      "Total count of ICMP replies that have been proxied from any origin",
+	})
+)
+
+func init() {
+	prometheus.MustRegister(
+		icmpRequests,
+		icmpReplies,
+	)
+}
+
+func incrementICMPRequest() {
+	icmpRequests.Inc()
+}
+
+func incrementICMPReply() {
+	icmpReplies.Inc()
+}

ingress/origin_icmp_proxy.go

@@ -105,6 +105,7 @@ func isEchoReply(msg *icmp.Message) bool {
 }
 
 func observeICMPRequest(logger *zerolog.Logger, span trace.Span, src string, dst string, echoID int, seq int) {
+	incrementICMPRequest()
 	logger.Debug().
 		Str("src", src).
 		Str("dst", dst).
@@ -118,6 +119,7 @@ func observeICMPRequest(logger *zerolog.Logger, span trace.Span, src string, dst
 }
 
 func observeICMPReply(logger *zerolog.Logger, span trace.Span, dst string, echoID int, seq int) {
+	incrementICMPReply()
 	logger.Debug().Str("dst", dst).Int("echoID", echoID).Int("seq", seq).Msg("Sent ICMP reply to edge")
 	span.SetAttributes(
 		attribute.String("dst", dst),

management/middleware_test.go

@@ -52,13 +52,11 @@ func testRequest(t *testing.T, ts *httptest.Server, method, path string, body io
 	req, err := http.NewRequest(method, ts.URL+path, body)
 	if err != nil {
 		t.Fatal(err)
-		return nil, nil
 	}
 
 	resp, err := ts.Client().Do(req)
 	if err != nil {
 		t.Fatal(err)
-		return nil, nil
 	}
 	var claims managementErrorResponse
 	err = json.NewDecoder(resp.Body).Decode(&claims)

orchestration/orchestrator.go

@@ -14,7 +14,7 @@ import (
 	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/ingress"
 	"github.com/cloudflare/cloudflared/proxy"
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
 // Orchestrator manages configurations, so they can be updatable during runtime
@@ -32,7 +32,7 @@ type Orchestrator struct {
 	internalRules []ingress.Rule
 	// cloudflared Configuration
 	config *Config
-	tags   []tunnelpogs.Tag
+	tags   []pogs.Tag
 	log    *zerolog.Logger
 
 	// orchestrator must not handle any more updates after shutdownC is closed
@@ -43,7 +43,7 @@ type Orchestrator struct {
 
 func NewOrchestrator(ctx context.Context,
 	config *Config,
-	tags []tunnelpogs.Tag,
+	tags []pogs.Tag,
 	internalRules []ingress.Rule,
 	log *zerolog.Logger) (*Orchestrator, error) {
 	o := &Orchestrator{
@@ -65,7 +65,7 @@ func NewOrchestrator(ctx context.Context,
 }
 
 // UpdateConfig creates a new proxy with the new ingress rules
-func (o *Orchestrator) UpdateConfig(version int32, config []byte) *tunnelpogs.UpdateConfigurationResponse {
+func (o *Orchestrator) UpdateConfig(version int32, config []byte) *pogs.UpdateConfigurationResponse {
 	o.lock.Lock()
 	defer o.lock.Unlock()
 
@@ -74,7 +74,7 @@ func (o *Orchestrator) UpdateConfig(version int32, config []byte) *tunnelpogs.Up
 			Int32("current_version", o.currentVersion).
 			Int32("received_version", version).
 			Msg("Current version is equal or newer than received version")
-		return &tunnelpogs.UpdateConfigurationResponse{
+		return &pogs.UpdateConfigurationResponse{
 			LastAppliedVersion: o.currentVersion,
 		}
 	}
@@ -84,7 +84,7 @@ func (o *Orchestrator) UpdateConfig(version int32, config []byte) *tunnelpogs.Up
 			Int32("version", version).
 			Str("config", string(config)).
 			Msgf("Failed to deserialize new configuration")
-		return &tunnelpogs.UpdateConfigurationResponse{
+		return &pogs.UpdateConfigurationResponse{
 			LastAppliedVersion: o.currentVersion,
 			Err:                err,
 		}
@@ -95,7 +95,7 @@ func (o *Orchestrator) UpdateConfig(version int32, config []byte) *tunnelpogs.Up
 			Int32("version", version).
 			Str("config", string(config)).
 			Msgf("Failed to update ingress")
-		return &tunnelpogs.UpdateConfigurationResponse{
+		return &pogs.UpdateConfigurationResponse{
 			LastAppliedVersion: o.currentVersion,
 			Err:                err,
 		}
@@ -107,7 +107,7 @@ func (o *Orchestrator) UpdateConfig(version int32, config []byte) *tunnelpogs.Up
 		Str("config", string(config)).
 		Msg("Updated to new configuration")
 	configVersion.Set(float64(version))
-	return &tunnelpogs.UpdateConfigurationResponse{
+	return &pogs.UpdateConfigurationResponse{
 		LastAppliedVersion: o.currentVersion,
 	}
 }

orchestration/orchestrator_test.go

@@ -23,12 +23,12 @@ import (
 	"github.com/cloudflare/cloudflared/ingress"
 	"github.com/cloudflare/cloudflared/management"
 	"github.com/cloudflare/cloudflared/tracing"
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
 var (
 	testLogger = zerolog.Nop()
-	testTags   = []tunnelpogs.Tag{
+	testTags   = []pogs.Tag{
 		{
 			Name:  "package",
 			Value: "orchestration",

proxy/proxy.go

@@ -19,7 +19,7 @@ import (
 	"github.com/cloudflare/cloudflared/ingress"
 	"github.com/cloudflare/cloudflared/stream"
 	"github.com/cloudflare/cloudflared/tracing"
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
 const (
@@ -33,7 +33,7 @@ type Proxy struct {
 	ingressRules ingress.Ingress
 	warpRouting  *ingress.WarpRoutingService
 	management   *ingress.ManagementService
-	tags         []tunnelpogs.Tag
+	tags         []pogs.Tag
 	log          *zerolog.Logger
 }
 
@@ -41,7 +41,7 @@ type Proxy struct {
 func NewOriginProxy(
 	ingressRules ingress.Ingress,
 	warpRouting ingress.WarpRoutingConfig,
-	tags []tunnelpogs.Tag,
+	tags []pogs.Tag,
 	writeTimeout time.Duration,
 	log *zerolog.Logger,
 ) *Proxy {

proxy/proxy_test.go

@@ -30,11 +30,11 @@ import (
 	"github.com/cloudflare/cloudflared/ingress"
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/tracing"
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
 var (
-	testTags        = []tunnelpogs.Tag{{Name: "Name", Value: "value"}}
+	testTags        = []pogs.Tag{{Name: "Name", Value: "value"}}
 	noWarpRouting   = ingress.WarpRoutingConfig{}
 	testWarpRouting = ingress.WarpRoutingConfig{
 		ConnectTimeout: config.CustomDuration{Duration: time.Second},

quic/constants.go

@@ -0,0 +1,12 @@
+package quic
+
+import "time"
+
+const (
+	HandshakeIdleTimeout = 5 * time.Second
+	MaxIdleTimeout       = 5 * time.Second
+	MaxIdlePingPeriod    = 1 * time.Second
+
+	// MaxIncomingStreams is 2^60, which is the maximum supported value by Quic-Go
+	MaxIncomingStreams = 1 << 60
+)

quic/metrics.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/quic-go/quic-go/logging"
+	"github.com/rs/zerolog"
 )
 
 const (
@@ -18,6 +19,7 @@ var (
 	clientMetrics    = struct {
 		totalConnections  prometheus.Counter
 		closedConnections prometheus.Counter
+		maxUDPPayloadSize *prometheus.GaugeVec
 		sentFrames        *prometheus.CounterVec
 		sentBytes         *prometheus.CounterVec
 		receivedFrames    *prometheus.CounterVec
@@ -28,6 +30,9 @@ var (
 		minRTT            *prometheus.GaugeVec
 		latestRTT         *prometheus.GaugeVec
 		smoothedRTT       *prometheus.GaugeVec
+		mtu               *prometheus.GaugeVec
+		congestionWindow  *prometheus.GaugeVec
+		congestionState   *prometheus.GaugeVec
 	}{
 		totalConnections: prometheus.NewCounter(
 			prometheus.CounterOpts{
@@ -45,6 +50,15 @@ var (
 				Help:      "Number of connections that has been closed",
 			},
 		),
+		maxUDPPayloadSize: prometheus.NewGaugeVec(
+			prometheus.GaugeOpts{
+				Namespace: namespace,
+				Subsystem: "client",
+				Name:      "max_udp_payload",
+				Help:      "Maximum UDP payload size in bytes for a QUIC packet",
+			},
+			clientConnLabels,
+		),
 		sentFrames: prometheus.NewCounterVec(
 			prometheus.CounterOpts{
 				Namespace: namespace,
@@ -135,6 +149,33 @@ var (
 			},
 			clientConnLabels,
 		),
+		mtu: prometheus.NewGaugeVec(
+			prometheus.GaugeOpts{
+				Namespace: namespace,
+				Subsystem: "client",
+				Name:      "mtu",
+				Help:      "Current maximum transmission unit (MTU) of a connection",
+			},
+			clientConnLabels,
+		),
+		congestionWindow: prometheus.NewGaugeVec(
+			prometheus.GaugeOpts{
+				Namespace: namespace,
+				Subsystem: "client",
+				Name:      "congestion_window",
+				Help:      "Current congestion window size",
+			},
+			clientConnLabels,
+		),
+		congestionState: prometheus.NewGaugeVec(
+			prometheus.GaugeOpts{
+				Namespace: namespace,
+				Subsystem: "client",
+				Name:      "congestion_state",
+				Help:      "Current congestion control state. See https://pkg.go.dev/github.com/quic-go/quic-go@v0.45.0/logging#CongestionState for what each value maps to",
+			},
+			clientConnLabels,
+		),
 	}
 
 	registerClient = sync.Once{}
@@ -148,14 +189,16 @@ var (
 )
 
 type clientCollector struct {
-	index string
+	index  string
+	logger *zerolog.Logger
 }
 
-func newClientCollector(index uint8) *clientCollector {
+func newClientCollector(index string, logger *zerolog.Logger) *clientCollector {
 	registerClient.Do(func() {
 		prometheus.MustRegister(
 			clientMetrics.totalConnections,
 			clientMetrics.closedConnections,
+			clientMetrics.maxUDPPayloadSize,
 			clientMetrics.sentFrames,
 			clientMetrics.sentBytes,
 			clientMetrics.receivedFrames,
@@ -166,11 +209,16 @@ func newClientCollector(index uint8) *clientCollector {
 			clientMetrics.minRTT,
 			clientMetrics.latestRTT,
 			clientMetrics.smoothedRTT,
+			clientMetrics.mtu,
+			clientMetrics.congestionWindow,
+			clientMetrics.congestionState,
 			packetTooBigDropped,
 		)
 	})
+
 	return &clientCollector{
-		index: uint8ToString(index),
+		index:  index,
+		logger: logger,
 	}
 }
 
@@ -178,16 +226,21 @@ func (cc *clientCollector) startedConnection() {
 	clientMetrics.totalConnections.Inc()
 }
 
-func (cc *clientCollector) closedConnection(err error) {
+func (cc *clientCollector) closedConnection(error) {
 	clientMetrics.closedConnections.Inc()
 }
 
+func (cc *clientCollector) receivedTransportParameters(params *logging.TransportParameters) {
+	clientMetrics.maxUDPPayloadSize.WithLabelValues(cc.index).Set(float64(params.MaxUDPPayloadSize))
+	cc.logger.Debug().Msgf("Received transport parameters: MaxUDPPayloadSize=%d, MaxIdleTimeout=%v, MaxDatagramFrameSize=%d", params.MaxUDPPayloadSize, params.MaxIdleTimeout, params.MaxDatagramFrameSize)
+}
+
 func (cc *clientCollector) sentPackets(size logging.ByteCount, frames []logging.Frame) {
-	cc.collectPackets(size, frames, clientMetrics.sentFrames, clientMetrics.sentBytes)
+	cc.collectPackets(size, frames, clientMetrics.sentFrames, clientMetrics.sentBytes, sent)
 }
 
 func (cc *clientCollector) receivedPackets(size logging.ByteCount, frames []logging.Frame) {
-	cc.collectPackets(size, frames, clientMetrics.receivedFrames, clientMetrics.receivedBytes)
+	cc.collectPackets(size, frames, clientMetrics.receivedFrames, clientMetrics.receivedBytes, received)
 }
 
 func (cc *clientCollector) bufferedPackets(packetType logging.PacketType) {
@@ -212,8 +265,27 @@ func (cc *clientCollector) updatedRTT(rtt *logging.RTTStats) {
 	clientMetrics.smoothedRTT.WithLabelValues(cc.index).Set(durationToPromGauge(rtt.SmoothedRTT()))
 }
 
-func (cc *clientCollector) collectPackets(size logging.ByteCount, frames []logging.Frame, counter, bandwidth *prometheus.CounterVec) {
+func (cc *clientCollector) updateCongestionWindow(size logging.ByteCount) {
+	clientMetrics.congestionWindow.WithLabelValues(cc.index).Set(float64(size))
+}
+
+func (cc *clientCollector) updatedCongestionState(state logging.CongestionState) {
+	clientMetrics.congestionState.WithLabelValues(cc.index).Set(float64(state))
+}
+
+func (cc *clientCollector) updateMTU(mtu logging.ByteCount) {
+	clientMetrics.mtu.WithLabelValues(cc.index).Set(float64(mtu))
+	cc.logger.Debug().Msgf("QUIC MTU updated to %d", mtu)
+}
+
+func (cc *clientCollector) collectPackets(size logging.ByteCount, frames []logging.Frame, counter, bandwidth *prometheus.CounterVec, direction direction) {
 	for _, frame := range frames {
+		switch f := frame.(type) {
+		case logging.DataBlockedFrame:
+			cc.logger.Debug().Msgf("%s data_blocked frame", direction)
+		case logging.StreamDataBlockedFrame:
+			cc.logger.Debug().Int64("streamID", int64(f.StreamID)).Msgf("%s stream_data_blocked frame", direction)
+		}
 		counter.WithLabelValues(cc.index, frameName(frame)).Inc()
 	}
 	bandwidth.WithLabelValues(cc.index).Add(byteCountToPromCount(size))
@@ -227,3 +299,17 @@ func frameName(frame logging.Frame) string {
 		return strings.TrimSuffix(name, "Frame")
 	}
 }
+
+type direction uint8
+
+const (
+	sent direction = iota
+	received
+)
+
+func (d direction) String() string {
+	if d == sent {
+		return "sent"
+	}
+	return "received"
+}

quic/quic_protocol.go

@@ -1,274 +0,0 @@
-package quic
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"net"
-	"time"
-
-	capnp "zombiezen.com/go/capnproto2"
-	"zombiezen.com/go/capnproto2/rpc"
-
-	"github.com/google/uuid"
-	"github.com/rs/zerolog"
-
-	"github.com/cloudflare/cloudflared/tunnelrpc"
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
-)
-
-// ProtocolSignature defines the first 6 bytes of the stream, which is used to distinguish the type of stream. It
-// ensures whoever performs a handshake does not write data before writing the metadata.
-type ProtocolSignature [6]byte
-
-var (
-	// DataStreamProtocolSignature is a custom protocol signature for data stream
-	DataStreamProtocolSignature = ProtocolSignature{0x0A, 0x36, 0xCD, 0x12, 0xA1, 0x3E}
-
-	// RPCStreamProtocolSignature is a custom protocol signature for RPC stream
-	RPCStreamProtocolSignature = ProtocolSignature{0x52, 0xBB, 0x82, 0x5C, 0xDB, 0x65}
-)
-
-type protocolVersion string
-
-const (
-	protocolV1 protocolVersion = "01"
-
-	protocolVersionLength = 2
-
-	HandshakeIdleTimeout = 5 * time.Second
-	MaxIdleTimeout       = 5 * time.Second
-	MaxIdlePingPeriod    = 1 * time.Second
-
-	// MaxIncomingStreams is 2^60, which is the maximum supported value by Quic-Go
-	MaxIncomingStreams = 1 << 60
-)
-
-// RequestServerStream is a stream to serve requests
-type RequestServerStream struct {
-	io.ReadWriteCloser
-}
-
-func NewRequestServerStream(stream io.ReadWriteCloser, signature ProtocolSignature) (*RequestServerStream, error) {
-	if signature != DataStreamProtocolSignature {
-		return nil, fmt.Errorf("RequestClientStream can only be created from data stream")
-	}
-	return &RequestServerStream{stream}, nil
-}
-
-// ReadConnectRequestData reads the handshake data from a QUIC stream.
-func (rss *RequestServerStream) ReadConnectRequestData() (*ConnectRequest, error) {
-	// This is a NO-OP for now. We could cause a branching if we wanted to use multiple versions.
-	if _, err := readVersion(rss); err != nil {
-		return nil, err
-	}
-
-	msg, err := capnp.NewDecoder(rss).Decode()
-	if err != nil {
-		return nil, err
-	}
-
-	r := &ConnectRequest{}
-	if err := r.fromPogs(msg); err != nil {
-		return nil, err
-	}
-	return r, nil
-}
-
-// WriteConnectResponseData writes response to a QUIC stream.
-func (rss *RequestServerStream) WriteConnectResponseData(respErr error, metadata ...Metadata) error {
-	var connectResponse *ConnectResponse
-	if respErr != nil {
-		connectResponse = &ConnectResponse{
-			Error: respErr.Error(),
-		}
-	} else {
-		connectResponse = &ConnectResponse{
-			Metadata: metadata,
-		}
-	}
-
-	msg, err := connectResponse.toPogs()
-	if err != nil {
-		return err
-	}
-
-	if err := writeDataStreamPreamble(rss); err != nil {
-		return err
-	}
-	return capnp.NewEncoder(rss).Encode(msg)
-}
-
-type RequestClientStream struct {
-	io.ReadWriteCloser
-}
-
-// WriteConnectRequestData writes requestMeta to a stream.
-func (rcs *RequestClientStream) WriteConnectRequestData(dest string, connectionType ConnectionType, metadata ...Metadata) error {
-	connectRequest := &ConnectRequest{
-		Dest:     dest,
-		Type:     connectionType,
-		Metadata: metadata,
-	}
-
-	msg, err := connectRequest.toPogs()
-	if err != nil {
-		return err
-	}
-
-	if err := writeDataStreamPreamble(rcs); err != nil {
-		return err
-	}
-	return capnp.NewEncoder(rcs).Encode(msg)
-}
-
-// ReadConnectResponseData reads the response to a RequestMeta in a stream.
-func (rcs *RequestClientStream) ReadConnectResponseData() (*ConnectResponse, error) {
-	signature, err := DetermineProtocol(rcs)
-	if err != nil {
-		return nil, err
-	}
-	if signature != DataStreamProtocolSignature {
-		return nil, fmt.Errorf("wrong protocol signature %v", signature)
-	}
-
-	// This is a NO-OP for now. We could cause a branching if we wanted to use multiple versions.
-	if _, err := readVersion(rcs); err != nil {
-		return nil, err
-	}
-
-	msg, err := capnp.NewDecoder(rcs).Decode()
-	if err != nil {
-		return nil, err
-	}
-
-	r := &ConnectResponse{}
-	if err := r.fromPogs(msg); err != nil {
-		return nil, err
-	}
-	return r, nil
-}
-
-// RPCServerStream is a stream to serve RPCs. It is closed when the RPC client is done
-type RPCServerStream struct {
-	io.ReadWriteCloser
-}
-
-func NewRPCServerStream(stream io.ReadWriteCloser, protocol ProtocolSignature) (*RPCServerStream, error) {
-	if protocol != RPCStreamProtocolSignature {
-		return nil, fmt.Errorf("RPCStream can only be created from rpc stream")
-	}
-	return &RPCServerStream{stream}, nil
-}
-
-func (s *RPCServerStream) Serve(sessionManager tunnelpogs.SessionManager, configManager tunnelpogs.ConfigurationManager, logger *zerolog.Logger) error {
-	// RPC logs are very robust, create a new logger that only logs error to reduce noise
-	rpcLogger := logger.Level(zerolog.ErrorLevel)
-	rpcTransport := tunnelrpc.NewTransportLogger(&rpcLogger, rpc.StreamTransport(s))
-	defer rpcTransport.Close()
-
-	main := tunnelpogs.CloudflaredServer_ServerToClient(sessionManager, configManager)
-	rpcConn := rpc.NewConn(
-		rpcTransport,
-		rpc.MainInterface(main.Client),
-		tunnelrpc.ConnLog(&rpcLogger),
-	)
-	defer rpcConn.Close()
-
-	return rpcConn.Wait()
-}
-
-func DetermineProtocol(stream io.Reader) (ProtocolSignature, error) {
-	signature, err := readSignature(stream)
-	if err != nil {
-		return ProtocolSignature{}, err
-	}
-	switch signature {
-	case DataStreamProtocolSignature:
-		return DataStreamProtocolSignature, nil
-	case RPCStreamProtocolSignature:
-		return RPCStreamProtocolSignature, nil
-	default:
-		return ProtocolSignature{}, fmt.Errorf("unknown signature %v", signature)
-	}
-}
-
-func writeDataStreamPreamble(stream io.Writer) error {
-	if err := writeSignature(stream, DataStreamProtocolSignature); err != nil {
-		return err
-	}
-
-	return writeVersion(stream)
-}
-
-func writeVersion(stream io.Writer) error {
-	_, err := stream.Write([]byte(protocolV1)[:protocolVersionLength])
-	return err
-}
-
-func readVersion(stream io.Reader) (string, error) {
-	version := make([]byte, protocolVersionLength)
-	_, err := stream.Read(version)
-	return string(version), err
-}
-
-func readSignature(stream io.Reader) (ProtocolSignature, error) {
-	var signature ProtocolSignature
-	if _, err := io.ReadFull(stream, signature[:]); err != nil {
-		return ProtocolSignature{}, err
-	}
-	return signature, nil
-}
-
-func writeSignature(stream io.Writer, signature ProtocolSignature) error {
-	_, err := stream.Write(signature[:])
-	return err
-}
-
-// RPCClientStream is a stream to call methods of SessionManager
-type RPCClientStream struct {
-	client    tunnelpogs.CloudflaredServer_PogsClient
-	transport rpc.Transport
-
-	// Time we wait for the server to respond to a request before we close the connection.
-	rpcUnregisterUDPSessionDeadline time.Duration
-}
-
-func NewRPCClientStream(ctx context.Context, stream io.ReadWriteCloser, rpcUnregisterUDPSessionDeadline time.Duration, logger *zerolog.Logger) (*RPCClientStream, error) {
-	n, err := stream.Write(RPCStreamProtocolSignature[:])
-	if err != nil {
-		return nil, err
-	}
-	if n != len(RPCStreamProtocolSignature) {
-		return nil, fmt.Errorf("expect to write %d bytes for RPC stream protocol signature, wrote %d", len(RPCStreamProtocolSignature), n)
-	}
-	transport := tunnelrpc.NewTransportLogger(logger, rpc.StreamTransport(stream))
-	conn := rpc.NewConn(
-		transport,
-		tunnelrpc.ConnLog(logger),
-	)
-	return &RPCClientStream{
-		client:                          tunnelpogs.NewCloudflaredServer_PogsClient(conn.Bootstrap(ctx), conn),
-		transport:                       transport,
-		rpcUnregisterUDPSessionDeadline: rpcUnregisterUDPSessionDeadline,
-	}, nil
-}
-
-func (rcs *RPCClientStream) RegisterUdpSession(ctx context.Context, sessionID uuid.UUID, dstIP net.IP, dstPort uint16, closeIdleAfterHint time.Duration, traceContext string) (*tunnelpogs.RegisterUdpSessionResponse, error) {
-	return rcs.client.RegisterUdpSession(ctx, sessionID, dstIP, dstPort, closeIdleAfterHint, traceContext)
-}
-
-func (rcs *RPCClientStream) UnregisterUdpSession(ctx context.Context, sessionID uuid.UUID, message string) error {
-	ctx, cancel := context.WithTimeout(ctx, rcs.rpcUnregisterUDPSessionDeadline)
-	defer cancel()
-	return rcs.client.UnregisterUdpSession(ctx, sessionID, message)
-}
-
-func (rcs *RPCClientStream) UpdateConfiguration(ctx context.Context, version int32, config []byte) (*tunnelpogs.UpdateConfigurationResponse, error) {
-	return rcs.client.UpdateConfiguration(ctx, version, config)
-}
-
-func (rcs *RPCClientStream) Close() {
-	_ = rcs.client.Close()
-	_ = rcs.transport.Close()
-}

quic/safe_stream_test.go

@@ -2,8 +2,13 @@ package quic
 
 import (
 	"context"
+	"crypto/rand"
+	"crypto/rsa"
 	"crypto/tls"
+	"crypto/x509"
+	"encoding/pem"
 	"io"
+	"math/big"
 	"net"
 	"sync"
 	"testing"
@@ -147,3 +152,27 @@ func serverRoundTrip(t *testing.T, stream io.ReadWriteCloser, mustWork bool) {
 	}
 	require.NoError(t, err)
 }
+
+// GenerateTLSConfig sets up a bare-bones TLS config for a QUIC server
+func GenerateTLSConfig() *tls.Config {
+	key, err := rsa.GenerateKey(rand.Reader, 1024)
+	if err != nil {
+		panic(err)
+	}
+	template := x509.Certificate{SerialNumber: big.NewInt(1)}
+	certDER, err := x509.CreateCertificate(rand.Reader, &template, &template, &key.PublicKey, key)
+	if err != nil {
+		panic(err)
+	}
+	keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)})
+	certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER})
+
+	tlsCert, err := tls.X509KeyPair(certPEM, keyPEM)
+	if err != nil {
+		panic(err)
+	}
+	return &tls.Config{
+		Certificates: []tls.Certificate{tlsCert},
+		NextProtos:   []string{"argotunnel"},
+	}
+}

quic/schema/quic_metadata_protocol.capnp

@@ -1,28 +0,0 @@
-using Go = import "/go.capnp";
-@0xb29021ef7421cc32;
-
-$Go.package("schema");
-$Go.import("schema");
-
-
-struct ConnectRequest{
-	dest @0 :Text;
-	type @1 :ConnectionType;
-	metadata @2 :List(Metadata);
-}
-
-enum ConnectionType{
-	http @0;
-	websocket @1;
-	tcp @2;
-}
-
-struct Metadata {
-    key @0 :Text;
-	val @1 :Text;
-}
-
-struct ConnectResponse{
-	error @0 :Text;
-	metadata @1 :List(Metadata);
-}

quic/test_utils.go

@@ -1,34 +0,0 @@
-package quic
-
-import (
-	"crypto/rand"
-	"crypto/rsa"
-	"crypto/tls"
-	"crypto/x509"
-	"encoding/pem"
-	"math/big"
-)
-
-// GenerateTLSConfig sets up a bare-bones TLS config for a QUIC server
-func GenerateTLSConfig() *tls.Config {
-	key, err := rsa.GenerateKey(rand.Reader, 1024)
-	if err != nil {
-		panic(err)
-	}
-	template := x509.Certificate{SerialNumber: big.NewInt(1)}
-	certDER, err := x509.CreateCertificate(rand.Reader, &template, &template, &key.PublicKey, key)
-	if err != nil {
-		panic(err)
-	}
-	keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)})
-	certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER})
-
-	tlsCert, err := tls.X509KeyPair(certPEM, keyPEM)
-	if err != nil {
-		panic(err)
-	}
-	return &tls.Config{
-		Certificates: []tls.Certificate{tlsCert},
-		NextProtos:   []string{"argotunnel"},
-	}
-}

quic/tracing.go

@@ -10,26 +10,20 @@ import (
 
 // QUICTracer is a wrapper to create new quicConnTracer
 type tracer struct {
+	index  string
 	logger *zerolog.Logger
-	config *tracerConfig
-}
-
-type tracerConfig struct {
-	index uint8
 }
 
 func NewClientTracer(logger *zerolog.Logger, index uint8) func(context.Context, logging.Perspective, logging.ConnectionID) *logging.ConnectionTracer {
 	t := &tracer{
+		index:  uint8ToString(index),
 		logger: logger,
-		config: &tracerConfig{
-			index: index,
-		},
 	}
 	return t.TracerForConnection
 }
 
 func (t *tracer) TracerForConnection(_ctx context.Context, _p logging.Perspective, _odcid logging.ConnectionID) *logging.ConnectionTracer {
-	return newConnTracer(newClientCollector(t.config.index))
+	return newConnTracer(newClientCollector(t.index, t.logger))
 }
 
 // connTracer collects connection level metrics
@@ -42,16 +36,19 @@ func newConnTracer(metricsCollector *clientCollector) *logging.ConnectionTracer
 		metricsCollector: metricsCollector,
 	}
 	return &logging.ConnectionTracer{
-		StartedConnection:         tracer.StartedConnection,
-		ClosedConnection:          tracer.ClosedConnection,
-		SentLongHeaderPacket:      tracer.SentLongHeaderPacket,
-		SentShortHeaderPacket:     tracer.SentShortHeaderPacket,
-		ReceivedLongHeaderPacket:  tracer.ReceivedLongHeaderPacket,
-		ReceivedShortHeaderPacket: tracer.ReceivedShortHeaderPacket,
-		BufferedPacket:            tracer.BufferedPacket,
-		DroppedPacket:             tracer.DroppedPacket,
-		UpdatedMetrics:            tracer.UpdatedMetrics,
-		LostPacket:                tracer.LostPacket,
+		StartedConnection:           tracer.StartedConnection,
+		ClosedConnection:            tracer.ClosedConnection,
+		ReceivedTransportParameters: tracer.ReceivedTransportParameters,
+		SentLongHeaderPacket:        tracer.SentLongHeaderPacket,
+		SentShortHeaderPacket:       tracer.SentShortHeaderPacket,
+		ReceivedLongHeaderPacket:    tracer.ReceivedLongHeaderPacket,
+		ReceivedShortHeaderPacket:   tracer.ReceivedShortHeaderPacket,
+		BufferedPacket:              tracer.BufferedPacket,
+		DroppedPacket:               tracer.DroppedPacket,
+		UpdatedMetrics:              tracer.UpdatedMetrics,
+		LostPacket:                  tracer.LostPacket,
+		UpdatedMTU:                  tracer.UpdatedMTU,
+		UpdatedCongestionState:      tracer.UpdatedCongestionState,
 	}
 }
 
@@ -63,6 +60,10 @@ func (ct *connTracer) ClosedConnection(err error) {
 	ct.metricsCollector.closedConnection(err)
 }
 
+func (ct *connTracer) ReceivedTransportParameters(params *logging.TransportParameters) {
+	ct.metricsCollector.receivedTransportParameters(params)
+}
+
 func (ct *connTracer) BufferedPacket(pt logging.PacketType, size logging.ByteCount) {
 	ct.metricsCollector.bufferedPackets(pt)
 }
@@ -77,6 +78,7 @@ func (ct *connTracer) LostPacket(level logging.EncryptionLevel, number logging.P
 
 func (ct *connTracer) UpdatedMetrics(rttStats *logging.RTTStats, cwnd, bytesInFlight logging.ByteCount, packetsInFlight int) {
 	ct.metricsCollector.updatedRTT(rttStats)
+	ct.metricsCollector.updateCongestionWindow(cwnd)
 }
 
 func (ct *connTracer) SentLongHeaderPacket(hdr *logging.ExtendedHeader, size logging.ByteCount, ecn logging.ECN, ack *logging.AckFrame, frames []logging.Frame) {
@@ -95,16 +97,10 @@ func (ct *connTracer) ReceivedShortHeaderPacket(hdr *logging.ShortHeader, size l
 	ct.metricsCollector.receivedPackets(size, frames)
 }
 
-type quicLogger struct {
-	logger       *zerolog.Logger
-	connectionID string
+func (ct *connTracer) UpdatedMTU(mtu logging.ByteCount, done bool) {
+	ct.metricsCollector.updateMTU(mtu)
 }
 
-func (qt *quicLogger) Write(p []byte) (n int, err error) {
-	qt.logger.Trace().Str("quicConnection", qt.connectionID).RawJSON("event", p).Msg("Quic event")
-	return len(p), nil
-}
-
-func (*quicLogger) Close() error {
-	return nil
+func (ct *connTracer) UpdatedCongestionState(state logging.CongestionState) {
+	ct.metricsCollector.updatedCongestionState(state)
 }

retry/backoffhandler.go

@@ -6,17 +6,16 @@ import (
 	"time"
 )
 
+const (
+	DefaultBaseTime time.Duration = time.Second
+)
+
 // Redeclare time functions so they can be overridden in tests.
-type clock struct {
+type Clock struct {
 	Now   func() time.Time
 	After func(d time.Duration) <-chan time.Time
 }
 
-var Clock = clock{
-	Now:   time.Now,
-	After: time.After,
-}
-
 // BackoffHandler manages exponential backoff and limits the maximum number of retries.
 // The base time period is 1 second, doubling with each retry.
 // After initial success, a grace period can be set to reset the backoff timer if
@@ -25,15 +24,26 @@ var Clock = clock{
 type BackoffHandler struct {
 	// MaxRetries sets the maximum number of retries to perform. The default value
 	// of 0 disables retry completely.
-	MaxRetries uint
+	maxRetries uint
 	// RetryForever caps the exponential backoff period according to MaxRetries
 	// but allows you to retry indefinitely.
-	RetryForever bool
+	retryForever bool
 	// BaseTime sets the initial backoff period.
-	BaseTime time.Duration
+	baseTime time.Duration
 
 	retries       uint
 	resetDeadline time.Time
+
+	Clock Clock
+}
+
+func NewBackoff(maxRetries uint, baseTime time.Duration, retryForever bool) BackoffHandler {
+	return BackoffHandler{
+		maxRetries:   maxRetries,
+		baseTime:     baseTime,
+		retryForever: retryForever,
+		Clock:        Clock{Now: time.Now, After: time.After},
+	}
 }
 
 func (b BackoffHandler) GetMaxBackoffDuration(ctx context.Context) (time.Duration, bool) {
@@ -44,11 +54,11 @@ func (b BackoffHandler) GetMaxBackoffDuration(ctx context.Context) (time.Duratio
 		return time.Duration(0), false
 	default:
 	}
-	if !b.resetDeadline.IsZero() && Clock.Now().After(b.resetDeadline) {
+	if !b.resetDeadline.IsZero() && b.Clock.Now().After(b.resetDeadline) {
 		// b.retries would be set to 0 at this point
 		return time.Second, true
 	}
-	if b.retries >= b.MaxRetries && !b.RetryForever {
+	if b.retries >= b.maxRetries && !b.retryForever {
 		return time.Duration(0), false
 	}
 	maxTimeToWait := b.GetBaseTime() * 1 << (b.retries + 1)
@@ -58,12 +68,12 @@ func (b BackoffHandler) GetMaxBackoffDuration(ctx context.Context) (time.Duratio
 // BackoffTimer returns a channel that sends the current time when the exponential backoff timeout expires.
 // Returns nil if the maximum number of retries have been used.
 func (b *BackoffHandler) BackoffTimer() <-chan time.Time {
-	if !b.resetDeadline.IsZero() && Clock.Now().After(b.resetDeadline) {
+	if !b.resetDeadline.IsZero() && b.Clock.Now().After(b.resetDeadline) {
 		b.retries = 0
 		b.resetDeadline = time.Time{}
 	}
-	if b.retries >= b.MaxRetries {
-		if !b.RetryForever {
+	if b.retries >= b.maxRetries {
+		if !b.retryForever {
 			return nil
 		}
 	} else {
@@ -71,7 +81,7 @@ func (b *BackoffHandler) BackoffTimer() <-chan time.Time {
 	}
 	maxTimeToWait := time.Duration(b.GetBaseTime() * 1 << (b.retries))
 	timeToWait := time.Duration(rand.Int63n(maxTimeToWait.Nanoseconds()))
-	return Clock.After(timeToWait)
+	return b.Clock.After(timeToWait)
 }
 
 // Backoff is used to wait according to exponential backoff. Returns false if the
@@ -94,16 +104,16 @@ func (b *BackoffHandler) Backoff(ctx context.Context) bool {
 func (b *BackoffHandler) SetGracePeriod() time.Duration {
 	maxTimeToWait := b.GetBaseTime() * 2 << (b.retries + 1)
 	timeToWait := time.Duration(rand.Int63n(maxTimeToWait.Nanoseconds()))
-	b.resetDeadline = Clock.Now().Add(timeToWait)
+	b.resetDeadline = b.Clock.Now().Add(timeToWait)
 
 	return timeToWait
 }
 
 func (b BackoffHandler) GetBaseTime() time.Duration {
-	if b.BaseTime == 0 {
-		return time.Second
+	if b.baseTime == 0 {
+		return DefaultBaseTime
 	}
-	return b.BaseTime
+	return b.baseTime
 }
 
 // Retries returns the number of retries consumed so far.
@@ -112,9 +122,10 @@ func (b *BackoffHandler) Retries() int {
 }
 
 func (b *BackoffHandler) ReachedMaxRetries() bool {
-	return b.retries == b.MaxRetries
+	return b.retries == b.maxRetries
 }
 
 func (b *BackoffHandler) ResetNow() {
-	b.resetDeadline = time.Now()
+	b.resetDeadline = b.Clock.Now()
+	b.retries = 0
 }

retry/backoffhandler_test.go

@@ -13,10 +13,9 @@ func immediateTimeAfter(time.Duration) <-chan time.Time {
 }
 
 func TestBackoffRetries(t *testing.T) {
-	// make backoff return immediately
-	Clock.After = immediateTimeAfter
 	ctx := context.Background()
-	backoff := BackoffHandler{MaxRetries: 3}
+	// make backoff return immediately
+	backoff := BackoffHandler{maxRetries: 3, Clock: Clock{time.Now, immediateTimeAfter}}
 	if !backoff.Backoff(ctx) {
 		t.Fatalf("backoff failed immediately")
 	}
@@ -32,10 +31,10 @@ func TestBackoffRetries(t *testing.T) {
 }
 
 func TestBackoffCancel(t *testing.T) {
-	// prevent backoff from returning normally
-	Clock.After = func(time.Duration) <-chan time.Time { return make(chan time.Time) }
 	ctx, cancelFunc := context.WithCancel(context.Background())
-	backoff := BackoffHandler{MaxRetries: 3}
+	// prevent backoff from returning normally
+	after := func(time.Duration) <-chan time.Time { return make(chan time.Time) }
+	backoff := BackoffHandler{maxRetries: 3, Clock: Clock{time.Now, after}}
 	cancelFunc()
 	if backoff.Backoff(ctx) {
 		t.Fatalf("backoff allowed after cancel")
@@ -46,13 +45,12 @@ func TestBackoffCancel(t *testing.T) {
 }
 
 func TestBackoffGracePeriod(t *testing.T) {
+	ctx := context.Background()
 	currentTime := time.Now()
 	// make Clock.Now return whatever we like
-	Clock.Now = func() time.Time { return currentTime }
+	now := func() time.Time { return currentTime }
 	// make backoff return immediately
-	Clock.After = immediateTimeAfter
-	ctx := context.Background()
-	backoff := BackoffHandler{MaxRetries: 1}
+	backoff := BackoffHandler{maxRetries: 1, Clock: Clock{now, immediateTimeAfter}}
 	if !backoff.Backoff(ctx) {
 		t.Fatalf("backoff failed immediately")
 	}
@@ -70,10 +68,9 @@ func TestBackoffGracePeriod(t *testing.T) {
 }
 
 func TestGetMaxBackoffDurationRetries(t *testing.T) {
-	// make backoff return immediately
-	Clock.After = immediateTimeAfter
 	ctx := context.Background()
-	backoff := BackoffHandler{MaxRetries: 3}
+	// make backoff return immediately
+	backoff := BackoffHandler{maxRetries: 3, Clock: Clock{time.Now, immediateTimeAfter}}
 	if _, ok := backoff.GetMaxBackoffDuration(ctx); !ok {
 		t.Fatalf("backoff failed immediately")
 	}
@@ -95,10 +92,9 @@ func TestGetMaxBackoffDurationRetries(t *testing.T) {
 }
 
 func TestGetMaxBackoffDuration(t *testing.T) {
-	// make backoff return immediately
-	Clock.After = immediateTimeAfter
 	ctx := context.Background()
-	backoff := BackoffHandler{MaxRetries: 3}
+	// make backoff return immediately
+	backoff := BackoffHandler{maxRetries: 3, Clock: Clock{time.Now, immediateTimeAfter}}
 	if duration, ok := backoff.GetMaxBackoffDuration(ctx); !ok || duration > time.Second*2 {
 		t.Fatalf("backoff (%s) didn't return < 2 seconds on first retry", duration)
 	}
@@ -117,10 +113,9 @@ func TestGetMaxBackoffDuration(t *testing.T) {
 }
 
 func TestBackoffRetryForever(t *testing.T) {
-	// make backoff return immediately
-	Clock.After = immediateTimeAfter
 	ctx := context.Background()
-	backoff := BackoffHandler{MaxRetries: 3, RetryForever: true}
+	// make backoff return immediately
+	backoff := BackoffHandler{maxRetries: 3, retryForever: true, Clock: Clock{time.Now, immediateTimeAfter}}
 	if duration, ok := backoff.GetMaxBackoffDuration(ctx); !ok || duration > time.Second*2 {
 		t.Fatalf("backoff (%s) didn't return < 2 seconds on first retry", duration)
 	}

supervisor/pqtunnels.go

@@ -12,7 +12,7 @@ import (
 // to https://pqtunnels.cloudflareresearch.com.
 
 const (
-	PQKex     = tls.CurveID(0xfe31) // X25519Kyber768Draft00
+	PQKex     = tls.CurveID(0x6399) // X25519Kyber768Draft00
 	PQKexName = "X25519Kyber768Draft00"
 )
 

supervisor/reconnect.go

@@ -1,138 +0,0 @@
-package supervisor
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"sync"
-	"time"
-
-	"github.com/prometheus/client_golang/prometheus"
-
-	"github.com/cloudflare/cloudflared/retry"
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
-)
-
-var (
-	errJWTUnset = errors.New("JWT unset")
-)
-
-// reconnectTunnelCredentialManager is invoked by functions in tunnel.go to
-// get/set parameters for ReconnectTunnel RPC calls.
-type reconnectCredentialManager struct {
-	mu          sync.RWMutex
-	jwt         []byte
-	eventDigest map[uint8][]byte
-	connDigest  map[uint8][]byte
-	authSuccess prometheus.Counter
-	authFail    *prometheus.CounterVec
-}
-
-func newReconnectCredentialManager(namespace, subsystem string, haConnections int) *reconnectCredentialManager {
-	authSuccess := prometheus.NewCounter(
-		prometheus.CounterOpts{
-			Namespace: namespace,
-			Subsystem: subsystem,
-			Name:      "tunnel_authenticate_success",
-			Help:      "Count of successful tunnel authenticate",
-		},
-	)
-	authFail := prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Namespace: namespace,
-			Subsystem: subsystem,
-			Name:      "tunnel_authenticate_fail",
-			Help:      "Count of tunnel authenticate errors by type",
-		},
-		[]string{"error"},
-	)
-	prometheus.MustRegister(authSuccess, authFail)
-	return &reconnectCredentialManager{
-		eventDigest: make(map[uint8][]byte, haConnections),
-		connDigest:  make(map[uint8][]byte, haConnections),
-		authSuccess: authSuccess,
-		authFail:    authFail,
-	}
-}
-
-func (cm *reconnectCredentialManager) ReconnectToken() ([]byte, error) {
-	cm.mu.RLock()
-	defer cm.mu.RUnlock()
-	if cm.jwt == nil {
-		return nil, errJWTUnset
-	}
-	return cm.jwt, nil
-}
-
-func (cm *reconnectCredentialManager) SetReconnectToken(jwt []byte) {
-	cm.mu.Lock()
-	defer cm.mu.Unlock()
-	cm.jwt = jwt
-}
-
-func (cm *reconnectCredentialManager) EventDigest(connID uint8) ([]byte, error) {
-	cm.mu.RLock()
-	defer cm.mu.RUnlock()
-	digest, ok := cm.eventDigest[connID]
-	if !ok {
-		return nil, fmt.Errorf("no event digest for connection %v", connID)
-	}
-	return digest, nil
-}
-
-func (cm *reconnectCredentialManager) SetEventDigest(connID uint8, digest []byte) {
-	cm.mu.Lock()
-	defer cm.mu.Unlock()
-	cm.eventDigest[connID] = digest
-}
-
-func (cm *reconnectCredentialManager) ConnDigest(connID uint8) ([]byte, error) {
-	cm.mu.RLock()
-	defer cm.mu.RUnlock()
-	digest, ok := cm.connDigest[connID]
-	if !ok {
-		return nil, fmt.Errorf("no connection digest for connection %v", connID)
-	}
-	return digest, nil
-}
-
-func (cm *reconnectCredentialManager) SetConnDigest(connID uint8, digest []byte) {
-	cm.mu.Lock()
-	defer cm.mu.Unlock()
-	cm.connDigest[connID] = digest
-}
-
-func (cm *reconnectCredentialManager) RefreshAuth(
-	ctx context.Context,
-	backoff *retry.BackoffHandler,
-	authenticate func(ctx context.Context, numPreviousAttempts int) (tunnelpogs.AuthOutcome, error),
-) (retryTimer <-chan time.Time, err error) {
-	authOutcome, err := authenticate(ctx, backoff.Retries())
-	if err != nil {
-		cm.authFail.WithLabelValues(err.Error()).Inc()
-		if _, ok := backoff.GetMaxBackoffDuration(ctx); ok {
-			return backoff.BackoffTimer(), nil
-		}
-		return nil, err
-	}
-	// clear backoff timer
-	backoff.SetGracePeriod()
-
-	switch outcome := authOutcome.(type) {
-	case tunnelpogs.AuthSuccess:
-		cm.SetReconnectToken(outcome.JWT())
-		cm.authSuccess.Inc()
-		return retry.Clock.After(outcome.RefreshAfter()), nil
-	case tunnelpogs.AuthUnknown:
-		duration := outcome.RefreshAfter()
-		cm.authFail.WithLabelValues(outcome.Error()).Inc()
-		return retry.Clock.After(duration), nil
-	case tunnelpogs.AuthFail:
-		cm.authFail.WithLabelValues(outcome.Error()).Inc()
-		return nil, outcome
-	default:
-		err := fmt.Errorf("refresh_auth: Unexpected outcome type %T", authOutcome)
-		cm.authFail.WithLabelValues(err.Error()).Inc()
-		return nil, err
-	}
-}

supervisor/reconnect_test.go

@@ -1,120 +0,0 @@
-package supervisor
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-
-	"github.com/cloudflare/cloudflared/retry"
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
-)
-
-func TestRefreshAuthBackoff(t *testing.T) {
-	rcm := newReconnectCredentialManager(t.Name(), t.Name(), 4)
-
-	var wait time.Duration
-	retry.Clock.After = func(d time.Duration) <-chan time.Time {
-		wait = d
-		return time.After(d)
-	}
-	backoff := &retry.BackoffHandler{MaxRetries: 3}
-	auth := func(ctx context.Context, n int) (tunnelpogs.AuthOutcome, error) {
-		return nil, fmt.Errorf("authentication failure")
-	}
-
-	// authentication failures should consume the backoff
-	for i := uint(0); i < backoff.MaxRetries; i++ {
-		retryChan, err := rcm.RefreshAuth(context.Background(), backoff, auth)
-		require.NoError(t, err)
-		require.NotNil(t, retryChan)
-		require.Greater(t, wait.Seconds(), 0.0)
-		require.Less(t, wait.Seconds(), float64((1<<(i+1))*time.Second))
-	}
-	retryChan, err := rcm.RefreshAuth(context.Background(), backoff, auth)
-	require.Error(t, err)
-	require.Nil(t, retryChan)
-
-	// now we actually make contact with the remote server
-	_, _ = rcm.RefreshAuth(context.Background(), backoff, func(ctx context.Context, n int) (tunnelpogs.AuthOutcome, error) {
-		return tunnelpogs.NewAuthUnknown(errors.New("auth unknown"), 19), nil
-	})
-
-	// The backoff timer should have been reset. To confirm this, make timeNow
-	// return a value after the backoff timer's grace period
-	retry.Clock.Now = func() time.Time {
-		expectedGracePeriod := time.Duration(time.Second * 2 << backoff.MaxRetries)
-		return time.Now().Add(expectedGracePeriod * 2)
-	}
-	_, ok := backoff.GetMaxBackoffDuration(context.Background())
-	require.True(t, ok)
-}
-
-func TestRefreshAuthSuccess(t *testing.T) {
-	rcm := newReconnectCredentialManager(t.Name(), t.Name(), 4)
-
-	var wait time.Duration
-	retry.Clock.After = func(d time.Duration) <-chan time.Time {
-		wait = d
-		return time.After(d)
-	}
-
-	backoff := &retry.BackoffHandler{MaxRetries: 3}
-	auth := func(ctx context.Context, n int) (tunnelpogs.AuthOutcome, error) {
-		return tunnelpogs.NewAuthSuccess([]byte("jwt"), 19), nil
-	}
-
-	retryChan, err := rcm.RefreshAuth(context.Background(), backoff, auth)
-	assert.NoError(t, err)
-	assert.NotNil(t, retryChan)
-	assert.Equal(t, 19*time.Hour, wait)
-
-	token, err := rcm.ReconnectToken()
-	assert.NoError(t, err)
-	assert.Equal(t, []byte("jwt"), token)
-}
-
-func TestRefreshAuthUnknown(t *testing.T) {
-	rcm := newReconnectCredentialManager(t.Name(), t.Name(), 4)
-
-	var wait time.Duration
-	retry.Clock.After = func(d time.Duration) <-chan time.Time {
-		wait = d
-		return time.After(d)
-	}
-
-	backoff := &retry.BackoffHandler{MaxRetries: 3}
-	auth := func(ctx context.Context, n int) (tunnelpogs.AuthOutcome, error) {
-		return tunnelpogs.NewAuthUnknown(errors.New("auth unknown"), 19), nil
-	}
-
-	retryChan, err := rcm.RefreshAuth(context.Background(), backoff, auth)
-	assert.NoError(t, err)
-	assert.NotNil(t, retryChan)
-	assert.Equal(t, 19*time.Hour, wait)
-
-	token, err := rcm.ReconnectToken()
-	assert.Equal(t, errJWTUnset, err)
-	assert.Nil(t, token)
-}
-
-func TestRefreshAuthFail(t *testing.T) {
-	rcm := newReconnectCredentialManager(t.Name(), t.Name(), 4)
-
-	backoff := &retry.BackoffHandler{MaxRetries: 3}
-	auth := func(ctx context.Context, n int) (tunnelpogs.AuthOutcome, error) {
-		return tunnelpogs.NewAuthFail(errors.New("auth fail")), nil
-	}
-
-	retryChan, err := rcm.RefreshAuth(context.Background(), backoff, auth)
-	assert.Error(t, err)
-	assert.Nil(t, retryChan)
-
-	token, err := rcm.ReconnectToken()
-	assert.Equal(t, errJWTUnset, err)
-	assert.Nil(t, token)
-}

supervisor/supervisor.go

@@ -49,8 +49,6 @@ type Supervisor struct {
 	log          *ConnAwareLogger
 	logTransport *zerolog.Logger
 
-	reconnectCredentialManager *reconnectCredentialManager
-
 	reconnectCh       chan ReconnectSignal
 	gracefulShutdownC <-chan struct{}
 }
@@ -76,8 +74,6 @@ func NewSupervisor(config *TunnelConfig, orchestrator *orchestration.Orchestrato
 		return nil, err
 	}
 
-	reconnectCredentialManager := newReconnectCredentialManager(connection.MetricsNamespace, connection.TunnelSubsystem, config.HAConnections)
-
 	tracker := tunnelstate.NewConnTracker(config.Log)
 	log := NewConnAwareLogger(config.Log, tracker, config.Observer)
 
@@ -87,7 +83,6 @@ func NewSupervisor(config *TunnelConfig, orchestrator *orchestration.Orchestrato
 	edgeTunnelServer := EdgeTunnelServer{
 		config:            config,
 		orchestrator:      orchestrator,
-		credentialManager: reconnectCredentialManager,
 		edgeAddrs:         edgeIPs,
 		edgeAddrHandler:   edgeAddrHandler,
 		edgeBindAddr:      edgeBindAddr,
@@ -98,18 +93,17 @@ func NewSupervisor(config *TunnelConfig, orchestrator *orchestration.Orchestrato
 	}
 
 	return &Supervisor{
-		config:                     config,
-		orchestrator:               orchestrator,
-		edgeIPs:                    edgeIPs,
-		edgeTunnelServer:           &edgeTunnelServer,
-		tunnelErrors:               make(chan tunnelError),
-		tunnelsConnecting:          map[int]chan struct{}{},
-		tunnelsProtocolFallback:    map[int]*protocolFallback{},
-		log:                        log,
-		logTransport:               config.LogTransport,
-		reconnectCredentialManager: reconnectCredentialManager,
-		reconnectCh:                reconnectCh,
-		gracefulShutdownC:          gracefulShutdownC,
+		config:                  config,
+		orchestrator:            orchestrator,
+		edgeIPs:                 edgeIPs,
+		edgeTunnelServer:        &edgeTunnelServer,
+		tunnelErrors:            make(chan tunnelError),
+		tunnelsConnecting:       map[int]chan struct{}{},
+		tunnelsProtocolFallback: map[int]*protocolFallback{},
+		log:                     log,
+		logTransport:            config.LogTransport,
+		reconnectCh:             reconnectCh,
+		gracefulShutdownC:       gracefulShutdownC,
 	}, nil
 }
 
@@ -138,7 +132,7 @@ func (s *Supervisor) Run(
 	var tunnelsWaiting []int
 	tunnelsActive := s.config.HAConnections
 
-	backoff := retry.BackoffHandler{MaxRetries: s.config.Retries, BaseTime: tunnelRetryDuration, RetryForever: true}
+	backoff := retry.NewBackoff(s.config.Retries, tunnelRetryDuration, true)
 	var backoffTimer <-chan time.Time
 
 	shuttingDown := false
@@ -212,7 +206,7 @@ func (s *Supervisor) initialize(
 		s.config.HAConnections = availableAddrs
 	}
 	s.tunnelsProtocolFallback[0] = &protocolFallback{
-		retry.BackoffHandler{MaxRetries: s.config.Retries, RetryForever: true},
+		retry.NewBackoff(s.config.Retries, retry.DefaultBaseTime, true),
 		s.config.ProtocolSelector.Current(),
 		false,
 	}
@@ -234,7 +228,7 @@ func (s *Supervisor) initialize(
 	// At least one successful connection, so start the rest
 	for i := 1; i < s.config.HAConnections; i++ {
 		s.tunnelsProtocolFallback[i] = &protocolFallback{
-			retry.BackoffHandler{MaxRetries: s.config.Retries, RetryForever: true},
+			retry.NewBackoff(s.config.Retries, retry.DefaultBaseTime, true),
 			// Set the protocol we know the first tunnel connected with.
 			s.tunnelsProtocolFallback[0].protocol,
 			false,

supervisor/tunnel.go

@@ -10,7 +10,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/google/uuid"
 	"github.com/pkg/errors"
 	"github.com/quic-go/quic-go"
 	"github.com/rs/zerolog"
@@ -27,8 +26,7 @@ import (
 	quicpogs "github.com/cloudflare/cloudflared/quic"
 	"github.com/cloudflare/cloudflared/retry"
 	"github.com/cloudflare/cloudflared/signal"
-	"github.com/cloudflare/cloudflared/tunnelrpc"
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 	"github.com/cloudflare/cloudflared/tunnelstate"
 )
 
@@ -49,7 +47,7 @@ type TunnelConfig struct {
 	HAConnections      int
 	IsAutoupdated      bool
 	LBPool             string
-	Tags               []tunnelpogs.Tag
+	Tags               []pogs.Tag
 	Log                *zerolog.Logger
 	LogTransport       *zerolog.Logger
 	Observer           *connection.Observer
@@ -60,47 +58,27 @@ type TunnelConfig struct {
 
 	NeedPQ bool
 
-	NamedTunnel      *connection.NamedTunnelProperties
+	NamedTunnel      *connection.TunnelProperties
 	ProtocolSelector connection.ProtocolSelector
 	EdgeTLSConfigs   map[connection.Protocol]*tls.Config
 	PacketConfig     *ingress.GlobalRouterConfig
 
-	UDPUnregisterSessionTimeout time.Duration
-	WriteStreamTimeout          time.Duration
+	RPCTimeout         time.Duration
+	WriteStreamTimeout time.Duration
 
-	DisableQUICPathMTUDiscovery bool
+	DisableQUICPathMTUDiscovery         bool
+	QUICConnectionLevelFlowControlLimit uint64
+	QUICStreamLevelFlowControlLimit     uint64
 
 	FeatureSelector *features.FeatureSelector
 }
 
-func (c *TunnelConfig) registrationOptions(connectionID uint8, OriginLocalIP string, uuid uuid.UUID) *tunnelpogs.RegistrationOptions {
-	policy := tunnelrpc.ExistingTunnelPolicy_balance
-	if c.HAConnections <= 1 && c.LBPool == "" {
-		policy = tunnelrpc.ExistingTunnelPolicy_disconnect
-	}
-	return &tunnelpogs.RegistrationOptions{
-		ClientID:             c.ClientID,
-		Version:              c.ReportedVersion,
-		OS:                   c.OSArch,
-		ExistingTunnelPolicy: policy,
-		PoolName:             c.LBPool,
-		Tags:                 c.Tags,
-		ConnectionID:         connectionID,
-		OriginLocalIP:        OriginLocalIP,
-		IsAutoupdated:        c.IsAutoupdated,
-		RunFromTerminal:      c.RunFromTerminal,
-		CompressionQuality:   0,
-		UUID:                 uuid.String(),
-		Features:             c.SupportedFeatures(),
-	}
-}
-
-func (c *TunnelConfig) connectionOptions(originLocalAddr string, numPreviousAttempts uint8) *tunnelpogs.ConnectionOptions {
+func (c *TunnelConfig) connectionOptions(originLocalAddr string, numPreviousAttempts uint8) *pogs.ConnectionOptions {
 	// attempt to parse out origin IP, but don't fail since it's informational field
 	host, _, _ := net.SplitHostPort(originLocalAddr)
 	originIP := net.ParseIP(host)
 
-	return &tunnelpogs.ConnectionOptions{
+	return &pogs.ConnectionOptions{
 		Client:              c.NamedTunnel.Client,
 		OriginLocalIP:       originIP,
 		ReplaceExisting:     c.ReplaceExisting,
@@ -203,7 +181,6 @@ func (f *ipAddrFallback) ShouldGetNewAddress(connIndex uint8, err error) (needsN
 type EdgeTunnelServer struct {
 	config            *TunnelConfig
 	orchestrator      *orchestration.Orchestrator
-	credentialManager *reconnectCredentialManager
 	edgeAddrHandler   EdgeAddrHandler
 	edgeAddrs         *edgediscovery.Edge
 	edgeBindAddr      net.IP
@@ -479,6 +456,7 @@ func (e *EdgeTunnelServer) serveConnection(
 		connIndex,
 		addr.UDP.IP,
 		nil,
+		e.config.RPCTimeout,
 		e.gracefulShutdownC,
 		e.config.GracePeriod,
 		protocol,
@@ -531,7 +509,7 @@ func (e *EdgeTunnelServer) serveHTTP2(
 	ctx context.Context,
 	connLog *ConnAwareLogger,
 	tlsServerConn net.Conn,
-	connOptions *tunnelpogs.ConnectionOptions,
+	connOptions *pogs.ConnectionOptions,
 	controlStreamHandler connection.ControlStreamHandler,
 	connIndex uint8,
 ) error {
@@ -573,7 +551,7 @@ func (e *EdgeTunnelServer) serveQUIC(
 	ctx context.Context,
 	edgeAddr *net.UDPAddr,
 	connLogger *ConnAwareLogger,
-	connOptions *tunnelpogs.ConnectionOptions,
+	connOptions *pogs.ConnectionOptions,
 	controlStreamHandler connection.ControlStreamHandler,
 	connIndex uint8,
 ) (err error, recoverable bool) {
@@ -591,15 +569,25 @@ func (e *EdgeTunnelServer) serveQUIC(
 
 	tlsConfig.CurvePreferences = curvePref
 
+	// quic-go 0.44 increases the initial packet size to 1280 by default. That breaks anyone running tunnel through WARP
+	// because WARP MTU is 1280.
+	var initialPacketSize uint16 = 1252
+	if edgeAddr.IP.To4() == nil {
+		initialPacketSize = 1232
+	}
+
 	quicConfig := &quic.Config{
-		HandshakeIdleTimeout:    quicpogs.HandshakeIdleTimeout,
-		MaxIdleTimeout:          quicpogs.MaxIdleTimeout,
-		KeepAlivePeriod:         quicpogs.MaxIdlePingPeriod,
-		MaxIncomingStreams:      quicpogs.MaxIncomingStreams,
-		MaxIncomingUniStreams:   quicpogs.MaxIncomingStreams,
-		EnableDatagrams:         true,
-		Tracer:                  quicpogs.NewClientTracer(connLogger.Logger(), connIndex),
-		DisablePathMTUDiscovery: e.config.DisableQUICPathMTUDiscovery,
+		HandshakeIdleTimeout:       quicpogs.HandshakeIdleTimeout,
+		MaxIdleTimeout:             quicpogs.MaxIdleTimeout,
+		KeepAlivePeriod:            quicpogs.MaxIdlePingPeriod,
+		MaxIncomingStreams:         quicpogs.MaxIncomingStreams,
+		MaxIncomingUniStreams:      quicpogs.MaxIncomingStreams,
+		EnableDatagrams:            true,
+		Tracer:                     quicpogs.NewClientTracer(connLogger.Logger(), connIndex),
+		DisablePathMTUDiscovery:    e.config.DisableQUICPathMTUDiscovery,
+		MaxConnectionReceiveWindow: e.config.QUICConnectionLevelFlowControlLimit,
+		MaxStreamReceiveWindow:     e.config.QUICStreamLevelFlowControlLimit,
+		InitialPacketSize:          initialPacketSize,
 	}
 
 	quicConn, err := connection.NewQUICConnection(
@@ -614,8 +602,9 @@ func (e *EdgeTunnelServer) serveQUIC(
 		controlStreamHandler,
 		connLogger.Logger(),
 		e.config.PacketConfig,
-		e.config.UDPUnregisterSessionTimeout,
+		e.config.RPCTimeout,
 		e.config.WriteStreamTimeout,
+		e.config.GracePeriod,
 	)
 	if err != nil {
 		connLogger.ConnAwareLogger().Err(err).Msgf("Failed to create new quic connection")

supervisor/tunnel_test.go

@@ -24,14 +24,18 @@ func (dmf *dynamicMockFetcher) fetch() edgediscovery.PercentageFetcher {
 	}
 }
 
+func immediateTimeAfter(time.Duration) <-chan time.Time {
+	c := make(chan time.Time, 1)
+	c <- time.Now()
+	return c
+}
+
 func TestWaitForBackoffFallback(t *testing.T) {
 	maxRetries := uint(3)
-	backoff := retry.BackoffHandler{
-		MaxRetries: maxRetries,
-		BaseTime:   time.Millisecond * 10,
-	}
+	backoff := retry.NewBackoff(maxRetries, 40*time.Millisecond, false)
+	backoff.Clock.After = immediateTimeAfter
 	log := zerolog.Nop()
-	resolveTTL := time.Duration(0)
+	resolveTTL := 10 * time.Second
 	mockFetcher := dynamicMockFetcher{
 		protocolPercents: edgediscovery.ProtocolPercents{edgediscovery.ProtocolPercent{Protocol: "quic", Percentage: 100}},
 	}
@@ -64,21 +68,23 @@ func TestWaitForBackoffFallback(t *testing.T) {
 	}
 
 	// Retry fallback protocol
-	for i := 0; i < int(maxRetries); i++ {
-		protoFallback.BackoffTimer() // simulate retry
-		ok := selectNextProtocol(&log, protoFallback, protocolSelector, nil)
-		assert.True(t, ok)
-		fallback, ok := protocolSelector.Fallback()
-		assert.True(t, ok)
-		assert.Equal(t, fallback, protoFallback.protocol)
-	}
+	protoFallback.BackoffTimer() // simulate retry
+	ok := selectNextProtocol(&log, protoFallback, protocolSelector, nil)
+	assert.True(t, ok)
+	fallback, ok := protocolSelector.Fallback()
+	assert.True(t, ok)
+	assert.Equal(t, fallback, protoFallback.protocol)
+	assert.Equal(t, connection.HTTP2, protoFallback.protocol)
 
 	currentGlobalProtocol := protocolSelector.Current()
 	assert.Equal(t, initProtocol, currentGlobalProtocol)
 
+	// Simulate max retries again (retries reset after protocol switch)
+	for i := 0; i < int(maxRetries); i++ {
+		protoFallback.BackoffTimer()
+	}
 	// No protocol to fallback, return error
-	protoFallback.BackoffTimer() // simulate retry
-	ok := selectNextProtocol(&log, protoFallback, protocolSelector, nil)
+	ok = selectNextProtocol(&log, protoFallback, protocolSelector, nil)
 	assert.False(t, ok)
 
 	protoFallback.reset()

token/token.go

@@ -94,9 +94,10 @@ func errDeleteTokenFailed(lockFilePath string) error {
 // newLock will get a new file lock
 func newLock(path string) *lock {
 	lockPath := path + ".lock"
+	backoff := retry.NewBackoff(uint(7), retry.DefaultBaseTime, false)
 	return &lock{
 		lockFilePath: lockPath,
-		backoff:      &retry.BackoffHandler{MaxRetries: 7},
+		backoff:      &backoff,
 		sigHandler: &signalHandler{
 			signals: []os.Signal{syscall.SIGINT, syscall.SIGTERM},
 		},

tunnelrpc/go.capnp

@@ -1,15 +0,0 @@
-# Generate go.capnp.out with:
-# capnp compile -o- go.capnp > go.capnp.out
-# Must run inside this directory to preserve paths.
-
-@0xd12a1c51fedd6c88;
-
-annotation package(file) :Text;
-annotation import(file) :Text;
-annotation doc(struct, field, enum) :Text;
-annotation tag(enumerant) :Text;
-annotation notag(enumerant) :Void;
-annotation customtype(field) :Text;
-annotation name(struct, field, union, enum, enumerant, interface, method, param, annotation, const, group) :Text;
-
-$package("capnp");

tunnelrpc/log.go

@@ -1,43 +0,0 @@
-package tunnelrpc
-
-import (
-	"context"
-
-	"github.com/rs/zerolog"
-	"golang.org/x/net/trace"
-	"zombiezen.com/go/capnproto2/rpc"
-)
-
-// ConnLogger wraps a Zerolog Logger for a connection.
-type ConnLogger struct {
-	Log *zerolog.Logger
-}
-
-func (c ConnLogger) Infof(ctx context.Context, format string, args ...interface{}) {
-	c.Log.Info().Msgf(format, args...)
-}
-
-func (c ConnLogger) Errorf(ctx context.Context, format string, args ...interface{}) {
-	c.Log.Error().Msgf(format, args...)
-}
-
-func ConnLog(log *zerolog.Logger) rpc.ConnOption {
-	return rpc.ConnLog(ConnLogger{log})
-}
-
-// ConnTracer wraps a trace.EventLog for a connection.
-type ConnTracer struct {
-	Events trace.EventLog
-}
-
-func (c ConnTracer) Infof(ctx context.Context, format string, args ...interface{}) {
-	c.Events.Printf(format, args...)
-}
-
-func (c ConnTracer) Errorf(ctx context.Context, format string, args ...interface{}) {
-	c.Events.Errorf(format, args...)
-}
-
-func ConnTrace(events trace.EventLog) rpc.ConnOption {
-	return rpc.ConnLog(ConnTracer{events})
-}

tunnelrpc/logtransport.go

@@ -1,45 +0,0 @@
-// Package logtransport provides a transport that logs all of its messages.
-package tunnelrpc
-
-import (
-	"bytes"
-	"context"
-
-	"github.com/rs/zerolog"
-	"zombiezen.com/go/capnproto2/encoding/text"
-	"zombiezen.com/go/capnproto2/rpc"
-	rpccapnp "zombiezen.com/go/capnproto2/std/capnp/rpc"
-)
-
-type transport struct {
-	rpc.Transport
-	log *zerolog.Logger
-}
-
-// NewTransportLogger creates a new logger that proxies messages to and from t and
-// logs them to log.  If log is nil, then the log package's default
-// logger is used.
-func NewTransportLogger(log *zerolog.Logger, t rpc.Transport) rpc.Transport {
-	return &transport{Transport: t, log: log}
-}
-
-func (t *transport) SendMessage(ctx context.Context, msg rpccapnp.Message) error {
-	t.log.Trace().Msgf("rpc tx: %s", formatMsg(msg))
-	return t.Transport.SendMessage(ctx, msg)
-}
-
-func (t *transport) RecvMessage(ctx context.Context) (rpccapnp.Message, error) {
-	msg, err := t.Transport.RecvMessage(ctx)
-	if err != nil {
-		t.log.Debug().Msgf("rpc rx error: %s", err)
-		return msg, err
-	}
-	t.log.Trace().Msgf("rpc rx: %s", formatMsg(msg))
-	return msg, nil
-}
-
-func formatMsg(m rpccapnp.Message) string {
-	var buf bytes.Buffer
-	_ = text.NewEncoder(&buf).Encode(0x91b79f1f808db032, m.Struct)
-	return buf.String()
-}

tunnelrpc/metrics/metrics.go

@@ -0,0 +1,143 @@
+package metrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const (
+	metricsNamespace = "cloudflared"
+	rpcSubsystem     = "rpc"
+)
+
+// CloudflaredServer operation labels
+// CloudflaredServer is an extension of SessionManager with additional methods, but it's helpful
+// to visualize it separately in the metrics since they are technically different client/servers.
+const (
+	Cloudflared = "cloudflared"
+)
+
+// ConfigurationManager operation labels
+const (
+	ConfigurationManager = "config"
+
+	OperationUpdateConfiguration = "update_configuration"
+)
+
+// SessionManager operation labels
+const (
+	SessionManager = "session"
+
+	OperationRegisterUdpSession   = "register_udp_session"
+	OperationUnregisterUdpSession = "unregister_udp_session"
+)
+
+// RegistrationServer operation labels
+const (
+	Registration = "registration"
+
+	OperationRegisterConnection       = "register_connection"
+	OperationUnregisterConnection     = "unregister_connection"
+	OperationUpdateLocalConfiguration = "update_local_configuration"
+)
+
+type rpcMetrics struct {
+	serverOperations        *prometheus.CounterVec
+	serverFailures          *prometheus.CounterVec
+	serverOperationsLatency *prometheus.HistogramVec
+
+	ClientOperations        *prometheus.CounterVec
+	ClientFailures          *prometheus.CounterVec
+	ClientOperationsLatency *prometheus.HistogramVec
+}
+
+var CapnpMetrics *rpcMetrics = &rpcMetrics{
+	serverOperations: prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: metricsNamespace,
+			Subsystem: rpcSubsystem,
+			Name:      "server_operations",
+			Help:      "Number of rpc methods by handler served",
+		},
+		[]string{"handler", "method"},
+	),
+	serverFailures: prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: metricsNamespace,
+			Subsystem: rpcSubsystem,
+			Name:      "server_failures",
+			Help:      "Number of rpc methods failures by handler served",
+		},
+		[]string{"handler", "method"},
+	),
+	serverOperationsLatency: prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Namespace: metricsNamespace,
+			Subsystem: rpcSubsystem,
+			Name:      "server_latency_secs",
+			Help:      "Latency of rpc methods by handler served",
+			// Bucket starts at 50ms, each bucket grows by a factor of 3, up to 5 buckets and is expressed as seconds:
+			// 50ms, 150ms, 450ms, 1350ms, 4050ms
+			Buckets: prometheus.ExponentialBuckets(0.05, 3, 5),
+		},
+		[]string{"handler", "method"},
+	),
+	ClientOperations: prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: metricsNamespace,
+			Subsystem: rpcSubsystem,
+			Name:      "client_operations",
+			Help:      "Number of rpc methods by handler requested",
+		},
+		[]string{"handler", "method"},
+	),
+	ClientFailures: prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: metricsNamespace,
+			Subsystem: rpcSubsystem,
+			Name:      "client_failures",
+			Help:      "Number of rpc method failures by handler requested",
+		},
+		[]string{"handler", "method"},
+	),
+	ClientOperationsLatency: prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Namespace: metricsNamespace,
+			Subsystem: rpcSubsystem,
+			Name:      "client_latency_secs",
+			Help:      "Latency of rpc methods by handler requested",
+			// Bucket starts at 50ms, each bucket grows by a factor of 3, up to 5 buckets and is expressed as seconds:
+			// 50ms, 150ms, 450ms, 1350ms, 4050ms
+			Buckets: prometheus.ExponentialBuckets(0.05, 3, 5),
+		},
+		[]string{"handler", "method"},
+	),
+}
+
+func ObserveServerHandler(inner func() error, handler, method string) error {
+	defer CapnpMetrics.serverOperations.WithLabelValues(handler, method).Inc()
+	timer := prometheus.NewTimer(prometheus.ObserverFunc(func(s float64) {
+		CapnpMetrics.serverOperationsLatency.WithLabelValues(handler, method).Observe(s)
+	}))
+	defer timer.ObserveDuration()
+
+	err := inner()
+	if err != nil {
+		CapnpMetrics.serverFailures.WithLabelValues(handler, method).Inc()
+	}
+	return err
+}
+
+func NewClientOperationLatencyObserver(server string, method string) *prometheus.Timer {
+	return prometheus.NewTimer(prometheus.ObserverFunc(func(s float64) {
+		CapnpMetrics.ClientOperationsLatency.WithLabelValues(server, method).Observe(s)
+	}))
+}
+
+func init() {
+	prometheus.MustRegister(CapnpMetrics.serverOperations)
+	prometheus.MustRegister(CapnpMetrics.serverFailures)
+	prometheus.MustRegister(CapnpMetrics.serverOperationsLatency)
+	prometheus.MustRegister(CapnpMetrics.ClientOperations)
+	prometheus.MustRegister(CapnpMetrics.ClientFailures)
+	prometheus.MustRegister(CapnpMetrics.ClientOperationsLatency)
+}

tunnelrpc/pogs/auth_outcome.go

@@ -1,131 +0,0 @@
-package pogs
-
-import (
-	"errors"
-	"time"
-)
-
-// AuthenticateResponse is the serialized response from the Authenticate RPC.
-// It's a 1:1 representation of the capnp message, so it's not very useful for programmers.
-// Instead, you should call the `Outcome()` method to get a programmer-friendly sum type, with one
-// case for each possible outcome.
-type AuthenticateResponse struct {
-	PermanentErr      string
-	RetryableErr      string
-	Jwt               []byte
-	HoursUntilRefresh uint8
-}
-
-// Outcome turns the deserialized response of Authenticate into a programmer-friendly sum type.
-func (ar AuthenticateResponse) Outcome() AuthOutcome {
-	// If the user's authentication was unsuccessful, the server will return an error explaining why.
-	// cloudflared should fatal with this error.
-	if ar.PermanentErr != "" {
-		return NewAuthFail(errors.New(ar.PermanentErr))
-	}
-
-	// If there was a network error, then cloudflared should retry later,
-	// because origintunneld couldn't prove whether auth was correct or not.
-	if ar.RetryableErr != "" {
-		return NewAuthUnknown(errors.New(ar.RetryableErr), ar.HoursUntilRefresh)
-	}
-
-	// If auth succeeded, return the token and refresh it when instructed.
-	if len(ar.Jwt) > 0 {
-		return NewAuthSuccess(ar.Jwt, ar.HoursUntilRefresh)
-	}
-
-	// Otherwise the state got messed up.
-	return nil
-}
-
-// AuthOutcome is a programmer-friendly sum type denoting the possible outcomes of Authenticate.
-type AuthOutcome interface {
-	isAuthOutcome()
-	// Serialize into an AuthenticateResponse which can be sent via Capnp
-	Serialize() AuthenticateResponse
-}
-
-// AuthSuccess means the backend successfully authenticated this cloudflared.
-type AuthSuccess struct {
-	jwt               []byte
-	hoursUntilRefresh uint8
-}
-
-func NewAuthSuccess(jwt []byte, hoursUntilRefresh uint8) AuthSuccess {
-	return AuthSuccess{jwt: jwt, hoursUntilRefresh: hoursUntilRefresh}
-}
-
-func (ao AuthSuccess) JWT() []byte {
-	return ao.jwt
-}
-
-// RefreshAfter is how long cloudflared should wait before rerunning Authenticate.
-func (ao AuthSuccess) RefreshAfter() time.Duration {
-	return hoursToTime(ao.hoursUntilRefresh)
-}
-
-// Serialize into an AuthenticateResponse which can be sent via Capnp
-func (ao AuthSuccess) Serialize() AuthenticateResponse {
-	return AuthenticateResponse{
-		Jwt:               ao.jwt,
-		HoursUntilRefresh: ao.hoursUntilRefresh,
-	}
-}
-
-func (ao AuthSuccess) isAuthOutcome() {}
-
-// AuthFail means this cloudflared has the wrong auth and should exit.
-type AuthFail struct {
-	err error
-}
-
-func NewAuthFail(err error) AuthFail {
-	return AuthFail{err: err}
-}
-
-func (ao AuthFail) Error() string {
-	return ao.err.Error()
-}
-
-// Serialize into an AuthenticateResponse which can be sent via Capnp
-func (ao AuthFail) Serialize() AuthenticateResponse {
-	return AuthenticateResponse{
-		PermanentErr: ao.err.Error(),
-	}
-}
-
-func (ao AuthFail) isAuthOutcome() {}
-
-// AuthUnknown means the backend couldn't finish checking authentication. Try again later.
-type AuthUnknown struct {
-	err               error
-	hoursUntilRefresh uint8
-}
-
-func NewAuthUnknown(err error, hoursUntilRefresh uint8) AuthUnknown {
-	return AuthUnknown{err: err, hoursUntilRefresh: hoursUntilRefresh}
-}
-
-func (ao AuthUnknown) Error() string {
-	return ao.err.Error()
-}
-
-// RefreshAfter is how long cloudflared should wait before rerunning Authenticate.
-func (ao AuthUnknown) RefreshAfter() time.Duration {
-	return hoursToTime(ao.hoursUntilRefresh)
-}
-
-// Serialize into an AuthenticateResponse which can be sent via Capnp
-func (ao AuthUnknown) Serialize() AuthenticateResponse {
-	return AuthenticateResponse{
-		RetryableErr:      ao.err.Error(),
-		HoursUntilRefresh: ao.hoursUntilRefresh,
-	}
-}
-
-func (ao AuthUnknown) isAuthOutcome() {}
-
-func hoursToTime(hours uint8) time.Duration {
-	return time.Duration(hours) * time.Hour
-}

tunnelrpc/pogs/auth_serialize.go

@@ -1,78 +0,0 @@
-package pogs
-
-import (
-	"context"
-
-	"zombiezen.com/go/capnproto2/pogs"
-	"zombiezen.com/go/capnproto2/server"
-
-	"github.com/cloudflare/cloudflared/tunnelrpc"
-)
-
-func (i TunnelServer_PogsImpl) Authenticate(p tunnelrpc.TunnelServer_authenticate) error {
-	originCert, err := p.Params.OriginCert()
-	if err != nil {
-		return err
-	}
-	hostname, err := p.Params.Hostname()
-	if err != nil {
-		return err
-	}
-	options, err := p.Params.Options()
-	if err != nil {
-		return err
-	}
-	pogsOptions, err := UnmarshalRegistrationOptions(options)
-	if err != nil {
-		return err
-	}
-
-	server.Ack(p.Options)
-	resp, err := i.impl.Authenticate(p.Ctx, originCert, hostname, pogsOptions)
-	if err != nil {
-		return err
-	}
-	result, err := p.Results.NewResult()
-	if err != nil {
-		return err
-	}
-	return MarshalAuthenticateResponse(result, resp)
-}
-
-func MarshalAuthenticateResponse(s tunnelrpc.AuthenticateResponse, p *AuthenticateResponse) error {
-	return pogs.Insert(tunnelrpc.AuthenticateResponse_TypeID, s.Struct, p)
-}
-
-func (c TunnelServer_PogsClient) Authenticate(ctx context.Context, originCert []byte, hostname string, options *RegistrationOptions) (*AuthenticateResponse, error) {
-	client := tunnelrpc.TunnelServer{Client: c.Client}
-	promise := client.Authenticate(ctx, func(p tunnelrpc.TunnelServer_authenticate_Params) error {
-		err := p.SetOriginCert(originCert)
-		if err != nil {
-			return err
-		}
-		err = p.SetHostname(hostname)
-		if err != nil {
-			return err
-		}
-		registrationOptions, err := p.NewOptions()
-		if err != nil {
-			return err
-		}
-		err = MarshalRegistrationOptions(registrationOptions, options)
-		if err != nil {
-			return err
-		}
-		return nil
-	})
-	retval, err := promise.Result().Struct()
-	if err != nil {
-		return nil, err
-	}
-	return UnmarshalAuthenticateResponse(retval)
-}
-
-func UnmarshalAuthenticateResponse(s tunnelrpc.AuthenticateResponse) (*AuthenticateResponse, error) {
-	p := new(AuthenticateResponse)
-	err := pogs.Extract(p, tunnelrpc.AuthenticateResponse_TypeID, s.Struct)
-	return p, err
-}

tunnelrpc/pogs/auth_test.go

@@ -1,136 +0,0 @@
-package pogs
-
-import (
-	"fmt"
-	"reflect"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-	capnp "zombiezen.com/go/capnproto2"
-
-	"github.com/cloudflare/cloudflared/tunnelrpc"
-)
-
-// Ensure the AuthOutcome sum is correct
-var _ AuthOutcome = &AuthSuccess{}
-var _ AuthOutcome = &AuthFail{}
-var _ AuthOutcome = &AuthUnknown{}
-
-// Unit tests for AuthenticateResponse.Outcome()
-func TestAuthenticateResponseOutcome(t *testing.T) {
-	type fields struct {
-		PermanentErr      string
-		RetryableErr      string
-		Jwt               []byte
-		HoursUntilRefresh uint8
-	}
-	tests := []struct {
-		name   string
-		fields fields
-		want   AuthOutcome
-	}{
-		{"success",
-			fields{Jwt: []byte("asdf"), HoursUntilRefresh: 6},
-			AuthSuccess{jwt: []byte("asdf"), hoursUntilRefresh: 6},
-		},
-		{"fail",
-			fields{PermanentErr: "bad creds"},
-			AuthFail{err: fmt.Errorf("bad creds")},
-		},
-		{"error",
-			fields{RetryableErr: "bad conn", HoursUntilRefresh: 6},
-			AuthUnknown{err: fmt.Errorf("bad conn"), hoursUntilRefresh: 6},
-		},
-		{"nil (no fields are set)",
-			fields{},
-			nil,
-		},
-		{"nil (too few fields are set)",
-			fields{HoursUntilRefresh: 6},
-			nil,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			ar := AuthenticateResponse{
-				PermanentErr:      tt.fields.PermanentErr,
-				RetryableErr:      tt.fields.RetryableErr,
-				Jwt:               tt.fields.Jwt,
-				HoursUntilRefresh: tt.fields.HoursUntilRefresh,
-			}
-			got := ar.Outcome()
-			if !reflect.DeepEqual(got, tt.want) {
-				t.Errorf("AuthenticateResponse.Outcome() = %T, want %v", got, tt.want)
-			}
-			if got != nil && !reflect.DeepEqual(got.Serialize(), ar) {
-				t.Errorf(".Outcome() and .Serialize() should be inverses but weren't. Expected %v, got %v", ar, got.Serialize())
-			}
-		})
-	}
-}
-
-func TestAuthSuccess(t *testing.T) {
-	input := NewAuthSuccess([]byte("asdf"), 6)
-	output, ok := input.Serialize().Outcome().(AuthSuccess)
-	assert.True(t, ok)
-	assert.Equal(t, input, output)
-}
-
-func TestAuthUnknown(t *testing.T) {
-	input := NewAuthUnknown(fmt.Errorf("pdx unreachable"), 6)
-	output, ok := input.Serialize().Outcome().(AuthUnknown)
-	assert.True(t, ok)
-	assert.Equal(t, input, output)
-}
-
-func TestAuthFail(t *testing.T) {
-	input := NewAuthFail(fmt.Errorf("wrong creds"))
-	output, ok := input.Serialize().Outcome().(AuthFail)
-	assert.True(t, ok)
-	assert.Equal(t, input, output)
-}
-
-func TestWhenToRefresh(t *testing.T) {
-	expected := 4 * time.Hour
-	actual := hoursToTime(4)
-	if expected != actual {
-		t.Fatalf("expected %v hours, got %v", expected, actual)
-	}
-}
-
-// Test that serializing and deserializing AuthenticationResponse undo each other.
-func TestSerializeAuthenticationResponse(t *testing.T) {
-
-	tests := []*AuthenticateResponse{
-		{
-			Jwt:               []byte("\xbd\xb2\x3d\xbc\x20\xe2\x8c\x98"),
-			HoursUntilRefresh: 24,
-		},
-		{
-			PermanentErr: "bad auth",
-		},
-		{
-			RetryableErr:      "bad connection",
-			HoursUntilRefresh: 24,
-		},
-	}
-
-	for i, testCase := range tests {
-		_, seg, err := capnp.NewMessage(capnp.SingleSegment(nil))
-		assert.NoError(t, err)
-		capnpEntity, err := tunnelrpc.NewAuthenticateResponse(seg)
-		if !assert.NoError(t, err) {
-			t.Fatal("Couldn't initialize a new message")
-		}
-		err = MarshalAuthenticateResponse(capnpEntity, testCase)
-		if !assert.NoError(t, err, "testCase index %v failed to marshal", i) {
-			continue
-		}
-		result, err := UnmarshalAuthenticateResponse(capnpEntity)
-		if !assert.NoError(t, err, "testCase index %v failed to unmarshal", i) {
-			continue
-		}
-		assert.Equal(t, testCase, result, "testCase index %v didn't preserve struct through marshalling and unmarshalling", i)
-	}
-}

tunnelrpc/pogs/cloudflared_server.go

@@ -1,9 +1,10 @@
 package pogs
 
 import (
-	"github.com/cloudflare/cloudflared/tunnelrpc"
 	capnp "zombiezen.com/go/capnproto2"
 	"zombiezen.com/go/capnproto2/rpc"
+
+	"github.com/cloudflare/cloudflared/tunnelrpc/proto"
 )
 
 type CloudflaredServer interface {
@@ -16,8 +17,8 @@ type CloudflaredServer_PogsImpl struct {
 	ConfigurationManager_PogsImpl
 }
 
-func CloudflaredServer_ServerToClient(s SessionManager, c ConfigurationManager) tunnelrpc.CloudflaredServer {
-	return tunnelrpc.CloudflaredServer_ServerToClient(CloudflaredServer_PogsImpl{
+func CloudflaredServer_ServerToClient(s SessionManager, c ConfigurationManager) proto.CloudflaredServer {
+	return proto.CloudflaredServer_ServerToClient(CloudflaredServer_PogsImpl{
 		SessionManager_PogsImpl:       SessionManager_PogsImpl{s},
 		ConfigurationManager_PogsImpl: ConfigurationManager_PogsImpl{c},
 	})

tunnelrpc/pogs/configuration_manager.go

@@ -4,13 +4,16 @@ import (
 	"context"
 	"fmt"
 
-	"github.com/cloudflare/cloudflared/tunnelrpc"
 	capnp "zombiezen.com/go/capnproto2"
 	"zombiezen.com/go/capnproto2/rpc"
 	"zombiezen.com/go/capnproto2/server"
+
+	"github.com/cloudflare/cloudflared/tunnelrpc/metrics"
+	"github.com/cloudflare/cloudflared/tunnelrpc/proto"
 )
 
 type ConfigurationManager interface {
+	// UpdateConfiguration is the call provided to cloudflared to load the latest remote configuration.
 	UpdateConfiguration(ctx context.Context, version int32, config []byte) *UpdateConfigurationResponse
 }
 
@@ -18,11 +21,15 @@ type ConfigurationManager_PogsImpl struct {
 	impl ConfigurationManager
 }
 
-func ConfigurationManager_ServerToClient(c ConfigurationManager) tunnelrpc.ConfigurationManager {
-	return tunnelrpc.ConfigurationManager_ServerToClient(ConfigurationManager_PogsImpl{c})
+func ConfigurationManager_ServerToClient(c ConfigurationManager) proto.ConfigurationManager {
+	return proto.ConfigurationManager_ServerToClient(ConfigurationManager_PogsImpl{c})
 }
 
-func (i ConfigurationManager_PogsImpl) UpdateConfiguration(p tunnelrpc.ConfigurationManager_updateConfiguration) error {
+func (i ConfigurationManager_PogsImpl) UpdateConfiguration(p proto.ConfigurationManager_updateConfiguration) error {
+	return metrics.ObserveServerHandler(func() error { return i.updateConfiguration(p) }, metrics.ConfigurationManager, metrics.OperationUpdateConfiguration)
+}
+
+func (i ConfigurationManager_PogsImpl) updateConfiguration(p proto.ConfigurationManager_updateConfiguration) error {
 	server.Ack(p.Options)
 
 	version := p.Params.Version()
@@ -51,8 +58,8 @@ func (c ConfigurationManager_PogsClient) Close() error {
 }
 
 func (c ConfigurationManager_PogsClient) UpdateConfiguration(ctx context.Context, version int32, config []byte) (*UpdateConfigurationResponse, error) {
-	client := tunnelrpc.ConfigurationManager{Client: c.Client}
-	promise := client.UpdateConfiguration(ctx, func(p tunnelrpc.ConfigurationManager_updateConfiguration_Params) error {
+	client := proto.ConfigurationManager{Client: c.Client}
+	promise := client.UpdateConfiguration(ctx, func(p proto.ConfigurationManager_updateConfiguration_Params) error {
 		p.SetVersion(version)
 		return p.SetConfig(config)
 	})
@@ -74,7 +81,7 @@ type UpdateConfigurationResponse struct {
 	Err                error `json:"err"`
 }
 
-func (p *UpdateConfigurationResponse) Marshal(s tunnelrpc.UpdateConfigurationResponse) error {
+func (p *UpdateConfigurationResponse) Marshal(s proto.UpdateConfigurationResponse) error {
 	s.SetLatestAppliedVersion(p.LastAppliedVersion)
 	if p.Err != nil {
 		return s.SetErr(p.Err.Error())
@@ -82,7 +89,7 @@ func (p *UpdateConfigurationResponse) Marshal(s tunnelrpc.UpdateConfigurationRes
 	return nil
 }
 
-func (p *UpdateConfigurationResponse) Unmarshal(s tunnelrpc.UpdateConfigurationResponse) error {
+func (p *UpdateConfigurationResponse) Unmarshal(s proto.UpdateConfigurationResponse) error {
 	p.LastAppliedVersion = s.LatestAppliedVersion()
 	respErr, err := s.Err()
 	if err != nil {

tunnelrpc/pogs/quic_metadata_protocol.go

@@ -1,4 +1,4 @@
-package quic
+package pogs
 
 import (
 	"fmt"
@@ -6,7 +6,7 @@ import (
 	capnp "zombiezen.com/go/capnproto2"
 	"zombiezen.com/go/capnproto2/pogs"
 
-	"github.com/cloudflare/cloudflared/quic/schema"
+	"github.com/cloudflare/cloudflared/tunnelrpc/proto"
 )
 
 // ConnectionType indicates the type of underlying connection proxied within the QUIC stream.
@@ -52,26 +52,26 @@ func (r *ConnectRequest) MetadataMap() map[string]string {
 	return metadataMap
 }
 
-func (r *ConnectRequest) fromPogs(msg *capnp.Message) error {
-	metadata, err := schema.ReadRootConnectRequest(msg)
+func (r *ConnectRequest) FromPogs(msg *capnp.Message) error {
+	metadata, err := proto.ReadRootConnectRequest(msg)
 	if err != nil {
 		return err
 	}
-	return pogs.Extract(r, schema.ConnectRequest_TypeID, metadata.Struct)
+	return pogs.Extract(r, proto.ConnectRequest_TypeID, metadata.Struct)
 }
 
-func (r *ConnectRequest) toPogs() (*capnp.Message, error) {
+func (r *ConnectRequest) ToPogs() (*capnp.Message, error) {
 	msg, seg, err := capnp.NewMessage(capnp.SingleSegment(nil))
 	if err != nil {
 		return nil, err
 	}
 
-	root, err := schema.NewRootConnectRequest(seg)
+	root, err := proto.NewRootConnectRequest(seg)
 	if err != nil {
 		return nil, err
 	}
 
-	if err := pogs.Insert(schema.ConnectRequest_TypeID, root.Struct, r); err != nil {
+	if err := pogs.Insert(proto.ConnectRequest_TypeID, root.Struct, r); err != nil {
 		return nil, err
 	}
 
@@ -84,26 +84,26 @@ type ConnectResponse struct {
 	Metadata []Metadata `capnp:"metadata"`
 }
 
-func (r *ConnectResponse) fromPogs(msg *capnp.Message) error {
-	metadata, err := schema.ReadRootConnectResponse(msg)
+func (r *ConnectResponse) FromPogs(msg *capnp.Message) error {
+	metadata, err := proto.ReadRootConnectResponse(msg)
 	if err != nil {
 		return err
 	}
-	return pogs.Extract(r, schema.ConnectResponse_TypeID, metadata.Struct)
+	return pogs.Extract(r, proto.ConnectResponse_TypeID, metadata.Struct)
 }
 
-func (r *ConnectResponse) toPogs() (*capnp.Message, error) {
+func (r *ConnectResponse) ToPogs() (*capnp.Message, error) {
 	msg, seg, err := capnp.NewMessage(capnp.SingleSegment(nil))
 	if err != nil {
 		return nil, err
 	}
 
-	root, err := schema.NewRootConnectResponse(seg)
+	root, err := proto.NewRootConnectResponse(seg)
 	if err != nil {
 		return nil, err
 	}
 
-	if err := pogs.Insert(schema.ConnectResponse_TypeID, root.Struct, r); err != nil {
+	if err := pogs.Insert(proto.ConnectResponse_TypeID, root.Struct, r); err != nil {
 		return nil, err
 	}
 

tunnelrpc/pogs/reconnect_tunnel.go

@@ -1,93 +0,0 @@
-package pogs
-
-import (
-	"context"
-
-	"zombiezen.com/go/capnproto2/server"
-
-	"github.com/cloudflare/cloudflared/tunnelrpc"
-)
-
-func (i TunnelServer_PogsImpl) ReconnectTunnel(p tunnelrpc.TunnelServer_reconnectTunnel) error {
-	jwt, err := p.Params.Jwt()
-	if err != nil {
-		return err
-	}
-	eventDigest, err := p.Params.EventDigest()
-	if err != nil {
-		return err
-	}
-	connDigest, err := p.Params.ConnDigest()
-	if err != nil {
-		return err
-	}
-	hostname, err := p.Params.Hostname()
-	if err != nil {
-		return err
-	}
-	options, err := p.Params.Options()
-	if err != nil {
-		return err
-	}
-	pogsOptions, err := UnmarshalRegistrationOptions(options)
-	if err != nil {
-		return err
-	}
-	server.Ack(p.Options)
-	registration, err := i.impl.ReconnectTunnel(p.Ctx, jwt, eventDigest, connDigest, hostname, pogsOptions)
-	if err != nil {
-		return err
-	}
-	result, err := p.Results.NewResult()
-	if err != nil {
-		return err
-	}
-	return MarshalTunnelRegistration(result, registration)
-}
-
-func (c TunnelServer_PogsClient) ReconnectTunnel(
-	ctx context.Context,
-	jwt,
-	eventDigest []byte,
-	connDigest []byte,
-	hostname string,
-	options *RegistrationOptions,
-) *TunnelRegistration {
-	client := tunnelrpc.TunnelServer{Client: c.Client}
-	promise := client.ReconnectTunnel(ctx, func(p tunnelrpc.TunnelServer_reconnectTunnel_Params) error {
-		err := p.SetJwt(jwt)
-		if err != nil {
-			return err
-		}
-		err = p.SetEventDigest(eventDigest)
-		if err != nil {
-			return err
-		}
-		err = p.SetConnDigest(connDigest)
-		if err != nil {
-			return err
-		}
-		err = p.SetHostname(hostname)
-		if err != nil {
-			return err
-		}
-		registrationOptions, err := p.NewOptions()
-		if err != nil {
-			return err
-		}
-		err = MarshalRegistrationOptions(registrationOptions, options)
-		if err != nil {
-			return err
-		}
-		return nil
-	})
-	retval, err := promise.Result().Struct()
-	if err != nil {
-		return NewRetryableRegistrationError(err, defaultRetryAfterSeconds).Serialize()
-	}
-	registration, err := UnmarshalTunnelRegistration(retval)
-	if err != nil {
-		return NewRetryableRegistrationError(err, defaultRetryAfterSeconds).Serialize()
-	}
-	return registration
-}

tunnelrpc/pogs/registration_server.go

@@ -12,12 +12,18 @@ import (
 	"zombiezen.com/go/capnproto2/rpc"
 	"zombiezen.com/go/capnproto2/server"
 
-	"github.com/cloudflare/cloudflared/tunnelrpc"
+	"github.com/cloudflare/cloudflared/tunnelrpc/metrics"
+	"github.com/cloudflare/cloudflared/tunnelrpc/proto"
 )
 
 type RegistrationServer interface {
+	// RegisterConnection is the call typically handled by the edge to initiate and authenticate a new connection
+	// for cloudflared.
 	RegisterConnection(ctx context.Context, auth TunnelAuth, tunnelID uuid.UUID, connIndex byte, options *ConnectionOptions) (*ConnectionDetails, error)
+	// UnregisterConnection is the call typically handled by the edge to close an existing connection for cloudflared.
 	UnregisterConnection(ctx context.Context)
+	// UpdateLocalConfiguration is the call typically handled by the edge for cloudflared to provide the current
+	// configuration it is operating with.
 	UpdateLocalConfiguration(ctx context.Context, config []byte) error
 }
 
@@ -25,11 +31,15 @@ type RegistrationServer_PogsImpl struct {
 	impl RegistrationServer
 }
 
-func RegistrationServer_ServerToClient(s RegistrationServer) tunnelrpc.RegistrationServer {
-	return tunnelrpc.RegistrationServer_ServerToClient(RegistrationServer_PogsImpl{s})
+func RegistrationServer_ServerToClient(s RegistrationServer) proto.RegistrationServer {
+	return proto.RegistrationServer_ServerToClient(RegistrationServer_PogsImpl{s})
 }
 
-func (i RegistrationServer_PogsImpl) RegisterConnection(p tunnelrpc.RegistrationServer_registerConnection) error {
+func (i RegistrationServer_PogsImpl) RegisterConnection(p proto.RegistrationServer_registerConnection) error {
+	return metrics.ObserveServerHandler(func() error { return i.registerConnection(p) }, metrics.Registration, metrics.OperationRegisterConnection)
+}
+
+func (i RegistrationServer_PogsImpl) registerConnection(p proto.RegistrationServer_registerConnection) error {
 	server.Ack(p.Options)
 
 	auth, err := p.Params.Auth()
@@ -82,14 +92,19 @@ func (i RegistrationServer_PogsImpl) RegisterConnection(p tunnelrpc.Registration
 	}
 }
 
-func (i RegistrationServer_PogsImpl) UnregisterConnection(p tunnelrpc.RegistrationServer_unregisterConnection) error {
-	server.Ack(p.Options)
-
-	i.impl.UnregisterConnection(p.Ctx)
-	return nil
+func (i RegistrationServer_PogsImpl) UnregisterConnection(p proto.RegistrationServer_unregisterConnection) error {
+	return metrics.ObserveServerHandler(func() error {
+		server.Ack(p.Options)
+		i.impl.UnregisterConnection(p.Ctx)
+		return nil // No metrics will be reported for failure as this method has no return value
+	}, metrics.Registration, metrics.OperationUnregisterConnection)
 }
 
-func (i RegistrationServer_PogsImpl) UpdateLocalConfiguration(c tunnelrpc.RegistrationServer_updateLocalConfiguration) error {
+func (i RegistrationServer_PogsImpl) UpdateLocalConfiguration(p proto.RegistrationServer_updateLocalConfiguration) error {
+	return metrics.ObserveServerHandler(func() error { return i.updateLocalConfiguration(p) }, metrics.Registration, metrics.OperationUpdateLocalConfiguration)
+}
+
+func (i RegistrationServer_PogsImpl) updateLocalConfiguration(c proto.RegistrationServer_updateLocalConfiguration) error {
 	server.Ack(c.Options)
 
 	configBytes, err := c.Params.Config()
@@ -105,14 +120,21 @@ type RegistrationServer_PogsClient struct {
 	Conn   *rpc.Conn
 }
 
+func NewRegistrationServer_PogsClient(client capnp.Client, conn *rpc.Conn) RegistrationServer_PogsClient {
+	return RegistrationServer_PogsClient{
+		Client: client,
+		Conn:   conn,
+	}
+}
+
 func (c RegistrationServer_PogsClient) Close() error {
 	c.Client.Close()
 	return c.Conn.Close()
 }
 
 func (c RegistrationServer_PogsClient) RegisterConnection(ctx context.Context, auth TunnelAuth, tunnelID uuid.UUID, connIndex byte, options *ConnectionOptions) (*ConnectionDetails, error) {
-	client := tunnelrpc.TunnelServer{Client: c.Client}
-	promise := client.RegisterConnection(ctx, func(p tunnelrpc.RegistrationServer_registerConnection_Params) error {
+	client := proto.TunnelServer{Client: c.Client}
+	promise := client.RegisterConnection(ctx, func(p proto.RegistrationServer_registerConnection_Params) error {
 		tunnelAuth, err := p.NewAuth()
 		if err != nil {
 			return err
@@ -145,7 +167,7 @@ func (c RegistrationServer_PogsClient) RegisterConnection(ctx context.Context, a
 	}
 	result := response.Result()
 	switch result.Which() {
-	case tunnelrpc.ConnectionResponse_result_Which_error:
+	case proto.ConnectionResponse_result_Which_error:
 		resultError, err := result.Error()
 		if err != nil {
 			return nil, wrapRPCError(err)
@@ -160,7 +182,7 @@ func (c RegistrationServer_PogsClient) RegisterConnection(ctx context.Context, a
 		}
 		return nil, err
 
-	case tunnelrpc.ConnectionResponse_result_Which_connectionDetails:
+	case proto.ConnectionResponse_result_Which_connectionDetails:
 		connDetails, err := result.ConnectionDetails()
 		if err != nil {
 			return nil, wrapRPCError(err)
@@ -176,8 +198,8 @@ func (c RegistrationServer_PogsClient) RegisterConnection(ctx context.Context, a
 }
 
 func (c RegistrationServer_PogsClient) SendLocalConfiguration(ctx context.Context, config []byte) error {
-	client := tunnelrpc.TunnelServer{Client: c.Client}
-	promise := client.UpdateLocalConfiguration(ctx, func(p tunnelrpc.RegistrationServer_updateLocalConfiguration_Params) error {
+	client := proto.TunnelServer{Client: c.Client}
+	promise := client.UpdateLocalConfiguration(ctx, func(p proto.RegistrationServer_updateLocalConfiguration_Params) error {
 		if err := p.SetConfig(config); err != nil {
 			return err
 		}
@@ -194,8 +216,8 @@ func (c RegistrationServer_PogsClient) SendLocalConfiguration(ctx context.Contex
 }
 
 func (c RegistrationServer_PogsClient) UnregisterConnection(ctx context.Context) error {
-	client := tunnelrpc.TunnelServer{Client: c.Client}
-	promise := client.UnregisterConnection(ctx, func(p tunnelrpc.RegistrationServer_unregisterConnection_Params) error {
+	client := proto.TunnelServer{Client: c.Client}
+	promise := client.UnregisterConnection(ctx, func(p proto.RegistrationServer_unregisterConnection_Params) error {
 		return nil
 	})
 	_, err := promise.Struct()
@@ -225,20 +247,20 @@ type TunnelAuth struct {
 	TunnelSecret []byte
 }
 
-func (p *ConnectionOptions) MarshalCapnproto(s tunnelrpc.ConnectionOptions) error {
-	return pogs.Insert(tunnelrpc.ConnectionOptions_TypeID, s.Struct, p)
+func (p *ConnectionOptions) MarshalCapnproto(s proto.ConnectionOptions) error {
+	return pogs.Insert(proto.ConnectionOptions_TypeID, s.Struct, p)
 }
 
-func (p *ConnectionOptions) UnmarshalCapnproto(s tunnelrpc.ConnectionOptions) error {
-	return pogs.Extract(p, tunnelrpc.ConnectionOptions_TypeID, s.Struct)
+func (p *ConnectionOptions) UnmarshalCapnproto(s proto.ConnectionOptions) error {
+	return pogs.Extract(p, proto.ConnectionOptions_TypeID, s.Struct)
 }
 
-func (a *TunnelAuth) MarshalCapnproto(s tunnelrpc.TunnelAuth) error {
-	return pogs.Insert(tunnelrpc.TunnelAuth_TypeID, s.Struct, a)
+func (a *TunnelAuth) MarshalCapnproto(s proto.TunnelAuth) error {
+	return pogs.Insert(proto.TunnelAuth_TypeID, s.Struct, a)
 }
 
-func (a *TunnelAuth) UnmarshalCapnproto(s tunnelrpc.TunnelAuth) error {
-	return pogs.Extract(a, tunnelrpc.TunnelAuth_TypeID, s.Struct)
+func (a *TunnelAuth) UnmarshalCapnproto(s proto.TunnelAuth) error {
+	return pogs.Extract(a, proto.TunnelAuth_TypeID, s.Struct)
 }
 
 type ConnectionDetails struct {
@@ -247,7 +269,7 @@ type ConnectionDetails struct {
 	TunnelIsRemotelyManaged bool
 }
 
-func (details *ConnectionDetails) MarshalCapnproto(s tunnelrpc.ConnectionDetails) error {
+func (details *ConnectionDetails) MarshalCapnproto(s proto.ConnectionDetails) error {
 	if err := s.SetUuid(details.UUID[:]); err != nil {
 		return err
 	}
@@ -259,7 +281,7 @@ func (details *ConnectionDetails) MarshalCapnproto(s tunnelrpc.ConnectionDetails
 	return nil
 }
 
-func (details *ConnectionDetails) UnmarshalCapnproto(s tunnelrpc.ConnectionDetails) error {
+func (details *ConnectionDetails) UnmarshalCapnproto(s proto.ConnectionDetails) error {
 	uuidBytes, err := s.Uuid()
 	if err != nil {
 		return err
@@ -277,7 +299,7 @@ func (details *ConnectionDetails) UnmarshalCapnproto(s tunnelrpc.ConnectionDetai
 	return err
 }
 
-func MarshalError(s tunnelrpc.ConnectionError, err error) error {
+func MarshalError(s proto.ConnectionError, err error) error {
 	if err := s.SetCause(err.Error()); err != nil {
 		return err
 	}

tunnelrpc/pogs/registration_server_test.go

@@ -13,7 +13,7 @@ import (
 	capnp "zombiezen.com/go/capnproto2"
 	"zombiezen.com/go/capnproto2/rpc"
 
-	"github.com/cloudflare/cloudflared/tunnelrpc"
+	"github.com/cloudflare/cloudflared/tunnelrpc/proto"
 )
 
 const testAccountTag = "abc123"
@@ -34,7 +34,7 @@ func TestMarshalConnectionOptions(t *testing.T) {
 
 	_, seg, err := capnp.NewMessage(capnp.SingleSegment(nil))
 	require.NoError(t, err)
-	capnpOpts, err := tunnelrpc.NewConnectionOptions(seg)
+	capnpOpts, err := proto.NewConnectionOptions(seg)
 	require.NoError(t, err)
 
 	err = orig.MarshalCapnproto(capnpOpts)
@@ -54,18 +54,14 @@ func TestConnectionRegistrationRPC(t *testing.T) {
 
 	// Server-side
 	testImpl := testConnectionRegistrationServer{}
-	srv := TunnelServer_ServerToClient(&testImpl)
+	srv := RegistrationServer_ServerToClient(&testImpl)
 	serverConn := rpc.NewConn(t1, rpc.MainInterface(srv.Client))
 	defer serverConn.Wait()
 
 	ctx := context.Background()
 	clientConn := rpc.NewConn(t2)
 	defer clientConn.Close()
-	client := TunnelServer_PogsClient{
-		RegistrationServer_PogsClient: RegistrationServer_PogsClient{
-			Client: clientConn.Bootstrap(ctx),
-			Conn:   clientConn,
-		},
+	client := RegistrationServer_PogsClient{
 		Client: clientConn.Bootstrap(ctx),
 		Conn:   clientConn,
 	}
@@ -123,8 +119,6 @@ func TestConnectionRegistrationRPC(t *testing.T) {
 }
 
 type testConnectionRegistrationServer struct {
-	mockTunnelServerBase
-
 	details *ConnectionDetails
 	err     error
 }
@@ -147,3 +141,7 @@ func (t *testConnectionRegistrationServer) RegisterConnection(ctx context.Contex
 
 	panic("either details or err mush be set")
 }
+
+func (t *testConnectionRegistrationServer) UnregisterConnection(ctx context.Context) {
+	panic("unimplemented: UnregisterConnection")
+}

tunnelrpc/pogs/session_manager.go

@@ -6,11 +6,13 @@ import (
 	"net"
 	"time"
 
-	"github.com/cloudflare/cloudflared/tunnelrpc"
 	"github.com/google/uuid"
 	capnp "zombiezen.com/go/capnproto2"
 	"zombiezen.com/go/capnproto2/rpc"
 	"zombiezen.com/go/capnproto2/server"
+
+	"github.com/cloudflare/cloudflared/tunnelrpc/metrics"
+	"github.com/cloudflare/cloudflared/tunnelrpc/proto"
 )
 
 type SessionManager interface {
@@ -26,11 +28,15 @@ type SessionManager_PogsImpl struct {
 	impl SessionManager
 }
 
-func SessionManager_ServerToClient(s SessionManager) tunnelrpc.SessionManager {
-	return tunnelrpc.SessionManager_ServerToClient(SessionManager_PogsImpl{s})
+func SessionManager_ServerToClient(s SessionManager) proto.SessionManager {
+	return proto.SessionManager_ServerToClient(SessionManager_PogsImpl{s})
 }
 
-func (i SessionManager_PogsImpl) RegisterUdpSession(p tunnelrpc.SessionManager_registerUdpSession) error {
+func (i SessionManager_PogsImpl) RegisterUdpSession(p proto.SessionManager_registerUdpSession) error {
+	return metrics.ObserveServerHandler(func() error { return i.registerUdpSession(p) }, metrics.SessionManager, metrics.OperationRegisterUdpSession)
+}
+
+func (i SessionManager_PogsImpl) registerUdpSession(p proto.SessionManager_registerUdpSession) error {
 	server.Ack(p.Options)
 
 	sessionIDRaw, err := p.Params.SessionId()
@@ -76,7 +82,11 @@ func (i SessionManager_PogsImpl) RegisterUdpSession(p tunnelrpc.SessionManager_r
 	return resp.Marshal(result)
 }
 
-func (i SessionManager_PogsImpl) UnregisterUdpSession(p tunnelrpc.SessionManager_unregisterUdpSession) error {
+func (i SessionManager_PogsImpl) UnregisterUdpSession(p proto.SessionManager_unregisterUdpSession) error {
+	return metrics.ObserveServerHandler(func() error { return i.unregisterUdpSession(p) }, metrics.SessionManager, metrics.OperationUnregisterUdpSession)
+}
+
+func (i SessionManager_PogsImpl) unregisterUdpSession(p proto.SessionManager_unregisterUdpSession) error {
 	server.Ack(p.Options)
 
 	sessionIDRaw, err := p.Params.SessionId()
@@ -101,7 +111,7 @@ type RegisterUdpSessionResponse struct {
 	Spans []byte // Spans in protobuf format
 }
 
-func (p *RegisterUdpSessionResponse) Marshal(s tunnelrpc.RegisterUdpSessionResponse) error {
+func (p *RegisterUdpSessionResponse) Marshal(s proto.RegisterUdpSessionResponse) error {
 	if p.Err != nil {
 		return s.SetErr(p.Err.Error())
 	}
@@ -111,7 +121,7 @@ func (p *RegisterUdpSessionResponse) Marshal(s tunnelrpc.RegisterUdpSessionRespo
 	return nil
 }
 
-func (p *RegisterUdpSessionResponse) Unmarshal(s tunnelrpc.RegisterUdpSessionResponse) error {
+func (p *RegisterUdpSessionResponse) Unmarshal(s proto.RegisterUdpSessionResponse) error {
 	respErr, err := s.Err()
 	if err != nil {
 		return err
@@ -131,14 +141,21 @@ type SessionManager_PogsClient struct {
 	Conn   *rpc.Conn
 }
 
+func NewSessionManager_PogsClient(client capnp.Client, conn *rpc.Conn) SessionManager_PogsClient {
+	return SessionManager_PogsClient{
+		Client: client,
+		Conn:   conn,
+	}
+}
+
 func (c SessionManager_PogsClient) Close() error {
 	c.Client.Close()
 	return c.Conn.Close()
 }
 
 func (c SessionManager_PogsClient) RegisterUdpSession(ctx context.Context, sessionID uuid.UUID, dstIP net.IP, dstPort uint16, closeAfterIdleHint time.Duration, traceContext string) (*RegisterUdpSessionResponse, error) {
-	client := tunnelrpc.SessionManager{Client: c.Client}
-	promise := client.RegisterUdpSession(ctx, func(p tunnelrpc.SessionManager_registerUdpSession_Params) error {
+	client := proto.SessionManager{Client: c.Client}
+	promise := client.RegisterUdpSession(ctx, func(p proto.SessionManager_registerUdpSession_Params) error {
 		if err := p.SetSessionId(sessionID[:]); err != nil {
 			return err
 		}
@@ -164,8 +181,8 @@ func (c SessionManager_PogsClient) RegisterUdpSession(ctx context.Context, sessi
 }
 
 func (c SessionManager_PogsClient) UnregisterUdpSession(ctx context.Context, sessionID uuid.UUID, message string) error {
-	client := tunnelrpc.SessionManager{Client: c.Client}
-	promise := client.UnregisterUdpSession(ctx, func(p tunnelrpc.SessionManager_unregisterUdpSession_Params) error {
+	client := proto.SessionManager{Client: c.Client}
+	promise := client.UnregisterUdpSession(ctx, func(p proto.SessionManager_unregisterUdpSession_Params) error {
 		if err := p.SetSessionId(sessionID[:]); err != nil {
 			return err
 		}

tunnelrpc/pogs/support_test.go

@@ -1,40 +0,0 @@
-package pogs
-
-import (
-	"context"
-
-	"github.com/google/uuid"
-)
-
-// mockTunnelServerBase provides a placeholder implementation
-// for TunnelServer interface that can be used to build
-// mocks for specific unit tests without having to implement every method
-type mockTunnelServerBase struct{}
-
-func (mockTunnelServerBase) RegisterConnection(ctx context.Context, auth TunnelAuth, tunnelID uuid.UUID, connIndex byte, options *ConnectionOptions) (*ConnectionDetails, error) {
-	panic("unexpected call to RegisterConnection")
-}
-
-func (mockTunnelServerBase) UnregisterConnection(ctx context.Context) {
-	panic("unexpected call to UnregisterConnection")
-}
-
-func (mockTunnelServerBase) RegisterTunnel(ctx context.Context, originCert []byte, hostname string, options *RegistrationOptions) *TunnelRegistration {
-	panic("unexpected call to RegisterTunnel")
-}
-
-func (mockTunnelServerBase) GetServerInfo(ctx context.Context) (*ServerInfo, error) {
-	panic("unexpected call to GetServerInfo")
-}
-
-func (mockTunnelServerBase) UnregisterTunnel(ctx context.Context, gracePeriodNanoSec int64) error {
-	panic("unexpected call to UnregisterTunnel")
-}
-
-func (mockTunnelServerBase) Authenticate(ctx context.Context, originCert []byte, hostname string, options *RegistrationOptions) (*AuthenticateResponse, error) {
-	panic("unexpected call to Authenticate")
-}
-
-func (mockTunnelServerBase) ReconnectTunnel(ctx context.Context, jwt, eventDigest, connDigest []byte, hostname string, options *RegistrationOptions) (*TunnelRegistration, error) {
-	panic("unexpected call to ReconnectTunnel")
-}

tunnelrpc/pogs/tag.go

@@ -0,0 +1,8 @@
+package pogs
+
+// Tag previously was a legacy tunnel capnp struct but was deprecated. To help reduce the amount of changes imposed
+// by removing this simple struct, it was copied out of the capnp and provided here instead.
+type Tag struct {
+	Name  string
+	Value string
+}

tunnelrpc/pogs/tunnelrpc.go

@@ -1,334 +0,0 @@
-package pogs
-
-import (
-	"context"
-	"fmt"
-
-	capnp "zombiezen.com/go/capnproto2"
-	"zombiezen.com/go/capnproto2/pogs"
-	"zombiezen.com/go/capnproto2/rpc"
-	"zombiezen.com/go/capnproto2/server"
-
-	"github.com/cloudflare/cloudflared/tunnelrpc"
-)
-
-const (
-	defaultRetryAfterSeconds = 15
-)
-
-type Authentication struct {
-	Key         string
-	Email       string
-	OriginCAKey string
-}
-
-func MarshalAuthentication(s tunnelrpc.Authentication, p *Authentication) error {
-	return pogs.Insert(tunnelrpc.Authentication_TypeID, s.Struct, p)
-}
-
-func UnmarshalAuthentication(s tunnelrpc.Authentication) (*Authentication, error) {
-	p := new(Authentication)
-	err := pogs.Extract(p, tunnelrpc.Authentication_TypeID, s.Struct)
-	return p, err
-}
-
-type TunnelRegistration struct {
-	SuccessfulTunnelRegistration
-	Err               string
-	PermanentFailure  bool
-	RetryAfterSeconds uint16
-}
-
-type SuccessfulTunnelRegistration struct {
-	Url         string
-	LogLines    []string
-	TunnelID    string `capnp:"tunnelID"`
-	EventDigest []byte
-	ConnDigest  []byte
-}
-
-func NewSuccessfulTunnelRegistration(
-	url string,
-	logLines []string,
-	tunnelID string,
-	eventDigest []byte,
-	connDigest []byte,
-) *TunnelRegistration {
-	// Marshal nil will result in an error
-	if logLines == nil {
-		logLines = []string{}
-	}
-	return &TunnelRegistration{
-		SuccessfulTunnelRegistration: SuccessfulTunnelRegistration{
-			Url:         url,
-			LogLines:    logLines,
-			TunnelID:    tunnelID,
-			EventDigest: eventDigest,
-			ConnDigest:  connDigest,
-		},
-	}
-}
-
-// Not calling this function Error() to avoid confusion with implementing error interface
-func (tr TunnelRegistration) DeserializeError() TunnelRegistrationError {
-	if tr.Err != "" {
-		err := fmt.Errorf(tr.Err)
-		if tr.PermanentFailure {
-			return NewPermanentRegistrationError(err)
-		}
-		retryAfterSeconds := tr.RetryAfterSeconds
-		if retryAfterSeconds < defaultRetryAfterSeconds {
-			retryAfterSeconds = defaultRetryAfterSeconds
-		}
-		return NewRetryableRegistrationError(err, retryAfterSeconds)
-	}
-	return nil
-}
-
-type TunnelRegistrationError interface {
-	error
-	Serialize() *TunnelRegistration
-	IsPermanent() bool
-}
-
-type PermanentRegistrationError struct {
-	err string
-}
-
-func NewPermanentRegistrationError(err error) TunnelRegistrationError {
-	return &PermanentRegistrationError{
-		err: err.Error(),
-	}
-}
-
-func (pre *PermanentRegistrationError) Error() string {
-	return pre.err
-}
-
-func (pre *PermanentRegistrationError) Serialize() *TunnelRegistration {
-	return &TunnelRegistration{
-		Err:              pre.err,
-		PermanentFailure: true,
-	}
-}
-
-func (*PermanentRegistrationError) IsPermanent() bool {
-	return true
-}
-
-type RetryableRegistrationError struct {
-	err               string
-	retryAfterSeconds uint16
-}
-
-func NewRetryableRegistrationError(err error, retryAfterSeconds uint16) TunnelRegistrationError {
-	return &RetryableRegistrationError{
-		err:               err.Error(),
-		retryAfterSeconds: retryAfterSeconds,
-	}
-}
-
-func (rre *RetryableRegistrationError) Error() string {
-	return rre.err
-}
-
-func (rre *RetryableRegistrationError) Serialize() *TunnelRegistration {
-	return &TunnelRegistration{
-		Err:               rre.err,
-		PermanentFailure:  false,
-		RetryAfterSeconds: rre.retryAfterSeconds,
-	}
-}
-
-func (*RetryableRegistrationError) IsPermanent() bool {
-	return false
-}
-
-func MarshalTunnelRegistration(s tunnelrpc.TunnelRegistration, p *TunnelRegistration) error {
-	return pogs.Insert(tunnelrpc.TunnelRegistration_TypeID, s.Struct, p)
-}
-
-func UnmarshalTunnelRegistration(s tunnelrpc.TunnelRegistration) (*TunnelRegistration, error) {
-	p := new(TunnelRegistration)
-	err := pogs.Extract(p, tunnelrpc.TunnelRegistration_TypeID, s.Struct)
-	return p, err
-}
-
-type RegistrationOptions struct {
-	ClientID             string `capnp:"clientId"`
-	Version              string
-	OS                   string `capnp:"os"`
-	ExistingTunnelPolicy tunnelrpc.ExistingTunnelPolicy
-	PoolName             string `capnp:"poolName"`
-	Tags                 []Tag
-	ConnectionID         uint8  `capnp:"connectionId"`
-	OriginLocalIP        string `capnp:"originLocalIp"`
-	IsAutoupdated        bool   `capnp:"isAutoupdated"`
-	RunFromTerminal      bool   `capnp:"runFromTerminal"`
-	CompressionQuality   uint64 `capnp:"compressionQuality"`
-	UUID                 string `capnp:"uuid"`
-	NumPreviousAttempts  uint8
-	Features             []string
-}
-
-func MarshalRegistrationOptions(s tunnelrpc.RegistrationOptions, p *RegistrationOptions) error {
-	return pogs.Insert(tunnelrpc.RegistrationOptions_TypeID, s.Struct, p)
-}
-
-func UnmarshalRegistrationOptions(s tunnelrpc.RegistrationOptions) (*RegistrationOptions, error) {
-	p := new(RegistrationOptions)
-	err := pogs.Extract(p, tunnelrpc.RegistrationOptions_TypeID, s.Struct)
-	return p, err
-}
-
-type Tag struct {
-	Name  string `json:"name"`
-	Value string `json:"value"`
-}
-
-type ServerInfo struct {
-	LocationName string
-}
-
-func MarshalServerInfo(s tunnelrpc.ServerInfo, p *ServerInfo) error {
-	return pogs.Insert(tunnelrpc.ServerInfo_TypeID, s.Struct, p)
-}
-
-func UnmarshalServerInfo(s tunnelrpc.ServerInfo) (*ServerInfo, error) {
-	p := new(ServerInfo)
-	err := pogs.Extract(p, tunnelrpc.ServerInfo_TypeID, s.Struct)
-	return p, err
-}
-
-type TunnelServer interface {
-	RegistrationServer
-	RegisterTunnel(ctx context.Context, originCert []byte, hostname string, options *RegistrationOptions) *TunnelRegistration
-	GetServerInfo(ctx context.Context) (*ServerInfo, error)
-	UnregisterTunnel(ctx context.Context, gracePeriodNanoSec int64) error
-	Authenticate(ctx context.Context, originCert []byte, hostname string, options *RegistrationOptions) (*AuthenticateResponse, error)
-	ReconnectTunnel(ctx context.Context, jwt, eventDigest, connDigest []byte, hostname string, options *RegistrationOptions) (*TunnelRegistration, error)
-}
-
-func TunnelServer_ServerToClient(s TunnelServer) tunnelrpc.TunnelServer {
-	return tunnelrpc.TunnelServer_ServerToClient(TunnelServer_PogsImpl{RegistrationServer_PogsImpl{s}, s})
-}
-
-type TunnelServer_PogsImpl struct {
-	RegistrationServer_PogsImpl
-	impl TunnelServer
-}
-
-func (i TunnelServer_PogsImpl) RegisterTunnel(p tunnelrpc.TunnelServer_registerTunnel) error {
-	originCert, err := p.Params.OriginCert()
-	if err != nil {
-		return err
-	}
-	hostname, err := p.Params.Hostname()
-	if err != nil {
-		return err
-	}
-	options, err := p.Params.Options()
-	if err != nil {
-		return err
-	}
-	pogsOptions, err := UnmarshalRegistrationOptions(options)
-	if err != nil {
-		return err
-	}
-	server.Ack(p.Options)
-	registration := i.impl.RegisterTunnel(p.Ctx, originCert, hostname, pogsOptions)
-
-	result, err := p.Results.NewResult()
-	if err != nil {
-		return err
-	}
-	return MarshalTunnelRegistration(result, registration)
-}
-
-func (i TunnelServer_PogsImpl) GetServerInfo(p tunnelrpc.TunnelServer_getServerInfo) error {
-	server.Ack(p.Options)
-	serverInfo, err := i.impl.GetServerInfo(p.Ctx)
-	if err != nil {
-		return err
-	}
-	result, err := p.Results.NewResult()
-	if err != nil {
-		return err
-	}
-	return MarshalServerInfo(result, serverInfo)
-}
-
-func (i TunnelServer_PogsImpl) UnregisterTunnel(p tunnelrpc.TunnelServer_unregisterTunnel) error {
-	gracePeriodNanoSec := p.Params.GracePeriodNanoSec()
-	server.Ack(p.Options)
-	return i.impl.UnregisterTunnel(p.Ctx, gracePeriodNanoSec)
-}
-
-func (i TunnelServer_PogsImpl) ObsoleteDeclarativeTunnelConnect(p tunnelrpc.TunnelServer_obsoleteDeclarativeTunnelConnect) error {
-	return fmt.Errorf("RPC to create declarative tunnel connection has been deprecated")
-}
-
-type TunnelServer_PogsClient struct {
-	RegistrationServer_PogsClient
-	Client capnp.Client
-	Conn   *rpc.Conn
-}
-
-func (c TunnelServer_PogsClient) Close() error {
-	c.Client.Close()
-	return c.Conn.Close()
-}
-
-func (c TunnelServer_PogsClient) RegisterTunnel(ctx context.Context, originCert []byte, hostname string, options *RegistrationOptions) *TunnelRegistration {
-	client := tunnelrpc.TunnelServer{Client: c.Client}
-	promise := client.RegisterTunnel(ctx, func(p tunnelrpc.TunnelServer_registerTunnel_Params) error {
-		err := p.SetOriginCert(originCert)
-		if err != nil {
-			return err
-		}
-		err = p.SetHostname(hostname)
-		if err != nil {
-			return err
-		}
-		registrationOptions, err := p.NewOptions()
-		if err != nil {
-			return err
-		}
-		err = MarshalRegistrationOptions(registrationOptions, options)
-		if err != nil {
-			return err
-		}
-		return nil
-	})
-	retval, err := promise.Result().Struct()
-	if err != nil {
-		return NewRetryableRegistrationError(err, defaultRetryAfterSeconds).Serialize()
-	}
-	registration, err := UnmarshalTunnelRegistration(retval)
-	if err != nil {
-		return NewRetryableRegistrationError(err, defaultRetryAfterSeconds).Serialize()
-	}
-	return registration
-}
-
-func (c TunnelServer_PogsClient) GetServerInfo(ctx context.Context) (*ServerInfo, error) {
-	client := tunnelrpc.TunnelServer{Client: c.Client}
-	promise := client.GetServerInfo(ctx, func(p tunnelrpc.TunnelServer_getServerInfo_Params) error {
-		return nil
-	})
-	retval, err := promise.Result().Struct()
-	if err != nil {
-		return nil, err
-	}
-	return UnmarshalServerInfo(retval)
-}
-
-func (c TunnelServer_PogsClient) UnregisterTunnel(ctx context.Context, gracePeriodNanoSec int64) error {
-	client := tunnelrpc.TunnelServer{Client: c.Client}
-	promise := client.UnregisterTunnel(ctx, func(p tunnelrpc.TunnelServer_unregisterTunnel_Params) error {
-		p.SetGracePeriodNanoSec(gracePeriodNanoSec)
-		return nil
-	})
-	_, err := promise.Struct()
-	return err
-}

tunnelrpc/pogs/tunnelrpc_test.go

@@ -1,57 +0,0 @@
-package pogs
-
-import (
-	"fmt"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	capnp "zombiezen.com/go/capnproto2"
-
-	"github.com/cloudflare/cloudflared/tunnelrpc"
-)
-
-const (
-	testURL               = "tunnel.example.com"
-	testTunnelID          = "asdfghjkl;"
-	testRetryAfterSeconds = 19
-)
-
-var (
-	testErr         = fmt.Errorf("Invalid credential")
-	testLogLines    = []string{"all", "working"}
-	testEventDigest = []byte("asdf")
-	testConnDigest  = []byte("lkjh")
-)
-
-// *PermanentRegistrationError implements TunnelRegistrationError
-var _ TunnelRegistrationError = (*PermanentRegistrationError)(nil)
-
-// *RetryableRegistrationError implements TunnelRegistrationError
-var _ TunnelRegistrationError = (*RetryableRegistrationError)(nil)
-
-func TestTunnelRegistration(t *testing.T) {
-	testCases := []*TunnelRegistration{
-		NewSuccessfulTunnelRegistration(testURL, testLogLines, testTunnelID, testEventDigest, testConnDigest),
-		NewSuccessfulTunnelRegistration(testURL, nil, testTunnelID, testEventDigest, testConnDigest),
-		NewPermanentRegistrationError(testErr).Serialize(),
-		NewRetryableRegistrationError(testErr, testRetryAfterSeconds).Serialize(),
-	}
-	for i, testCase := range testCases {
-		_, seg, err := capnp.NewMessage(capnp.SingleSegment(nil))
-		assert.NoError(t, err)
-		capnpEntity, err := tunnelrpc.NewTunnelRegistration(seg)
-		if !assert.NoError(t, err) {
-			t.Fatal("Couldn't initialize a new message")
-		}
-		err = MarshalTunnelRegistration(capnpEntity, testCase)
-		if !assert.NoError(t, err, "testCase #%v failed to marshal", i) {
-			continue
-		}
-		result, err := UnmarshalTunnelRegistration(capnpEntity)
-		if !assert.NoError(t, err, "testCase #%v failed to unmarshal", i) {
-			continue
-		}
-		assert.Equal(t, testCase, result, "testCase index %v didn't preserve struct through marshalling and unmarshalling", i)
-	}
-
-}

tunnelrpc/proto/go.capnp

@@ -0,0 +1,31 @@
+# Generate go.capnp.out with:
+# capnp compile -o- go.capnp > go.capnp.out
+# Must run inside this directory to preserve paths.
+
+@0xd12a1c51fedd6c88;
+
+annotation package(file) :Text;
+# The Go package name for the generated file.
+
+annotation import(file) :Text;
+# The Go import path that the generated file is accessible from.
+# Used to generate import statements and check if two types are in the
+# same package.
+
+annotation doc(struct, field, enum) :Text;
+# Adds a doc comment to the generated code.
+
+annotation tag(enumerant) :Text;
+# Changes the string representation of the enum in the generated code.
+
+annotation notag(enumerant) :Void;
+# Removes the string representation of the enum in the generated code.
+
+annotation customtype(field) :Text;
+# OBSOLETE, not used by code generator.
+
+annotation name(struct, field, union, enum, enumerant, interface, method, param, annotation, const, group) :Text;
+# Used to rename the element in the generated code.
+
+$package("capnp");
+$import("zombiezen.com/go/capnproto2");

tunnelrpc/proto/quic_metadata_protocol.capnp

@@ -0,0 +1,28 @@
+using Go = import "go.capnp";
+@0xb29021ef7421cc32;
+
+$Go.package("proto");
+$Go.import("github.com/cloudflare/cloudflared/tunnelrpc");
+
+
+struct ConnectRequest @0xc47116a1045e4061 {
+	dest @0 :Text;
+	type @1 :ConnectionType;
+	metadata @2 :List(Metadata);
+}
+
+enum ConnectionType @0xc52e1bac26d379c8 {
+	http @0;
+	websocket @1;
+	tcp @2;
+}
+
+struct Metadata @0xe1446b97bfd1cd37 {
+    key @0 :Text;
+	val @1 :Text;
+}
+
+struct ConnectResponse @0xb1032ec91cef8727 {
+	error @0 :Text;
+	metadata @1 :List(Metadata);
+}

tunnelrpc/proto/quic_metadata_protocol.capnp.go

@@ -1,6 +1,6 @@
 // Code generated by capnpc-go. DO NOT EDIT.
 
-package schema
+package proto
 
 import (
 	capnp "zombiezen.com/go/capnproto2"
@@ -357,34 +357,34 @@ func (p ConnectResponse_Promise) Struct() (ConnectResponse, error) {
 	return ConnectResponse{s}, err
 }
 
-const schema_b29021ef7421cc32 = "x\xda\xb4\x91\xcfk\x13A\x1c\xc5\xdf\x9bI\xba\x1e\xa2" +
-	"\x9b!\xd5\x8b\x8a\xa4\xf8+ES\xdb(\xa2\xa7\x80\x15" +
-	"TZ\xcc\x14\xcf\x96u;\x98\x92vw\x92\x9dZ\xf2" +
-	"\x17x\x15/\xe2\xd1\xbb \x15<\x0b\xa2\xa0\xa2\x07\x11" +
-	"\xff\x80\xfe\x05=y\xf0\xb42)\xdb@)\x08Bo" +
-	"\xdfy<\xe6}\xbe\xdfW\xfd\xd5\x16\xb3\xe5\xc7\x04t" +
-	"\xb5<\x91_x\xbas\xeaKSnA5\x98\xcf}" +
-	"\xab\xbb\x9d\xfa\xb3\xb7(\x8b\x00\x98}\xf9\x95\xea]\x00" +
-	"\xa8\xadM0\x8f\xda\x0fK\xafN\xf4?B7\xb8\xdf" +
-	"\xda\xaa\xf3\x03k7\x18\x00\xb5k|\x03\xe6\x9f\x87?" +
-	"\xcf\xbf>\xd9\xfc\x04\xd5\x10c3\xd8\xda\xf6\xce?#" +
-	"\xe7o\xde\x07\xf3\xeb\xdf\x7f\xbc\x7f\xd1\x9b\xdf>\x80\xa0" +
-	"uT<g\xed\x9c\x1fku\xe1!\xfa\x1b\xab\xf1L" +
-	"\x16w'\xccz4\xe3\x1f\xcb\xeb\xc6E+\x91\x8b\x96" +
-	"\xed ui\x9c\xae5\xe3\xc8&\xf6\xe6\xad4IL" +
-	"\xec\x96Lf\xd3$3@\x87\xd4Gd\x09(\x11P" +
-	"\x8d9@\x9f\x95\xd4W\x04\x159I/^\xbe\x07\xe8" +
-	"K\x92\xfa\x8e\xe0\x193\x18\xa4\x03V X\x01\xf3\"" +
-	"\x06\x00\x8f\x81\x1dIV\xc7\xe8\xa0\x17\xff\x87\xae\xbfa" +
-	"27b\xab\xec\xb1\xdd\x9e\x06t[R/\x08\x16h" +
-	"w\xbd6/\xa9;\x82Jp\x92\x02P\x8b\x9ewA" +
-	"Rw\x05\xc3\x15\x93\xb9\x027tCk\x18\x8e[\x00" +
-	"\x19\x1e\xda\x16\xabi\xf2`h\xcd\xee\x16#\xb0\xd3\xd3" +
-	"\xfe3u|\x09\xa0Pj\x0a\x08\xbb\xce\xd9|\xd3<" +
-	"\xca\xd2\xb8g@\x17\xb8\xd8\xeeE\x95\xff\x19\xb5\xb8\xab" +
-	"3\xdaW\xe3\xd4A5z\xf1\xa2\xa4\xbe*\x18\xf4\xcc" +
-	"\xb0\xb8J\xf0$Z+\xe6\xbf\x01\x00\x00\xff\xff\xf5\xed" +
-	"\xc9\xfe"
+const schema_b29021ef7421cc32 = "x\xda\xb4\x911k\x14A\x1c\xc5\xdf\x9b\xcde-\x0e" +
+	"\xf7\x86KlT\xc2\x05Q\x13\xdc\x8b\xc9\x09\xa2 \x1c" +
+	"\x18A%\xc1\x9b`mX7\x83\x09w\xee\xce\xed\xce" +
+	"\x19\xee\x13\xd8\xda\x89\xa5\xbd \x09X\xdb((h!" +
+	"\x16\xd6\x0a66\xf9\x04\xb22\x0b\x9b\x83\x90B\x04\xbb" +
+	"\xe1\xcd\x9by\xbf\xff\xff5\xbeu\xc5r\xed\x11\x01\xd5" +
+	"\xa8M\x17\x17\x9e\x1e\x9c\xf9\xd8\xf6\xf6 C\x16+\x9f" +
+	"Z\xf6\xa0\xf5l\x1f5\xe1\x03\xcb/~Q\xbe\xf1\x01" +
+	"\xb9\xb7\x0b\x16Q\xf7\xc1\xd4\xcbS\xc3wP!\x8fZ" +
+	";-\xfe`\xf3\x06}\xa0y\x8d\xaf\xc1\xe2\xc3\xf8\xeb" +
+	"\xf9W\xa7\xdb\xef!C11\x83\x9d\x9f\xceI\xf7\xa8" +
+	"\xf9\x9b\xf7\xc0\xe2\xea\xe7/o\x9f\xf7W\xbf\x1fC\xd0" +
+	"\x99\x15\xfbl\x86\xa5yA8\x08;J\x12=\xc8\xcc" +
+	"t\xbcd\xb2\xd4\xa6K\xc3\xd1N\xbc\xf9X\xdbh+" +
+	"\xb2\xd1f\xa9\xc5\xe9\xa0\x1dG&1\xd7o\xa6I\xa2" +
+	"c\xbb\xa1s\x13\xa4I\xae{\xa4:\xe1M\x01S\x04" +
+	"\xe4\xc2\x0a\xa0\xceyT\x97\x05%9C'\x86w\x01" +
+	"u\xc9\xa3\xba-8\xa7\xb3,\xcdX\x87`\x1d,\xaa" +
+	"\x14\x00<\x09\xf6<\xb21\xa1\x07\x9d\xf8\xaf\x80\xc3\x91" +
+	"\xafs\xeb\xf8\xea\x87|\xb7\x16\x01\xd5\xf5\xa8\xd6\x04+" +
+	"\xbc;N[\xf5\xa8z\x82Rp\x86\x02\x90\xeb\x8ey" +
+	"\xcd\xa3\xda\x16\x0c\xb6tn+\xe4\xc0\x8e\x8df0)" +
+	"\x03d\xf0_'\xd9I\x93\xfb\xfe\xd8\x94\x9b\xae\x97p" +
+	"g\x17\xdd\x87rv\x03\xa0\x90r\x1e\x08\xb6\xad5\xc5" +
+	"\xae~\x98\xa7q_\x83\xd6\xb7\xb19\x8c\xab\xfdU\xdc" +
+	"\xba\xb6s\xe5\xc5\x91J\xe7\x8f\xab\xd4\x89\x17=\xaa+" +
+	"\x82~_\x8f\xab\xed\xf8O\xa2Au\xfe\x13\x00\x00\xff" +
+	"\xff\x1d\xce\xd1\xb0"
 
 func init() {
 	schemas.Register(schema_b29021ef7421cc32,

tunnelrpc/proto/tunnelrpc.capnp

@@ -1,15 +1,23 @@
 using Go = import "go.capnp";
 @0xdb8274f9144abc7e;
-$Go.package("tunnelrpc");
+$Go.package("proto");
 $Go.import("github.com/cloudflare/cloudflared/tunnelrpc");
 
-struct Authentication {
+# === DEPRECATED Legacy Tunnel Authentication and Registration methods/servers ===
+# 
+# These structs and interfaces are no longer used but it is important to keep
+# them around to make sure backwards compatibility within the rpc protocol is
+# maintained.
+
+struct Authentication @0xc082ef6e0d42ed1d {
+    # DEPRECATED: Legacy tunnel authentication mechanism
     key @0 :Text;
     email @1 :Text;
     originCAKey @2 :Text;
 }
 
-struct TunnelRegistration {
+struct TunnelRegistration @0xf41a0f001ad49e46 {
+    # DEPRECATED: Legacy tunnel authentication mechanism
     err @0 :Text;
     # the url to access the tunnel
     url @1 :Text;
@@ -27,7 +35,9 @@ struct TunnelRegistration {
     connDigest @7 :Data;
 }
 
-struct RegistrationOptions {
+struct RegistrationOptions @0xc793e50592935b4a {
+    # DEPRECATED: Legacy tunnel authentication mechanism
+    
     # The tunnel client's unique identifier, used to verify a reconnection.
     clientId @0 :Text;
     # Information about the running binary.
@@ -56,29 +66,51 @@ struct RegistrationOptions {
     features @13 :List(Text);
 }
 
-struct Tag {
-    name @0 :Text;
-    value @1 :Text;
-}
+enum ExistingTunnelPolicy @0x84cb9536a2cf6d3c {
+    # DEPRECATED: Legacy tunnel registration mechanism
 
-enum ExistingTunnelPolicy {
     ignore @0;
     disconnect @1;
     balance @2;
 }
 
-struct ServerInfo {
+struct ServerInfo @0xf2c68e2547ec3866 {
+    # DEPRECATED: Legacy tunnel registration mechanism
+
     locationName @0 :Text;
 }
 
-struct AuthenticateResponse {
+struct AuthenticateResponse @0x82c325a07ad22a65 {
+    # DEPRECATED: Legacy tunnel registration mechanism
+
     permanentErr @0 :Text;
     retryableErr @1 :Text;
     jwt @2 :Data;
     hoursUntilRefresh @3 :UInt8;
 }
 
-struct ClientInfo {
+interface TunnelServer @0xea58385c65416035 extends (RegistrationServer) {
+    # DEPRECATED: Legacy tunnel authentication server
+
+    registerTunnel @0 (originCert :Data, hostname :Text, options :RegistrationOptions) -> (result :TunnelRegistration);
+    getServerInfo @1 () -> (result :ServerInfo);
+    unregisterTunnel @2 (gracePeriodNanoSec :Int64) -> ();
+    # obsoleteDeclarativeTunnelConnect RPC deprecated in TUN-3019
+    obsoleteDeclarativeTunnelConnect @3 () -> ();
+    authenticate @4 (originCert :Data, hostname :Text, options :RegistrationOptions) -> (result :AuthenticateResponse);
+    reconnectTunnel @5 (jwt :Data, eventDigest :Data, connDigest :Data, hostname :Text, options :RegistrationOptions) -> (result :TunnelRegistration);
+}
+
+struct Tag @0xcbd96442ae3bb01a {
+    # DEPRECATED: Legacy tunnel additional HTTP header mechanism 
+
+    name @0 :Text;
+    value @1 :Text;
+}
+
+# === End DEPRECATED Objects ===
+
+struct ClientInfo @0x83ced0145b2f114b {
     # The tunnel client's unique identifier, used to verify a reconnection.
     clientId @0 :Data;
     # Set of features this cloudflared knows it supports
@@ -89,7 +121,7 @@ struct ClientInfo {
     arch @3 :Text;
 }
 
-struct ConnectionOptions {
+struct ConnectionOptions @0xb4bf9861fe035d04 {
     # client details
     client @0 :ClientInfo;
     # origin LAN IP
@@ -102,21 +134,21 @@ struct ConnectionOptions {
     numPreviousAttempts @4 :UInt8;
 }
 
-struct ConnectionResponse {
+struct ConnectionResponse @0xdbaa9d03d52b62dc {
     result :union {
         error @0 :ConnectionError;
         connectionDetails @1 :ConnectionDetails;
     }
 }
 
-struct ConnectionError {
+struct ConnectionError @0xf5f383d2785edb86 {
     cause @0 :Text;
     # How long should this connection wait to retry in ns
     retryAfter @1 :Int64;
     shouldRetry @2 :Bool;
 }
 
-struct ConnectionDetails {
+struct ConnectionDetails @0xb5f39f082b9ac18a {
     # identifier of this connection
     uuid @0 :Data;
     # airport code of the colo where this connection landed
@@ -125,39 +157,29 @@ struct ConnectionDetails {
     tunnelIsRemotelyManaged @2: Bool;
 }
 
-struct TunnelAuth {
+struct TunnelAuth @0x9496331ab9cd463f {
   accountTag @0 :Text;
   tunnelSecret @1 :Data;
 }
 
-interface RegistrationServer {
+interface RegistrationServer @0xf71695ec7fe85497 {
     registerConnection @0 (auth :TunnelAuth, tunnelId :Data, connIndex :UInt8, options :ConnectionOptions) -> (result :ConnectionResponse);
     unregisterConnection @1 () -> ();
     updateLocalConfiguration @2 (config :Data) -> ();
 }
 
-interface TunnelServer extends (RegistrationServer) {
-    registerTunnel @0 (originCert :Data, hostname :Text, options :RegistrationOptions) -> (result :TunnelRegistration);
-    getServerInfo @1 () -> (result :ServerInfo);
-    unregisterTunnel @2 (gracePeriodNanoSec :Int64) -> ();
-    # obsoleteDeclarativeTunnelConnect RPC deprecated in TUN-3019
-    obsoleteDeclarativeTunnelConnect @3 () -> ();
-    authenticate @4 (originCert :Data, hostname :Text, options :RegistrationOptions) -> (result :AuthenticateResponse);
-    reconnectTunnel @5 (jwt :Data, eventDigest :Data, connDigest :Data, hostname :Text, options :RegistrationOptions) -> (result :TunnelRegistration);
-}
-
-struct RegisterUdpSessionResponse {
+struct RegisterUdpSessionResponse @0xab6d5210c1f26687 {
     err @0 :Text;
     spans @1 :Data;
 }
 
-interface SessionManager {
+interface SessionManager @0x839445a59fb01686 {
     # Let the edge decide closeAfterIdle to make sure cloudflared doesn't close session before the edge closes its side
     registerUdpSession @0 (sessionId :Data, dstIp :Data, dstPort :UInt16, closeAfterIdleHint :Int64, traceContext :Text = "") -> (result :RegisterUdpSessionResponse);
     unregisterUdpSession @1 (sessionId :Data, message :Text) -> ();
 }
 
-struct UpdateConfigurationResponse {
+struct UpdateConfigurationResponse @0xdb58ff694ba05cf9 {
     # Latest configuration that was applied successfully. The err field might be populated at the same time to indicate
     # that cloudflared is using an older configuration because the latest cannot be applied
     latestAppliedVersion @0 :Int32;
@@ -166,8 +188,8 @@ struct UpdateConfigurationResponse {
 }
 
 # ConfigurationManager defines RPC to manage cloudflared configuration remotely
-interface ConfigurationManager {
+interface ConfigurationManager @0xb48edfbdaa25db04 {
     updateConfiguration @0 (version :Int32, config :Data) -> (result: UpdateConfigurationResponse);
 }
 
-interface CloudflaredServer extends(SessionManager, ConfigurationManager) {}
+interface CloudflaredServer @0xf548cef9dea2a4a1 extends(SessionManager, ConfigurationManager) {}

tunnelrpc/quic/cloudflared_client.go

@@ -0,0 +1,89 @@
+package quic
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net"
+	"time"
+
+	"zombiezen.com/go/capnproto2/rpc"
+
+	"github.com/google/uuid"
+
+	"github.com/cloudflare/cloudflared/tunnelrpc"
+	"github.com/cloudflare/cloudflared/tunnelrpc/metrics"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+)
+
+// CloudflaredClient calls capnp rpc methods of SessionManager and ConfigurationManager.
+type CloudflaredClient struct {
+	client         pogs.CloudflaredServer_PogsClient
+	transport      rpc.Transport
+	requestTimeout time.Duration
+}
+
+func NewCloudflaredClient(ctx context.Context, stream io.ReadWriteCloser, requestTimeout time.Duration) (*CloudflaredClient, error) {
+	n, err := stream.Write(rpcStreamProtocolSignature[:])
+	if err != nil {
+		return nil, err
+	}
+	if n != len(rpcStreamProtocolSignature) {
+		return nil, fmt.Errorf("expect to write %d bytes for RPC stream protocol signature, wrote %d", len(rpcStreamProtocolSignature), n)
+	}
+	transport := tunnelrpc.SafeTransport(stream)
+	conn := tunnelrpc.NewClientConn(transport)
+	client := pogs.NewCloudflaredServer_PogsClient(conn.Bootstrap(ctx), conn)
+	return &CloudflaredClient{
+		client:         client,
+		transport:      transport,
+		requestTimeout: requestTimeout,
+	}, nil
+}
+
+func (c *CloudflaredClient) RegisterUdpSession(ctx context.Context, sessionID uuid.UUID, dstIP net.IP, dstPort uint16, closeIdleAfterHint time.Duration, traceContext string) (*pogs.RegisterUdpSessionResponse, error) {
+	ctx, cancel := context.WithTimeout(ctx, c.requestTimeout)
+	defer cancel()
+	defer metrics.CapnpMetrics.ClientOperations.WithLabelValues(metrics.Cloudflared, metrics.OperationRegisterUdpSession).Inc()
+	timer := metrics.NewClientOperationLatencyObserver(metrics.Cloudflared, metrics.OperationRegisterUdpSession)
+	defer timer.ObserveDuration()
+
+	resp, err := c.client.RegisterUdpSession(ctx, sessionID, dstIP, dstPort, closeIdleAfterHint, traceContext)
+	if err != nil {
+		metrics.CapnpMetrics.ClientFailures.WithLabelValues(metrics.Cloudflared, metrics.OperationRegisterUdpSession).Inc()
+	}
+	return resp, err
+}
+
+func (c *CloudflaredClient) UnregisterUdpSession(ctx context.Context, sessionID uuid.UUID, message string) error {
+	ctx, cancel := context.WithTimeout(ctx, c.requestTimeout)
+	defer cancel()
+	defer metrics.CapnpMetrics.ClientOperations.WithLabelValues(metrics.Cloudflared, metrics.OperationUnregisterUdpSession).Inc()
+	timer := metrics.NewClientOperationLatencyObserver(metrics.Cloudflared, metrics.OperationUnregisterUdpSession)
+	defer timer.ObserveDuration()
+
+	err := c.client.UnregisterUdpSession(ctx, sessionID, message)
+	if err != nil {
+		metrics.CapnpMetrics.ClientFailures.WithLabelValues(metrics.Cloudflared, metrics.OperationUnregisterUdpSession).Inc()
+	}
+	return err
+}
+
+func (c *CloudflaredClient) UpdateConfiguration(ctx context.Context, version int32, config []byte) (*pogs.UpdateConfigurationResponse, error) {
+	ctx, cancel := context.WithTimeout(ctx, c.requestTimeout)
+	defer cancel()
+	defer metrics.CapnpMetrics.ClientOperations.WithLabelValues(metrics.Cloudflared, metrics.OperationUpdateConfiguration).Inc()
+	timer := metrics.NewClientOperationLatencyObserver(metrics.Cloudflared, metrics.OperationUpdateConfiguration)
+	defer timer.ObserveDuration()
+
+	resp, err := c.client.UpdateConfiguration(ctx, version, config)
+	if err != nil {
+		metrics.CapnpMetrics.ClientFailures.WithLabelValues(metrics.Cloudflared, metrics.OperationUpdateConfiguration).Inc()
+	}
+	return resp, err
+}
+
+func (c *CloudflaredClient) Close() {
+	_ = c.client.Close()
+	_ = c.transport.Close()
+}

tunnelrpc/quic/cloudflared_server.go

@@ -0,0 +1,68 @@
+package quic
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"time"
+
+	"github.com/cloudflare/cloudflared/tunnelrpc"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+)
+
+// HandleRequestFunc wraps the proxied request from the upstream and also provides methods on the stream to
+// handle the response back.
+type HandleRequestFunc = func(ctx context.Context, stream *RequestServerStream) error
+
+// CloudflaredServer provides a handler interface for a client to provide methods to handle the different types of
+// requests that can be communicated by the stream.
+type CloudflaredServer struct {
+	handleRequest   HandleRequestFunc
+	sessionManager  pogs.SessionManager
+	configManager   pogs.ConfigurationManager
+	responseTimeout time.Duration
+}
+
+func NewCloudflaredServer(handleRequest HandleRequestFunc, sessionManager pogs.SessionManager, configManager pogs.ConfigurationManager, responseTimeout time.Duration) *CloudflaredServer {
+	return &CloudflaredServer{
+		handleRequest:   handleRequest,
+		sessionManager:  sessionManager,
+		configManager:   configManager,
+		responseTimeout: responseTimeout,
+	}
+}
+
+// Serve executes the defined handlers in ServerStream on the provided stream if it is a proper RPC stream with the
+// correct preamble protocol signature.
+func (s *CloudflaredServer) Serve(ctx context.Context, stream io.ReadWriteCloser) error {
+	signature, err := determineProtocol(stream)
+	if err != nil {
+		return err
+	}
+	switch signature {
+	case dataStreamProtocolSignature:
+		return s.handleRequest(ctx, &RequestServerStream{stream})
+	case rpcStreamProtocolSignature:
+		return s.handleRPC(ctx, stream)
+	default:
+		return fmt.Errorf("unknown protocol %v", signature)
+	}
+}
+
+func (s *CloudflaredServer) handleRPC(ctx context.Context, stream io.ReadWriteCloser) error {
+	ctx, cancel := context.WithTimeout(ctx, s.responseTimeout)
+	defer cancel()
+	transport := tunnelrpc.SafeTransport(stream)
+	defer transport.Close()
+
+	main := pogs.CloudflaredServer_ServerToClient(s.sessionManager, s.configManager)
+	rpcConn := tunnelrpc.NewServerConn(transport, main.Client)
+	defer rpcConn.Close()
+
+	// We ignore the errors here because if cloudflared fails to handle a request, we will just move on.
+	select {
+	case <-rpcConn.Done():
+	case <-ctx.Done():
+	}
+	return nil
+}

tunnelrpc/quic/protocol.go

@@ -0,0 +1,78 @@
+package quic
+
+import (
+	"fmt"
+	"io"
+)
+
+// protocolSignature defines the first 6 bytes of the stream, which is used to distinguish the type of stream. It
+// ensures whoever performs a handshake does not write data before writing the metadata.
+type protocolSignature [6]byte
+
+var (
+	// dataStreamProtocolSignature is a custom protocol signature for data stream
+	dataStreamProtocolSignature = protocolSignature{0x0A, 0x36, 0xCD, 0x12, 0xA1, 0x3E}
+
+	// rpcStreamProtocolSignature is a custom protocol signature for RPC stream
+	rpcStreamProtocolSignature = protocolSignature{0x52, 0xBB, 0x82, 0x5C, 0xDB, 0x65}
+
+	errDataStreamNotSupported = fmt.Errorf("data protocol not supported")
+	errRPCStreamNotSupported  = fmt.Errorf("rpc protocol not supported")
+)
+
+type protocolVersion string
+
+const (
+	protocolV1 protocolVersion = "01"
+
+	protocolVersionLength = 2
+)
+
+// determineProtocol reads the first 6 bytes from the stream to determine which protocol is spoken by the client.
+// The protocols are magic byte arrays understood by both sides of the stream.
+func determineProtocol(stream io.Reader) (protocolSignature, error) {
+	signature, err := readSignature(stream)
+	if err != nil {
+		return protocolSignature{}, err
+	}
+	switch signature {
+	case dataStreamProtocolSignature:
+		return dataStreamProtocolSignature, nil
+	case rpcStreamProtocolSignature:
+		return rpcStreamProtocolSignature, nil
+	default:
+		return protocolSignature{}, fmt.Errorf("unknown signature %v", signature)
+	}
+}
+
+func writeDataStreamPreamble(stream io.Writer) error {
+	if err := writeSignature(stream, dataStreamProtocolSignature); err != nil {
+		return err
+	}
+
+	return writeVersion(stream)
+}
+
+func writeVersion(stream io.Writer) error {
+	_, err := stream.Write([]byte(protocolV1)[:protocolVersionLength])
+	return err
+}
+
+func readVersion(stream io.Reader) (string, error) {
+	version := make([]byte, protocolVersionLength)
+	_, err := stream.Read(version)
+	return string(version), err
+}
+
+func readSignature(stream io.Reader) (protocolSignature, error) {
+	var signature protocolSignature
+	if _, err := io.ReadFull(stream, signature[:]); err != nil {
+		return protocolSignature{}, err
+	}
+	return signature, nil
+}
+
+func writeSignature(stream io.Writer, signature protocolSignature) error {
+	_, err := stream.Write(signature[:])
+	return err
+}

tunnelrpc/quic/request_client_stream.go

@@ -0,0 +1,61 @@
+package quic
+
+import (
+	"fmt"
+	"io"
+
+	capnp "zombiezen.com/go/capnproto2"
+
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+)
+
+// RequestClientStream is a stream to provide requests to the server. This operation is typically driven by the edge service.
+type RequestClientStream struct {
+	io.ReadWriteCloser
+}
+
+// WriteConnectRequestData writes requestMeta to a stream.
+func (rcs *RequestClientStream) WriteConnectRequestData(dest string, connectionType pogs.ConnectionType, metadata ...pogs.Metadata) error {
+	connectRequest := &pogs.ConnectRequest{
+		Dest:     dest,
+		Type:     connectionType,
+		Metadata: metadata,
+	}
+
+	msg, err := connectRequest.ToPogs()
+	if err != nil {
+		return err
+	}
+
+	if err := writeDataStreamPreamble(rcs); err != nil {
+		return err
+	}
+	return capnp.NewEncoder(rcs).Encode(msg)
+}
+
+// ReadConnectResponseData reads the response from the rpc stream to a ConnectResponse.
+func (rcs *RequestClientStream) ReadConnectResponseData() (*pogs.ConnectResponse, error) {
+	signature, err := determineProtocol(rcs)
+	if err != nil {
+		return nil, err
+	}
+	if signature != dataStreamProtocolSignature {
+		return nil, fmt.Errorf("wrong protocol signature %v", signature)
+	}
+
+	// This is a NO-OP for now. We could cause a branching if we wanted to use multiple versions.
+	if _, err := readVersion(rcs); err != nil {
+		return nil, err
+	}
+
+	msg, err := capnp.NewDecoder(rcs).Decode()
+	if err != nil {
+		return nil, err
+	}
+
+	r := &pogs.ConnectResponse{}
+	if err := r.FromPogs(msg); err != nil {
+		return nil, err
+	}
+	return r, nil
+}

tunnelrpc/quic/request_server_stream.go

@@ -0,0 +1,57 @@
+package quic
+
+import (
+	"io"
+
+	capnp "zombiezen.com/go/capnproto2"
+
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+)
+
+// RequestServerStream is a stream to serve requests
+type RequestServerStream struct {
+	io.ReadWriteCloser
+}
+
+// ReadConnectRequestData reads the handshake data from a QUIC stream.
+func (rss *RequestServerStream) ReadConnectRequestData() (*pogs.ConnectRequest, error) {
+	// This is a NO-OP for now. We could cause a branching if we wanted to use multiple versions.
+	if _, err := readVersion(rss); err != nil {
+		return nil, err
+	}
+
+	msg, err := capnp.NewDecoder(rss).Decode()
+	if err != nil {
+		return nil, err
+	}
+
+	r := &pogs.ConnectRequest{}
+	if err := r.FromPogs(msg); err != nil {
+		return nil, err
+	}
+	return r, nil
+}
+
+// WriteConnectResponseData writes response to a QUIC stream.
+func (rss *RequestServerStream) WriteConnectResponseData(respErr error, metadata ...pogs.Metadata) error {
+	var connectResponse *pogs.ConnectResponse
+	if respErr != nil {
+		connectResponse = &pogs.ConnectResponse{
+			Error: respErr.Error(),
+		}
+	} else {
+		connectResponse = &pogs.ConnectResponse{
+			Metadata: metadata,
+		}
+	}
+
+	msg, err := connectResponse.ToPogs()
+	if err != nil {
+		return err
+	}
+
+	if err := writeDataStreamPreamble(rss); err != nil {
+		return err
+	}
+	return capnp.NewEncoder(rss).Encode(msg)
+}

tunnelrpc/quic/request_server_stream_test.go

@@ -11,11 +11,10 @@ import (
 	"time"
 
 	"github.com/google/uuid"
-	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
 const (
@@ -26,14 +25,14 @@ func TestConnectRequestData(t *testing.T) {
 	var tests = []struct {
 		name           string
 		hostname       string
-		connectionType ConnectionType
-		metadata       []Metadata
+		connectionType pogs.ConnectionType
+		metadata       []pogs.Metadata
 	}{
 		{
 			name:           "Signature verified and request metadata is unmarshaled and read correctly",
 			hostname:       "tunnel.com",
-			connectionType: ConnectionTypeHTTP,
-			metadata: []Metadata{
+			connectionType: pogs.ConnectionTypeHTTP,
+			metadata: []pogs.Metadata{
 				{
 					Key: "key",
 					Val: "1234",
@@ -47,10 +46,10 @@ func TestConnectRequestData(t *testing.T) {
 			reqClientStream := RequestClientStream{noopCloser{b}}
 			err := reqClientStream.WriteConnectRequestData(test.hostname, test.connectionType, test.metadata...)
 			require.NoError(t, err)
-			protocol, err := DetermineProtocol(b)
-			require.NoError(t, err)
-			reqServerStream, err := NewRequestServerStream(noopCloser{b}, protocol)
+			protocol, err := determineProtocol(b)
 			require.NoError(t, err)
+			require.Equal(t, dataStreamProtocolSignature, protocol)
+			reqServerStream := RequestServerStream{&noopCloser{b}}
 
 			reqMeta, err := reqServerStream.ReadConnectRequestData()
 			require.NoError(t, err)
@@ -66,11 +65,11 @@ func TestConnectResponseMeta(t *testing.T) {
 	var tests = []struct {
 		name     string
 		err      error
-		metadata []Metadata
+		metadata []pogs.Metadata
 	}{
 		{
 			name: "Signature verified and response metadata is unmarshaled and read correctly",
-			metadata: []Metadata{
+			metadata: []pogs.Metadata{
 				{
 					Key: "key",
 					Val: "1234",
@@ -80,7 +79,7 @@ func TestConnectResponseMeta(t *testing.T) {
 		{
 			name: "If error is not empty, other fields should be blank",
 			err:  errors.New("something happened"),
-			metadata: []Metadata{
+			metadata: []pogs.Metadata{
 				{
 					Key: "key",
 					Val: "1234",
@@ -142,22 +141,18 @@ func TestRegisterUdpSession(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			logger := zerolog.Nop()
 			clientStream, serverStream := newMockRPCStreams()
 			sessionRegisteredChan := make(chan struct{})
 			go func() {
-				protocol, err := DetermineProtocol(serverStream)
-				assert.NoError(t, err)
-				rpcServerStream, err := NewRPCServerStream(serverStream, protocol)
-				assert.NoError(t, err)
-				err = rpcServerStream.Serve(test.sessionRPCServer, nil, &logger)
+				ss := NewCloudflaredServer(nil, test.sessionRPCServer, nil, 10*time.Second)
+				err := ss.Serve(context.Background(), serverStream)
 				assert.NoError(t, err)
 
 				serverStream.Close()
 				close(sessionRegisteredChan)
 			}()
 
-			rpcClientStream, err := NewRPCClientStream(context.Background(), clientStream, 5*time.Second, &logger)
+			rpcClientStream, err := NewCloudflaredClient(context.Background(), clientStream, 5*time.Second)
 			assert.NoError(t, err)
 
 			reg, err := rpcClientStream.RegisterUdpSession(context.Background(), test.sessionRPCServer.sessionID, test.sessionRPCServer.dstIP, test.sessionRPCServer.dstPort, testCloseIdleAfterHint, test.sessionRPCServer.traceContext)
@@ -192,14 +187,10 @@ func TestManageConfiguration(t *testing.T) {
 		config:  config,
 	}
 
-	logger := zerolog.Nop()
 	updatedChan := make(chan struct{})
 	go func() {
-		protocol, err := DetermineProtocol(serverStream)
-		assert.NoError(t, err)
-		rpcServerStream, err := NewRPCServerStream(serverStream, protocol)
-		assert.NoError(t, err)
-		err = rpcServerStream.Serve(nil, configRPCServer, &logger)
+		server := NewCloudflaredServer(nil, nil, configRPCServer, 10*time.Second)
+		err := server.Serve(context.Background(), serverStream)
 		assert.NoError(t, err)
 
 		serverStream.Close()
@@ -208,7 +199,7 @@ func TestManageConfiguration(t *testing.T) {
 
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
 	defer cancel()
-	rpcClientStream, err := NewRPCClientStream(ctx, clientStream, 5*time.Second, &logger)
+	rpcClientStream, err := NewCloudflaredClient(ctx, clientStream, 5*time.Second)
 	assert.NoError(t, err)
 
 	result, err := rpcClientStream.UpdateConfiguration(ctx, version, config)
@@ -230,7 +221,7 @@ type mockSessionRPCServer struct {
 	traceContext      string
 }
 
-func (s mockSessionRPCServer) RegisterUdpSession(_ context.Context, sessionID uuid.UUID, dstIP net.IP, dstPort uint16, closeIdleAfter time.Duration, traceContext string) (*tunnelpogs.RegisterUdpSessionResponse, error) {
+func (s mockSessionRPCServer) RegisterUdpSession(_ context.Context, sessionID uuid.UUID, dstIP net.IP, dstPort uint16, closeIdleAfter time.Duration, traceContext string) (*pogs.RegisterUdpSessionResponse, error) {
 	if s.sessionID != sessionID {
 		return nil, fmt.Errorf("expect session ID %s, got %s", s.sessionID, sessionID)
 	}
@@ -246,7 +237,7 @@ func (s mockSessionRPCServer) RegisterUdpSession(_ context.Context, sessionID uu
 	if s.traceContext != traceContext {
 		return nil, fmt.Errorf("expect traceContext %s, got %s", s.traceContext, traceContext)
 	}
-	return &tunnelpogs.RegisterUdpSessionResponse{}, nil
+	return &pogs.RegisterUdpSessionResponse{}, nil
 }
 
 func (s mockSessionRPCServer) UnregisterUdpSession(_ context.Context, sessionID uuid.UUID, message string) error {
@@ -264,18 +255,18 @@ type mockConfigRPCServer struct {
 	config  []byte
 }
 
-func (s mockConfigRPCServer) UpdateConfiguration(_ context.Context, version int32, config []byte) *tunnelpogs.UpdateConfigurationResponse {
+func (s mockConfigRPCServer) UpdateConfiguration(_ context.Context, version int32, config []byte) *pogs.UpdateConfigurationResponse {
 	if s.version != version {
-		return &tunnelpogs.UpdateConfigurationResponse{
+		return &pogs.UpdateConfigurationResponse{
 			Err: fmt.Errorf("expect version %d, got %d", s.version, version),
 		}
 	}
 	if !bytes.Equal(s.config, config) {
-		return &tunnelpogs.UpdateConfigurationResponse{
+		return &pogs.UpdateConfigurationResponse{
 			Err: fmt.Errorf("expect config %v, got %v", s.config, config),
 		}
 	}
-	return &tunnelpogs.UpdateConfigurationResponse{LastAppliedVersion: version}
+	return &pogs.UpdateConfigurationResponse{LastAppliedVersion: version}
 }
 
 type mockRPCStream struct {

tunnelrpc/quic/session_client.go

@@ -0,0 +1,73 @@
+package quic
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net"
+	"time"
+
+	"github.com/google/uuid"
+	"zombiezen.com/go/capnproto2/rpc"
+
+	"github.com/cloudflare/cloudflared/tunnelrpc"
+	"github.com/cloudflare/cloudflared/tunnelrpc/metrics"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+)
+
+// SessionClient calls capnp rpc methods of SessionManager.
+type SessionClient struct {
+	client         pogs.SessionManager_PogsClient
+	transport      rpc.Transport
+	requestTimeout time.Duration
+}
+
+func NewSessionClient(ctx context.Context, stream io.ReadWriteCloser, requestTimeout time.Duration) (*SessionClient, error) {
+	n, err := stream.Write(rpcStreamProtocolSignature[:])
+	if err != nil {
+		return nil, err
+	}
+	if n != len(rpcStreamProtocolSignature) {
+		return nil, fmt.Errorf("expect to write %d bytes for RPC stream protocol signature, wrote %d", len(rpcStreamProtocolSignature), n)
+	}
+	transport := tunnelrpc.SafeTransport(stream)
+	conn := tunnelrpc.NewClientConn(transport)
+	return &SessionClient{
+		client:         pogs.NewSessionManager_PogsClient(conn.Bootstrap(ctx), conn),
+		transport:      transport,
+		requestTimeout: requestTimeout,
+	}, nil
+}
+
+func (c *SessionClient) RegisterUdpSession(ctx context.Context, sessionID uuid.UUID, dstIP net.IP, dstPort uint16, closeIdleAfterHint time.Duration, traceContext string) (*pogs.RegisterUdpSessionResponse, error) {
+	ctx, cancel := context.WithTimeout(ctx, c.requestTimeout)
+	defer cancel()
+	defer metrics.CapnpMetrics.ClientOperations.WithLabelValues(metrics.SessionManager, metrics.OperationRegisterUdpSession).Inc()
+	timer := metrics.NewClientOperationLatencyObserver(metrics.SessionManager, metrics.OperationRegisterUdpSession)
+	defer timer.ObserveDuration()
+
+	resp, err := c.client.RegisterUdpSession(ctx, sessionID, dstIP, dstPort, closeIdleAfterHint, traceContext)
+	if err != nil {
+		metrics.CapnpMetrics.ClientFailures.WithLabelValues(metrics.SessionManager, metrics.OperationRegisterUdpSession).Inc()
+	}
+	return resp, err
+}
+
+func (c *SessionClient) UnregisterUdpSession(ctx context.Context, sessionID uuid.UUID, message string) error {
+	ctx, cancel := context.WithTimeout(ctx, c.requestTimeout)
+	defer cancel()
+	defer metrics.CapnpMetrics.ClientOperations.WithLabelValues(metrics.SessionManager, metrics.OperationUnregisterUdpSession).Inc()
+	timer := metrics.NewClientOperationLatencyObserver(metrics.SessionManager, metrics.OperationUnregisterUdpSession)
+	defer timer.ObserveDuration()
+
+	err := c.client.UnregisterUdpSession(ctx, sessionID, message)
+	if err != nil {
+		metrics.CapnpMetrics.ClientFailures.WithLabelValues(metrics.SessionManager, metrics.OperationUnregisterUdpSession).Inc()
+	}
+	return err
+}
+
+func (c *SessionClient) Close() {
+	_ = c.client.Close()
+	_ = c.transport.Close()
+}

tunnelrpc/quic/session_server.go

@@ -0,0 +1,58 @@
+package quic
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"time"
+
+	"github.com/cloudflare/cloudflared/tunnelrpc"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+)
+
+// SessionManagerServer handles streams with the SessionManager RPCs.
+type SessionManagerServer struct {
+	sessionManager  pogs.SessionManager
+	responseTimeout time.Duration
+}
+
+func NewSessionManagerServer(sessionManager pogs.SessionManager, responseTimeout time.Duration) *SessionManagerServer {
+	return &SessionManagerServer{
+		sessionManager:  sessionManager,
+		responseTimeout: responseTimeout,
+	}
+}
+
+func (s *SessionManagerServer) Serve(ctx context.Context, stream io.ReadWriteCloser) error {
+	signature, err := determineProtocol(stream)
+	if err != nil {
+		return err
+	}
+	switch signature {
+	case rpcStreamProtocolSignature:
+		break
+	case dataStreamProtocolSignature:
+		return errDataStreamNotSupported
+	default:
+		return fmt.Errorf("unknown protocol %v", signature)
+	}
+
+	// Every new quic.Stream request aligns to a new RPC request, this is why there is a timeout for the server-side
+	// of the RPC request.
+	ctx, cancel := context.WithTimeout(ctx, s.responseTimeout)
+	defer cancel()
+
+	transport := tunnelrpc.SafeTransport(stream)
+	defer transport.Close()
+
+	main := pogs.SessionManager_ServerToClient(s.sessionManager)
+	rpcConn := tunnelrpc.NewServerConn(transport, main.Client)
+	defer rpcConn.Close()
+
+	select {
+	case <-rpcConn.Done():
+		return rpcConn.Err()
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+}

tunnelrpc/registration_client.go

@@ -0,0 +1,101 @@
+package tunnelrpc
+
+import (
+	"context"
+	"io"
+	"net"
+	"time"
+
+	"github.com/google/uuid"
+	"zombiezen.com/go/capnproto2/rpc"
+
+	"github.com/cloudflare/cloudflared/tunnelrpc/metrics"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+)
+
+type RegistrationClient interface {
+	RegisterConnection(
+		ctx context.Context,
+		auth pogs.TunnelAuth,
+		tunnelID uuid.UUID,
+		options *pogs.ConnectionOptions,
+		connIndex uint8,
+		edgeAddress net.IP,
+	) (*pogs.ConnectionDetails, error)
+	SendLocalConfiguration(ctx context.Context, config []byte) error
+	GracefulShutdown(ctx context.Context, gracePeriod time.Duration) error
+	Close()
+}
+
+type registrationClient struct {
+	client         pogs.RegistrationServer_PogsClient
+	transport      rpc.Transport
+	requestTimeout time.Duration
+}
+
+func NewRegistrationClient(ctx context.Context, stream io.ReadWriteCloser, requestTimeout time.Duration) RegistrationClient {
+	transport := SafeTransport(stream)
+	conn := NewClientConn(transport)
+	client := pogs.NewRegistrationServer_PogsClient(conn.Bootstrap(ctx), conn)
+	return &registrationClient{
+		client:         client,
+		transport:      transport,
+		requestTimeout: requestTimeout,
+	}
+}
+
+func (r *registrationClient) RegisterConnection(
+	ctx context.Context,
+	auth pogs.TunnelAuth,
+	tunnelID uuid.UUID,
+	options *pogs.ConnectionOptions,
+	connIndex uint8,
+	edgeAddress net.IP,
+) (*pogs.ConnectionDetails, error) {
+	ctx, cancel := context.WithTimeout(ctx, r.requestTimeout)
+	defer cancel()
+	defer metrics.CapnpMetrics.ClientOperations.WithLabelValues(metrics.Registration, metrics.OperationRegisterConnection).Inc()
+	timer := metrics.NewClientOperationLatencyObserver(metrics.Registration, metrics.OperationRegisterConnection)
+	defer timer.ObserveDuration()
+
+	conn, err := r.client.RegisterConnection(ctx, auth, tunnelID, connIndex, options)
+	if err != nil {
+		metrics.CapnpMetrics.ClientFailures.WithLabelValues(metrics.Registration, metrics.OperationRegisterConnection).Inc()
+	}
+	return conn, err
+}
+
+func (r *registrationClient) SendLocalConfiguration(ctx context.Context, config []byte) error {
+	ctx, cancel := context.WithTimeout(ctx, r.requestTimeout)
+	defer cancel()
+	defer metrics.CapnpMetrics.ClientOperations.WithLabelValues(metrics.Registration, metrics.OperationUpdateLocalConfiguration).Inc()
+	timer := metrics.NewClientOperationLatencyObserver(metrics.Registration, metrics.OperationUpdateLocalConfiguration)
+	defer timer.ObserveDuration()
+
+	err := r.client.SendLocalConfiguration(ctx, config)
+	if err != nil {
+		metrics.CapnpMetrics.ClientFailures.WithLabelValues(metrics.Registration, metrics.OperationUpdateLocalConfiguration).Inc()
+	}
+	return err
+}
+
+func (r *registrationClient) GracefulShutdown(ctx context.Context, gracePeriod time.Duration) error {
+	ctx, cancel := context.WithTimeout(ctx, gracePeriod)
+	defer cancel()
+	defer metrics.CapnpMetrics.ClientOperations.WithLabelValues(metrics.Registration, metrics.OperationUnregisterConnection).Inc()
+	timer := metrics.NewClientOperationLatencyObserver(metrics.Registration, metrics.OperationUnregisterConnection)
+	defer timer.ObserveDuration()
+	err := r.client.UnregisterConnection(ctx)
+	if err != nil {
+		metrics.CapnpMetrics.ClientFailures.WithLabelValues(metrics.Registration, metrics.OperationUnregisterConnection).Inc()
+		return err
+	}
+	return nil
+}
+
+func (r *registrationClient) Close() {
+	// Closing the client will also close the connection
+	_ = r.client.Close()
+	// Closing the transport also closes the stream
+	_ = r.transport.Close()
+}

tunnelrpc/registration_server.go

@@ -0,0 +1,37 @@
+package tunnelrpc
+
+import (
+	"context"
+	"io"
+
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+)
+
+// RegistrationServer provides a handler interface for a client to provide methods to handle the different types of
+// requests that can be communicated by the stream.
+type RegistrationServer struct {
+	registrationServer pogs.RegistrationServer
+}
+
+func NewRegistrationServer(registrationServer pogs.RegistrationServer) *RegistrationServer {
+	return &RegistrationServer{
+		registrationServer: registrationServer,
+	}
+}
+
+// Serve listens for all RegistrationServer RPCs, including UnregisterConnection until the underlying connection
+// is terminated.
+func (s *RegistrationServer) Serve(ctx context.Context, stream io.ReadWriteCloser) error {
+	transport := SafeTransport(stream)
+	defer transport.Close()
+
+	main := pogs.RegistrationServer_ServerToClient(s.registrationServer)
+	rpcConn := NewServerConn(transport, main.Client)
+
+	select {
+	case <-rpcConn.Done():
+		return rpcConn.Wait()
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+}

tunnelrpc/utils.go

@@ -0,0 +1,89 @@
+package tunnelrpc
+
+import (
+	"context"
+	"io"
+	"time"
+
+	"github.com/pkg/errors"
+	capnp "zombiezen.com/go/capnproto2"
+	"zombiezen.com/go/capnproto2/rpc"
+)
+
+const (
+	// These default values are here so that we give some time for the underlying connection/stream
+	// to recover in the face of what we believe to be temporarily errors.
+	// We don't want to be too aggressive, as the end result of giving a final error (non-temporary)
+	// will result in the connection to be dropped.
+	// In turn, the other side will probably reconnect, which will put again more pressure in the overall system.
+	// So, the best solution is to give it some conservative time to recover.
+	defaultSleepBetweenTemporaryError = 500 * time.Millisecond
+	defaultMaxRetries                 = 3
+)
+
+type readWriterSafeTemporaryErrorCloser struct {
+	io.ReadWriteCloser
+
+	retries             int
+	sleepBetweenRetries time.Duration
+	maxRetries          int
+}
+
+func (r *readWriterSafeTemporaryErrorCloser) Read(p []byte) (n int, err error) {
+	n, err = r.ReadWriteCloser.Read(p)
+
+	// if there was a failure reading from the read closer, and the error is temporary, try again in some seconds
+	// otherwise, just fail without a temporary error.
+	if n == 0 && err != nil && isTemporaryError(err) {
+		if r.retries >= r.maxRetries {
+			return 0, errors.Wrap(err, "failed read from capnproto ReaderWriter after multiple temporary errors")
+		} else {
+			r.retries += 1
+
+			// sleep for some time to prevent quick read loops that cause exhaustion of CPU resources
+			time.Sleep(r.sleepBetweenRetries)
+		}
+	}
+
+	if err == nil {
+		r.retries = 0
+	}
+
+	return n, err
+}
+
+func SafeTransport(rw io.ReadWriteCloser) rpc.Transport {
+	return rpc.StreamTransport(&readWriterSafeTemporaryErrorCloser{
+		ReadWriteCloser:     rw,
+		maxRetries:          defaultMaxRetries,
+		sleepBetweenRetries: defaultSleepBetweenTemporaryError,
+	})
+}
+
+// isTemporaryError reports whether e has a Temporary() method that
+// returns true.
+func isTemporaryError(e error) bool {
+	type temp interface {
+		Temporary() bool
+	}
+	t, ok := e.(temp)
+	return ok && t.Temporary()
+}
+
+// NoopCapnpLogger provides a logger to discard all capnp rpc internal logging messages as
+// they are by default provided to stdout if no logger interface is provided. These logging
+// messages in cloudflared have typically not provided a high amount of pratical value
+// as the messages are extremely verbose and don't provide a good insight into the message
+// contents or rpc method names.
+type noopCapnpLogger struct{}
+
+func (noopCapnpLogger) Infof(ctx context.Context, format string, args ...interface{})  {}
+func (noopCapnpLogger) Errorf(ctx context.Context, format string, args ...interface{}) {}
+
+func NewClientConn(transport rpc.Transport) *rpc.Conn {
+	return rpc.NewConn(transport, rpc.ConnLog(noopCapnpLogger{}))
+}
+
+func NewServerConn(transport rpc.Transport, client capnp.Client) *rpc.Conn {
+	return rpc.NewConn(transport, rpc.MainInterface(client), rpc.ConnLog(noopCapnpLogger{}))
+}

vendor/github.com/golang/protobuf/ptypes/timestamp/timestamp.pb.go

@@ -1,64 +0,0 @@
-// Code generated by protoc-gen-go. DO NOT EDIT.
-// source: github.com/golang/protobuf/ptypes/timestamp/timestamp.proto
-
-package timestamp
-
-import (
-	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
-	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
-	timestamppb "google.golang.org/protobuf/types/known/timestamppb"
-	reflect "reflect"
-)
-
-// Symbols defined in public import of google/protobuf/timestamp.proto.
-
-type Timestamp = timestamppb.Timestamp
-
-var File_github_com_golang_protobuf_ptypes_timestamp_timestamp_proto protoreflect.FileDescriptor
-
-var file_github_com_golang_protobuf_ptypes_timestamp_timestamp_proto_rawDesc = []byte{
-	0x0a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x6c,
-	0x61, 0x6e, 0x67, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x70, 0x74, 0x79,
-	0x70, 0x65, 0x73, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2f, 0x74, 0x69,
-	0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1f, 0x67,
-	0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74,
-	0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x42, 0x37,
-	0x5a, 0x35, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x6c,
-	0x61, 0x6e, 0x67, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x70, 0x74, 0x79,
-	0x70, 0x65, 0x73, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x3b, 0x74, 0x69,
-	0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x50, 0x00, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f,
-	0x33,
-}
-
-var file_github_com_golang_protobuf_ptypes_timestamp_timestamp_proto_goTypes = []interface{}{}
-var file_github_com_golang_protobuf_ptypes_timestamp_timestamp_proto_depIdxs = []int32{
-	0, // [0:0] is the sub-list for method output_type
-	0, // [0:0] is the sub-list for method input_type
-	0, // [0:0] is the sub-list for extension type_name
-	0, // [0:0] is the sub-list for extension extendee
-	0, // [0:0] is the sub-list for field type_name
-}
-
-func init() { file_github_com_golang_protobuf_ptypes_timestamp_timestamp_proto_init() }
-func file_github_com_golang_protobuf_ptypes_timestamp_timestamp_proto_init() {
-	if File_github_com_golang_protobuf_ptypes_timestamp_timestamp_proto != nil {
-		return
-	}
-	type x struct{}
-	out := protoimpl.TypeBuilder{
-		File: protoimpl.DescBuilder{
-			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
-			RawDescriptor: file_github_com_golang_protobuf_ptypes_timestamp_timestamp_proto_rawDesc,
-			NumEnums:      0,
-			NumMessages:   0,
-			NumExtensions: 0,
-			NumServices:   0,
-		},
-		GoTypes:           file_github_com_golang_protobuf_ptypes_timestamp_timestamp_proto_goTypes,
-		DependencyIndexes: file_github_com_golang_protobuf_ptypes_timestamp_timestamp_proto_depIdxs,
-	}.Build()
-	File_github_com_golang_protobuf_ptypes_timestamp_timestamp_proto = out.File
-	file_github_com_golang_protobuf_ptypes_timestamp_timestamp_proto_rawDesc = nil
-	file_github_com_golang_protobuf_ptypes_timestamp_timestamp_proto_goTypes = nil
-	file_github_com_golang_protobuf_ptypes_timestamp_timestamp_proto_depIdxs = nil
-}

vendor/github.com/prometheus/client_golang/prometheus/counter.go

@@ -20,6 +20,7 @@ import (
 	"time"
 
 	dto "github.com/prometheus/client_model/go"
+	"google.golang.org/protobuf/types/known/timestamppb"
 )
 
 // Counter is a Metric that represents a single numerical value that only ever
@@ -59,6 +60,18 @@ type ExemplarAdder interface {
 // CounterOpts is an alias for Opts. See there for doc comments.
 type CounterOpts Opts
 
+// CounterVecOpts bundles the options to create a CounterVec metric.
+// It is mandatory to set CounterOpts, see there for mandatory fields. VariableLabels
+// is optional and can safely be left to its default value.
+type CounterVecOpts struct {
+	CounterOpts
+
+	// VariableLabels are used to partition the metric vector by the given set
+	// of labels. Each label value will be constrained with the optional Constraint
+	// function, if provided.
+	VariableLabels ConstrainableLabels
+}
+
 // NewCounter creates a new Counter based on the provided CounterOpts.
 //
 // The returned implementation also implements ExemplarAdder. It is safe to
@@ -78,8 +91,12 @@ func NewCounter(opts CounterOpts) Counter {
 		nil,
 		opts.ConstLabels,
 	)
-	result := &counter{desc: desc, labelPairs: desc.constLabelPairs, now: time.Now}
+	if opts.now == nil {
+		opts.now = time.Now
+	}
+	result := &counter{desc: desc, labelPairs: desc.constLabelPairs, now: opts.now}
 	result.init(result) // Init self-collection.
+	result.createdTs = timestamppb.New(opts.now())
 	return result
 }
 
@@ -94,10 +111,12 @@ type counter struct {
 	selfCollector
 	desc *Desc
 
+	createdTs  *timestamppb.Timestamp
 	labelPairs []*dto.LabelPair
 	exemplar   atomic.Value // Containing nil or a *dto.Exemplar.
 
-	now func() time.Time // To mock out time.Now() for testing.
+	// now is for testing purposes, by default it's time.Now.
+	now func() time.Time
 }
 
 func (c *counter) Desc() *Desc {
@@ -140,14 +159,14 @@ func (c *counter) get() float64 {
 }
 
 func (c *counter) Write(out *dto.Metric) error {
-	val := c.get()
-
+	// Read the Exemplar first and the value second. This is to avoid a race condition
+	// where users see an exemplar for a not-yet-existing observation.
 	var exemplar *dto.Exemplar
 	if e := c.exemplar.Load(); e != nil {
 		exemplar = e.(*dto.Exemplar)
 	}
-
-	return populateMetric(CounterValue, val, c.labelPairs, exemplar, out)
+	val := c.get()
+	return populateMetric(CounterValue, val, c.labelPairs, exemplar, out, c.createdTs)
 }
 
 func (c *counter) updateExemplar(v float64, l Labels) {
@@ -173,19 +192,31 @@ type CounterVec struct {
 // NewCounterVec creates a new CounterVec based on the provided CounterOpts and
 // partitioned by the given label names.
 func NewCounterVec(opts CounterOpts, labelNames []string) *CounterVec {
-	desc := NewDesc(
+	return V2.NewCounterVec(CounterVecOpts{
+		CounterOpts:    opts,
+		VariableLabels: UnconstrainedLabels(labelNames),
+	})
+}
+
+// NewCounterVec creates a new CounterVec based on the provided CounterVecOpts.
+func (v2) NewCounterVec(opts CounterVecOpts) *CounterVec {
+	desc := V2.NewDesc(
 		BuildFQName(opts.Namespace, opts.Subsystem, opts.Name),
 		opts.Help,
-		labelNames,
+		opts.VariableLabels,
 		opts.ConstLabels,
 	)
+	if opts.now == nil {
+		opts.now = time.Now
+	}
 	return &CounterVec{
 		MetricVec: NewMetricVec(desc, func(lvs ...string) Metric {
-			if len(lvs) != len(desc.variableLabels) {
-				panic(makeInconsistentCardinalityError(desc.fqName, desc.variableLabels, lvs))
+			if len(lvs) != len(desc.variableLabels.names) {
+				panic(makeInconsistentCardinalityError(desc.fqName, desc.variableLabels.names, lvs))
 			}
-			result := &counter{desc: desc, labelPairs: MakeLabelPairs(desc, lvs), now: time.Now}
+			result := &counter{desc: desc, labelPairs: MakeLabelPairs(desc, lvs), now: opts.now}
 			result.init(result) // Init self-collection.
+			result.createdTs = timestamppb.New(opts.now())
 			return result
 		}),
 	}
@@ -245,7 +276,8 @@ func (v *CounterVec) GetMetricWith(labels Labels) (Counter, error) {
 // WithLabelValues works as GetMetricWithLabelValues, but panics where
 // GetMetricWithLabelValues would have returned an error. Not returning an
 // error allows shortcuts like
-//     myVec.WithLabelValues("404", "GET").Add(42)
+//
+//	myVec.WithLabelValues("404", "GET").Add(42)
 func (v *CounterVec) WithLabelValues(lvs ...string) Counter {
 	c, err := v.GetMetricWithLabelValues(lvs...)
 	if err != nil {
@@ -256,7 +288,8 @@ func (v *CounterVec) WithLabelValues(lvs ...string) Counter {
 
 // With works as GetMetricWith, but panics where GetMetricWithLabels would have
 // returned an error. Not returning an error allows shortcuts like
-//     myVec.With(prometheus.Labels{"code": "404", "method": "GET"}).Add(42)
+//
+//	myVec.With(prometheus.Labels{"code": "404", "method": "GET"}).Add(42)
 func (v *CounterVec) With(labels Labels) Counter {
 	c, err := v.GetMetricWith(labels)
 	if err != nil {

vendor/github.com/prometheus/client_golang/prometheus/desc.go

@@ -14,20 +14,16 @@
 package prometheus
 
 import (
-	"errors"
 	"fmt"
 	"sort"
 	"strings"
 
 	"github.com/cespare/xxhash/v2"
+	dto "github.com/prometheus/client_model/go"
+	"github.com/prometheus/common/model"
+	"google.golang.org/protobuf/proto"
 
 	"github.com/prometheus/client_golang/prometheus/internal"
-
-	//nolint:staticcheck // Ignore SA1019. Need to keep deprecated package for compatibility.
-	"github.com/golang/protobuf/proto"
-	"github.com/prometheus/common/model"
-
-	dto "github.com/prometheus/client_model/go"
 )
 
 // Desc is the descriptor used by every Prometheus Metric. It is essentially
@@ -54,9 +50,9 @@ type Desc struct {
 	// constLabelPairs contains precalculated DTO label pairs based on
 	// the constant labels.
 	constLabelPairs []*dto.LabelPair
-	// variableLabels contains names of labels for which the metric
-	// maintains variable values.
-	variableLabels []string
+	// variableLabels contains names of labels and normalization function for
+	// which the metric maintains variable values.
+	variableLabels *compiledLabels
 	// id is a hash of the values of the ConstLabels and fqName. This
 	// must be unique among all registered descriptors and can therefore be
 	// used as an identifier of the descriptor.
@@ -80,10 +76,24 @@ type Desc struct {
 // For constLabels, the label values are constant. Therefore, they are fully
 // specified in the Desc. See the Collector example for a usage pattern.
 func NewDesc(fqName, help string, variableLabels []string, constLabels Labels) *Desc {
+	return V2.NewDesc(fqName, help, UnconstrainedLabels(variableLabels), constLabels)
+}
+
+// NewDesc allocates and initializes a new Desc. Errors are recorded in the Desc
+// and will be reported on registration time. variableLabels and constLabels can
+// be nil if no such labels should be set. fqName must not be empty.
+//
+// variableLabels only contain the label names and normalization functions. Their
+// label values are variable and therefore not part of the Desc. (They are managed
+// within the Metric.)
+//
+// For constLabels, the label values are constant. Therefore, they are fully
+// specified in the Desc. See the Collector example for a usage pattern.
+func (v2) NewDesc(fqName, help string, variableLabels ConstrainableLabels, constLabels Labels) *Desc {
 	d := &Desc{
 		fqName:         fqName,
 		help:           help,
-		variableLabels: variableLabels,
+		variableLabels: variableLabels.compile(),
 	}
 	if !model.IsValidMetricName(model.LabelValue(fqName)) {
 		d.err = fmt.Errorf("%q is not a valid metric name", fqName)
@@ -93,7 +103,7 @@ func NewDesc(fqName, help string, variableLabels []string, constLabels Labels) *
 	// their sorted label names) plus the fqName (at position 0).
 	labelValues := make([]string, 1, len(constLabels)+1)
 	labelValues[0] = fqName
-	labelNames := make([]string, 0, len(constLabels)+len(variableLabels))
+	labelNames := make([]string, 0, len(constLabels)+len(d.variableLabels.names))
 	labelNameSet := map[string]struct{}{}
 	// First add only the const label names and sort them...
 	for labelName := range constLabels {
@@ -118,16 +128,16 @@ func NewDesc(fqName, help string, variableLabels []string, constLabels Labels) *
 	// Now add the variable label names, but prefix them with something that
 	// cannot be in a regular label name. That prevents matching the label
 	// dimension with a different mix between preset and variable labels.
-	for _, labelName := range variableLabels {
-		if !checkLabelName(labelName) {
-			d.err = fmt.Errorf("%q is not a valid label name for metric %q", labelName, fqName)
+	for _, label := range d.variableLabels.names {
+		if !checkLabelName(label) {
+			d.err = fmt.Errorf("%q is not a valid label name for metric %q", label, fqName)
 			return d
 		}
-		labelNames = append(labelNames, "$"+labelName)
-		labelNameSet[labelName] = struct{}{}
+		labelNames = append(labelNames, "$"+label)
+		labelNameSet[label] = struct{}{}
 	}
 	if len(labelNames) != len(labelNameSet) {
-		d.err = errors.New("duplicate label names")
+		d.err = fmt.Errorf("duplicate label names in constant and variable labels for metric %q", fqName)
 		return d
 	}
 
@@ -179,11 +189,19 @@ func (d *Desc) String() string {
 			fmt.Sprintf("%s=%q", lp.GetName(), lp.GetValue()),
 		)
 	}
+	vlStrings := make([]string, 0, len(d.variableLabels.names))
+	for _, vl := range d.variableLabels.names {
+		if fn, ok := d.variableLabels.labelConstraints[vl]; ok && fn != nil {
+			vlStrings = append(vlStrings, fmt.Sprintf("c(%s)", vl))
+		} else {
+			vlStrings = append(vlStrings, vl)
+		}
+	}
 	return fmt.Sprintf(
-		"Desc{fqName: %q, help: %q, constLabels: {%s}, variableLabels: %v}",
+		"Desc{fqName: %q, help: %q, constLabels: {%s}, variableLabels: {%s}}",
 		d.fqName,
 		d.help,
 		strings.Join(lpStrings, ","),
-		d.variableLabels,
+		strings.Join(vlStrings, ","),
 	)
 }

vendor/github.com/prometheus/client_golang/prometheus/doc.go

@@ -21,55 +21,66 @@
 // All exported functions and methods are safe to be used concurrently unless
 // specified otherwise.
 //
-// A Basic Example
+// # A Basic Example
 //
 // As a starting point, a very basic usage example:
 //
-//    package main
+//	package main
 //
-//    import (
-//    	"log"
-//    	"net/http"
+//	import (
+//		"log"
+//		"net/http"
 //
-//    	"github.com/prometheus/client_golang/prometheus"
-//    	"github.com/prometheus/client_golang/prometheus/promhttp"
-//    )
+//		"github.com/prometheus/client_golang/prometheus"
+//		"github.com/prometheus/client_golang/prometheus/promhttp"
+//	)
 //
-//    var (
-//    	cpuTemp = prometheus.NewGauge(prometheus.GaugeOpts{
-//    		Name: "cpu_temperature_celsius",
-//    		Help: "Current temperature of the CPU.",
-//    	})
-//    	hdFailures = prometheus.NewCounterVec(
-//    		prometheus.CounterOpts{
-//    			Name: "hd_errors_total",
-//    			Help: "Number of hard-disk errors.",
-//    		},
-//    		[]string{"device"},
-//    	)
-//    )
+//	type metrics struct {
+//		cpuTemp  prometheus.Gauge
+//		hdFailures *prometheus.CounterVec
+//	}
 //
-//    func init() {
-//    	// Metrics have to be registered to be exposed:
-//    	prometheus.MustRegister(cpuTemp)
-//    	prometheus.MustRegister(hdFailures)
-//    }
+//	func NewMetrics(reg prometheus.Registerer) *metrics {
+//		m := &metrics{
+//			cpuTemp: prometheus.NewGauge(prometheus.GaugeOpts{
+//				Name: "cpu_temperature_celsius",
+//				Help: "Current temperature of the CPU.",
+//			}),
+//			hdFailures: prometheus.NewCounterVec(
+//				prometheus.CounterOpts{
+//					Name: "hd_errors_total",
+//					Help: "Number of hard-disk errors.",
+//				},
+//				[]string{"device"},
+//			),
+//		}
+//		reg.MustRegister(m.cpuTemp)
+//		reg.MustRegister(m.hdFailures)
+//		return m
+//	}
 //
-//    func main() {
-//    	cpuTemp.Set(65.3)
-//    	hdFailures.With(prometheus.Labels{"device":"/dev/sda"}).Inc()
+//	func main() {
+//		// Create a non-global registry.
+//		reg := prometheus.NewRegistry()
 //
-//    	// The Handler function provides a default handler to expose metrics
-//    	// via an HTTP server. "/metrics" is the usual endpoint for that.
-//    	http.Handle("/metrics", promhttp.Handler())
-//    	log.Fatal(http.ListenAndServe(":8080", nil))
-//    }
+//		// Create new metrics and register them using the custom registry.
+//		m := NewMetrics(reg)
+//		// Set values for the new created metrics.
+//		m.cpuTemp.Set(65.3)
+//		m.hdFailures.With(prometheus.Labels{"device":"/dev/sda"}).Inc()
 //
+//		// Expose metrics and custom registry via an HTTP server
+//		// using the HandleFor function. "/metrics" is the usual endpoint for that.
+//		http.Handle("/metrics", promhttp.HandlerFor(reg, promhttp.HandlerOpts{Registry: reg}))
+//		log.Fatal(http.ListenAndServe(":8080", nil))
+//	}
 //
 // This is a complete program that exports two metrics, a Gauge and a Counter,
 // the latter with a label attached to turn it into a (one-dimensional) vector.
+// It register the metrics using a custom registry and exposes them via an HTTP server
+// on the /metrics endpoint.
 //
-// Metrics
+// # Metrics
 //
 // The number of exported identifiers in this package might appear a bit
 // overwhelming. However, in addition to the basic plumbing shown in the example
@@ -100,7 +111,7 @@
 // To create instances of Metrics and their vector versions, you need a suitable
 // …Opts struct, i.e. GaugeOpts, CounterOpts, SummaryOpts, or HistogramOpts.
 //
-// Custom Collectors and constant Metrics
+// # Custom Collectors and constant Metrics
 //
 // While you could create your own implementations of Metric, most likely you
 // will only ever implement the Collector interface on your own. At a first
@@ -141,7 +152,7 @@
 // a metric, GaugeFunc, CounterFunc, or UntypedFunc might be interesting
 // shortcuts.
 //
-// Advanced Uses of the Registry
+// # Advanced Uses of the Registry
 //
 // While MustRegister is the by far most common way of registering a Collector,
 // sometimes you might want to handle the errors the registration might cause.
@@ -176,23 +187,23 @@
 // NewProcessCollector). With a custom registry, you are in control and decide
 // yourself about the Collectors to register.
 //
-// HTTP Exposition
+// # HTTP Exposition
 //
 // The Registry implements the Gatherer interface. The caller of the Gather
 // method can then expose the gathered metrics in some way. Usually, the metrics
 // are served via HTTP on the /metrics endpoint. That's happening in the example
 // above. The tools to expose metrics via HTTP are in the promhttp sub-package.
 //
-// Pushing to the Pushgateway
+// # Pushing to the Pushgateway
 //
 // Function for pushing to the Pushgateway can be found in the push sub-package.
 //
-// Graphite Bridge
+// # Graphite Bridge
 //
 // Functions and examples to push metrics from a Gatherer to Graphite can be
 // found in the graphite sub-package.
 //
-// Other Means of Exposition
+// # Other Means of Exposition
 //
 // More ways of exposing metrics can easily be added by following the approaches
 // of the existing implementations.

vendor/github.com/prometheus/client_golang/prometheus/expvar_collector.go

@@ -48,7 +48,7 @@ func (e *expvarCollector) Collect(ch chan<- Metric) {
 			continue
 		}
 		var v interface{}
-		labels := make([]string, len(desc.variableLabels))
+		labels := make([]string, len(desc.variableLabels.names))
 		if err := json.Unmarshal([]byte(expVar.String()), &v); err != nil {
 			ch <- NewInvalidMetric(desc, err)
 			continue

vendor/github.com/prometheus/client_golang/prometheus/gauge.go

@@ -55,6 +55,18 @@ type Gauge interface {
 // GaugeOpts is an alias for Opts. See there for doc comments.
 type GaugeOpts Opts
 
+// GaugeVecOpts bundles the options to create a GaugeVec metric.
+// It is mandatory to set GaugeOpts, see there for mandatory fields. VariableLabels
+// is optional and can safely be left to its default value.
+type GaugeVecOpts struct {
+	GaugeOpts
+
+	// VariableLabels are used to partition the metric vector by the given set
+	// of labels. Each label value will be constrained with the optional Constraint
+	// function, if provided.
+	VariableLabels ConstrainableLabels
+}
+
 // NewGauge creates a new Gauge based on the provided GaugeOpts.
 //
 // The returned implementation is optimized for a fast Set method. If you have a
@@ -123,7 +135,7 @@ func (g *gauge) Sub(val float64) {
 
 func (g *gauge) Write(out *dto.Metric) error {
 	val := math.Float64frombits(atomic.LoadUint64(&g.valBits))
-	return populateMetric(GaugeValue, val, g.labelPairs, nil, out)
+	return populateMetric(GaugeValue, val, g.labelPairs, nil, out, nil)
 }
 
 // GaugeVec is a Collector that bundles a set of Gauges that all share the same
@@ -138,16 +150,24 @@ type GaugeVec struct {
 // NewGaugeVec creates a new GaugeVec based on the provided GaugeOpts and
 // partitioned by the given label names.
 func NewGaugeVec(opts GaugeOpts, labelNames []string) *GaugeVec {
-	desc := NewDesc(
+	return V2.NewGaugeVec(GaugeVecOpts{
+		GaugeOpts:      opts,
+		VariableLabels: UnconstrainedLabels(labelNames),
+	})
+}
+
+// NewGaugeVec creates a new GaugeVec based on the provided GaugeVecOpts.
+func (v2) NewGaugeVec(opts GaugeVecOpts) *GaugeVec {
+	desc := V2.NewDesc(
 		BuildFQName(opts.Namespace, opts.Subsystem, opts.Name),
 		opts.Help,
-		labelNames,
+		opts.VariableLabels,
 		opts.ConstLabels,
 	)
 	return &GaugeVec{
 		MetricVec: NewMetricVec(desc, func(lvs ...string) Metric {
-			if len(lvs) != len(desc.variableLabels) {
-				panic(makeInconsistentCardinalityError(desc.fqName, desc.variableLabels, lvs))
+			if len(lvs) != len(desc.variableLabels.names) {
+				panic(makeInconsistentCardinalityError(desc.fqName, desc.variableLabels.names, lvs))
 			}
 			result := &gauge{desc: desc, labelPairs: MakeLabelPairs(desc, lvs)}
 			result.init(result) // Init self-collection.
@@ -210,7 +230,8 @@ func (v *GaugeVec) GetMetricWith(labels Labels) (Gauge, error) {
 // WithLabelValues works as GetMetricWithLabelValues, but panics where
 // GetMetricWithLabelValues would have returned an error. Not returning an
 // error allows shortcuts like
-//     myVec.WithLabelValues("404", "GET").Add(42)
+//
+//	myVec.WithLabelValues("404", "GET").Add(42)
 func (v *GaugeVec) WithLabelValues(lvs ...string) Gauge {
 	g, err := v.GetMetricWithLabelValues(lvs...)
 	if err != nil {
@@ -221,7 +242,8 @@ func (v *GaugeVec) WithLabelValues(lvs ...string) Gauge {
 
 // With works as GetMetricWith, but panics where GetMetricWithLabels would have
 // returned an error. Not returning an error allows shortcuts like
-//     myVec.With(prometheus.Labels{"code": "404", "method": "GET"}).Add(42)
+//
+//	myVec.With(prometheus.Labels{"code": "404", "method": "GET"}).Add(42)
 func (v *GaugeVec) With(labels Labels) Gauge {
 	g, err := v.GetMetricWith(labels)
 	if err != nil {

vendor/github.com/prometheus/client_golang/prometheus/go_collector_latest.go

@@ -23,11 +23,10 @@ import (
 	"strings"
 	"sync"
 
-	//nolint:staticcheck // Ignore SA1019. Need to keep deprecated package for compatibility.
-	"github.com/golang/protobuf/proto"
-	dto "github.com/prometheus/client_model/go"
-
 	"github.com/prometheus/client_golang/prometheus/internal"
+
+	dto "github.com/prometheus/client_model/go"
+	"google.golang.org/protobuf/proto"
 )
 
 const (

vendor/github.com/prometheus/client_golang/prometheus/internal/almost_equal.go

@@ -0,0 +1,60 @@
+// Copyright (c) 2015 Björn Rabenstein
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+//
+// The code in this package is copy/paste to avoid a dependency. Hence this file
+// carries the copyright of the original repo.
+// https://github.com/beorn7/floats
+package internal
+
+import (
+	"math"
+)
+
+// minNormalFloat64 is the smallest positive normal value of type float64.
+var minNormalFloat64 = math.Float64frombits(0x0010000000000000)
+
+// AlmostEqualFloat64 returns true if a and b are equal within a relative error
+// of epsilon. See http://floating-point-gui.de/errors/comparison/ for the
+// details of the applied method.
+func AlmostEqualFloat64(a, b, epsilon float64) bool {
+	if a == b {
+		return true
+	}
+	absA := math.Abs(a)
+	absB := math.Abs(b)
+	diff := math.Abs(a - b)
+	if a == 0 || b == 0 || absA+absB < minNormalFloat64 {
+		return diff < epsilon*minNormalFloat64
+	}
+	return diff/math.Min(absA+absB, math.MaxFloat64) < epsilon
+}
+
+// AlmostEqualFloat64s is the slice form of AlmostEqualFloat64.
+func AlmostEqualFloat64s(a, b []float64, epsilon float64) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i := range a {
+		if !AlmostEqualFloat64(a[i], b[i], epsilon) {
+			return false
+		}
+	}
+	return true
+}

vendor/github.com/prometheus/client_golang/prometheus/internal/difflib.go

@@ -14,7 +14,7 @@
 // It provides tools to compare sequences of strings and generate textual diffs.
 //
 // Maintaining `GetUnifiedDiffString` here because original repository
-// (https://github.com/pmezard/go-difflib) is no loger maintained.
+// (https://github.com/pmezard/go-difflib) is no longer maintained.
 package internal
 
 import (
@@ -201,12 +201,15 @@ func (m *SequenceMatcher) isBJunk(s string) bool {
 // If IsJunk is not defined:
 //
 // Return (i,j,k) such that a[i:i+k] is equal to b[j:j+k], where
-//     alo <= i <= i+k <= ahi
-//     blo <= j <= j+k <= bhi
+//
+//	alo <= i <= i+k <= ahi
+//	blo <= j <= j+k <= bhi
+//
 // and for all (i',j',k') meeting those conditions,
-//     k >= k'
-//     i <= i'
-//     and if i == i', j <= j'
+//
+//	k >= k'
+//	i <= i'
+//	and if i == i', j <= j'
 //
 // In other words, of all maximal matching blocks, return one that
 // starts earliest in a, and of all those maximal matching blocks that

vendor/github.com/prometheus/client_golang/prometheus/labels.go

@@ -25,12 +25,111 @@ import (
 // Labels represents a collection of label name -> value mappings. This type is
 // commonly used with the With(Labels) and GetMetricWith(Labels) methods of
 // metric vector Collectors, e.g.:
-//     myVec.With(Labels{"code": "404", "method": "GET"}).Add(42)
+//
+//	myVec.With(Labels{"code": "404", "method": "GET"}).Add(42)
 //
 // The other use-case is the specification of constant label pairs in Opts or to
 // create a Desc.
 type Labels map[string]string
 
+// LabelConstraint normalizes label values.
+type LabelConstraint func(string) string
+
+// ConstrainedLabels represents a label name and its constrain function
+// to normalize label values. This type is commonly used when constructing
+// metric vector Collectors.
+type ConstrainedLabel struct {
+	Name       string
+	Constraint LabelConstraint
+}
+
+// ConstrainableLabels is an interface that allows creating of labels that can
+// be optionally constrained.
+//
+//	prometheus.V2().NewCounterVec(CounterVecOpts{
+//	  CounterOpts: {...}, // Usual CounterOpts fields
+//	  VariableLabels: []ConstrainedLabels{
+//	    {Name: "A"},
+//	    {Name: "B", Constraint: func(v string) string { ... }},
+//	  },
+//	})
+type ConstrainableLabels interface {
+	compile() *compiledLabels
+	labelNames() []string
+}
+
+// ConstrainedLabels represents a collection of label name -> constrain function
+// to normalize label values. This type is commonly used when constructing
+// metric vector Collectors.
+type ConstrainedLabels []ConstrainedLabel
+
+func (cls ConstrainedLabels) compile() *compiledLabels {
+	compiled := &compiledLabels{
+		names:            make([]string, len(cls)),
+		labelConstraints: map[string]LabelConstraint{},
+	}
+
+	for i, label := range cls {
+		compiled.names[i] = label.Name
+		if label.Constraint != nil {
+			compiled.labelConstraints[label.Name] = label.Constraint
+		}
+	}
+
+	return compiled
+}
+
+func (cls ConstrainedLabels) labelNames() []string {
+	names := make([]string, len(cls))
+	for i, label := range cls {
+		names[i] = label.Name
+	}
+	return names
+}
+
+// UnconstrainedLabels represents collection of label without any constraint on
+// their value. Thus, it is simply a collection of label names.
+//
+//	UnconstrainedLabels([]string{ "A", "B" })
+//
+// is equivalent to
+//
+//	ConstrainedLabels {
+//	  { Name: "A" },
+//	  { Name: "B" },
+//	}
+type UnconstrainedLabels []string
+
+func (uls UnconstrainedLabels) compile() *compiledLabels {
+	return &compiledLabels{
+		names: uls,
+	}
+}
+
+func (uls UnconstrainedLabels) labelNames() []string {
+	return uls
+}
+
+type compiledLabels struct {
+	names            []string
+	labelConstraints map[string]LabelConstraint
+}
+
+func (cls *compiledLabels) compile() *compiledLabels {
+	return cls
+}
+
+func (cls *compiledLabels) labelNames() []string {
+	return cls.names
+}
+
+func (cls *compiledLabels) constrain(labelName, value string) string {
+	if fn, ok := cls.labelConstraints[labelName]; ok && fn != nil {
+		return fn(value)
+	}
+	return value
+}
+
 // reservedLabelPrefix is a prefix which is not legal in user-supplied
 // label names.
 const reservedLabelPrefix = "__"
@@ -66,6 +165,8 @@ func validateValuesInLabels(labels Labels, expectedNumberOfValues int) error {
 
 func validateLabelValues(vals []string, expectedNumberOfValues int) error {
 	if len(vals) != expectedNumberOfValues {
+		// The call below makes vals escape, copy them to avoid that.
+		vals := append([]string(nil), vals...)
 		return fmt.Errorf(
 			"%w: expected %d label values but got %d in %#v",
 			errInconsistentCardinality, expectedNumberOfValues,

vendor/github.com/prometheus/client_golang/prometheus/metric.go

@@ -20,11 +20,9 @@ import (
 	"strings"
 	"time"
 
-	//nolint:staticcheck // Ignore SA1019. Need to keep deprecated package for compatibility.
-	"github.com/golang/protobuf/proto"
-	"github.com/prometheus/common/model"
-
 	dto "github.com/prometheus/client_model/go"
+	"github.com/prometheus/common/model"
+	"google.golang.org/protobuf/proto"
 )
 
 var separatorByteSlice = []byte{model.SeparatorByte} // For convenient use with xxhash.
@@ -94,6 +92,9 @@ type Opts struct {
 	// machine_role metric). See also
 	// https://prometheus.io/docs/instrumenting/writing_exporters/#target-labels-not-static-scraped-labels
 	ConstLabels Labels
+
+	// now is for testing purposes, by default it's time.Now.
+	now func() time.Time
 }
 
 // BuildFQName joins the given three name components by "_". Empty name
@@ -187,7 +188,7 @@ func (m *withExemplarsMetric) Write(pb *dto.Metric) error {
 			} else {
 				// The +Inf bucket should be explicitly added if there is an exemplar for it, similar to non-const histogram logic in https://github.com/prometheus/client_golang/blob/main/prometheus/histogram.go#L357-L365.
 				b := &dto.Bucket{
-					CumulativeCount: proto.Uint64(pb.Histogram.Bucket[len(pb.Histogram.GetBucket())-1].GetCumulativeCount()),
+					CumulativeCount: proto.Uint64(pb.Histogram.GetSampleCount()),
 					UpperBound:      proto.Float64(math.Inf(1)),
 					Exemplar:        e,
 				}