Release 2022.8.0

This reverts commit d4d9a43dd7.

We revert this change because the value this configuration addition
brings is small (it only stops an explicit cyclic configuration versus
not accounting for local hosts and ip based cycles amongst other things)
whilst the potential inconvenience it may cause is high (for example,
someone had a cyclic configuration as an ingress rule that they weren't
even using).

.github/ISSUE_TEMPLATE/---bug-report.md

@@ -1,9 +1,8 @@
 ---
-name: "Bug report \U0001F41B"
+name: "\U0001F41B Bug report"
 about: Create a report to help us improve cloudflared
-title: ''
-labels: awaiting reply, bug
-assignees: ''
+title: "\U0001F41B"
+labels: 'Priority: Normal, Type: Bug'
 
 ---
 

.github/ISSUE_TEMPLATE/---documentation.md

@@ -0,0 +1,16 @@
+---
+name: "\U0001F4DD Documentation"
+about: Request new or updated documentation for cloudflared
+title: "\U0001F4DD"
+labels: 'Priority: Normal, Type: Documentation'
+
+---
+
+**Available Documentation**
+A link to the documentation that is available today and the areas which could be improved. 
+
+**Suggested Documentation**
+A clear and concise description of the documentation, tutorial, or guide that should be added. 
+
+**Additional context**
+Add any other context or screenshots about the documentation request here.

.github/ISSUE_TEMPLATE/---feature-request.md

@@ -1,9 +1,8 @@
 ---
-name: "Feature request \U0001F4A1"
+name: "\U0001F4A1 Feature request"
 about: Suggest a feature or enhancement for cloudflared
-title: ''
-labels: awaiting reply, feature-request
-assignees: ''
+title: "\U0001F4A1"
+labels: 'Priority: Normal, Type: Feature Request'
 
 ---
 

.github/workflows/check.yaml

@@ -12,8 +12,6 @@ jobs:
       uses: actions/setup-go@v2
       with:
         go-version: ${{ matrix.go-version }}
-    - name: Install go-sumtype
-      run: go get github.com/sudarshan-reddy/go-sumtype
     - name: Checkout code
       uses: actions/checkout@v2
     - name: Test

CHANGES.md

@@ -1,3 +1,10 @@
+## 2022.7.1
+### New Features
+- It is now possible to connect cloudflared tunnel to Cloudflare Global Network with IPv6. See `cloudflared tunnel --help` and look for `edge-ip-version` for more information. For now, the default behavior is to still connect with IPv4 only.
+
+### Bug Fixes
+- Several bug fixes related with QUIC transport (used between cloudflared tunnel and Cloudflare Global Network). Updating to this version is highly recommended.
+
 ## 2022.4.0
 ### Bug Fixes
 - `cloudflared tunnel run` no longer logs the Tunnel token or JSON credentials in clear text as those are the secret

Dockerfile

@@ -1,11 +1,13 @@
 # use a builder image for building cloudflare
 ARG TARGET_GOOS
 ARG TARGET_GOARCH
-FROM golang:1.17.1 as builder
+FROM golang:1.18.1 as builder
 ENV GO111MODULE=on \
     CGO_ENABLED=0 \
     TARGET_GOOS=${TARGET_GOOS} \
     TARGET_GOARCH=${TARGET_GOARCH}
+    
+LABEL org.opencontainers.image.source="https://github.com/cloudflare/cloudflared"
 
 WORKDIR /go/src/github.com/cloudflare/cloudflared/
 

Makefile

@@ -98,6 +98,16 @@ else
 	TARGET_PUBLIC_REPO ?= $(FLAVOR)
 endif
 
+ifneq ($(TARGET_ARM), )
+	ARM_COMMAND := GOARM=$(TARGET_ARM)
+endif
+
+ifeq ($(TARGET_ARM), 7) 
+	PACKAGE_ARCH := armhf
+else
+	PACKAGE_ARCH := $(TARGET_ARCH)
+endif
+
 .PHONY: all
 all: cloudflared test
 
@@ -111,7 +121,7 @@ ifeq ($(FIPS), true)
 	$(info Building cloudflared with go-fips)
 	cp -f fips/fips.go.linux-amd64 cmd/cloudflared/fips.go
 endif
-	GOOS=$(TARGET_OS) GOARCH=$(TARGET_ARCH) go build -v -mod=vendor $(GO_BUILD_TAGS) $(LDFLAGS) $(IMPORT_PATH)/cmd/cloudflared
+	GOOS=$(TARGET_OS) GOARCH=$(TARGET_ARCH) $(ARM_COMMAND) go build -v -mod=vendor $(GO_BUILD_TAGS) $(LDFLAGS) $(IMPORT_PATH)/cmd/cloudflared
 ifeq ($(FIPS), true)
 	rm -f cmd/cloudflared/fips.go
 	./check-fips.sh cloudflared
@@ -171,7 +181,7 @@ define build_package
 		--license 'Apache License Version 2.0' \
 		--url 'https://github.com/cloudflare/cloudflared' \
 		-m 'Cloudflare <support@cloudflare.com>' \
-		-a $(TARGET_ARCH) -v $(VERSION) -n $(DEB_PACKAGE_NAME) $(NIGHTLY_FLAGS) --after-install postinst.sh --after-remove postrm.sh \
+	    -a $(PACKAGE_ARCH) -v $(VERSION) -n $(DEB_PACKAGE_NAME) $(NIGHTLY_FLAGS) --after-install postinst.sh --after-remove postrm.sh \
 		cloudflared=$(INSTALL_BINDIR) cloudflared.1=$(INSTALL_MANDIR)
 endef
 
@@ -293,11 +303,6 @@ quic-deps:
 .PHONY: vet
 vet:
 	go vet -mod=vendor ./...
-	# go get github.com/sudarshan-reddy/go-sumtype (don't do this in build directory or this will cause vendor issues)
-	# Note: If you have github.com/BurntSushi/go-sumtype then you might have to use the repo above instead
-	# for now because it uses an older version of golang.org/x/tools.
-	which go-sumtype
-	go-sumtype $$(go list -mod=vendor ./...)
 
 .PHONY: goimports
 goimports:

README.md

@@ -43,7 +43,7 @@ Once installed, you can authenticate `cloudflared` into your Cloudflare account
 * Route traffic to that Tunnel:
   * Via public [DNS records in Cloudflare](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/routing-to-tunnel/dns)
   * Or via a public hostname guided by a [Cloudflare Load Balancer](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/routing-to-tunnel/lb)
-  * Or from [WARP client private traffic](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/configuration/private-networks)
+  * Or from [WARP client private traffic](https://developers.cloudflare.com/cloudflare-one/connections/connect-networks/private-net/)
 
 
 ## TryCloudflare

RELEASE_NOTES

@@ -1,3 +1,72 @@
+2022.8.0
+- 2022-08-10 TUN-6637: Upgrade quic-go
+- 2022-08-10 TUN-6646: Add support to SafeStreamCloser to close only write side of stream
+- 2022-08-09 TUN-6642: Fix unexpected close of quic stream triggered by upstream origin close
+- 2022-08-09 TUN-6639: Validate cyclic ingress configuration
+- 2022-08-08 TUN-6637: Upgrade go version and quic-go
+- 2022-08-08 TUN-6639: Validate cyclic ingress configuration
+- 2022-08-04 EDGEPLAT-3918: build cloudflared for Bookworm
+- 2022-08-02 Revert "TUN-6576: Consume cf-trace-id from incoming TCP requests to create root span"
+- 2022-07-27 TUN-6601: Update gopkg.in/yaml.v3 references in modules
+- 2022-07-26 TUN-6576: Consume cf-trace-id from incoming TCP requests to create root span
+- 2022-07-26 TUN-6576: Consume cf-trace-id from incoming TCP requests to create root span
+- 2022-07-25 TUN-6598: Remove auto assignees on github issues
+- 2022-07-20 TUN-6583: Remove legacy --ui flag
+- 2022-07-20 cURL supports stdin and uses os pipes directly without copying
+- 2022-07-07 TUN-6517: Use QUIC stream context while proxying HTTP requests and TCP connections
+2022.7.1
+- 2022-07-06 TUN-6503: Fix transport fallback from QUIC in face of dial error "no network activity"
+
+2022.7.0
+- 2022-07-05 TUN-6499: Remove log that is per datagram
+- 2022-06-24 TUN-6460: Rename metric label location to edge_location
+- 2022-06-24 TUN-6459: Add cloudflared user-agent to access calls
+- 2022-06-17 TUN-6427: Differentiate between upstream request closed/canceled and failed origin requests
+- 2022-06-17 TUN-6388: Fix first tunnel connection not retrying
+- 2022-06-13 TUN-6384: Correct duplicate connection error to fetch new IP first
+- 2022-06-13 TUN-6373: Add edge-ip-version to remotely pushed configuration
+- 2022-06-07 TUN-6010: Add component tests for --edge-ip-version
+- 2022-05-20 TUN-6007: Implement new edge discovery algorithm
+- 2022-02-18 Ensure service install directories are created before writing file
+
+2022.6.3
+- 2022-06-20 TUN-6362: Add armhf support to cloudflare packaging
+
+2022.6.2
+- 2022-06-13 TUN-6381: Write error data on QUIC stream when we fail to talk to the origin; separate logging for protocol errors vs. origin errors.
+- 2022-06-17 TUN-6414: Remove go-sumtype from cloudflared build process
+- 2022-06-01 Add Http2Origin option to force HTTP/2 origin connections
+- 2022-06-02 fix ingress rules unit test
+- 2022-06-09 Update remaining OriginRequestConfig functions for Http2Origins
+- 2022-05-31 Add image source label to docker container.
+- 2022-05-10 Warp Private Network link updated
+
+2022.6.1
+- 2022-06-14 TUN-6395: Fix writing RPM repo data
+
+2022.6.0
+- 2022-06-14 Revert "TUN-6010: Add component tests for --edge-ip-version"
+- 2022-06-14 Revert "TUN-6373: Add edge-ip-version to remotely pushed configuration"
+- 2022-06-14 Revert "TUN-6384: Correct duplicate connection error to fetch new IP first"
+- 2022-06-14 Revert "TUN-6007: Implement new edge discovery algorithm"
+- 2022-06-13 TUN-6385: Don't share err between acceptStream loop and per-stream goroutines
+- 2022-06-13 TUN-6384: Correct duplicate connection error to fetch new IP first
+- 2022-06-13 TUN-6373: Add edge-ip-version to remotely pushed configuration
+- 2022-06-13 TUN-6380: Enforce connect and keep-alive timeouts for TCP connections in both WARP routing and websocket based TCP proxy.
+- 2022-06-11 Update issue templates
+- 2022-06-11 Amendment to previous PR
+- 2022-06-09 TUN-6347: Add TCP stream logs with FlowID
+- 2022-06-08 TUN-6361: Add cloudflared arm builds to pkging as well
+- 2022-06-07 TUN-6357: Add connector id to ready check endpoint
+- 2022-06-07 TUN-6010: Add component tests for --edge-ip-version
+- 2022-06-06 TUN-6191: Update quic-go to v0.27.1 and with custom patch to allow keep alive period to be configurable
+- 2022-06-03 TUN-6343: Fix QUIC->HTTP2 fallback
+- 2022-06-02 TUN-6339: Add config for IPv6 support
+- 2022-06-02 TUN-6341: Fix default config value for edge-ip-version
+- 2022-06-01 TUN-6323: Add Xenial and Trusty for Ubuntu pkging
+- 2022-05-31 TUN-6210: Add cloudflared.repo to make it easy for yum installs
+- 2022-05-30 TUN-6293: Update yaml v3 to latest hotfix
+- 2022-05-20 TUN-6007: Implement new edge discovery algorithm
 2022.5.3
 - 2022-05-30 TUN-6308: Add debug logs to see if packets are sent/received from edge
 - 2022-05-30 TUN-6301: Allow to update logger used by UDP session manager

build-packages.sh

@@ -17,10 +17,18 @@ for arch in ${windowsArchs[@]}; do
 done
 
 
-linuxArchs=("386" "amd64" "arm" "arm64")
+linuxArchs=("386" "amd64" "arm" "armhf" "arm64")
 export TARGET_OS=linux
 for arch in ${linuxArchs[@]}; do
+    unset TARGET_ARM
     export TARGET_ARCH=$arch
+    
+    ## Support for armhf builds 
+    if [[ $arch == armhf ]] ; then
+        export TARGET_ARCH=arm
+        export TARGET_ARM=7 
+    fi
+    
     make cloudflared-deb
     mv cloudflared\_$VERSION\_$arch.deb $ARTIFACT_DIR/cloudflared-linux-$arch.deb
 

cfsetup.yaml

@@ -1,8 +1,8 @@
-pinned_go: &pinned_go go=1.17.10-1
-pinned_go_fips: &pinned_go_fips go-boring=1.17.9-1
+pinned_go: &pinned_go go=1.18-1
+pinned_go_fips: &pinned_go_fips go-boring=1.18-1
 
 build_dir: &build_dir /cfsetup_build
-default-flavor: buster
+default-flavor: bullseye
 stretch: &stretch
   build:
     build_dir: *build_dir
@@ -152,8 +152,7 @@ stretch: &stretch
       - build-essential
       - gotest-to-teamcity
     pre-cache: &test_pre_cache
-      - go get golang.org/x/tools/cmd/goimports
-      - go get github.com/sudarshan-reddy/go-sumtype@v0.0.0-20210827105221-82eca7e5abb1
+      - go install golang.org/x/tools/cmd/goimports@latest
     post-cache:
       - export GOOS=linux
       - export GOARCH=amd64
@@ -247,6 +246,7 @@ stretch: &stretch
 
 buster: *stretch
 bullseye: *stretch
+bookworm: *stretch
 centos-7:
   publish-rpm:
     build_dir: *build_dir
@@ -255,8 +255,8 @@ centos-7:
     pre-cache:
       - yum install -y fakeroot
       - yum upgrade -y binutils-2.27-44.base.el7.x86_64
-      - wget https://go.dev/dl/go1.17.10.linux-amd64.tar.gz -P /tmp/
-      - tar -C /usr/local -xzf /tmp/go1.17.10.linux-amd64.tar.gz
+      - wget https://go.dev/dl/go1.18.1.linux-amd64.tar.gz -P /tmp/
+      - tar -C /usr/local -xzf /tmp/go1.18.1.linux-amd64.tar.gz
     post-cache:
       - export PATH=$PATH:/usr/local/go/bin
       - export GOOS=linux

cmd/cloudflared/access/cmd.go

@@ -56,11 +56,13 @@ const sentryDSN = "https://56a9c9fa5c364ab28f34b14f35ea0f1b@sentry.io/189878"
 
 var (
 	shutdownC chan struct{}
+	userAgent = "DEV"
 )
 
 // Init will initialize and store vars from the main program
-func Init(shutdown chan struct{}) {
+func Init(shutdown chan struct{}, version string) {
 	shutdownC = shutdown
+	userAgent = fmt.Sprintf("cloudflared/%s", version)
 }
 
 // Flags return the global flags for Access related commands (hopefully none)
@@ -294,6 +296,7 @@ func curl(c *cli.Context) error {
 // run kicks off a shell task and pipe the results to the respective std pipes
 func run(cmd string, args ...string) error {
 	c := exec.Command(cmd, args...)
+	c.Stdin = os.Stdin
 	stderr, err := c.StderrPipe()
 	if err != nil {
 		return err
@@ -505,7 +508,7 @@ func isTokenValid(options *carrier.StartOptions, log *zerolog.Logger) (bool, err
 	if err != nil {
 		return false, errors.Wrap(err, "Could not create access request")
 	}
-
+	req.Header.Set("User-Agent", userAgent)
 	// Do not follow redirects
 	client := &http.Client{
 		CheckRedirect: func(req *http.Request, via []*http.Request) error {

cmd/cloudflared/main.go

@@ -21,6 +21,7 @@ import (
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/metrics"
 	"github.com/cloudflare/cloudflared/overwatch"
+	"github.com/cloudflare/cloudflared/token"
 	"github.com/cloudflare/cloudflared/tracing"
 	"github.com/cloudflare/cloudflared/watcher"
 )
@@ -85,9 +86,10 @@ func main() {
 	app.Commands = commands(cli.ShowVersion)
 
 	tunnel.Init(bInfo, graceShutdownC) // we need this to support the tunnel sub command...
-	access.Init(graceShutdownC)
+	access.Init(graceShutdownC, Version)
 	updater.Init(Version)
 	tracing.Init(Version)
+	token.Init(Version)
 	runApp(app, graceShutdownC)
 }
 

cmd/cloudflared/service_template.go

@@ -8,6 +8,7 @@ import (
 	"io/ioutil"
 	"os"
 	"os/exec"
+	"path"
 	"text/template"
 
 	homedir "github.com/mitchellh/go-homedir"
@@ -52,10 +53,17 @@ func (st *ServiceTemplate) Generate(args *ServiceTemplateArgs) error {
 	if err != nil {
 		return fmt.Errorf("error generating %s: %v", st.Path, err)
 	}
-	fileMode := os.FileMode(0644)
+	fileMode := os.FileMode(0o644)
 	if st.FileMode != 0 {
 		fileMode = st.FileMode
 	}
+
+	plistFolder := path.Dir(resolvedPath)
+	err = os.MkdirAll(plistFolder, 0o755)
+	if err != nil {
+		return fmt.Errorf("error creating %s: %v", plistFolder, err)
+	}
+
 	err = ioutil.WriteFile(resolvedPath, buffer.Bytes(), fileMode)
 	if err != nil {
 		return fmt.Errorf("error writing %s: %v", resolvedPath, err)

cmd/cloudflared/tunnel/cmd.go

@@ -15,6 +15,7 @@ import (
 	"github.com/coreos/go-systemd/daemon"
 	"github.com/facebookgo/grace/gracenet"
 	"github.com/getsentry/raven-go"
+	"github.com/google/uuid"
 	homedir "github.com/mitchellh/go-homedir"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
@@ -24,7 +25,6 @@ import (
 	"github.com/cloudflare/cloudflared/cfapi"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/proxydns"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/ui"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/updater"
 	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/connection"
@@ -217,7 +217,7 @@ func runAdhocNamedTunnel(sc *subcommandContext, name, credentialsOutputPath stri
 
 // runClassicTunnel creates a "classic" non-named tunnel
 func runClassicTunnel(sc *subcommandContext) error {
-	return StartServer(sc.c, buildInfo, nil, sc.log, sc.isUIEnabled)
+	return StartServer(sc.c, buildInfo, nil, sc.log)
 }
 
 func routeFromFlag(c *cli.Context) (route cfapi.HostnameRoute, ok bool) {
@@ -235,7 +235,6 @@ func StartServer(
 	info *cliutil.BuildInfo,
 	namedTunnel *connection.NamedTunnelProperties,
 	log *zerolog.Logger,
-	isUIEnabled bool,
 ) error {
 	_ = raven.SetDSN(sentryDSN)
 	var wg sync.WaitGroup
@@ -330,9 +329,9 @@ func StartServer(
 		return fmt.Errorf(errText)
 	}
 
-	logTransport := logger.CreateTransportLoggerFromContext(c, isUIEnabled)
+	logTransport := logger.CreateTransportLoggerFromContext(c, logger.EnableTerminalLog)
 
-	observer := connection.NewObserver(log, logTransport, isUIEnabled)
+	observer := connection.NewObserver(log, logTransport)
 
 	// Send Quick Tunnel URL to UI if applicable
 	var quickTunnelURL string
@@ -348,6 +347,14 @@ func StartServer(
 		log.Err(err).Msg("Couldn't start tunnel")
 		return err
 	}
+	var clientID uuid.UUID
+	if tunnelConfig.NamedTunnel != nil {
+		clientID, err = uuid.FromBytes(tunnelConfig.NamedTunnel.Client.ClientID)
+		if err != nil {
+			// set to nil for classic tunnels
+			clientID = uuid.Nil
+		}
+	}
 
 	orchestrator, err := orchestration.NewOrchestrator(ctx, orchestratorConfig, tunnelConfig.Tags, tunnelConfig.Log)
 	if err != nil {
@@ -363,7 +370,7 @@ func StartServer(
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
-		readinessServer := metrics.NewReadyServer(log)
+		readinessServer := metrics.NewReadyServer(log, clientID)
 		observer.RegisterSink(readinessServer)
 		errC <- metrics.ServeMetrics(metricsListener, ctx.Done(), readinessServer, quickTunnelURL, orchestrator, log)
 	}()
@@ -383,18 +390,6 @@ func StartServer(
 		errC <- supervisor.StartTunnelDaemon(ctx, tunnelConfig, orchestrator, connectedSignal, reconnectCh, graceShutdownC)
 	}()
 
-	if isUIEnabled {
-		tunnelUI := ui.NewUIModel(
-			info.Version(),
-			hostname,
-			metricsListener.Addr().String(),
-			orchestratorConfig.Ingress,
-			tunnelConfig.HAConnections,
-		)
-		app := tunnelUI.Launch(ctx, log, logTransport)
-		observer.RegisterSink(app)
-	}
-
 	gracePeriod, err := gracePeriod(c)
 	if err != nil {
 		return err
@@ -514,6 +509,13 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			Usage:   "Cloudflare Edge region to connect to. Omit or set to empty to connect to the global region.",
 			EnvVars: []string{"TUNNEL_REGION"},
 		}),
+		altsrc.NewStringFlag(&cli.StringFlag{
+			Name:    "edge-ip-version",
+			Usage:   "Cloudflare Edge ip address version to connect with. {4, 6, auto}",
+			EnvVars: []string{"TUNNEL_EDGE_IP_VERSION"},
+			Value:   "4",
+			Hidden:  false,
+		}),
 		altsrc.NewStringFlag(&cli.StringFlag{
 			Name:    tlsconfig.CaCertFlag,
 			Usage:   "Certificate Authority authenticating connections with Cloudflare's edge network.",
@@ -647,9 +649,9 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 		}),
 		altsrc.NewBoolFlag(&cli.BoolFlag{
 			Name:   uiFlag,
-			Usage:  "Launch tunnel UI. Tunnel logs are scrollable via 'j', 'k', or arrow keys.",
+			Usage:  "(depreciated) Launch tunnel UI. Tunnel logs are scrollable via 'j', 'k', or arrow keys.",
 			Value:  false,
-			Hidden: shouldHide,
+			Hidden: true,
 		}),
 		altsrc.NewStringFlag(&cli.StringFlag{
 			Name:   "quick-service",
@@ -821,6 +823,13 @@ func configureProxyFlags(shouldHide bool) []cli.Flag {
 			EnvVars: []string{"TUNNEL_NO_CHUNKED_ENCODING"},
 			Hidden:  shouldHide,
 		}),
+		altsrc.NewBoolFlag(&cli.BoolFlag{
+			Name:    ingress.Http2OriginFlag,
+			Usage:   "Enables HTTP/2 origin servers.",
+			EnvVars: []string{"TUNNEL_ORIGIN_ENABLE_HTTP2"},
+			Hidden:  shouldHide,
+			Value:   false,
+		}),
 	}
 	return append(flags, sshFlags(shouldHide)...)
 }

cmd/cloudflared/tunnel/configuration.go

@@ -18,6 +18,7 @@ import (
 	"golang.org/x/crypto/ssh/terminal"
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
+	"github.com/cloudflare/cloudflared/edgediscovery/allregions"
 
 	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/connection"
@@ -44,7 +45,7 @@ var (
 	secretFlags     = [2]*altsrc.StringFlag{credentialsContentsFlag, tunnelTokenFlag}
 	defaultFeatures = []string{supervisor.FeatureAllowRemoteConfig, supervisor.FeatureSerializedHeaders}
 
-	configFlags = []string{"autoupdate-freq", "no-autoupdate", "retries", "protocol", "loglevel", "transport-loglevel", "origincert", "metrics", "metrics-update-freq"}
+	configFlags = []string{"autoupdate-freq", "no-autoupdate", "retries", "protocol", "loglevel", "transport-loglevel", "origincert", "metrics", "metrics-update-freq", "edge-ip-version"}
 )
 
 // returns the first path that contains a cert.pem file. If none of the DefaultConfigSearchDirectories
@@ -324,6 +325,10 @@ func prepareTunnelConfig(
 		CompressionSetting: h2mux.CompressionSetting(uint64(c.Int("compression-quality"))),
 		MetricsUpdateFreq:  c.Duration("metrics-update-freq"),
 	}
+	edgeIPVersion, err := parseConfigIPVersion(c.String("edge-ip-version"))
+	if err != nil {
+		return nil, nil, err
+	}
 
 	tunnelConfig := &supervisor.TunnelConfig{
 		GracePeriod:     gracePeriod,
@@ -332,6 +337,7 @@ func prepareTunnelConfig(
 		ClientID:        clientID,
 		EdgeAddrs:       c.StringSlice("edge"),
 		Region:          c.String("region"),
+		EdgeIPVersion:   edgeIPVersion,
 		HAConnections:   c.Int("ha-connections"),
 		IncidentLookup:  supervisor.NewIncidentLookup(),
 		IsAutoupdated:   c.Bool("is-autoupdated"),
@@ -352,7 +358,7 @@ func prepareTunnelConfig(
 	}
 	orchestratorConfig := &orchestration.Config{
 		Ingress:            &ingressRules,
-		WarpRoutingEnabled: warpRoutingEnabled,
+		WarpRouting:        ingress.NewWarpRoutingConfig(&cfg.WarpRouting),
 		ConfigurationFlags: parseConfigFlags(c),
 	}
 	return tunnelConfig, orchestratorConfig, nil
@@ -404,3 +410,18 @@ func dedup(slice []string) []string {
 	}
 	return keys
 }
+
+// ParseConfigIPVersion returns the IP version from possible expected values from config
+func parseConfigIPVersion(version string) (v allregions.ConfigIPVersion, err error) {
+	switch version {
+	case "4":
+		v = allregions.IPv4Only
+	case "6":
+		v = allregions.IPv6Only
+	case "auto":
+		v = allregions.Auto
+	default: // unspecified or invalid
+		err = fmt.Errorf("invalid value for edge-ip-version: %s", version)
+	}
+	return
+}

cmd/cloudflared/tunnel/quick_tunnel.go

@@ -78,7 +78,6 @@ func RunQuickTunnel(sc *subcommandContext) error {
 		buildInfo,
 		&connection.NamedTunnelProperties{Credentials: credentials, QuickTunnelUrl: data.Result.Hostname},
 		sc.log,
-		sc.isUIEnabled,
 	)
 }
 

cmd/cloudflared/tunnel/subcommand_context.go

@@ -31,10 +31,9 @@ func (e errInvalidJSONCredential) Error() string {
 // subcommandContext carries structs shared between subcommands, to reduce number of arguments needed to
 // pass between subcommands, and make sure they are only initialized once
 type subcommandContext struct {
-	c           *cli.Context
-	log         *zerolog.Logger
-	isUIEnabled bool
-	fs          fileSystem
+	c   *cli.Context
+	log *zerolog.Logger
+	fs  fileSystem
 
 	// These fields should be accessed using their respective Getter
 	tunnelstoreClient cfapi.Client
@@ -42,16 +41,10 @@ type subcommandContext struct {
 }
 
 func newSubcommandContext(c *cli.Context) (*subcommandContext, error) {
-	isUIEnabled := c.IsSet(uiFlag) && c.String("name") != ""
-
-	// If UI is enabled, terminal log output should be disabled -- log should be written into a UI log window instead
-	log := logger.CreateLoggerFromContext(c, isUIEnabled)
-
 	return &subcommandContext{
-		c:           c,
-		log:         log,
-		isUIEnabled: isUIEnabled,
-		fs:          realFileSystem{},
+		c:   c,
+		log: logger.CreateLoggerFromContext(c, logger.EnableTerminalLog),
+		fs:  realFileSystem{},
 	}, nil
 }
 
@@ -312,7 +305,6 @@ func (sc *subcommandContext) runWithCredentials(credentials connection.Credentia
 		buildInfo,
 		&connection.NamedTunnelProperties{Credentials: credentials},
 		sc.log,
-		sc.isUIEnabled,
 	)
 }
 

cmd/cloudflared/tunnel/subcommand_context_test.go

@@ -35,7 +35,6 @@ func Test_subcommandContext_findCredentials(t *testing.T) {
 	type fields struct {
 		c                 *cli.Context
 		log               *zerolog.Logger
-		isUIEnabled       bool
 		fs                fileSystem
 		tunnelstoreClient cfapi.Client
 		userCredential    *userCredential
@@ -168,7 +167,6 @@ func Test_subcommandContext_findCredentials(t *testing.T) {
 			sc := &subcommandContext{
 				c:                 tt.fields.c,
 				log:               tt.fields.log,
-				isUIEnabled:       tt.fields.isUIEnabled,
 				fs:                tt.fields.fs,
 				tunnelstoreClient: tt.fields.tunnelstoreClient,
 				userCredential:    tt.fields.userCredential,
@@ -307,7 +305,6 @@ func Test_subcommandContext_Delete(t *testing.T) {
 			sc := &subcommandContext{
 				c:                 tt.fields.c,
 				log:               tt.fields.log,
-				isUIEnabled:       tt.fields.isUIEnabled,
 				fs:                tt.fields.fs,
 				tunnelstoreClient: tt.fields.tunnelstoreClient,
 				userCredential:    tt.fields.userCredential,

cmd/cloudflared/tunnel/subcommands.go

@@ -20,7 +20,7 @@ import (
 	"github.com/urfave/cli/v2"
 	"github.com/urfave/cli/v2/altsrc"
 	"golang.org/x/net/idna"
-	yaml "gopkg.in/yaml.v2"
+	yaml "gopkg.in/yaml.v3"
 
 	"github.com/cloudflare/cloudflared/cfapi"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"

cmd/cloudflared/ui/launch_ui.go

@@ -1,223 +0,0 @@
-package ui
-
-import (
-	"context"
-	"fmt"
-	"strings"
-
-	"github.com/gdamore/tcell"
-	"github.com/rivo/tview"
-	"github.com/rs/zerolog"
-
-	"github.com/cloudflare/cloudflared/connection"
-	"github.com/cloudflare/cloudflared/ingress"
-)
-
-type connState struct {
-	location string
-}
-
-type uiModel struct {
-	version       string
-	edgeURL       string
-	metricsURL    string
-	localServices []string
-	connections   []connState
-}
-
-type palette struct {
-	url          string
-	connected    string
-	defaultText  string
-	disconnected string
-	reconnecting string
-	unregistered string
-}
-
-func NewUIModel(version, hostname, metricsURL string, ing *ingress.Ingress, haConnections int) *uiModel {
-	localServices := make([]string, len(ing.Rules))
-	for i, rule := range ing.Rules {
-		localServices[i] = rule.Service.String()
-	}
-	return &uiModel{
-		version:       version,
-		edgeURL:       hostname,
-		metricsURL:    metricsURL,
-		localServices: localServices,
-		connections:   make([]connState, haConnections),
-	}
-}
-
-func (data *uiModel) Launch(
-	ctx context.Context,
-	log, transportLog *zerolog.Logger,
-) connection.EventSink {
-	// Configure the logger to stream logs into the textview
-
-	// Add TextView as a group to write output to
-	logTextView := NewDynamicColorTextView()
-	// TODO: Format log for UI
-	//log.Add(logTextView, logger.NewUIFormatter(time.RFC3339), logLevels...)
-	//transportLog.Add(logTextView, logger.NewUIFormatter(time.RFC3339), logLevels...)
-
-	// Construct the UI
-	palette := palette{
-		url:          "lightblue",
-		connected:    "lime",
-		defaultText:  "white",
-		disconnected: "red",
-		reconnecting: "orange",
-		unregistered: "orange",
-	}
-
-	app := tview.NewApplication()
-
-	grid := tview.NewGrid().SetGap(1, 0)
-	frame := tview.NewFrame(grid)
-	header := fmt.Sprintf("cloudflared [::b]%s", data.version)
-
-	frame.AddText(header, true, tview.AlignLeft, tcell.ColorWhite)
-
-	// Create table to store connection info and status
-	connTable := tview.NewTable()
-	// SetColumns takes a value for each column, representing the size of the column
-	// Numbers <= 0 represent proportional widths and positive numbers represent absolute widths
-	grid.SetColumns(20, 0)
-
-	// SetRows takes a value for each row, representing the size of the row
-	grid.SetRows(1, 1, len(data.connections), 1, 0)
-
-	// AddItem takes a primitive tview type, row, column, rowSpan, columnSpan, minGridHeight, minGridWidth, and focus
-	grid.AddItem(tview.NewTextView().SetText("Tunnel:"), 0, 0, 1, 1, 0, 0, false)
-	grid.AddItem(tview.NewTextView().SetText("Status:"), 1, 0, 1, 1, 0, 0, false)
-	grid.AddItem(tview.NewTextView().SetText("Connections:"), 2, 0, 1, 1, 0, 0, false)
-
-	grid.AddItem(tview.NewTextView().SetText("Metrics:"), 3, 0, 1, 1, 0, 0, false)
-
-	tunnelHostText := tview.NewTextView().SetText(data.edgeURL)
-
-	grid.AddItem(tunnelHostText, 0, 1, 1, 1, 0, 0, false)
-	status := fmt.Sprintf("[%s]\u2022[%s] Proxying to [%s::b]%s", palette.connected, palette.defaultText, palette.url, strings.Join(data.localServices, ", "))
-	grid.AddItem(NewDynamicColorTextView().SetText(status), 1, 1, 1, 1, 0, 0, false)
-
-	grid.AddItem(connTable, 2, 1, 1, 1, 0, 0, false)
-
-	grid.AddItem(NewDynamicColorTextView().SetText(fmt.Sprintf("Metrics at [%s::b]http://%s/metrics", palette.url, data.metricsURL)), 3, 1, 1, 1, 0, 0, false)
-
-	// Add TextView to stream logs
-	// Logs are displayed in a new grid so a border can be set around them
-	logGrid := tview.NewGrid().SetBorders(true).AddItem(logTextView.SetChangedFunc(handleNewText(app, logTextView)), 0, 0, 5, 2, 0, 0, false)
-	// LogFrame holds the Logs header as well as the grid with the textView for streamed logs
-	logFrame := tview.NewFrame(logGrid).AddText("[::b]Logs:[::-]", true, tview.AlignLeft, tcell.ColorWhite).SetBorders(0, 0, 0, 0, 0, 0)
-	// Footer for log frame
-	logFrame.AddText("[::d]Use Ctrl+C to exit[::-]", false, tview.AlignRight, tcell.ColorWhite)
-	grid.AddItem(logFrame, 4, 0, 5, 2, 0, 0, false)
-
-	go func() {
-		<-ctx.Done()
-		app.Stop()
-		return
-	}()
-
-	go func() {
-		if err := app.SetRoot(frame, true).Run(); err != nil {
-			log.Error().Msgf("Error launching UI: %s", err)
-		}
-	}()
-
-	return connection.EventSinkFunc(func(event connection.Event) {
-		switch event.EventType {
-		case connection.Connected:
-			data.setConnTableCell(event, connTable, palette)
-		case connection.Disconnected, connection.Reconnecting, connection.Unregistering:
-			data.changeConnStatus(event, connTable, log, palette)
-		case connection.SetURL:
-			tunnelHostText.SetText(event.URL)
-			data.edgeURL = event.URL
-		case connection.RegisteringTunnel:
-			if data.edgeURL == "" {
-				tunnelHostText.SetText(fmt.Sprintf("Registering tunnel connection %d...", event.Index))
-			}
-		}
-		app.Draw()
-	})
-}
-
-func NewDynamicColorTextView() *tview.TextView {
-	return tview.NewTextView().SetDynamicColors(true)
-}
-
-// Re-draws application when new logs are streamed to UI
-func handleNewText(app *tview.Application, logTextView *tview.TextView) func() {
-	return func() {
-		app.Draw()
-		// SetFocus to enable scrolling in textview
-		app.SetFocus(logTextView)
-	}
-}
-
-func (data *uiModel) changeConnStatus(event connection.Event, table *tview.Table, log *zerolog.Logger, palette palette) {
-	index := int(event.Index)
-	// Get connection location and state
-	connState := data.getConnState(index)
-	// Check if connection is already displayed in UI
-	if connState == nil {
-		log.Info().Msg("Connection is not in the UI table")
-		return
-	}
-
-	locationState := event.Location
-
-	if event.EventType == connection.Reconnecting {
-		locationState = "Reconnecting..."
-	}
-
-	connectionNum := index + 1
-	// Get table cell
-	cell := table.GetCell(index, 0)
-	// Change dot color in front of text as well as location state
-	text := newCellText(palette, connectionNum, locationState, event.EventType)
-	cell.SetText(text)
-}
-
-// Return connection location and row in UI table
-func (data *uiModel) getConnState(connID int) *connState {
-	if connID < len(data.connections) {
-		return &data.connections[connID]
-	}
-
-	return nil
-}
-
-func (data *uiModel) setConnTableCell(event connection.Event, table *tview.Table, palette palette) {
-	index := int(event.Index)
-	connectionNum := index + 1
-
-	// Update slice to keep track of connection location and state in UI table
-	data.connections[index].location = event.Location
-
-	// Update text in table cell to show disconnected state
-	text := newCellText(palette, connectionNum, event.Location, event.EventType)
-	cell := tview.NewTableCell(text)
-	table.SetCell(index, 0, cell)
-}
-
-func newCellText(palette palette, connectionNum int, location string, connectedStatus connection.Status) string {
-	// HA connection indicator formatted as: "• #<CONNECTION_INDEX>: <COLO>",
-	//  where the left middle dot's color depends on the status of the connection
-	const connFmtString = "[%s]\u2022[%s] #%d: %s"
-
-	var dotColor string
-	switch connectedStatus {
-	case connection.Connected:
-		dotColor = palette.connected
-	case connection.Disconnected:
-		dotColor = palette.disconnected
-	case connection.Reconnecting:
-		dotColor = palette.reconnecting
-	case connection.Unregistering:
-		dotColor = palette.unregistered
-	}
-
-	return fmt.Sprintf(connFmtString, dotColor, palette.defaultText, connectionNum, location)
-}

component-tests/cli.py

@@ -0,0 +1,95 @@
+import json
+import subprocess
+from time import sleep
+
+from setup import get_config_from_file
+
+SINGLE_CASE_TIMEOUT = 600
+
+class CloudflaredCli:
+    def __init__(self, config, config_path, logger):
+        self.basecmd = [config.cloudflared_binary, "tunnel"]
+        if config_path is not None:
+            self.basecmd += ["--config", str(config_path)]
+        origincert = get_config_from_file()["origincert"]
+        if origincert:
+            self.basecmd += ["--origincert", origincert]
+        self.logger = logger
+
+    def _run_command(self, subcmd, subcmd_name, needs_to_pass=True):
+        cmd = self.basecmd + subcmd
+        # timeout limits the time a subprocess can run. This is useful to guard against running a tunnel when
+        # command/args are in wrong order.
+        result = run_subprocess(cmd, subcmd_name, self.logger, check=needs_to_pass, capture_output=True, timeout=15)
+        return result
+
+    def list_tunnels(self):
+        cmd_args = ["list", "--output", "json"]
+        listed = self._run_command(cmd_args, "list")
+        return json.loads(listed.stdout)
+
+    def get_tunnel_info(self, tunnel_id):
+        info = self._run_command(["info", "--output", "json", tunnel_id], "info")
+        return json.loads(info.stdout)
+
+    def __enter__(self):
+        self.basecmd += ["run"]
+        self.process = subprocess.Popen(self.basecmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        self.logger.info(f"Run cmd {self.basecmd}")
+        return self.process
+
+    def __exit__(self, exc_type, exc_value, exc_traceback):
+        terminate_gracefully(self.process, self.logger, self.basecmd)
+        self.logger.debug(f"{self.basecmd} logs: {self.process.stderr.read()}")
+
+
+def terminate_gracefully(process, logger, cmd):
+    process.terminate()
+    process_terminated = wait_for_terminate(process)
+    if not process_terminated:
+        process.kill()
+        logger.warning(f"{cmd}: cloudflared did not terminate within wait period. Killing process. logs: \
+                stdout: {process.stdout.read()}, stderr: {process.stderr.read()}")
+
+
+def wait_for_terminate(opened_subprocess, attempts=10, poll_interval=1):
+    """
+        wait_for_terminate polls the opened_subprocess every x seconds for a given number of attempts.
+        It returns true if the subprocess was terminated and false if it didn't.
+    """
+    for _ in range(attempts):
+        if _is_process_stopped(opened_subprocess):
+            return True
+        sleep(poll_interval)
+    return False
+
+
+def _is_process_stopped(process):
+    return process.poll() is not None
+
+
+def cert_path():
+    return get_config_from_file()["origincert"]
+
+
+class SubprocessError(Exception):
+    def __init__(self, program, exit_code, cause):
+        self.program = program
+        self.exit_code = exit_code
+        self.cause = cause
+
+
+def run_subprocess(cmd, cmd_name, logger, timeout=SINGLE_CASE_TIMEOUT, **kargs):
+    kargs["timeout"] = timeout
+    try:
+        result = subprocess.run(cmd, **kargs)
+        logger.debug(f"{cmd} log: {result.stdout}", extra={"cmd": cmd_name})
+        return result
+    except subprocess.CalledProcessError as e:
+        err = f"{cmd} return exit code {e.returncode}, stderr" + e.stderr.decode("utf-8")
+        logger.error(err, extra={"cmd": cmd_name, "return_code": e.returncode})
+        raise SubprocessError(cmd[0], e.returncode, e)
+    except subprocess.TimeoutExpired as e:
+        err = f"{cmd} timeout after {e.timeout} seconds, stdout: {e.stdout}, stderr: {e.stderr}"
+        logger.error(err, extra={"cmd": cmd_name, "return_code": "timeout"})
+        raise e

component-tests/test_edge_discovery.py

@@ -0,0 +1,165 @@
+import ipaddress
+import socket
+
+import pytest
+
+from constants import protocols
+from cli import CloudflaredCli
+from util import get_tunnel_connector_id, LOGGER, wait_tunnel_ready, write_config
+
+
+class TestEdgeDiscovery:
+    def _extra_config(self, protocol, edge_ip_version):
+        config = {
+            "protocol": protocol,
+        }
+        if edge_ip_version:
+            config["edge-ip-version"] = edge_ip_version
+        return config
+
+    @pytest.mark.parametrize("protocol", protocols())
+    def test_default_only(self, tmp_path, component_tests_config, protocol):
+        """
+        This test runs a tunnel to connect via IPv4-only edge addresses (default is unset "--edge-ip-version 4")
+        """
+        if self.has_ipv6_only():
+            pytest.skip("Host has IPv6 only support and current default is IPv4 only")
+        self.expect_address_connections(
+            tmp_path, component_tests_config, protocol, None, self.expect_ipv4_address)
+
+    @pytest.mark.parametrize("protocol", protocols())
+    def test_ipv4_only(self, tmp_path, component_tests_config, protocol):
+        """
+        This test runs a tunnel to connect via IPv4-only edge addresses
+        """
+        if self.has_ipv6_only():
+            pytest.skip("Host has IPv6 only support")
+        self.expect_address_connections(
+            tmp_path, component_tests_config, protocol, "4", self.expect_ipv4_address)
+
+    @pytest.mark.parametrize("protocol", protocols())
+    def test_ipv6_only(self, tmp_path, component_tests_config, protocol):
+        """
+        This test runs a tunnel to connect via IPv6-only edge addresses
+        """
+        if self.has_ipv4_only():
+            pytest.skip("Host has IPv4 only support")
+        self.expect_address_connections(
+            tmp_path, component_tests_config, protocol, "6", self.expect_ipv6_address)
+
+    @pytest.mark.parametrize("protocol", protocols())
+    def test_auto_ip64(self, tmp_path, component_tests_config, protocol):
+        """
+        This test runs a tunnel to connect via auto with a preference of IPv6 then IPv4 addresses for a dual stack host
+
+        This test also assumes that the host has IPv6 preference.
+        """
+        if not self.has_dual_stack(address_family_preference=socket.AddressFamily.AF_INET6):
+            pytest.skip("Host does not support dual stack with IPv6 preference")
+        self.expect_address_connections(
+            tmp_path, component_tests_config, protocol, "auto", self.expect_ipv6_address)
+
+    @pytest.mark.parametrize("protocol", protocols())
+    def test_auto_ip46(self, tmp_path, component_tests_config, protocol):
+        """
+        This test runs a tunnel to connect via auto with a preference of IPv4 then IPv6 addresses for a dual stack host
+
+        This test also assumes that the host has IPv4 preference.
+        """
+        if not self.has_dual_stack(address_family_preference=socket.AddressFamily.AF_INET):
+            pytest.skip("Host does not support dual stack with IPv4 preference")
+        self.expect_address_connections(
+            tmp_path, component_tests_config, protocol, "auto", self.expect_ipv4_address)
+
+    def expect_address_connections(self, tmp_path, component_tests_config, protocol, edge_ip_version, assert_address_type):
+        config = component_tests_config(
+            self._extra_config(protocol, edge_ip_version))
+        config_path = write_config(tmp_path, config.full_config)
+        LOGGER.debug(config)
+        with CloudflaredCli(config, config_path, LOGGER):
+            wait_tunnel_ready(tunnel_url=config.get_url(),
+                              require_min_connections=4)
+            cfd_cli = CloudflaredCli(config, config_path, LOGGER)
+            tunnel_id = config.get_tunnel_id()
+            info = cfd_cli.get_tunnel_info(tunnel_id)
+            connector_id = get_tunnel_connector_id()
+            connector = next(
+                (c for c in info["conns"] if c["id"] == connector_id), None)
+            assert connector, f"Expected connection info from get tunnel info for the connected instance: {info}"
+            conns = connector["conns"]
+            assert conns == None or len(
+                conns) == 4, f"There should be 4 connections registered: {conns}"
+            for conn in conns:
+                origin_ip = conn["origin_ip"]
+                assert origin_ip, f"No available origin_ip for this connection: {conn}"
+                assert_address_type(origin_ip)
+
+    def expect_ipv4_address(self, address):
+        assert type(ipaddress.ip_address(
+            address)) is ipaddress.IPv4Address, f"Expected connection from origin to be a valid IPv4 address: {address}"
+
+    def expect_ipv6_address(self, address):
+        assert type(ipaddress.ip_address(
+            address)) is ipaddress.IPv6Address, f"Expected connection from origin to be a valid IPv6 address: {address}"
+
+    def get_addresses(self):
+        """
+        Returns a list of addresses for the host.
+        """
+        host_addresses = socket.getaddrinfo(
+            "region1.v2.argotunnel.com", 7844, socket.AF_UNSPEC, socket.SOCK_STREAM)
+        assert len(
+            host_addresses) > 0, "No addresses returned from getaddrinfo"
+        return host_addresses
+
+    def has_dual_stack(self, address_family_preference=None):
+        """
+        Returns true if the host has dual stack support and can optionally check 
+        the provided IP family preference.
+        """
+        dual_stack = not self.has_ipv6_only() and not self.has_ipv4_only()
+        if address_family_preference:
+            address = self.get_addresses()[0]
+            return dual_stack and address[0] == address_family_preference
+
+        return dual_stack
+
+    def has_ipv6_only(self):
+        """
+        Returns True if the host has only IPv6 address support.
+        """
+        return self.attempt_connection(socket.AddressFamily.AF_INET6) and not self.attempt_connection(socket.AddressFamily.AF_INET)
+
+    def has_ipv4_only(self):
+        """
+        Returns True if the host has only IPv4 address support.
+        """
+        return self.attempt_connection(socket.AddressFamily.AF_INET) and not self.attempt_connection(socket.AddressFamily.AF_INET6)
+
+    def attempt_connection(self, address_family):
+        """
+        Returns True if a successful socket connection can be made to the 
+        remote host with the provided address family to validate host support 
+        for the provided address family.
+        """
+        address = None
+        for a in self.get_addresses():
+            if a[0] == address_family:
+                address = a
+                break
+        if address is None:
+            # Couldn't even lookup the address family so we can't connect
+            return False
+        af, socktype, proto, canonname, sockaddr = address
+        s = None
+        try:
+            s = socket.socket(af, socktype, proto)
+        except OSError:
+            return False
+        try:
+            s.connect(sockaddr)
+        except OSError:
+            s.close()
+            return False
+        s.close()
+        return True

component-tests/test_service.py

@@ -1,7 +1,6 @@
 #!/usr/bin/env python
 import os
 import pathlib
-import platform
 import subprocess
 from contextlib import contextmanager
 from pathlib import Path
@@ -10,12 +9,7 @@ import pytest
 
 import test_logging
 from conftest import CfdModes
-from util import start_cloudflared, wait_tunnel_ready, write_config
-
-
-def select_platform(plat):
-    return pytest.mark.skipif(
-        platform.system() != plat, reason=f"Only runs on {plat}")
+from util import select_platform, start_cloudflared, wait_tunnel_ready, write_config
 
 
 def default_config_dir():

component-tests/util.py

@@ -1,9 +1,12 @@
 import logging
 import os
+import platform
 import subprocess
 from contextlib import contextmanager
 from time import sleep
 
+import pytest
+
 import requests
 import yaml
 from retrying import retry
@@ -12,6 +15,10 @@ from constants import METRICS_PORT, MAX_RETRIES, BACKOFF_SECS
 
 LOGGER = logging.getLogger(__name__)
 
+def select_platform(plat):
+    return pytest.mark.skipif(
+        platform.system() != plat, reason=f"Only runs on {plat}")
+
 
 def write_config(directory, config):
     config_path = directory / "config.yml"
@@ -111,6 +118,17 @@ def check_tunnel_not_connected():
         LOGGER.warning(f"Failed to connect to {url}, error: {e}")
 
 
+def get_tunnel_connector_id():
+    url = f'http://localhost:{METRICS_PORT}/ready'
+
+    try:
+        resp = requests.get(url, timeout=1)
+        return resp.json()["connectorId"]
+    # cloudflared might already terminated
+    except requests.exceptions.ConnectionError as e:
+        LOGGER.warning(f"Failed to connect to {url}, error: {e}")
+
+
 # In some cases we don't need to check response status, such as when sending batch requests to generate logs
 def send_requests(url, count, require_ok=True):
     errors = 0

config/configuration.go

@@ -227,6 +227,8 @@ type OriginRequestConfig struct {
 	ProxyType *string `yaml:"proxyType" json:"proxyType,omitempty"`
 	// IP rules for the proxy service
 	IPRules []IngressIPRule `yaml:"ipRules" json:"ipRules,omitempty"`
+	// Attempt to connect to origin with HTTP/2
+	Http2Origin *bool `yaml:"http2Origin" json:"http2Origin,omitempty"`
 }
 
 type IngressIPRule struct {
@@ -244,7 +246,9 @@ type Configuration struct {
 }
 
 type WarpRoutingConfig struct {
-	Enabled bool `yaml:"enabled" json:"enabled"`
+	Enabled        bool            `yaml:"enabled" json:"enabled"`
+	ConnectTimeout *CustomDuration `yaml:"connectTimeout" json:"connectTimeout,omitempty"`
+	TCPKeepAlive   *CustomDuration `yaml:"tcpKeepAlive" json:"tcpKeepAlive,omitempty"`
 }
 
 type configFileSettings struct {

config/configuration_test.go

@@ -7,7 +7,7 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	yaml "gopkg.in/yaml.v2"
+	yaml "gopkg.in/yaml.v3"
 )
 
 func TestConfigFileSettings(t *testing.T) {
@@ -23,7 +23,9 @@ func TestConfigFileSettings(t *testing.T) {
 			Service:  "https://localhost:8001",
 		}
 		warpRouting = WarpRoutingConfig{
-			Enabled: true,
+			Enabled:        true,
+			ConnectTimeout: &CustomDuration{Duration: 2 * time.Second},
+			TCPKeepAlive:   &CustomDuration{Duration: 10 * time.Second},
 		}
 	)
 	rawYAML := `
@@ -48,6 +50,9 @@ ingress:
    service: https://localhost:8001
 warp-routing: 
   enabled: true
+  connectTimeout: 2s
+  tcpKeepAlive: 10s
+
 retries: 5
 grace-period: 30s
 percentage: 3.14
@@ -111,7 +116,7 @@ counters:
 
 }
 
-var rawConfig = []byte(`
+var rawJsonConfig = []byte(`
 {
 	"connectTimeout": 10,
 	"tlsTimeout": 30,
@@ -139,7 +144,8 @@ var rawConfig = []byte(`
 			"ports": [443, 4443],
 			"allow": true
 		}
-	]
+	],
+	"http2Origin": true
 }
 `)
 
@@ -148,15 +154,14 @@ func TestMarshalUnmarshalOriginRequest(t *testing.T) {
 		name          string
 		marshalFunc   func(in interface{}) (out []byte, err error)
 		unMarshalFunc func(in []byte, out interface{}) (err error)
-		baseUnit      time.Duration
 	}{
-		{"json", json.Marshal, json.Unmarshal, time.Second},
-		{"yaml", yaml.Marshal, yaml.Unmarshal, time.Nanosecond},
+		{"json", json.Marshal, json.Unmarshal},
+		{"yaml", yaml.Marshal, yaml.Unmarshal},
 	}
 
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
-			assertConfig(t, tc.marshalFunc, tc.unMarshalFunc, tc.baseUnit)
+			assertConfig(t, tc.marshalFunc, tc.unMarshalFunc)
 		})
 	}
 }
@@ -165,18 +170,17 @@ func assertConfig(
 	t *testing.T,
 	marshalFunc func(in interface{}) (out []byte, err error),
 	unMarshalFunc func(in []byte, out interface{}) (err error),
-	baseUnit time.Duration,
 ) {
 	var config OriginRequestConfig
 	var config2 OriginRequestConfig
 
-	assert.NoError(t, unMarshalFunc(rawConfig, &config))
+	assert.NoError(t, json.Unmarshal(rawJsonConfig, &config))
 
-	assert.Equal(t, baseUnit*10, config.ConnectTimeout.Duration)
-	assert.Equal(t, baseUnit*30, config.TLSTimeout.Duration)
-	assert.Equal(t, baseUnit*30, config.TCPKeepAlive.Duration)
+	assert.Equal(t, time.Second*10, config.ConnectTimeout.Duration)
+	assert.Equal(t, time.Second*30, config.TLSTimeout.Duration)
+	assert.Equal(t, time.Second*30, config.TCPKeepAlive.Duration)
 	assert.Equal(t, true, *config.NoHappyEyeballs)
-	assert.Equal(t, baseUnit*60, config.KeepAliveTimeout.Duration)
+	assert.Equal(t, time.Second*60, config.KeepAliveTimeout.Duration)
 	assert.Equal(t, 10, *config.KeepAliveConnections)
 	assert.Equal(t, "app.tunnel.com", *config.HTTPHostHeader)
 	assert.Equal(t, "app.tunnel.com", *config.OriginServerName)
@@ -188,6 +192,7 @@ func assertConfig(
 	assert.Equal(t, true, *config.NoTLSVerify)
 	assert.Equal(t, uint(9000), *config.ProxyPort)
 	assert.Equal(t, "socks", *config.ProxyType)
+	assert.Equal(t, true, *config.Http2Origin)
 
 	privateV4 := "10.0.0.0/8"
 	privateV6 := "fc00::/7"

config/manager.go

@@ -6,7 +6,7 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
-	yaml "gopkg.in/yaml.v2"
+	yaml "gopkg.in/yaml.v3"
 
 	"github.com/cloudflare/cloudflared/watcher"
 )

connection/connection.go

@@ -123,22 +123,24 @@ func (t Type) String() string {
 
 // OriginProxy is how data flows from cloudflared to the origin services running behind it.
 type OriginProxy interface {
-	ProxyHTTP(w ResponseWriter, tr *tracing.TracedRequest, isWebsocket bool) error
+	ProxyHTTP(w ResponseWriter, tr *tracing.TracedHTTPRequest, isWebsocket bool) error
 	ProxyTCP(ctx context.Context, rwa ReadWriteAcker, req *TCPRequest) error
 }
 
 // TCPRequest defines the input format needed to perform a TCP proxy.
 type TCPRequest struct {
-	Dest    string
-	CFRay   string
-	LBProbe bool
+	Dest      string
+	CFRay     string
+	LBProbe   bool
+	FlowID    string
+	CfTraceID string
 }
 
 // ReadWriteAcker is a readwriter with the ability to Acknowledge to the downstream (edge) that the origin has
 // accepted the connection.
 type ReadWriteAcker interface {
 	io.ReadWriter
-	AckConnection() error
+	AckConnection(tracePropagation string) error
 }
 
 // HTTPResponseReadWriteAcker is an HTTP implementation of ReadWriteAcker.
@@ -167,7 +169,7 @@ func (h *HTTPResponseReadWriteAcker) Write(p []byte) (int, error) {
 
 // AckConnection acks an HTTP connection by sending a switch protocols status code that enables the caller to
 // upgrade to streams.
-func (h *HTTPResponseReadWriteAcker) AckConnection() error {
+func (h *HTTPResponseReadWriteAcker) AckConnection(tracePropagation string) error {
 	resp := &http.Response{
 		Status:        switchingProtocolText,
 		StatusCode:    http.StatusSwitchingProtocols,
@@ -178,6 +180,10 @@ func (h *HTTPResponseReadWriteAcker) AckConnection() error {
 		resp.Header = websocket.NewResponseHeader(h.req)
 	}
 
+	if tracePropagation != "" {
+		resp.Header.Add(tracing.CanonicalCloudflaredTracingHeader, tracePropagation)
+	}
+
 	return h.w.WriteRespHeaders(resp.StatusCode, resp.Header)
 }
 

connection/connection_test.go

@@ -30,6 +30,8 @@ var (
 	testLargeResp = make([]byte, largeFileSize)
 )
 
+var _ ReadWriteAcker = (*HTTPResponseReadWriteAcker)(nil)
+
 type testRequest struct {
 	name           string
 	endpoint       string
@@ -60,7 +62,7 @@ type mockOriginProxy struct{}
 
 func (moc *mockOriginProxy) ProxyHTTP(
 	w ResponseWriter,
-	tr *tracing.TracedRequest,
+	tr *tracing.TracedHTTPRequest,
 	isWebsocket bool,
 ) error {
 	req := tr.Request

connection/control.go

@@ -3,6 +3,7 @@ package connection
 import (
 	"context"
 	"io"
+	"net"
 	"time"
 
 	"github.com/rs/zerolog"
@@ -19,6 +20,7 @@ type controlStream struct {
 	connectedFuse         ConnectedFuse
 	namedTunnelProperties *NamedTunnelProperties
 	connIndex             uint8
+	edgeAddress           net.IP
 
 	newRPCClientFunc RPCClientFunc
 
@@ -45,6 +47,7 @@ func NewControlStream(
 	connectedFuse ConnectedFuse,
 	namedTunnelConfig *NamedTunnelProperties,
 	connIndex uint8,
+	edgeAddress net.IP,
 	newRPCClientFunc RPCClientFunc,
 	gracefulShutdownC <-chan struct{},
 	gracePeriod time.Duration,
@@ -58,6 +61,7 @@ func NewControlStream(
 		namedTunnelProperties: namedTunnelConfig,
 		newRPCClientFunc:      newRPCClientFunc,
 		connIndex:             connIndex,
+		edgeAddress:           edgeAddress,
 		gracefulShutdownC:     gracefulShutdownC,
 		gracePeriod:           gracePeriod,
 	}
@@ -71,7 +75,7 @@ func (c *controlStream) ServeControlStream(
 ) error {
 	rpcClient := c.newRPCClientFunc(ctx, rw, c.observer.log)
 
-	registrationDetails, err := rpcClient.RegisterConnection(ctx, c.namedTunnelProperties, connOptions, c.connIndex, c.observer)
+	registrationDetails, err := rpcClient.RegisterConnection(ctx, c.namedTunnelProperties, connOptions, c.connIndex, c.edgeAddress, c.observer)
 	if err != nil {
 		rpcClient.Close()
 		return err

connection/errors.go

@@ -18,6 +18,15 @@ func (e DupConnRegisterTunnelError) Error() string {
 	return "already connected to this server, trying another address"
 }
 
+// Dial to edge server with quic failed
+type EdgeQuicDialError struct {
+	Cause error
+}
+
+func (e *EdgeQuicDialError) Error() string {
+	return "failed to dial to edge with quic: " + e.Cause.Error()
+}
+
 // RegisterTunnel error from server
 type ServerRegisterTunnelError struct {
 	Cause     error

connection/h2mux.go

@@ -69,6 +69,7 @@ func NewH2muxConnection(
 	connIndex uint8,
 	observer *Observer,
 	gracefulShutdownC <-chan struct{},
+	log *zerolog.Logger,
 ) (*h2muxConnection, error, bool) {
 	h := &h2muxConnection{
 		orchestrator:      orchestrator,
@@ -79,6 +80,7 @@ func NewH2muxConnection(
 		observer:          observer,
 		gracefulShutdownC: gracefulShutdownC,
 		newRPCClientFunc:  newRegistrationRPCClient,
+		log:               log,
 	}
 
 	// Establish a muxed connection with the edge
@@ -234,7 +236,7 @@ func (h *h2muxConnection) ServeStream(stream *h2mux.MuxedStream) error {
 		return err
 	}
 
-	err = originProxy.ProxyHTTP(respWriter, tracing.NewTracedRequest(req), sourceConnectionType == TypeWebsocket)
+	err = originProxy.ProxyHTTP(respWriter, tracing.NewTracedHTTPRequest(req, h.log), sourceConnectionType == TypeWebsocket)
 	if err != nil {
 		respWriter.WriteErrorResponse()
 	}

connection/h2mux_test.go

@@ -47,8 +47,8 @@ func newH2MuxConnection(t require.TestingT) (*h2muxConnection, *h2mux.Muxer) {
 		edgeMuxChan <- edgeMux
 	}()
 	var connIndex = uint8(0)
-	testObserver := NewObserver(&log, &log, false)
-	h2muxConn, err, _ := NewH2muxConnection(testOrchestrator, testGracePeriod, testMuxerConfig, originConn, connIndex, testObserver, nil)
+	testObserver := NewObserver(&log, &log)
+	h2muxConn, err, _ := NewH2muxConnection(testOrchestrator, testGracePeriod, testMuxerConfig, originConn, connIndex, testObserver, nil, &log)
 	require.NoError(t, err)
 	return h2muxConn, <-edgeMuxChan
 }

connection/http2.go

@@ -132,7 +132,7 @@ func (c *HTTP2Connection) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	case TypeWebsocket, TypeHTTP:
 		stripWebsocketUpgradeHeader(r)
 		// Check for tracing on request
-		tr := tracing.NewTracedRequest(r)
+		tr := tracing.NewTracedHTTPRequest(r, c.log)
 		if err := originProxy.ProxyHTTP(respWriter, tr, connType == TypeWebsocket); err != nil {
 			err := fmt.Errorf("Failed to proxy HTTP: %w", err)
 			c.log.Error().Err(err)

connection/http2_test.go

@@ -33,7 +33,7 @@ func newTestHTTP2Connection() (*HTTP2Connection, net.Conn) {
 	edgeConn, cfdConn := net.Pipe()
 	var connIndex = uint8(0)
 	log := zerolog.Nop()
-	obs := NewObserver(&log, &log, false)
+	obs := NewObserver(&log, &log)
 	controlStream := NewControlStream(
 		obs,
 		mockConnectedFuse{},
@@ -41,6 +41,7 @@ func newTestHTTP2Connection() (*HTTP2Connection, net.Conn) {
 		connIndex,
 		nil,
 		nil,
+		nil,
 		1*time.Second,
 	)
 	return NewHTTP2Connection(
@@ -176,6 +177,7 @@ func (mc mockNamedTunnelRPCClient) RegisterConnection(
 	properties *NamedTunnelProperties,
 	options *tunnelpogs.ConnectionOptions,
 	connIndex uint8,
+	edgeAddress net.IP,
 	observer *Observer,
 ) (*tunnelpogs.ConnectionDetails, error) {
 	if mc.shouldFail != nil {
@@ -354,12 +356,13 @@ func TestServeControlStream(t *testing.T) {
 		unregistered: make(chan struct{}),
 	}
 
-	obs := NewObserver(&log, &log, false)
+	obs := NewObserver(&log, &log)
 	controlStream := NewControlStream(
 		obs,
 		mockConnectedFuse{},
 		&NamedTunnelProperties{},
 		1,
+		nil,
 		rpcClientFactory.newMockRPCClient,
 		nil,
 		1*time.Second,
@@ -404,12 +407,13 @@ func TestFailRegistration(t *testing.T) {
 		unregistered: make(chan struct{}),
 	}
 
-	obs := NewObserver(&log, &log, false)
+	obs := NewObserver(&log, &log)
 	controlStream := NewControlStream(
 		obs,
 		mockConnectedFuse{},
 		&NamedTunnelProperties{},
 		http2Conn.connIndex,
+		nil,
 		rpcClientFactory.newMockRPCClient,
 		nil,
 		1*time.Second,
@@ -449,13 +453,14 @@ func TestGracefulShutdownHTTP2(t *testing.T) {
 	events := &eventCollectorSink{}
 
 	shutdownC := make(chan struct{})
-	obs := NewObserver(&log, &log, false)
+	obs := NewObserver(&log, &log)
 	obs.RegisterSink(events)
 	controlStream := NewControlStream(
 		obs,
 		mockConnectedFuse{},
 		&NamedTunnelProperties{},
 		http2Conn.connIndex,
+		nil,
 		rpcClientFactory.newMockRPCClient,
 		shutdownC,
 		1*time.Second,

connection/metrics.go

@@ -367,7 +367,7 @@ func initTunnelMetrics() *tunnelMetrics {
 			Name:      "server_locations",
 			Help:      "Where each tunnel is connected to. 1 means current location, 0 means previous locations.",
 		},
-		[]string{"connection_id", "location"},
+		[]string{"connection_id", "edge_location"},
 	)
 	prometheus.MustRegister(serverLocations)
 

connection/observer.go

@@ -1,6 +1,7 @@
 package connection
 
 import (
+	"net"
 	"strings"
 
 	"github.com/rs/zerolog"
@@ -8,6 +9,7 @@ import (
 
 const (
 	LogFieldLocation          = "location"
+	LogFieldIPAddress         = "ip"
 	observerChannelBufferSize = 16
 )
 
@@ -16,7 +18,6 @@ type Observer struct {
 	logTransport    *zerolog.Logger
 	metrics         *tunnelMetrics
 	tunnelEventChan chan Event
-	uiEnabled       bool
 	addSinkChan     chan EventSink
 }
 
@@ -24,12 +25,11 @@ type EventSink interface {
 	OnTunnelEvent(event Event)
 }
 
-func NewObserver(log, logTransport *zerolog.Logger, uiEnabled bool) *Observer {
+func NewObserver(log, logTransport *zerolog.Logger) *Observer {
 	o := &Observer{
 		log:             log,
 		logTransport:    logTransport,
 		metrics:         newTunnelMetrics(),
-		uiEnabled:       uiEnabled,
 		tunnelEventChan: make(chan Event, observerChannelBufferSize),
 		addSinkChan:     make(chan EventSink, observerChannelBufferSize),
 	}
@@ -41,11 +41,12 @@ func (o *Observer) RegisterSink(sink EventSink) {
 	o.addSinkChan <- sink
 }
 
-func (o *Observer) logServerInfo(connIndex uint8, location, msg string) {
+func (o *Observer) logServerInfo(connIndex uint8, location string, address net.IP, msg string) {
 	o.sendEvent(Event{Index: connIndex, EventType: Connected, Location: location})
 	o.log.Info().
 		Uint8(LogFieldConnIndex, connIndex).
 		Str(LogFieldLocation, location).
+		IPAddr(LogFieldIPAddress, address).
 		Msg(msg)
 	o.metrics.registerServerLocation(uint8ToString(connIndex), location)
 }

connection/observer_test.go

@@ -12,7 +12,7 @@ import (
 )
 
 func TestSendUrl(t *testing.T) {
-	observer := NewObserver(&log, &log, false)
+	observer := NewObserver(&log, &log)
 
 	observer.SendURL("my-url.com")
 	assert.Equal(t, 1.0, getCounterValue(t, observer.metrics.userHostnamesCounts, "https://my-url.com"))
@@ -63,7 +63,7 @@ func TestRegisterServerLocation(t *testing.T) {
 }
 
 func TestObserverEventsDontBlock(t *testing.T) {
-	observer := NewObserver(&log, &log, false)
+	observer := NewObserver(&log, &log)
 	var mu sync.Mutex
 	observer.RegisterSink(EventSinkFunc(func(_ Event) {
 		// callback will block if lock is already held

connection/quic.go

@@ -31,11 +31,13 @@ const (
 	HTTPMethodKey = "HttpMethod"
 	// HTTPHostKey is used to get or set http Method in QUIC ALPN if the underlying proxy connection type is HTTP.
 	HTTPHostKey = "HttpHost"
+
+	QUICMetadataFlowID = "FlowID"
 )
 
 // QUICConnection represents the type that facilitates Proxying via QUIC streams.
 type QUICConnection struct {
-	session              quic.Session
+	session              quic.Connection
 	logger               *zerolog.Logger
 	orchestrator         Orchestrator
 	sessionManager       datagramsession.Manager
@@ -55,7 +57,7 @@ func NewQUICConnection(
 ) (*QUICConnection, error) {
 	session, err := quic.DialAddr(edgeAddr.String(), tlsConfig, quicConfig)
 	if err != nil {
-		return nil, fmt.Errorf("failed to dial to edge: %w", err)
+		return nil, &EdgeQuicDialError{Cause: err}
 	}
 
 	datagramMuxer, err := quicpogs.NewDatagramMuxer(session, logger)
@@ -120,6 +122,11 @@ func (q *QUICConnection) serveControlStream(ctx context.Context, controlStream q
 	return nil
 }
 
+// Close closes the session with no errors specified.
+func (q *QUICConnection) Close() {
+	q.session.CloseWithError(0, "")
+}
+
 func (q *QUICConnection) acceptStream(ctx context.Context) error {
 	defer q.Close()
 	for {
@@ -131,23 +138,25 @@ func (q *QUICConnection) acceptStream(ctx context.Context) error {
 			}
 			return fmt.Errorf("failed to accept QUIC stream: %w", err)
 		}
-		go func() {
-			stream := quicpogs.NewSafeStreamCloser(quicStream)
-			defer stream.Close()
-
-			if err = q.handleStream(stream); err != nil {
-				q.logger.Err(err).Msg("Failed to handle QUIC stream")
-			}
-		}()
+		go q.runStream(quicStream)
 	}
 }
 
-// Close closes the session with no errors specified.
-func (q *QUICConnection) Close() {
-	q.session.CloseWithError(0, "")
+func (q *QUICConnection) runStream(quicStream quic.Stream) {
+	ctx := quicStream.Context()
+	stream := quicpogs.NewSafeStreamCloser(quicStream)
+	defer stream.Close()
+
+	// we are going to fuse readers/writers from stream <- cloudflared -> origin, and we want to guarantee that
+	// code executed in the code path of handleStream don't trigger an earlier close to the downstream stream.
+	// So, we wrap the stream with a no-op closer and only this method can actually close the stream.
+	noCloseStream := &nopCloserReadWriter{stream}
+	if err := q.handleStream(ctx, noCloseStream); err != nil {
+		q.logger.Err(err).Msg("Failed to handle QUIC stream")
+	}
 }
 
-func (q *QUICConnection) handleStream(stream io.ReadWriteCloser) error {
+func (q *QUICConnection) handleStream(ctx context.Context, stream io.ReadWriteCloser) error {
 	signature, err := quicpogs.DetermineProtocol(stream)
 	if err != nil {
 		return err
@@ -156,9 +165,9 @@ func (q *QUICConnection) handleStream(stream io.ReadWriteCloser) error {
 	case quicpogs.DataStreamProtocolSignature:
 		reqServerStream, err := quicpogs.NewRequestServerStream(stream, signature)
 		if err != nil {
-			return nil
+			return err
 		}
-		return q.handleDataStream(reqServerStream)
+		return q.handleDataStream(ctx, reqServerStream)
 	case quicpogs.RPCStreamProtocolSignature:
 		rpcStream, err := quicpogs.NewRPCServerStream(stream, signature)
 		if err != nil {
@@ -170,28 +179,43 @@ func (q *QUICConnection) handleStream(stream io.ReadWriteCloser) error {
 	}
 }
 
-func (q *QUICConnection) handleDataStream(stream *quicpogs.RequestServerStream) error {
-	connectRequest, err := stream.ReadConnectRequestData()
+func (q *QUICConnection) handleDataStream(ctx context.Context, stream *quicpogs.RequestServerStream) error {
+	request, err := stream.ReadConnectRequestData()
 	if err != nil {
 		return err
 	}
 
+	if err := q.dispatchRequest(ctx, stream, err, request); err != nil {
+		_ = stream.WriteConnectResponseData(err)
+		q.logger.Err(err).Str("type", request.Type.String()).Str("dest", request.Dest).Msg("Request failed")
+	}
+
+	return nil
+}
+
+func (q *QUICConnection) dispatchRequest(ctx context.Context, stream *quicpogs.RequestServerStream, err error, request *quicpogs.ConnectRequest) error {
 	originProxy, err := q.orchestrator.GetOriginProxy()
 	if err != nil {
 		return err
 	}
-	switch connectRequest.Type {
+
+	switch request.Type {
 	case quicpogs.ConnectionTypeHTTP, quicpogs.ConnectionTypeWebsocket:
-		tracedReq, err := buildHTTPRequest(connectRequest, stream)
+		tracedReq, err := buildHTTPRequest(ctx, request, stream, q.logger)
 		if err != nil {
 			return err
 		}
-
 		w := newHTTPResponseAdapter(stream)
-		return originProxy.ProxyHTTP(w, tracedReq, connectRequest.Type == quicpogs.ConnectionTypeWebsocket)
+		return originProxy.ProxyHTTP(w, tracedReq, request.Type == quicpogs.ConnectionTypeWebsocket)
+
 	case quicpogs.ConnectionTypeTCP:
 		rwa := &streamReadWriteAcker{stream}
-		return originProxy.ProxyTCP(context.Background(), rwa, &TCPRequest{Dest: connectRequest.Dest})
+		metadata := request.MetadataMap()
+		return originProxy.ProxyTCP(ctx, rwa, &TCPRequest{
+			Dest:      request.Dest,
+			FlowID:    metadata[QUICMetadataFlowID],
+			CfTraceID: metadata[tracing.TracerContextName],
+		})
 	}
 	return nil
 }
@@ -277,8 +301,12 @@ type streamReadWriteAcker struct {
 }
 
 // AckConnection acks response back to the proxy.
-func (s *streamReadWriteAcker) AckConnection() error {
-	return s.WriteConnectResponseData(nil)
+func (s *streamReadWriteAcker) AckConnection(tracePropagation string) error {
+	metadata := quicpogs.Metadata{
+		Key: tracing.CanonicalCloudflaredTracingHeader,
+		Val: tracePropagation,
+	}
+	return s.WriteConnectResponseData(nil, metadata)
 }
 
 // httpResponseAdapter translates responses written by the HTTP Proxy into ones that can be used in QUIC.
@@ -306,14 +334,19 @@ func (hrw httpResponseAdapter) WriteErrorResponse(err error) {
 	hrw.WriteConnectResponseData(err, quicpogs.Metadata{Key: "HttpStatus", Val: strconv.Itoa(http.StatusBadGateway)})
 }
 
-func buildHTTPRequest(connectRequest *quicpogs.ConnectRequest, body io.ReadCloser) (*tracing.TracedRequest, error) {
+func buildHTTPRequest(
+	ctx context.Context,
+	connectRequest *quicpogs.ConnectRequest,
+	body io.ReadCloser,
+	log *zerolog.Logger,
+) (*tracing.TracedHTTPRequest, error) {
 	metadata := connectRequest.MetadataMap()
 	dest := connectRequest.Dest
 	method := metadata[HTTPMethodKey]
 	host := metadata[HTTPHostKey]
 	isWebsocket := connectRequest.Type == quicpogs.ConnectionTypeWebsocket
 
-	req, err := http.NewRequest(method, dest, body)
+	req, err := http.NewRequestWithContext(ctx, method, dest, body)
 	if err != nil {
 		return nil, err
 	}
@@ -348,7 +381,7 @@ func buildHTTPRequest(connectRequest *quicpogs.ConnectRequest, body io.ReadClose
 	stripWebsocketUpgradeHeader(req)
 
 	// Check for tracing on request
-	tracedReq := tracing.NewTracedRequest(req)
+	tracedReq := tracing.NewTracedHTTPRequest(req, log)
 	return tracedReq, err
 }
 
@@ -366,3 +399,11 @@ func isTransferEncodingChunked(req *http.Request) bool {
 	// separated value as well.
 	return strings.Contains(strings.ToLower(transferEncodingVal), "chunked")
 }
+
+type nopCloserReadWriter struct {
+	io.ReadWriteCloser
+}
+
+func (n *nopCloserReadWriter) Close() error {
+	return nil
+}

connection/quic_test.go

@@ -31,11 +31,14 @@ import (
 var (
 	testTLSServerConfig = quicpogs.GenerateTLSConfig()
 	testQUICConfig      = &quic.Config{
-		KeepAlive:       true,
-		EnableDatagrams: true,
+		ConnectionIDLength: 16,
+		KeepAlivePeriod:    5 * time.Second,
+		EnableDatagrams:    true,
 	}
 )
 
+var _ ReadWriteAcker = (*streamReadWriteAcker)(nil)
+
 // TestQUICServer tests if a quic server accepts and responds to a quic client with the acceptance protocol.
 // It also serves as a demonstration for communication with the QUIC connection started by a cloudflared.
 func TestQUICServer(t *testing.T) {
@@ -78,63 +81,63 @@ func TestQUICServer(t *testing.T) {
 			},
 			expectedResponse: []byte("OK"),
 		},
-		{
-			desc:           "test http body request streaming",
-			dest:           "/slow_echo_body",
-			connectionType: quicpogs.ConnectionTypeHTTP,
-			metadata: []quicpogs.Metadata{
-				{
-					Key: "HttpHeader:Cf-Ray",
-					Val: "123123123",
-				},
-				{
-					Key: "HttpHost",
-					Val: "cf.host",
-				},
-				{
-					Key: "HttpMethod",
-					Val: "POST",
-				},
-				{
-					Key: "HttpHeader:Content-Length",
-					Val: "24",
-				},
-			},
-			message:          []byte("This is the message body"),
-			expectedResponse: []byte("This is the message body"),
-		},
-		{
-			desc:           "test ws proxy",
-			dest:           "/ws/echo",
-			connectionType: quicpogs.ConnectionTypeWebsocket,
-			metadata: []quicpogs.Metadata{
-				{
-					Key: "HttpHeader:Cf-Cloudflared-Proxy-Connection-Upgrade",
-					Val: "Websocket",
-				},
-				{
-					Key: "HttpHeader:Another-Header",
-					Val: "Misc",
-				},
-				{
-					Key: "HttpHost",
-					Val: "cf.host",
-				},
-				{
-					Key: "HttpMethod",
-					Val: "get",
-				},
-			},
-			message:          wsBuf.Bytes(),
-			expectedResponse: []byte{0x82, 0x5, 0x48, 0x65, 0x6c, 0x6c, 0x6f},
-		},
-		{
-			desc:             "test tcp proxy",
-			connectionType:   quicpogs.ConnectionTypeTCP,
-			metadata:         []quicpogs.Metadata{},
-			message:          []byte("Here is some tcp data"),
-			expectedResponse: []byte("Here is some tcp data"),
-		},
+		//{
+		//	desc:           "test http body request streaming",
+		//	dest:           "/slow_echo_body",
+		//	connectionType: quicpogs.ConnectionTypeHTTP,
+		//	metadata: []quicpogs.Metadata{
+		//		{
+		//			Key: "HttpHeader:Cf-Ray",
+		//			Val: "123123123",
+		//		},
+		//		{
+		//			Key: "HttpHost",
+		//			Val: "cf.host",
+		//		},
+		//		{
+		//			Key: "HttpMethod",
+		//			Val: "POST",
+		//		},
+		//		{
+		//			Key: "HttpHeader:Content-Length",
+		//			Val: "24",
+		//		},
+		//	},
+		//	message:          []byte("This is the message body"),
+		//	expectedResponse: []byte("This is the message body"),
+		//},
+		//{
+		//	desc:           "test ws proxy",
+		//	dest:           "/ws/echo",
+		//	connectionType: quicpogs.ConnectionTypeWebsocket,
+		//	metadata: []quicpogs.Metadata{
+		//		{
+		//			Key: "HttpHeader:Cf-Cloudflared-Proxy-Connection-Upgrade",
+		//			Val: "Websocket",
+		//		},
+		//		{
+		//			Key: "HttpHeader:Another-Header",
+		//			Val: "Misc",
+		//		},
+		//		{
+		//			Key: "HttpHost",
+		//			Val: "cf.host",
+		//		},
+		//		{
+		//			Key: "HttpMethod",
+		//			Val: "get",
+		//		},
+		//	},
+		//	message:          wsBuf.Bytes(),
+		//	expectedResponse: []byte{0x82, 0x5, 0x48, 0x65, 0x6c, 0x6c, 0x6f},
+		//},
+		//{
+		//	desc:             "test tcp proxy",
+		//	connectionType:   quicpogs.ConnectionTypeTCP,
+		//	metadata:         []quicpogs.Metadata{},
+		//	message:          []byte("Here is some tcp data"),
+		//	expectedResponse: []byte("Here is some tcp data"),
+		//},
 	}
 
 	for _, test := range tests {
@@ -220,7 +223,7 @@ func quicServer(
 
 type mockOriginProxyWithRequest struct{}
 
-func (moc *mockOriginProxyWithRequest) ProxyHTTP(w ResponseWriter, tr *tracing.TracedRequest, isWebsocket bool) error {
+func (moc *mockOriginProxyWithRequest) ProxyHTTP(w ResponseWriter, tr *tracing.TracedHTTPRequest, isWebsocket bool) error {
 	// These are a series of crude tests to ensure the headers and http related data is transferred from
 	// metadata.
 	r := tr.Request
@@ -475,9 +478,10 @@ func TestBuildHTTPRequest(t *testing.T) {
 		},
 	}
 
+	log := zerolog.Nop()
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			req, err := buildHTTPRequest(test.connectRequest, test.body)
+			req, err := buildHTTPRequest(context.Background(), test.connectRequest, test.body, &log)
 			assert.NoError(t, err)
 			test.req = test.req.WithContext(req.Context())
 			assert.Equal(t, test.req, req.Request)
@@ -486,7 +490,7 @@ func TestBuildHTTPRequest(t *testing.T) {
 }
 
 func (moc *mockOriginProxyWithRequest) ProxyTCP(ctx context.Context, rwa ReadWriteAcker, tcpRequest *TCPRequest) error {
-	rwa.AckConnection()
+	rwa.AckConnection("")
 	io.Copy(rwa, rwa)
 	return nil
 }
@@ -500,9 +504,10 @@ func TestServeUDPSession(t *testing.T) {
 	defer udpListener.Close()
 
 	ctx, cancel := context.WithCancel(context.Background())
+	val := udpListener.LocalAddr()
 
 	// Establish QUIC connection with edge
-	edgeQUICSessionChan := make(chan quic.Session)
+	edgeQUICSessionChan := make(chan quic.Connection)
 	go func() {
 		earlyListener, err := quic.Listen(udpListener, testTLSServerConfig, testQUICConfig)
 		require.NoError(t, err)
@@ -512,7 +517,7 @@ func TestServeUDPSession(t *testing.T) {
 		edgeQUICSessionChan <- edgeQUICSession
 	}()
 
-	qc := testQUICConnection(udpListener.LocalAddr(), t)
+	qc := testQUICConnection(val, t)
 	go qc.Serve(ctx)
 
 	edgeQUICSession := <-edgeQUICSessionChan
@@ -522,7 +527,7 @@ func TestServeUDPSession(t *testing.T) {
 	cancel()
 }
 
-func serveSession(ctx context.Context, qc *QUICConnection, edgeQUICSession quic.Session, closeType closeReason, expectedReason string, t *testing.T) {
+func serveSession(ctx context.Context, qc *QUICConnection, edgeQUICSession quic.Connection, closeType closeReason, expectedReason string, t *testing.T) {
 	var (
 		payload = []byte(t.Name())
 	)
@@ -583,7 +588,7 @@ const (
 	closedByTimeout
 )
 
-func runRPCServer(ctx context.Context, session quic.Session, sessionRPCServer tunnelpogs.SessionManager, configRPCServer tunnelpogs.ConfigurationManager, t *testing.T) {
+func runRPCServer(ctx context.Context, session quic.Connection, sessionRPCServer tunnelpogs.SessionManager, configRPCServer tunnelpogs.ConfigurationManager, t *testing.T) {
 	stream, err := session.AcceptStream(ctx)
 	require.NoError(t, err)
 

connection/rpc.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"net"
 	"time"
 
 	"github.com/rs/zerolog"
@@ -57,6 +58,7 @@ type NamedTunnelRPCClient interface {
 		config *NamedTunnelProperties,
 		options *tunnelpogs.ConnectionOptions,
 		connIndex uint8,
+		edgeAddress net.IP,
 		observer *Observer,
 	) (*tunnelpogs.ConnectionDetails, error)
 	SendLocalConfiguration(
@@ -94,6 +96,7 @@ func (rsc *registrationServerClient) RegisterConnection(
 	properties *NamedTunnelProperties,
 	options *tunnelpogs.ConnectionOptions,
 	connIndex uint8,
+	edgeAddress net.IP,
 	observer *Observer,
 ) (*tunnelpogs.ConnectionDetails, error) {
 	conn, err := rsc.client.RegisterConnection(
@@ -114,7 +117,7 @@ func (rsc *registrationServerClient) RegisterConnection(
 
 	observer.metrics.regSuccess.WithLabelValues("registerConnection").Inc()
 
-	observer.logServerInfo(connIndex, conn.Location, fmt.Sprintf("Connection %s registered", conn.UUID))
+	observer.logServerInfo(connIndex, conn.Location, edgeAddress, fmt.Sprintf("Connection %s registered", conn.UUID))
 	observer.sendConnectedEvent(connIndex, conn.Location)
 
 	return conn, nil
@@ -274,7 +277,7 @@ func (h *h2muxConnection) logServerInfo(ctx context.Context, rpcClient *tunnelSe
 		h.observer.log.Err(err).Msg("Failed to retrieve server information")
 		return err
 	}
-	h.observer.logServerInfo(h.connIndex, serverInfo.LocationName, "Connection established")
+	h.observer.logServerInfo(h.connIndex, serverInfo.LocationName, net.IP{}, "Connection established")
 	return nil
 }
 
@@ -290,7 +293,7 @@ func (h *h2muxConnection) registerNamedTunnel(
 	rpcClient := h.newRPCClientFunc(ctx, stream, h.observer.log)
 	defer rpcClient.Close()
 
-	if _, err = rpcClient.RegisterConnection(ctx, namedTunnel, connOptions, h.connIndex, h.observer); err != nil {
+	if _, err = rpcClient.RegisterConnection(ctx, namedTunnel, connOptions, h.connIndex, nil, h.observer); err != nil {
 		return err
 	}
 	return nil

dev.Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.17.10 as builder
+FROM golang:1.18.1 as builder
 ENV GO111MODULE=on \
     CGO_ENABLED=0
 WORKDIR /go/src/github.com/cloudflare/cloudflared/

edgediscovery/allregions/address.go

@@ -0,0 +1,64 @@
+package allregions
+
+// Region contains cloudflared edge addresses. The edge is partitioned into several regions for
+// redundancy purposes.
+type AddrSet map[*EdgeAddr]UsedBy
+
+// AddrUsedBy finds the address used by the given connection in this region.
+// Returns nil if the connection isn't using any IP.
+func (a AddrSet) AddrUsedBy(connID int) *EdgeAddr {
+	for addr, used := range a {
+		if used.Used && used.ConnID == connID {
+			return addr
+		}
+	}
+	return nil
+}
+
+// AvailableAddrs counts how many unused addresses this region contains.
+func (a AddrSet) AvailableAddrs() int {
+	n := 0
+	for _, usedby := range a {
+		if !usedby.Used {
+			n++
+		}
+	}
+	return n
+}
+
+// GetUnusedIP returns a random unused address in this region.
+// Returns nil if all addresses are in use.
+func (a AddrSet) GetUnusedIP(excluding *EdgeAddr) *EdgeAddr {
+	for addr, usedby := range a {
+		if !usedby.Used && addr != excluding {
+			return addr
+		}
+	}
+	return nil
+}
+
+// Use the address, assigning it to a proxy connection.
+func (a AddrSet) Use(addr *EdgeAddr, connID int) {
+	if addr == nil {
+		return
+	}
+	a[addr] = InUse(connID)
+}
+
+// GetAnyAddress returns an arbitrary address from the region.
+func (a AddrSet) GetAnyAddress() *EdgeAddr {
+	for addr := range a {
+		return addr
+	}
+	return nil
+}
+
+// GiveBack the address, ensuring it is no longer assigned to an IP.
+// Returns true if the address is in this region.
+func (a AddrSet) GiveBack(addr *EdgeAddr) (ok bool) {
+	if _, ok := a[addr]; !ok {
+		return false
+	}
+	a[addr] = Unused()
+	return true
+}

edgediscovery/allregions/address_test.go

@@ -0,0 +1,247 @@
+package allregions
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestAddrSet_AddrUsedBy(t *testing.T) {
+	type args struct {
+		connID int
+	}
+	tests := []struct {
+		name    string
+		addrSet AddrSet
+		args    args
+		want    *EdgeAddr
+	}{
+		{
+			name: "happy trivial test",
+			addrSet: AddrSet{
+				&addr0: InUse(0),
+			},
+			args: args{connID: 0},
+			want: &addr0,
+		},
+		{
+			name: "sad trivial test",
+			addrSet: AddrSet{
+				&addr0: InUse(0),
+			},
+			args: args{connID: 1},
+			want: nil,
+		},
+		{
+			name: "sad test",
+			addrSet: AddrSet{
+				&addr0: InUse(0),
+				&addr1: InUse(1),
+				&addr2: InUse(2),
+			},
+			args: args{connID: 3},
+			want: nil,
+		},
+		{
+			name: "happy test",
+			addrSet: AddrSet{
+				&addr0: InUse(0),
+				&addr1: InUse(1),
+				&addr2: InUse(2),
+			},
+			args: args{connID: 1},
+			want: &addr1,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := tt.addrSet.AddrUsedBy(tt.args.connID); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("Region.AddrUsedBy() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestAddrSet_AvailableAddrs(t *testing.T) {
+	tests := []struct {
+		name    string
+		addrSet AddrSet
+		want    int
+	}{
+		{
+			name: "contains addresses",
+			addrSet: AddrSet{
+				&addr0: InUse(0),
+				&addr1: Unused(),
+				&addr2: InUse(2),
+			},
+			want: 1,
+		},
+		{
+			name: "all free",
+			addrSet: AddrSet{
+				&addr0: Unused(),
+				&addr1: Unused(),
+				&addr2: Unused(),
+			},
+			want: 3,
+		},
+		{
+			name: "all used",
+			addrSet: AddrSet{
+				&addr0: InUse(0),
+				&addr1: InUse(1),
+				&addr2: InUse(2),
+			},
+			want: 0,
+		},
+		{
+			name:    "empty",
+			addrSet: AddrSet{},
+			want:    0,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := tt.addrSet.AvailableAddrs(); got != tt.want {
+				t.Errorf("Region.AvailableAddrs() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestAddrSet_GetUnusedIP(t *testing.T) {
+	type args struct {
+		excluding *EdgeAddr
+	}
+	tests := []struct {
+		name    string
+		addrSet AddrSet
+		args    args
+		want    *EdgeAddr
+	}{
+		{
+			name: "happy test with excluding set",
+			addrSet: AddrSet{
+				&addr0: Unused(),
+				&addr1: Unused(),
+				&addr2: InUse(2),
+			},
+			args: args{excluding: &addr0},
+			want: &addr1,
+		},
+		{
+			name: "happy test with no excluding",
+			addrSet: AddrSet{
+				&addr0: InUse(0),
+				&addr1: Unused(),
+				&addr2: InUse(2),
+			},
+			args: args{excluding: nil},
+			want: &addr1,
+		},
+		{
+			name: "sad test with no excluding",
+			addrSet: AddrSet{
+				&addr0: InUse(0),
+				&addr1: InUse(1),
+				&addr2: InUse(2),
+			},
+			args: args{excluding: nil},
+			want: nil,
+		},
+		{
+			name: "sad test with excluding",
+			addrSet: AddrSet{
+				&addr0: Unused(),
+				&addr1: InUse(1),
+				&addr2: InUse(2),
+			},
+			args: args{excluding: &addr0},
+			want: nil,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := tt.addrSet.GetUnusedIP(tt.args.excluding); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("Region.GetUnusedIP() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestAddrSet_GiveBack(t *testing.T) {
+	type args struct {
+		addr *EdgeAddr
+	}
+	tests := []struct {
+		name           string
+		addrSet        AddrSet
+		args           args
+		wantOk         bool
+		availableAfter int
+	}{
+		{
+			name: "sad test with excluding",
+			addrSet: AddrSet{
+				&addr1: InUse(1),
+			},
+			args:           args{addr: &addr1},
+			wantOk:         true,
+			availableAfter: 1,
+		},
+		{
+			name: "sad test with excluding",
+			addrSet: AddrSet{
+				&addr1: InUse(1),
+			},
+			args:           args{addr: &addr2},
+			wantOk:         false,
+			availableAfter: 0,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if gotOk := tt.addrSet.GiveBack(tt.args.addr); gotOk != tt.wantOk {
+				t.Errorf("Region.GiveBack() = %v, want %v", gotOk, tt.wantOk)
+			}
+			if tt.availableAfter != tt.addrSet.AvailableAddrs() {
+				t.Errorf("Region.AvailableAddrs() = %v, want %v", tt.addrSet.AvailableAddrs(), tt.availableAfter)
+			}
+		})
+	}
+}
+
+func TestAddrSet_GetAnyAddress(t *testing.T) {
+	tests := []struct {
+		name    string
+		addrSet AddrSet
+		wantNil bool
+	}{
+		{
+			name:    "Sad test -- GetAnyAddress should only fail if the region is empty",
+			addrSet: AddrSet{},
+			wantNil: true,
+		},
+		{
+			name: "Happy test (all addresses unused)",
+			addrSet: AddrSet{
+				&addr0: Unused(),
+			},
+			wantNil: false,
+		},
+		{
+			name: "Happy test (GetAnyAddress can still return addresses used by proxy conns)",
+			addrSet: AddrSet{
+				&addr0: InUse(2),
+			},
+			wantNil: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := tt.addrSet.GetAnyAddress(); tt.wantNil != (got == nil) {
+				t.Errorf("Region.GetAnyAddress() = %v, but should it return nil? %v", got, tt.wantNil)
+			}
+		})
+	}
+}

edgediscovery/allregions/discovery.go

@@ -13,7 +13,7 @@ import (
 
 const (
 	// Used to discover HA origintunneld servers
-	srvService = "origintunneld"
+	srvService = "v2-origintunneld"
 	srvProto   = "tcp"
 	srvName    = "argotunnel.com"
 
@@ -32,10 +32,40 @@ var (
 	netLookupIP  = net.LookupIP
 )
 
+// ConfigIPVersion is the selection of IP versions from config
+type ConfigIPVersion int8
+
+const (
+	Auto     ConfigIPVersion = 2
+	IPv4Only ConfigIPVersion = 4
+	IPv6Only ConfigIPVersion = 6
+)
+
+// IPVersion is the IP version of an EdgeAddr
+type EdgeIPVersion int8
+
+const (
+	V4 EdgeIPVersion = 4
+	V6 EdgeIPVersion = 6
+)
+
+// String returns the enum's constant name.
+func (c EdgeIPVersion) String() string {
+	switch c {
+	case V4:
+		return "4"
+	case V6:
+		return "6"
+	default:
+		return ""
+	}
+}
+
 // EdgeAddr is a representation of possible ways to refer an edge location.
 type EdgeAddr struct {
-	TCP *net.TCPAddr
-	UDP *net.UDPAddr
+	TCP       *net.TCPAddr
+	UDP       *net.UDPAddr
+	IPVersion EdgeIPVersion
 }
 
 // If the call to net.LookupSRV fails, try to fall back to DoT from Cloudflare directly.
@@ -85,6 +115,9 @@ func edgeDiscovery(log *zerolog.Logger, srvService string) ([][]*EdgeAddr, error
 		if err != nil {
 			return nil, err
 		}
+		for _, e := range edgeAddrs {
+			log.Debug().Msgf("Edge Address: %+v", *e)
+		}
 		resolvedAddrPerCNAME = append(resolvedAddrPerCNAME, edgeAddrs)
 	}
 
@@ -120,9 +153,14 @@ func resolveSRV(srv *net.SRV) ([]*EdgeAddr, error) {
 	}
 	addrs := make([]*EdgeAddr, len(ips))
 	for i, ip := range ips {
+		version := V6
+		if ip.To4() != nil {
+			version = V4
+		}
 		addrs[i] = &EdgeAddr{
-			TCP: &net.TCPAddr{IP: ip, Port: int(srv.Port)},
-			UDP: &net.UDPAddr{IP: ip, Port: int(srv.Port)},
+			TCP:       &net.TCPAddr{IP: ip, Port: int(srv.Port)},
+			UDP:       &net.UDPAddr{IP: ip, Port: int(srv.Port)},
+			IPVersion: version,
 		}
 	}
 	return addrs, nil
@@ -143,11 +181,15 @@ func ResolveAddrs(addrs []string, log *zerolog.Logger) (resolved []*EdgeAddr) {
 			log.Error().Str(logFieldAddress, addr).Err(err).Msg("failed to resolve to UDP address")
 			continue
 		}
+		version := V6
+		if udpAddr.IP.To4() != nil {
+			version = V4
+		}
 		resolved = append(resolved, &EdgeAddr{
-			TCP: tcpAddr,
-			UDP: udpAddr,
+			TCP:       tcpAddr,
+			UDP:       udpAddr,
+			IPVersion: version,
 		})
-
 	}
 	return
 }

edgediscovery/allregions/mocks_for_test.go

@@ -9,6 +9,115 @@ import (
 	"testing/quick"
 )
 
+var (
+	v4Addrs = []*EdgeAddr{&addr0, &addr1, &addr2, &addr3}
+	v6Addrs = []*EdgeAddr{&addr4, &addr5, &addr6, &addr7}
+	addr0   = EdgeAddr{
+		TCP: &net.TCPAddr{
+			IP:   net.ParseIP("123.4.5.0"),
+			Port: 8000,
+			Zone: "",
+		},
+		UDP: &net.UDPAddr{
+			IP:   net.ParseIP("123.4.5.0"),
+			Port: 8000,
+			Zone: "",
+		},
+		IPVersion: V4,
+	}
+	addr1 = EdgeAddr{
+		TCP: &net.TCPAddr{
+			IP:   net.ParseIP("123.4.5.1"),
+			Port: 8000,
+			Zone: "",
+		},
+		UDP: &net.UDPAddr{
+			IP:   net.ParseIP("123.4.5.1"),
+			Port: 8000,
+			Zone: "",
+		},
+		IPVersion: V4,
+	}
+	addr2 = EdgeAddr{
+		TCP: &net.TCPAddr{
+			IP:   net.ParseIP("123.4.5.2"),
+			Port: 8000,
+			Zone: "",
+		},
+		UDP: &net.UDPAddr{
+			IP:   net.ParseIP("123.4.5.2"),
+			Port: 8000,
+			Zone: "",
+		},
+		IPVersion: V4,
+	}
+	addr3 = EdgeAddr{
+		TCP: &net.TCPAddr{
+			IP:   net.ParseIP("123.4.5.3"),
+			Port: 8000,
+			Zone: "",
+		},
+		UDP: &net.UDPAddr{
+			IP:   net.ParseIP("123.4.5.3"),
+			Port: 8000,
+			Zone: "",
+		},
+		IPVersion: V4,
+	}
+	addr4 = EdgeAddr{
+		TCP: &net.TCPAddr{
+			IP:   net.ParseIP("2606:4700:a0::1"),
+			Port: 8000,
+			Zone: "",
+		},
+		UDP: &net.UDPAddr{
+			IP:   net.ParseIP("2606:4700:a0::1"),
+			Port: 8000,
+			Zone: "",
+		},
+		IPVersion: V6,
+	}
+	addr5 = EdgeAddr{
+		TCP: &net.TCPAddr{
+			IP:   net.ParseIP("2606:4700:a0::2"),
+			Port: 8000,
+			Zone: "",
+		},
+		UDP: &net.UDPAddr{
+			IP:   net.ParseIP("2606:4700:a0::2"),
+			Port: 8000,
+			Zone: "",
+		},
+		IPVersion: V6,
+	}
+	addr6 = EdgeAddr{
+		TCP: &net.TCPAddr{
+			IP:   net.ParseIP("2606:4700:a0::3"),
+			Port: 8000,
+			Zone: "",
+		},
+		UDP: &net.UDPAddr{
+			IP:   net.ParseIP("2606:4700:a0::3"),
+			Port: 8000,
+			Zone: "",
+		},
+		IPVersion: V6,
+	}
+	addr7 = EdgeAddr{
+		TCP: &net.TCPAddr{
+			IP:   net.ParseIP("2606:4700:a0::4"),
+			Port: 8000,
+			Zone: "",
+		},
+		UDP: &net.UDPAddr{
+			IP:   net.ParseIP("2606:4700:a0::4"),
+			Port: 8000,
+			Zone: "",
+		},
+		IPVersion: V6,
+	}
+)
+
 type mockAddrs struct {
 	// a set of synthetic SRV records
 	addrMap map[net.SRV][]*EdgeAddr
@@ -36,7 +145,7 @@ func newMockAddrs(port uint16, numRegions uint8, numAddrsPerRegion uint8) mockAd
 				IP:   net.ParseIP(fmt.Sprintf("10.0.%v.%v", r, a)),
 				Port: int(port),
 			}
-			addrs = append(addrs, &EdgeAddr{tcpAddr, udpAddr})
+			addrs = append(addrs, &EdgeAddr{tcpAddr, udpAddr, V4})
 		}
 		addrMap[srv] = addrs
 		numAddrs += len(addrs)

edgediscovery/allregions/region.go

@@ -1,79 +1,155 @@
 package allregions
 
+import "time"
+
+const (
+	timeoutDuration = 10 * time.Minute
+)
+
 // Region contains cloudflared edge addresses. The edge is partitioned into several regions for
 // redundancy purposes.
 type Region struct {
-	connFor map[*EdgeAddr]UsedBy
+	primaryIsActive bool
+	active          AddrSet
+	primary         AddrSet
+	secondary       AddrSet
+	primaryTimeout  time.Time
+	timeoutDuration time.Duration
 }
 
 // NewRegion creates a region with the given addresses, which are all unused.
-func NewRegion(addrs []*EdgeAddr) Region {
+func NewRegion(addrs []*EdgeAddr, overrideIPVersion ConfigIPVersion) Region {
 	// The zero value of UsedBy is Unused(), so we can just initialize the map's values with their
 	// zero values.
-	connFor := make(map[*EdgeAddr]UsedBy)
-	for _, addr := range addrs {
-		connFor[addr] = Unused()
+	connForv4 := make(AddrSet)
+	connForv6 := make(AddrSet)
+	systemPreference := V6
+	for i, addr := range addrs {
+		if i == 0 {
+			// First family of IPs returned is system preference of IP
+			systemPreference = addr.IPVersion
+		}
+		switch addr.IPVersion {
+		case V4:
+			connForv4[addr] = Unused()
+		case V6:
+			connForv6[addr] = Unused()
+		}
 	}
+
+	// Process as system preference
+	var primary AddrSet
+	var secondary AddrSet
+	switch systemPreference {
+	case V4:
+		primary = connForv4
+		secondary = connForv6
+	case V6:
+		primary = connForv6
+		secondary = connForv4
+	}
+
+	// Override with provided preference
+	switch overrideIPVersion {
+	case IPv4Only:
+		primary = connForv4
+		secondary = make(AddrSet) // empty
+	case IPv6Only:
+		primary = connForv6
+		secondary = make(AddrSet) // empty
+	case Auto:
+		// no change
+	default:
+		// no change
+	}
+
 	return Region{
-		connFor: connFor,
+		primaryIsActive: true,
+		active:          primary,
+		primary:         primary,
+		secondary:       secondary,
+		timeoutDuration: timeoutDuration,
 	}
 }
 
 // AddrUsedBy finds the address used by the given connection in this region.
 // Returns nil if the connection isn't using any IP.
 func (r *Region) AddrUsedBy(connID int) *EdgeAddr {
-	for addr, used := range r.connFor {
-		if used.Used && used.ConnID == connID {
-			return addr
-		}
+	edgeAddr := r.primary.AddrUsedBy(connID)
+	if edgeAddr == nil {
+		edgeAddr = r.secondary.AddrUsedBy(connID)
 	}
-	return nil
+	return edgeAddr
 }
 
 // AvailableAddrs counts how many unused addresses this region contains.
 func (r Region) AvailableAddrs() int {
-	n := 0
-	for _, usedby := range r.connFor {
-		if !usedby.Used {
-			n++
-		}
-	}
-	return n
+	return r.active.AvailableAddrs()
 }
 
-// GetUnusedIP returns a random unused address in this region.
-// Returns nil if all addresses are in use.
-func (r Region) GetUnusedIP(excluding *EdgeAddr) *EdgeAddr {
-	for addr, usedby := range r.connFor {
-		if !usedby.Used && addr != excluding {
-			return addr
-		}
-	}
-	return nil
-}
-
-// Use the address, assigning it to a proxy connection.
-func (r Region) Use(addr *EdgeAddr, connID int) {
-	if addr == nil {
-		return
-	}
-	r.connFor[addr] = InUse(connID)
-}
-
-// GetAnyAddress returns an arbitrary address from the region.
-func (r Region) GetAnyAddress() *EdgeAddr {
-	for addr := range r.connFor {
+// AssignAnyAddress returns a random unused address in this region now
+// assigned to the connID excluding the provided EdgeAddr.
+// Returns nil if all addresses are in use for the region.
+func (r Region) AssignAnyAddress(connID int, excluding *EdgeAddr) *EdgeAddr {
+	if addr := r.active.GetUnusedIP(excluding); addr != nil {
+		r.active.Use(addr, connID)
 		return addr
 	}
 	return nil
 }
 
+// GetAnyAddress returns an arbitrary address from the region.
+func (r Region) GetAnyAddress() *EdgeAddr {
+	return r.active.GetAnyAddress()
+}
+
 // GiveBack the address, ensuring it is no longer assigned to an IP.
 // Returns true if the address is in this region.
-func (r Region) GiveBack(addr *EdgeAddr) (ok bool) {
-	if _, ok := r.connFor[addr]; !ok {
-		return false
+func (r *Region) GiveBack(addr *EdgeAddr, hasConnectivityError bool) (ok bool) {
+	if ok = r.primary.GiveBack(addr); !ok {
+		// Attempt to give back the address in the secondary set
+		if ok = r.secondary.GiveBack(addr); !ok {
+			// Address is not in this region
+			return
+		}
 	}
-	r.connFor[addr] = Unused()
-	return true
+
+	// No connectivity error: no worry
+	if !hasConnectivityError {
+		return
+	}
+
+	// If using primary and returned address is IPv6 and secondary is available
+	if r.primaryIsActive && addr.IPVersion == V6 && len(r.secondary) > 0 {
+		r.active = r.secondary
+		r.primaryIsActive = false
+		r.primaryTimeout = time.Now().Add(r.timeoutDuration)
+		return
+	}
+
+	// Do nothing for IPv4 or if secondary is empty
+	if r.primaryIsActive {
+		return
+	}
+
+	// Immediately return to primary pool, regardless of current primary timeout
+	if addr.IPVersion == V4 {
+		activatePrimary(r)
+		return
+	}
+
+	// Timeout exceeded and can be reset to primary pool
+	if r.primaryTimeout.Before(time.Now()) {
+		activatePrimary(r)
+		return
+	}
+
+	return
+}
+
+// activatePrimary sets the primary set to the active set and resets the timeout.
+func activatePrimary(r *Region) {
+	r.active = r.primary
+	r.primaryIsActive = true
+	r.primaryTimeout = time.Now() // reset timeout
 }

edgediscovery/allregions/region_test.go

@@ -1,284 +1,357 @@
 package allregions
 
 import (
-	"reflect"
+	"net"
 	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
 )
 
+func makeAddrSet(addrs []*EdgeAddr) AddrSet {
+	addrSet := make(AddrSet, len(addrs))
+	for _, addr := range addrs {
+		addrSet[addr] = Unused()
+	}
+	return addrSet
+}
+
 func TestRegion_New(t *testing.T) {
-	r := NewRegion([]*EdgeAddr{&addr0, &addr1, &addr2})
-	if r.AvailableAddrs() != 3 {
-		t.Errorf("r.AvailableAddrs() == %v but want 3", r.AvailableAddrs())
-	}
-}
-
-func TestRegion_AddrUsedBy(t *testing.T) {
-	type fields struct {
-		connFor map[*EdgeAddr]UsedBy
-	}
-	type args struct {
-		connID int
-	}
 	tests := []struct {
-		name   string
-		fields fields
-		args   args
-		want   *EdgeAddr
+		name          string
+		addrs         []*EdgeAddr
+		mode          ConfigIPVersion
+		expectedAddrs int
+		primary       AddrSet
+		secondary     AddrSet
 	}{
 		{
-			name: "happy trivial test",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{
-				&addr0: InUse(0),
-			}},
-			args: args{connID: 0},
-			want: &addr0,
+			name:          "IPv4 addresses with IPv4Only",
+			addrs:         v4Addrs,
+			mode:          IPv4Only,
+			expectedAddrs: len(v4Addrs),
+			primary:       makeAddrSet(v4Addrs),
+			secondary:     AddrSet{},
 		},
 		{
-			name: "sad trivial test",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{
-				&addr0: InUse(0),
-			}},
-			args: args{connID: 1},
-			want: nil,
+			name:          "IPv6 addresses with IPv4Only",
+			addrs:         v6Addrs,
+			mode:          IPv4Only,
+			expectedAddrs: 0,
+			primary:       AddrSet{},
+			secondary:     AddrSet{},
 		},
 		{
-			name: "sad test",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{
-				&addr0: InUse(0),
-				&addr1: InUse(1),
-				&addr2: InUse(2),
-			}},
-			args: args{connID: 3},
-			want: nil,
+			name:          "IPv6 addresses with IPv6Only",
+			addrs:         v6Addrs,
+			mode:          IPv6Only,
+			expectedAddrs: len(v6Addrs),
+			primary:       makeAddrSet(v6Addrs),
+			secondary:     AddrSet{},
 		},
 		{
-			name: "happy test",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{
-				&addr0: InUse(0),
-				&addr1: InUse(1),
-				&addr2: InUse(2),
-			}},
-			args: args{connID: 1},
-			want: &addr1,
+			name:          "IPv6 addresses with IPv4Only",
+			addrs:         v6Addrs,
+			mode:          IPv4Only,
+			expectedAddrs: 0,
+			primary:       AddrSet{},
+			secondary:     AddrSet{},
+		},
+		{
+			name:          "IPv4 (first) and IPv6 addresses with Auto",
+			addrs:         append(v4Addrs, v6Addrs...),
+			mode:          Auto,
+			expectedAddrs: len(v4Addrs),
+			primary:       makeAddrSet(v4Addrs),
+			secondary:     makeAddrSet(v6Addrs),
+		},
+		{
+			name:          "IPv6 (first) and IPv4 addresses with Auto",
+			addrs:         append(v6Addrs, v4Addrs...),
+			mode:          Auto,
+			expectedAddrs: len(v6Addrs),
+			primary:       makeAddrSet(v6Addrs),
+			secondary:     makeAddrSet(v4Addrs),
+		},
+		{
+			name:          "IPv4 addresses with Auto",
+			addrs:         v4Addrs,
+			mode:          Auto,
+			expectedAddrs: len(v4Addrs),
+			primary:       makeAddrSet(v4Addrs),
+			secondary:     AddrSet{},
+		},
+		{
+			name:          "IPv6 addresses with Auto",
+			addrs:         v6Addrs,
+			mode:          Auto,
+			expectedAddrs: len(v6Addrs),
+			primary:       makeAddrSet(v6Addrs),
+			secondary:     AddrSet{},
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			r := &Region{
-				connFor: tt.fields.connFor,
+			r := NewRegion(tt.addrs, tt.mode)
+			assert.Equal(t, tt.expectedAddrs, r.AvailableAddrs())
+			assert.Equal(t, tt.primary, r.primary)
+			assert.Equal(t, tt.secondary, r.secondary)
+		})
+	}
+}
+
+func TestRegion_AnyAddress_EmptyActiveSet(t *testing.T) {
+	tests := []struct {
+		name  string
+		addrs []*EdgeAddr
+		mode  ConfigIPVersion
+	}{
+		{
+			name:  "IPv6 addresses with IPv4Only",
+			addrs: v6Addrs,
+			mode:  IPv4Only,
+		},
+		{
+			name:  "IPv4 addresses with IPv6Only",
+			addrs: v4Addrs,
+			mode:  IPv6Only,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			r := NewRegion(tt.addrs, tt.mode)
+			addr := r.GetAnyAddress()
+			assert.Nil(t, addr)
+			addr = r.AssignAnyAddress(0, nil)
+			assert.Nil(t, addr)
+		})
+	}
+}
+
+func TestRegion_AssignAnyAddress_FullyUsedActiveSet(t *testing.T) {
+	tests := []struct {
+		name  string
+		addrs []*EdgeAddr
+		mode  ConfigIPVersion
+	}{
+		{
+			name:  "IPv6 addresses with IPv6Only",
+			addrs: v6Addrs,
+			mode:  IPv6Only,
+		},
+		{
+			name:  "IPv4 addresses with IPv4Only",
+			addrs: v4Addrs,
+			mode:  IPv4Only,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			r := NewRegion(tt.addrs, tt.mode)
+			total := r.active.AvailableAddrs()
+			for i := 0; i < total; i++ {
+				addr := r.AssignAnyAddress(i, nil)
+				assert.NotNil(t, addr)
 			}
-			if got := r.AddrUsedBy(tt.args.connID); !reflect.DeepEqual(got, tt.want) {
-				t.Errorf("Region.AddrUsedBy() = %v, want %v", got, tt.want)
+			addr := r.AssignAnyAddress(9, nil)
+			assert.Nil(t, addr)
+		})
+	}
+}
+
+var giveBackTests = []struct {
+	name          string
+	addrs         []*EdgeAddr
+	mode          ConfigIPVersion
+	expectedAddrs int
+	primary       AddrSet
+	secondary     AddrSet
+	primarySwap   bool
+}{
+	{
+		name:          "IPv4 addresses with IPv4Only",
+		addrs:         v4Addrs,
+		mode:          IPv4Only,
+		expectedAddrs: len(v4Addrs),
+		primary:       makeAddrSet(v4Addrs),
+		secondary:     AddrSet{},
+		primarySwap:   false,
+	},
+	{
+		name:          "IPv6 addresses with IPv6Only",
+		addrs:         v6Addrs,
+		mode:          IPv6Only,
+		expectedAddrs: len(v6Addrs),
+		primary:       makeAddrSet(v6Addrs),
+		secondary:     AddrSet{},
+		primarySwap:   false,
+	},
+	{
+		name:          "IPv4 (first) and IPv6 addresses with Auto",
+		addrs:         append(v4Addrs, v6Addrs...),
+		mode:          Auto,
+		expectedAddrs: len(v4Addrs),
+		primary:       makeAddrSet(v4Addrs),
+		secondary:     makeAddrSet(v6Addrs),
+		primarySwap:   false,
+	},
+	{
+		name:          "IPv6 (first) and IPv4 addresses with Auto",
+		addrs:         append(v6Addrs, v4Addrs...),
+		mode:          Auto,
+		expectedAddrs: len(v6Addrs),
+		primary:       makeAddrSet(v6Addrs),
+		secondary:     makeAddrSet(v4Addrs),
+		primarySwap:   true,
+	},
+	{
+		name:          "IPv4 addresses with Auto",
+		addrs:         v4Addrs,
+		mode:          Auto,
+		expectedAddrs: len(v4Addrs),
+		primary:       makeAddrSet(v4Addrs),
+		secondary:     AddrSet{},
+		primarySwap:   false,
+	},
+	{
+		name:          "IPv6 addresses with Auto",
+		addrs:         v6Addrs,
+		mode:          Auto,
+		expectedAddrs: len(v6Addrs),
+		primary:       makeAddrSet(v6Addrs),
+		secondary:     AddrSet{},
+		primarySwap:   false,
+	},
+}
+
+func TestRegion_GiveBack_NoConnectivityError(t *testing.T) {
+	for _, tt := range giveBackTests {
+		t.Run(tt.name, func(t *testing.T) {
+			r := NewRegion(tt.addrs, tt.mode)
+			addr := r.AssignAnyAddress(0, nil)
+			assert.NotNil(t, addr)
+			assert.True(t, r.GiveBack(addr, false))
+		})
+	}
+}
+
+func TestRegion_GiveBack_ForeignAddr(t *testing.T) {
+	invalid := EdgeAddr{
+		TCP: &net.TCPAddr{
+			IP:   net.ParseIP("123.4.5.0"),
+			Port: 8000,
+			Zone: "",
+		},
+		UDP: &net.UDPAddr{
+			IP:   net.ParseIP("123.4.5.0"),
+			Port: 8000,
+			Zone: "",
+		},
+		IPVersion: V4,
+	}
+	for _, tt := range giveBackTests {
+		t.Run(tt.name, func(t *testing.T) {
+			r := NewRegion(tt.addrs, tt.mode)
+			assert.False(t, r.GiveBack(&invalid, false))
+			assert.False(t, r.GiveBack(&invalid, true))
+		})
+	}
+}
+
+func TestRegion_GiveBack_SwapPrimary(t *testing.T) {
+	for _, tt := range giveBackTests {
+		t.Run(tt.name, func(t *testing.T) {
+			r := NewRegion(tt.addrs, tt.mode)
+			addr := r.AssignAnyAddress(0, nil)
+			assert.NotNil(t, addr)
+			assert.True(t, r.GiveBack(addr, true))
+			assert.Equal(t, tt.primarySwap, !r.primaryIsActive)
+			if tt.primarySwap {
+				assert.Equal(t, r.secondary, r.active)
+				assert.False(t, r.primaryTimeout.IsZero())
+			} else {
+				assert.Equal(t, r.primary, r.active)
+				assert.True(t, r.primaryTimeout.IsZero())
 			}
 		})
 	}
 }
 
-func TestRegion_AvailableAddrs(t *testing.T) {
-	type fields struct {
-		connFor map[*EdgeAddr]UsedBy
-	}
-	tests := []struct {
-		name   string
-		fields fields
-		want   int
-	}{
-		{
-			name: "contains addresses",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{
-				&addr0: InUse(0),
-				&addr1: Unused(),
-				&addr2: InUse(2),
-			}},
-			want: 1,
-		},
-		{
-			name: "all free",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{
-				&addr0: Unused(),
-				&addr1: Unused(),
-				&addr2: Unused(),
-			}},
-			want: 3,
-		},
-		{
-			name: "all used",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{
-				&addr0: InUse(0),
-				&addr1: InUse(1),
-				&addr2: InUse(2),
-			}},
-			want: 0,
-		},
-		{
-			name:   "empty",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{}},
-			want:   0,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			r := Region{
-				connFor: tt.fields.connFor,
-			}
-			if got := r.AvailableAddrs(); got != tt.want {
-				t.Errorf("Region.AvailableAddrs() = %v, want %v", got, tt.want)
-			}
-		})
-	}
+func TestRegion_GiveBack_IPv4_ResetPrimary(t *testing.T) {
+	r := NewRegion(append(v6Addrs, v4Addrs...), Auto)
+	// Exhaust all IPv6 addresses
+	a0 := r.AssignAnyAddress(0, nil)
+	a1 := r.AssignAnyAddress(1, nil)
+	a2 := r.AssignAnyAddress(2, nil)
+	a3 := r.AssignAnyAddress(3, nil)
+	assert.NotNil(t, a0)
+	assert.NotNil(t, a1)
+	assert.NotNil(t, a2)
+	assert.NotNil(t, a3)
+	// Give back the first IPv6 address to fallback to secondary IPv4 address set
+	assert.True(t, r.GiveBack(a0, true))
+	assert.False(t, r.primaryIsActive)
+	// Give back another IPv6 address
+	assert.True(t, r.GiveBack(a1, true))
+	// Primary shouldn't change
+	assert.False(t, r.primaryIsActive)
+	// Request an address (should be IPv4 from secondary)
+	a4_v4 := r.AssignAnyAddress(4, nil)
+	assert.NotNil(t, a4_v4)
+	assert.Equal(t, V4, a4_v4.IPVersion)
+	a5_v4 := r.AssignAnyAddress(5, nil)
+	assert.NotNil(t, a5_v4)
+	assert.Equal(t, V4, a5_v4.IPVersion)
+	a6_v4 := r.AssignAnyAddress(6, nil)
+	assert.NotNil(t, a6_v4)
+	assert.Equal(t, V4, a6_v4.IPVersion)
+	// Return IPv4 address (without failure)
+	// Primary shouldn't change because it is not a connectivity failure
+	assert.True(t, r.GiveBack(a4_v4, false))
+	assert.False(t, r.primaryIsActive)
+	// Return IPv4 address (with failure)
+	// Primary should change because it is a connectivity failure
+	assert.True(t, r.GiveBack(a5_v4, true))
+	assert.True(t, r.primaryIsActive)
+	// Return IPv4 address (with failure)
+	// Primary shouldn't change because the address is returned to the inactive
+	// secondary address set
+	assert.True(t, r.GiveBack(a6_v4, true))
+	assert.True(t, r.primaryIsActive)
+	// Return IPv6 address (without failure)
+	// Primary shoudn't change because it is not a connectivity failure
+	assert.True(t, r.GiveBack(a2, false))
+	assert.True(t, r.primaryIsActive)
 }
 
-func TestRegion_GetUnusedIP(t *testing.T) {
-	type fields struct {
-		connFor map[*EdgeAddr]UsedBy
-	}
-	type args struct {
-		excluding *EdgeAddr
-	}
-	tests := []struct {
-		name   string
-		fields fields
-		args   args
-		want   *EdgeAddr
-	}{
-		{
-			name: "happy test with excluding set",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{
-				&addr0: Unused(),
-				&addr1: Unused(),
-				&addr2: InUse(2),
-			}},
-			args: args{excluding: &addr0},
-			want: &addr1,
-		},
-		{
-			name: "happy test with no excluding",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{
-				&addr0: InUse(0),
-				&addr1: Unused(),
-				&addr2: InUse(2),
-			}},
-			args: args{excluding: nil},
-			want: &addr1,
-		},
-		{
-			name: "sad test with no excluding",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{
-				&addr0: InUse(0),
-				&addr1: InUse(1),
-				&addr2: InUse(2),
-			}},
-			args: args{excluding: nil},
-			want: nil,
-		},
-		{
-			name: "sad test with excluding",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{
-				&addr0: Unused(),
-				&addr1: InUse(1),
-				&addr2: InUse(2),
-			}},
-			args: args{excluding: &addr0},
-			want: nil,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			r := Region{
-				connFor: tt.fields.connFor,
-			}
-			if got := r.GetUnusedIP(tt.args.excluding); !reflect.DeepEqual(got, tt.want) {
-				t.Errorf("Region.GetUnusedIP() = %v, want %v", got, tt.want)
-			}
-		})
-	}
-}
-
-func TestRegion_GiveBack(t *testing.T) {
-	type fields struct {
-		connFor map[*EdgeAddr]UsedBy
-	}
-	type args struct {
-		addr *EdgeAddr
-	}
-	tests := []struct {
-		name           string
-		fields         fields
-		args           args
-		wantOk         bool
-		availableAfter int
-	}{
-		{
-			name: "sad test with excluding",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{
-				&addr1: InUse(1),
-			}},
-			args:           args{addr: &addr1},
-			wantOk:         true,
-			availableAfter: 1,
-		},
-		{
-			name: "sad test with excluding",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{
-				&addr1: InUse(1),
-			}},
-			args:           args{addr: &addr2},
-			wantOk:         false,
-			availableAfter: 0,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			r := Region{
-				connFor: tt.fields.connFor,
-			}
-			if gotOk := r.GiveBack(tt.args.addr); gotOk != tt.wantOk {
-				t.Errorf("Region.GiveBack() = %v, want %v", gotOk, tt.wantOk)
-			}
-			if tt.availableAfter != r.AvailableAddrs() {
-				t.Errorf("Region.AvailableAddrs() = %v, want %v", r.AvailableAddrs(), tt.availableAfter)
-			}
-		})
-	}
-}
-
-func TestRegion_GetAnyAddress(t *testing.T) {
-	type fields struct {
-		connFor map[*EdgeAddr]UsedBy
-	}
-	tests := []struct {
-		name    string
-		fields  fields
-		wantNil bool
-	}{
-		{
-			name:    "Sad test -- GetAnyAddress should only fail if the region is empty",
-			fields:  fields{connFor: map[*EdgeAddr]UsedBy{}},
-			wantNil: true,
-		},
-		{
-			name: "Happy test (all addresses unused)",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{
-				&addr0: Unused(),
-			}},
-			wantNil: false,
-		},
-		{
-			name: "Happy test (GetAnyAddress can still return addresses used by proxy conns)",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{
-				&addr0: InUse(2),
-			}},
-			wantNil: false,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			r := Region{
-				connFor: tt.fields.connFor,
-			}
-			if got := r.GetAnyAddress(); tt.wantNil != (got == nil) {
-				t.Errorf("Region.GetAnyAddress() = %v, but should it return nil? %v", got, tt.wantNil)
-			}
-		})
-	}
+func TestRegion_GiveBack_Timeout(t *testing.T) {
+	r := NewRegion(append(v6Addrs, v4Addrs...), Auto)
+	a0 := r.AssignAnyAddress(0, nil)
+	a1 := r.AssignAnyAddress(1, nil)
+	a2 := r.AssignAnyAddress(2, nil)
+	assert.NotNil(t, a0)
+	assert.NotNil(t, a1)
+	assert.NotNil(t, a2)
+	// Give back IPv6 address to set timeout
+	assert.True(t, r.GiveBack(a0, true))
+	assert.False(t, r.primaryIsActive)
+	assert.False(t, r.primaryTimeout.IsZero())
+	// Request an address (should be IPv4 from secondary)
+	a3_v4 := r.AssignAnyAddress(3, nil)
+	assert.NotNil(t, a3_v4)
+	assert.Equal(t, V4, a3_v4.IPVersion)
+	assert.False(t, r.primaryIsActive)
+	// Give back IPv6 address inside timeout (no change)
+	assert.True(t, r.GiveBack(a2, true))
+	assert.False(t, r.primaryIsActive)
+	assert.False(t, r.primaryTimeout.IsZero())
+	// Accelerate timeout
+	r.primaryTimeout = time.Now().Add(-time.Minute)
+	// Return IPv6 address
+	assert.True(t, r.GiveBack(a1, true))
+	assert.True(t, r.primaryIsActive)
+	// Returning an IPv4 address after primary is active shouldn't change primary
+	// even with a connectivity error
+	assert.True(t, r.GiveBack(a3_v4, true))
+	assert.True(t, r.primaryIsActive)
 }

edgediscovery/allregions/regions.go

@@ -18,7 +18,7 @@ type Regions struct {
 // ------------------------------------
 
 // ResolveEdge resolves the Cloudflare edge, returning all regions discovered.
-func ResolveEdge(log *zerolog.Logger, region string) (*Regions, error) {
+func ResolveEdge(log *zerolog.Logger, region string, overrideIPVersion ConfigIPVersion) (*Regions, error) {
 	edgeAddrs, err := edgeDiscovery(log, getRegionalServiceName(region))
 	if err != nil {
 		return nil, err
@@ -27,8 +27,8 @@ func ResolveEdge(log *zerolog.Logger, region string) (*Regions, error) {
 		return nil, fmt.Errorf("expected at least 2 Cloudflare Regions regions, but SRV only returned %v", len(edgeAddrs))
 	}
 	return &Regions{
-		region1: NewRegion(edgeAddrs[0]),
-		region2: NewRegion(edgeAddrs[1]),
+		region1: NewRegion(edgeAddrs[0], overrideIPVersion),
+		region2: NewRegion(edgeAddrs[1], overrideIPVersion),
 	}, nil
 }
 
@@ -56,8 +56,8 @@ func NewNoResolve(addrs []*EdgeAddr) *Regions {
 	}
 
 	return &Regions{
-		region1: NewRegion(region1),
-		region2: NewRegion(region2),
+		region1: NewRegion(region1, Auto),
+		region2: NewRegion(region2, Auto),
 	}
 }
 
@@ -95,14 +95,12 @@ func (rs *Regions) GetUnusedAddr(excluding *EdgeAddr, connID int) *EdgeAddr {
 // getAddrs tries to grab address form `first` region, then `second` region
 // this is an unrolled loop over 2 element array
 func getAddrs(excluding *EdgeAddr, connID int, first *Region, second *Region) *EdgeAddr {
-	addr := first.GetUnusedIP(excluding)
+	addr := first.AssignAnyAddress(connID, excluding)
 	if addr != nil {
-		first.Use(addr, connID)
 		return addr
 	}
-	addr = second.GetUnusedIP(excluding)
+	addr = second.AssignAnyAddress(connID, excluding)
 	if addr != nil {
-		second.Use(addr, connID)
 		return addr
 	}
 
@@ -116,18 +114,18 @@ func (rs *Regions) AvailableAddrs() int {
 
 // GiveBack the address so that other connections can use it.
 // Returns true if the address is in this edge.
-func (rs *Regions) GiveBack(addr *EdgeAddr) bool {
-	if found := rs.region1.GiveBack(addr); found {
+func (rs *Regions) GiveBack(addr *EdgeAddr, hasConnectivityError bool) bool {
+	if found := rs.region1.GiveBack(addr, hasConnectivityError); found {
 		return found
 	}
-	return rs.region2.GiveBack(addr)
+	return rs.region2.GiveBack(addr, hasConnectivityError)
 }
 
 // Return regionalized service name if `region` isn't empty, otherwise return the global service name for origintunneld
 func getRegionalServiceName(region string) string {
 	if region != "" {
-		return region + "-" + srvService // Example: `us-origintunneld`
+		return region + "-" + srvService // Example: `us-v2-origintunneld`
 	}
 
-	return srvService // Global service is just `origintunneld`
+	return srvService // Global service is just `v2-origintunneld`
 }

edgediscovery/allregions/regions_test.go

@@ -1,134 +1,215 @@
 package allregions
 
 import (
-	"net"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 )
 
-var (
-	addr0 = EdgeAddr{
-		TCP: &net.TCPAddr{
-			IP:   net.ParseIP("123.4.5.0"),
-			Port: 8000,
-			Zone: "",
-		},
-		UDP: &net.UDPAddr{
-			IP:   net.ParseIP("123.4.5.0"),
-			Port: 8000,
-			Zone: "",
-		},
+func makeRegions(addrs []*EdgeAddr, mode ConfigIPVersion) Regions {
+	r1addrs := make([]*EdgeAddr, 0)
+	r2addrs := make([]*EdgeAddr, 0)
+	for i, addr := range addrs {
+		if i%2 == 0 {
+			r1addrs = append(r1addrs, addr)
+		} else {
+			r2addrs = append(r2addrs, addr)
+		}
 	}
-	addr1 = EdgeAddr{
-		TCP: &net.TCPAddr{
-			IP:   net.ParseIP("123.4.5.1"),
-			Port: 8000,
-			Zone: "",
-		},
-		UDP: &net.UDPAddr{
-			IP:   net.ParseIP("123.4.5.1"),
-			Port: 8000,
-			Zone: "",
-		},
-	}
-	addr2 = EdgeAddr{
-		TCP: &net.TCPAddr{
-			IP:   net.ParseIP("123.4.5.2"),
-			Port: 8000,
-			Zone: "",
-		},
-		UDP: &net.UDPAddr{
-			IP:   net.ParseIP("123.4.5.2"),
-			Port: 8000,
-			Zone: "",
-		},
-	}
-	addr3 = EdgeAddr{
-		TCP: &net.TCPAddr{
-			IP:   net.ParseIP("123.4.5.3"),
-			Port: 8000,
-			Zone: "",
-		},
-		UDP: &net.UDPAddr{
-			IP:   net.ParseIP("123.4.5.3"),
-			Port: 8000,
-			Zone: "",
-		},
-	}
-)
-
-func makeRegions() Regions {
-	r1 := NewRegion([]*EdgeAddr{&addr0, &addr1})
-	r2 := NewRegion([]*EdgeAddr{&addr2, &addr3})
+	r1 := NewRegion(r1addrs, mode)
+	r2 := NewRegion(r2addrs, mode)
 	return Regions{region1: r1, region2: r2}
 }
 
 func TestRegions_AddrUsedBy(t *testing.T) {
-	rs := makeRegions()
-	addr1 := rs.GetUnusedAddr(nil, 1)
-	assert.Equal(t, addr1, rs.AddrUsedBy(1))
-	addr2 := rs.GetUnusedAddr(nil, 2)
-	assert.Equal(t, addr2, rs.AddrUsedBy(2))
-	addr3 := rs.GetUnusedAddr(nil, 3)
-	assert.Equal(t, addr3, rs.AddrUsedBy(3))
+	tests := []struct {
+		name  string
+		addrs []*EdgeAddr
+		mode  ConfigIPVersion
+	}{
+		{
+			name:  "IPv4 addresses with IPv4Only",
+			addrs: v4Addrs,
+			mode:  IPv4Only,
+		},
+		{
+			name:  "IPv6 addresses with IPv6Only",
+			addrs: v6Addrs,
+			mode:  IPv6Only,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			rs := makeRegions(tt.addrs, tt.mode)
+			addr1 := rs.GetUnusedAddr(nil, 1)
+			assert.Equal(t, addr1, rs.AddrUsedBy(1))
+			addr2 := rs.GetUnusedAddr(nil, 2)
+			assert.Equal(t, addr2, rs.AddrUsedBy(2))
+			addr3 := rs.GetUnusedAddr(nil, 3)
+			assert.Equal(t, addr3, rs.AddrUsedBy(3))
+		})
+	}
 }
 
 func TestRegions_Giveback_Region1(t *testing.T) {
-	rs := makeRegions()
-	rs.region1.Use(&addr0, 0)
-	rs.region1.Use(&addr1, 1)
-	rs.region2.Use(&addr2, 2)
-	rs.region2.Use(&addr3, 3)
+	tests := []struct {
+		name  string
+		addrs []*EdgeAddr
+		mode  ConfigIPVersion
+	}{
+		{
+			name:  "IPv4 addresses with IPv4Only",
+			addrs: v4Addrs,
+			mode:  IPv4Only,
+		},
+		{
+			name:  "IPv6 addresses with IPv6Only",
+			addrs: v6Addrs,
+			mode:  IPv6Only,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			rs := makeRegions(tt.addrs, tt.mode)
+			addr := rs.region1.AssignAnyAddress(0, nil)
+			rs.region1.AssignAnyAddress(1, nil)
+			rs.region2.AssignAnyAddress(2, nil)
+			rs.region2.AssignAnyAddress(3, nil)
 
-	assert.Equal(t, 0, rs.AvailableAddrs())
+			assert.Equal(t, 0, rs.AvailableAddrs())
 
-	rs.GiveBack(&addr0)
-	assert.Equal(t, &addr0, rs.GetUnusedAddr(nil, 3))
+			rs.GiveBack(addr, false)
+			assert.Equal(t, addr, rs.GetUnusedAddr(nil, 0))
+		})
+	}
 }
 
 func TestRegions_Giveback_Region2(t *testing.T) {
-	rs := makeRegions()
-	rs.region1.Use(&addr0, 0)
-	rs.region1.Use(&addr1, 1)
-	rs.region2.Use(&addr2, 2)
-	rs.region2.Use(&addr3, 3)
+	tests := []struct {
+		name  string
+		addrs []*EdgeAddr
+		mode  ConfigIPVersion
+	}{
+		{
+			name:  "IPv4 addresses with IPv4Only",
+			addrs: v4Addrs,
+			mode:  IPv4Only,
+		},
+		{
+			name:  "IPv6 addresses with IPv6Only",
+			addrs: v6Addrs,
+			mode:  IPv6Only,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			rs := makeRegions(tt.addrs, tt.mode)
+			rs.region1.AssignAnyAddress(0, nil)
+			rs.region1.AssignAnyAddress(1, nil)
+			addr := rs.region2.AssignAnyAddress(2, nil)
+			rs.region2.AssignAnyAddress(3, nil)
 
-	assert.Equal(t, 0, rs.AvailableAddrs())
+			assert.Equal(t, 0, rs.AvailableAddrs())
 
-	rs.GiveBack(&addr2)
-	assert.Equal(t, &addr2, rs.GetUnusedAddr(nil, 2))
+			rs.GiveBack(addr, false)
+			assert.Equal(t, addr, rs.GetUnusedAddr(nil, 2))
+		})
+	}
 }
 
 func TestRegions_GetUnusedAddr_OneAddrLeft(t *testing.T) {
-	rs := makeRegions()
+	tests := []struct {
+		name  string
+		addrs []*EdgeAddr
+		mode  ConfigIPVersion
+	}{
+		{
+			name:  "IPv4 addresses with IPv4Only",
+			addrs: v4Addrs,
+			mode:  IPv4Only,
+		},
+		{
+			name:  "IPv6 addresses with IPv6Only",
+			addrs: v6Addrs,
+			mode:  IPv6Only,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			rs := makeRegions(tt.addrs, tt.mode)
+			rs.region1.AssignAnyAddress(0, nil)
+			rs.region1.AssignAnyAddress(1, nil)
+			rs.region2.AssignAnyAddress(2, nil)
+			addr := rs.region2.active.GetUnusedIP(nil)
 
-	rs.region1.Use(&addr0, 0)
-	rs.region1.Use(&addr1, 1)
-	rs.region2.Use(&addr2, 2)
-
-	assert.Equal(t, 1, rs.AvailableAddrs())
-	assert.Equal(t, &addr3, rs.GetUnusedAddr(nil, 3))
+			assert.Equal(t, 1, rs.AvailableAddrs())
+			assert.Equal(t, addr, rs.GetUnusedAddr(nil, 3))
+		})
+	}
 }
 
 func TestRegions_GetUnusedAddr_Excluding_Region1(t *testing.T) {
-	rs := makeRegions()
+	tests := []struct {
+		name  string
+		addrs []*EdgeAddr
+		mode  ConfigIPVersion
+	}{
+		{
+			name:  "IPv4 addresses with IPv4Only",
+			addrs: v4Addrs,
+			mode:  IPv4Only,
+		},
+		{
+			name:  "IPv6 addresses with IPv6Only",
+			addrs: v6Addrs,
+			mode:  IPv6Only,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			rs := makeRegions(tt.addrs, tt.mode)
 
-	rs.region1.Use(&addr0, 0)
-	rs.region1.Use(&addr1, 1)
+			rs.region1.AssignAnyAddress(0, nil)
+			rs.region1.AssignAnyAddress(1, nil)
+			addr := rs.region2.active.GetUnusedIP(nil)
+			a2 := rs.region2.active.GetUnusedIP(addr)
 
-	assert.Equal(t, 2, rs.AvailableAddrs())
-	assert.Equal(t, &addr3, rs.GetUnusedAddr(&addr2, 3))
+			assert.Equal(t, 2, rs.AvailableAddrs())
+			assert.Equal(t, addr, rs.GetUnusedAddr(a2, 3))
+		})
+	}
 }
 
 func TestRegions_GetUnusedAddr_Excluding_Region2(t *testing.T) {
-	rs := makeRegions()
+	tests := []struct {
+		name  string
+		addrs []*EdgeAddr
+		mode  ConfigIPVersion
+	}{
+		{
+			name:  "IPv4 addresses with IPv4Only",
+			addrs: v4Addrs,
+			mode:  IPv4Only,
+		},
+		{
+			name:  "IPv6 addresses with IPv6Only",
+			addrs: v6Addrs,
+			mode:  IPv6Only,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			rs := makeRegions(tt.addrs, tt.mode)
 
-	rs.region2.Use(&addr2, 0)
-	rs.region2.Use(&addr3, 1)
+			rs.region2.AssignAnyAddress(0, nil)
+			rs.region2.AssignAnyAddress(1, nil)
+			addr := rs.region1.active.GetUnusedIP(nil)
+			a2 := rs.region1.active.GetUnusedIP(addr)
 
-	assert.Equal(t, 2, rs.AvailableAddrs())
-	assert.Equal(t, &addr1, rs.GetUnusedAddr(&addr0, 1))
+			assert.Equal(t, 2, rs.AvailableAddrs())
+			assert.Equal(t, addr, rs.GetUnusedAddr(a2, 1))
+		})
+	}
 }
 
 func TestNewNoResolveBalancesRegions(t *testing.T) {

edgediscovery/edgediscovery.go

@@ -1,7 +1,6 @@
 package edgediscovery
 
 import (
-	"fmt"
 	"sync"
 
 	"github.com/rs/zerolog"
@@ -11,9 +10,16 @@ import (
 
 const (
 	LogFieldConnIndex = "connIndex"
+	LogFieldIPAddress = "ip"
 )
 
-var errNoAddressesLeft = fmt.Errorf("there are no free edge addresses left")
+var errNoAddressesLeft = ErrNoAddressesLeft{}
+
+type ErrNoAddressesLeft struct{}
+
+func (e ErrNoAddressesLeft) Error() string {
+	return "there are no free edge addresses left to resolve to"
+}
 
 // Edge finds addresses on the Cloudflare edge and hands them out to connections.
 type Edge struct {
@@ -28,8 +34,8 @@ type Edge struct {
 
 // ResolveEdge runs the initial discovery of the Cloudflare edge, finding Addrs that can be allocated
 // to connections.
-func ResolveEdge(log *zerolog.Logger, region string) (*Edge, error) {
-	regions, err := allregions.ResolveEdge(log, region)
+func ResolveEdge(log *zerolog.Logger, region string, edgeIpVersion allregions.ConfigIPVersion) (*Edge, error) {
+	regions, err := allregions.ResolveEdge(log, region, edgeIpVersion)
 	if err != nil {
 		return new(Edge), err
 	}
@@ -51,15 +57,6 @@ func StaticEdge(log *zerolog.Logger, hostnames []string) (*Edge, error) {
 	}, nil
 }
 
-// MockEdge creates a Cloudflare Edge from arbitrary TCP addresses. Used for testing.
-func MockEdge(log *zerolog.Logger, addrs []*allregions.EdgeAddr) *Edge {
-	regions := allregions.NewNoResolve(addrs)
-	return &Edge{
-		log:     log,
-		regions: regions,
-	}
-}
-
 // ------------------------------------
 // Methods
 // ------------------------------------
@@ -93,12 +90,15 @@ func (ed *Edge) GetAddr(connIndex int) (*allregions.EdgeAddr, error) {
 		log.Debug().Msg("edgediscovery - GetAddr: No addresses left to give proxy connection")
 		return nil, errNoAddressesLeft
 	}
-	log.Debug().Msg("edgediscovery - GetAddr: Giving connection its new address")
+	log = ed.log.With().
+		Int(LogFieldConnIndex, connIndex).
+		IPAddr(LogFieldIPAddress, addr.UDP.IP).Logger()
+	log.Debug().Msgf("edgediscovery - GetAddr: Giving connection its new address")
 	return addr, nil
 }
 
 // GetDifferentAddr gives back the proxy connection's edge Addr and uses a new one.
-func (ed *Edge) GetDifferentAddr(connIndex int) (*allregions.EdgeAddr, error) {
+func (ed *Edge) GetDifferentAddr(connIndex int, hasConnectivityError bool) (*allregions.EdgeAddr, error) {
 	log := ed.log.With().Int(LogFieldConnIndex, connIndex).Logger()
 
 	ed.Lock()
@@ -106,7 +106,7 @@ func (ed *Edge) GetDifferentAddr(connIndex int) (*allregions.EdgeAddr, error) {
 
 	oldAddr := ed.regions.AddrUsedBy(connIndex)
 	if oldAddr != nil {
-		ed.regions.GiveBack(oldAddr)
+		ed.regions.GiveBack(oldAddr, hasConnectivityError)
 	}
 	addr := ed.regions.GetUnusedAddr(oldAddr, connIndex)
 	if addr == nil {
@@ -114,8 +114,10 @@ func (ed *Edge) GetDifferentAddr(connIndex int) (*allregions.EdgeAddr, error) {
 		// note: if oldAddr were not nil, it will become available on the next iteration
 		return nil, errNoAddressesLeft
 	}
-	log.Debug().Msgf("edgediscovery - GetDifferentAddr: Giving connection its new address: %v from the address list: %v",
-		addr, ed.regions.AvailableAddrs())
+	log = ed.log.With().
+		Int(LogFieldConnIndex, connIndex).
+		IPAddr(LogFieldIPAddress, addr.UDP.IP).Logger()
+	log.Debug().Msgf("edgediscovery - GetDifferentAddr: Giving connection its new address from the address list: %v", ed.regions.AvailableAddrs())
 	return addr, nil
 }
 
@@ -128,9 +130,11 @@ func (ed *Edge) AvailableAddrs() int {
 
 // GiveBack the address so that other connections can use it.
 // Returns true if the address is in this edge.
-func (ed *Edge) GiveBack(addr *allregions.EdgeAddr) bool {
+func (ed *Edge) GiveBack(addr *allregions.EdgeAddr, hasConnectivityError bool) bool {
 	ed.Lock()
 	defer ed.Unlock()
-	ed.log.Debug().Msg("edgediscovery - GiveBack: Address now unused")
-	return ed.regions.GiveBack(addr)
+	log := ed.log.With().
+		IPAddr(LogFieldIPAddress, addr.UDP.IP).Logger()
+	log.Debug().Msgf("edgediscovery - GiveBack: Address now unused")
+	return ed.regions.GiveBack(addr, hasConnectivityError)
 }

edgediscovery/edgediscovery_test.go

@@ -11,56 +11,113 @@ import (
 )
 
 var (
-	addr0 = allregions.EdgeAddr{
+	testLogger = zerolog.Nop()
+	v4Addrs    = []*allregions.EdgeAddr{&addr0, &addr1, &addr2, &addr3}
+	v6Addrs    = []*allregions.EdgeAddr{&addr4, &addr5, &addr6, &addr7}
+	addr0      = allregions.EdgeAddr{
 		TCP: &net.TCPAddr{
-			IP:   net.ParseIP("123.0.0.0"),
+			IP:   net.ParseIP("123.4.5.0"),
 			Port: 8000,
 			Zone: "",
 		},
 		UDP: &net.UDPAddr{
-			IP:   net.ParseIP("123.0.0.0"),
+			IP:   net.ParseIP("123.4.5.0"),
 			Port: 8000,
 			Zone: "",
 		},
+		IPVersion: allregions.V4,
 	}
 	addr1 = allregions.EdgeAddr{
 		TCP: &net.TCPAddr{
-			IP:   net.ParseIP("123.0.0.1"),
+			IP:   net.ParseIP("123.4.5.1"),
 			Port: 8000,
 			Zone: "",
 		},
 		UDP: &net.UDPAddr{
-			IP:   net.ParseIP("123.0.0.1"),
+			IP:   net.ParseIP("123.4.5.1"),
 			Port: 8000,
 			Zone: "",
 		},
+		IPVersion: allregions.V4,
 	}
 	addr2 = allregions.EdgeAddr{
 		TCP: &net.TCPAddr{
-			IP:   net.ParseIP("123.0.0.2"),
+			IP:   net.ParseIP("123.4.5.2"),
 			Port: 8000,
 			Zone: "",
 		},
 		UDP: &net.UDPAddr{
-			IP:   net.ParseIP("123.0.0.2"),
+			IP:   net.ParseIP("123.4.5.2"),
 			Port: 8000,
 			Zone: "",
 		},
+		IPVersion: allregions.V4,
 	}
 	addr3 = allregions.EdgeAddr{
 		TCP: &net.TCPAddr{
-			IP:   net.ParseIP("123.0.0.3"),
+			IP:   net.ParseIP("123.4.5.3"),
 			Port: 8000,
 			Zone: "",
 		},
 		UDP: &net.UDPAddr{
-			IP:   net.ParseIP("123.0.0.3"),
+			IP:   net.ParseIP("123.4.5.3"),
 			Port: 8000,
 			Zone: "",
 		},
+		IPVersion: allregions.V4,
+	}
+	addr4 = allregions.EdgeAddr{
+		TCP: &net.TCPAddr{
+			IP:   net.ParseIP("2606:4700:a0::1"),
+			Port: 8000,
+			Zone: "",
+		},
+		UDP: &net.UDPAddr{
+			IP:   net.ParseIP("2606:4700:a0::1"),
+			Port: 8000,
+			Zone: "",
+		},
+		IPVersion: allregions.V6,
+	}
+	addr5 = allregions.EdgeAddr{
+		TCP: &net.TCPAddr{
+			IP:   net.ParseIP("2606:4700:a0::2"),
+			Port: 8000,
+			Zone: "",
+		},
+		UDP: &net.UDPAddr{
+			IP:   net.ParseIP("2606:4700:a0::2"),
+			Port: 8000,
+			Zone: "",
+		},
+		IPVersion: allregions.V6,
+	}
+	addr6 = allregions.EdgeAddr{
+		TCP: &net.TCPAddr{
+			IP:   net.ParseIP("2606:4700:a0::3"),
+			Port: 8000,
+			Zone: "",
+		},
+		UDP: &net.UDPAddr{
+			IP:   net.ParseIP("2606:4700:a0::3"),
+			Port: 8000,
+			Zone: "",
+		},
+		IPVersion: allregions.V6,
+	}
+	addr7 = allregions.EdgeAddr{
+		TCP: &net.TCPAddr{
+			IP:   net.ParseIP("2606:4700:a0::4"),
+			Port: 8000,
+			Zone: "",
+		},
+		UDP: &net.UDPAddr{
+			IP:   net.ParseIP("2606:4700:a0::4"),
+			Port: 8000,
+			Zone: "",
+		},
+		IPVersion: allregions.V6,
 	}
-
-	testLogger = zerolog.Nop()
 )
 
 func TestGiveBack(t *testing.T) {
@@ -75,7 +132,7 @@ func TestGiveBack(t *testing.T) {
 	assert.Equal(t, 3, edge.AvailableAddrs())
 
 	// Get it back
-	edge.GiveBack(addr)
+	edge.GiveBack(addr, false)
 	assert.Equal(t, 4, edge.AvailableAddrs())
 }
 
@@ -107,7 +164,7 @@ func TestGetAddrForRPC(t *testing.T) {
 	assert.Equal(t, 4, edge.AvailableAddrs())
 
 	// Get it back
-	edge.GiveBack(addr)
+	edge.GiveBack(addr, false)
 	assert.Equal(t, 4, edge.AvailableAddrs())
 }
 
@@ -122,13 +179,13 @@ func TestOnePerRegion(t *testing.T) {
 	assert.NotNil(t, a1)
 
 	// if the first address is bad, get the second one
-	a2, err := edge.GetDifferentAddr(connID)
+	a2, err := edge.GetDifferentAddr(connID, false)
 	assert.NoError(t, err)
 	assert.NotNil(t, a2)
 	assert.NotEqual(t, a1, a2)
 
 	// now that second one is bad, get the first one again
-	a3, err := edge.GetDifferentAddr(connID)
+	a3, err := edge.GetDifferentAddr(connID, false)
 	assert.NoError(t, err)
 	assert.Equal(t, a1, a3)
 }
@@ -144,11 +201,11 @@ func TestOnlyOneAddrLeft(t *testing.T) {
 	assert.NotNil(t, addr)
 
 	// If that edge address is "bad", there's no alternative address.
-	_, err = edge.GetDifferentAddr(connID)
+	_, err = edge.GetDifferentAddr(connID, false)
 	assert.Error(t, err)
 
 	// previously bad address should become available again on next iteration.
-	addr, err = edge.GetDifferentAddr(connID)
+	addr, err = edge.GetDifferentAddr(connID, false)
 	assert.NoError(t, err)
 	assert.NotNil(t, addr)
 }
@@ -190,8 +247,17 @@ func TestGetDifferentAddr(t *testing.T) {
 	assert.Equal(t, 3, edge.AvailableAddrs())
 
 	// If the same connection requests another address, it should get the same one.
-	addr2, err := edge.GetDifferentAddr(connID)
+	addr2, err := edge.GetDifferentAddr(connID, false)
 	assert.NoError(t, err)
 	assert.NotEqual(t, addr, addr2)
 	assert.Equal(t, 3, edge.AvailableAddrs())
 }
+
+// MockEdge creates a Cloudflare Edge from arbitrary TCP addresses. Used for testing.
+func MockEdge(log *zerolog.Logger, addrs []*allregions.EdgeAddr) *Edge {
+	regions := allregions.NewNoResolve(addrs)
+	return &Edge{
+		log:     log,
+		regions: regions,
+	}
+}

go.mod

@@ -1,6 +1,6 @@
 module github.com/cloudflare/cloudflared
 
-go 1.17
+go 1.18
 
 require (
 	github.com/cloudflare/brotli-go v0.0.0-20191101163834-d34379f7ff93
@@ -9,7 +9,6 @@ require (
 	github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf
 	github.com/facebookgo/grace v0.0.0-20180706040059-75cf19382434
 	github.com/fsnotify/fsnotify v1.4.9
-	github.com/gdamore/tcell v1.3.0
 	github.com/getsentry/raven-go v0.0.0-20180517221441-ed7bcb39ff10
 	github.com/gobwas/ws v1.0.4
 	github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3
@@ -17,14 +16,13 @@ require (
 	github.com/gorilla/mux v1.8.0
 	github.com/gorilla/websocket v1.4.2
 	github.com/json-iterator/go v1.1.12
-	github.com/lucas-clemente/quic-go v0.24.0
+	github.com/lucas-clemente/quic-go v0.28.1
 	github.com/mattn/go-colorable v0.1.8
 	github.com/miekg/dns v1.1.45
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.12.1
 	github.com/prometheus/client_model v0.2.0
-	github.com/rivo/tview v0.0.0-20200712113419-c65badfc3d92
 	github.com/rs/zerolog v1.20.0
 	github.com/stretchr/testify v1.7.1
 	github.com/urfave/cli/v2 v2.3.0
@@ -36,16 +34,15 @@ require (
 	go.opentelemetry.io/proto/otlp v0.15.0
 	go.uber.org/automaxprocs v1.4.0
 	golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f
-	golang.org/x/net v0.0.0-20220114011407-0dd24b26b47d
+	golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
-	golang.org/x/sys v0.0.0-20220114195835-da31bd327af9
-	golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b
+	golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a
+	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211
 	google.golang.org/protobuf v1.28.0
 	gopkg.in/coreos/go-oidc.v2 v2.2.1
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
 	gopkg.in/square/go-jose.v2 v2.6.0
-	gopkg.in/yaml.v2 v2.4.0
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
+	gopkg.in/yaml.v3 v3.0.1
 	zombiezen.com/go/capnproto2 v2.18.0+incompatible
 )
 
@@ -64,8 +61,6 @@ require (
 	github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
 	github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870 // indirect
 	github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 // indirect
-	github.com/francoispqt/gojay v1.2.13 // indirect
-	github.com/gdamore/encoding v1.0.0 // indirect
 	github.com/go-logr/logr v1.2.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
@@ -75,12 +70,11 @@ require (
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect
 	github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
-	github.com/lucasb-eyer/go-colorful v1.0.3 // indirect
-	github.com/marten-seemann/qtls-go1-16 v0.1.4 // indirect
-	github.com/marten-seemann/qtls-go1-17 v0.1.0 // indirect
-	github.com/marten-seemann/qtls-go1-18 v0.1.0-beta.1 // indirect
+	github.com/marten-seemann/qtls-go1-16 v0.1.5 // indirect
+	github.com/marten-seemann/qtls-go1-17 v0.1.2 // indirect
+	github.com/marten-seemann/qtls-go1-18 v0.1.2 // indirect
+	github.com/marten-seemann/qtls-go1-19 v0.1.0-beta.1 // indirect
 	github.com/mattn/go-isatty v0.0.12 // indirect
-	github.com/mattn/go-runewidth v0.0.8 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
@@ -91,7 +85,6 @@ require (
 	github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 // indirect
 	github.com/prometheus/common v0.32.1 // indirect
 	github.com/prometheus/procfs v0.7.3 // indirect
-	github.com/rivo/uniseg v0.1.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	golang.org/x/mod v0.4.2 // indirect
 	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
@@ -102,11 +95,14 @@ require (
 	google.golang.org/genproto v0.0.0-20211223182754-3ac035c7e7cb // indirect
 	google.golang.org/grpc v1.45.0 // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
 )
 
 replace github.com/urfave/cli/v2 => github.com/ipostelnik/cli/v2 v2.3.1-0.20210324024421-b6ea8234fe3d
 
-replace github.com/lucas-clemente/quic-go => github.com/chungthuang/quic-go v0.24.1-0.20220110095058-981dc498cb62
+replace github.com/lucas-clemente/quic-go => github.com/chungthuang/quic-go v0.27.1-0.20220809135021-ca330f1dec9f
 
 // Avoid 'CVE-2022-21698'
 replace github.com/prometheus/golang_client => github.com/prometheus/golang_client v1.12.1
+
+replace gopkg.in/yaml.v3 => gopkg.in/yaml.v3 v3.0.1

go.sum

@@ -70,7 +70,6 @@ github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBp
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
-github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
 github.com/DataDog/datadog-go v4.4.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
 github.com/DataDog/gostackparse v0.5.0/go.mod h1:lTfqcJKqS9KnXQGnyQMCugq3u1FP6UZMfWR0aitKFMM=
 github.com/DataDog/sketches-go v1.0.0/go.mod h1:O+XkJHWk9w4hDwY2ZUDU31ZC9sNYlYo8DiFsxjYeo1k=
@@ -104,15 +103,14 @@ github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInq
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894 h1:JLaf/iINcLyjwbtTsCJjc6rtlASgHeIJPrB6QmwURnA=
 github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA=
-github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
 github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE=
 github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ=
-github.com/chungthuang/quic-go v0.24.1-0.20220110095058-981dc498cb62 h1:PLTB4iA6sOgAItzQY642tYdcGKfG/7i2gu93JQGgUcM=
-github.com/chungthuang/quic-go v0.24.1-0.20220110095058-981dc498cb62/go.mod h1:YtzP8bxRVCBlO77yRanE264+fY/T2U9ZlW1AaHOsMOg=
+github.com/chungthuang/quic-go v0.27.1-0.20220809135021-ca330f1dec9f h1:UWC3XjwZzocdNCzzXxq9j/1SdHMZXhcTOsh/+gNRBUQ=
+github.com/chungthuang/quic-go v0.27.1-0.20220809135021-ca330f1dec9f/go.mod h1:oGz5DKK41cJt5+773+BSO9BXDsREY4HLf7+0odGAPO0=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
@@ -184,16 +182,11 @@ github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
-github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk=
 github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY=
 github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
-github.com/gdamore/encoding v1.0.0 h1:+7OoQ1Bc6eTm5niUzBa0Ctsh6JbMW6Ra+YNuAtDBdko=
-github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo5dl+VrEg=
-github.com/gdamore/tcell v1.3.0 h1:r35w0JBADPZCVQijYebl6YMWWtHRqVEGt7kL2eBADRM=
-github.com/gdamore/tcell v1.3.0/go.mod h1:Hjvr+Ofd+gLglo7RYKxxnzCBmev3BzsS67MebKS4zMM=
 github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg=
 github.com/getsentry/raven-go v0.0.0-20180517221441-ed7bcb39ff10 h1:YO10pIIBftO/kkTFdWhctH96grJ7qiy7bMdiZcIvPKs=
 github.com/getsentry/raven-go v0.0.0-20180517221441-ed7bcb39ff10/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ=
@@ -344,7 +337,6 @@ github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
-github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo=
 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 h1:BZHcxBETFHIdVyhyEfOvn/RdU/QGdLI4y34qQGjGWO0=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks=
@@ -397,29 +389,24 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
-github.com/lucasb-eyer/go-colorful v1.0.2/go.mod h1:0MS4r+7BZKSJ5mw4/S5MPN+qHFF1fYclkSPilDOKW0s=
-github.com/lucasb-eyer/go-colorful v1.0.3 h1:QIbQXiugsb+q10B+MI+7DI1oQLdmnep86tWFlaaUAac=
-github.com/lucasb-eyer/go-colorful v1.0.3/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
 github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI=
 github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
 github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/marten-seemann/qpack v0.2.1/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc=
-github.com/marten-seemann/qtls-go1-15 v0.1.4/go.mod h1:GyFwywLKkRt+6mfU99csTEY1joMZz5vmB1WNZH3P81I=
-github.com/marten-seemann/qtls-go1-16 v0.1.4 h1:xbHbOGGhrenVtII6Co8akhLEdrawwB2iHl5yhJRpnco=
-github.com/marten-seemann/qtls-go1-16 v0.1.4/go.mod h1:gNpI2Ol+lRS3WwSOtIUUtRwZEQMXjYK+dQSBFbethAk=
-github.com/marten-seemann/qtls-go1-17 v0.1.0 h1:P9ggrs5xtwiqXv/FHNwntmuLMNq3KaSIG93AtAZ48xk=
-github.com/marten-seemann/qtls-go1-17 v0.1.0/go.mod h1:fz4HIxByo+LlWcreM4CZOYNuz3taBQ8rN2X6FqvaWo8=
-github.com/marten-seemann/qtls-go1-18 v0.1.0-beta.1 h1:EnzzN9fPUkUck/1CuY1FlzBaIYMoiBsdwTNmNGkwUUM=
-github.com/marten-seemann/qtls-go1-18 v0.1.0-beta.1/go.mod h1:PUhIQk19LoFt2174H4+an8TYvWOGjb/hHwphBeaDHwI=
+github.com/marten-seemann/qtls-go1-16 v0.1.5 h1:o9JrYPPco/Nukd/HpOHMHZoBDXQqoNtUCmny98/1uqQ=
+github.com/marten-seemann/qtls-go1-16 v0.1.5/go.mod h1:gNpI2Ol+lRS3WwSOtIUUtRwZEQMXjYK+dQSBFbethAk=
+github.com/marten-seemann/qtls-go1-17 v0.1.2 h1:JADBlm0LYiVbuSySCHeY863dNkcpMmDR7s0bLKJeYlQ=
+github.com/marten-seemann/qtls-go1-17 v0.1.2/go.mod h1:C2ekUKcDdz9SDWxec1N/MvcXBpaX9l3Nx67XaR84L5s=
+github.com/marten-seemann/qtls-go1-18 v0.1.2 h1:JH6jmzbduz0ITVQ7ShevK10Av5+jBEKAHMntXmIV7kM=
+github.com/marten-seemann/qtls-go1-18 v0.1.2/go.mod h1:mJttiymBAByA49mhlNZZGrH5u1uXYZJ+RW28Py7f4m4=
+github.com/marten-seemann/qtls-go1-19 v0.1.0-beta.1 h1:7m/WlWcSROrcK5NxuXaxYD32BZqe/LEEnBrWcH/cOqQ=
+github.com/marten-seemann/qtls-go1-19 v0.1.0-beta.1/go.mod h1:5HTDWtVudo/WFsHKRNuOhWlbdjrfs5JHrYb0wIJqGpI=
 github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8=
 github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
 github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
-github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
-github.com/mattn/go-runewidth v0.0.8 h1:3tS41NlGYSmhhe/8fhGRzc+z3AYCw1Fe1WAyLuujKs0=
-github.com/mattn/go-runewidth v0.0.8/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4=
@@ -515,16 +502,11 @@ github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1
 github.com/rabbitmq/amqp091-go v1.1.0/go.mod h1:ogQDLSOACsLPsIq0NpbtiifNZi2YOz0VTJ0kHRghqbM=
 github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
-github.com/rivo/tview v0.0.0-20200712113419-c65badfc3d92 h1:rqaqSUdaW+OBbjnsrOoiaJv43mSRARuvsAuirmdxu7E=
-github.com/rivo/tview v0.0.0-20200712113419-c65badfc3d92/go.mod h1:6lkG1x+13OShEf0EaOCaTQYyB7d5nSbb181KtjlS+84=
-github.com/rivo/uniseg v0.1.0 h1:+2KBaVoUmb9XzDsrx/Ct0W/EYOSFf/nWTauy++DprtY=
-github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
 github.com/rs/zerolog v1.20.0 h1:38k9hgtUBdxFwE34yS8rTHmHBa4eN16E4DJlv177LNs=
 github.com/rs/zerolog v1.20.0/go.mod h1:IzD0RJ65iWH0w97OQQebJEvTZYvsCUm9WVLWBQrJRjo=
-github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
@@ -724,11 +706,10 @@ golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qx
 golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210917221730-978cfadd31cf/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20220114011407-0dd24b26b47d h1:1n1fc535VhN8SYtD4cDUyNlfpAF2ROMM9+11equK3hs=
-golang.org/x/net v0.0.0-20220114011407-0dd24b26b47d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e h1:TsQ7F31D3bUCLeqPT0u+yjp1guoArKaNKmCr22PYgTQ=
+golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -775,7 +756,6 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190626150813-e07cf5db2756/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -833,11 +813,13 @@ golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 h1:XfKQ4OlFl8okEOr5UvAqFRVj8pY/4yfcXrddB8qAbU0=
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b h1:9zKuko04nR4gjZ4+DNjHqRlAJqbJETHwiNKDqTfOjfE=
 golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -1108,10 +1090,8 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o=
 honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

ingress/config.go

@@ -12,10 +12,11 @@ import (
 )
 
 var (
-	defaultConnectTimeout   = config.CustomDuration{Duration: 30 * time.Second}
-	defaultTLSTimeout       = config.CustomDuration{Duration: 10 * time.Second}
-	defaultTCPKeepAlive     = config.CustomDuration{Duration: 30 * time.Second}
-	defaultKeepAliveTimeout = config.CustomDuration{Duration: 90 * time.Second}
+	defaultHTTPConnectTimeout        = config.CustomDuration{Duration: 30 * time.Second}
+	defaultWarpRoutingConnectTimeout = config.CustomDuration{Duration: 5 * time.Second}
+	defaultTLSTimeout                = config.CustomDuration{Duration: 10 * time.Second}
+	defaultTCPKeepAlive              = config.CustomDuration{Duration: 30 * time.Second}
+	defaultKeepAliveTimeout          = config.CustomDuration{Duration: 90 * time.Second}
 )
 
 const (
@@ -35,16 +36,51 @@ const (
 	NoChunkedEncodingFlag         = "no-chunked-encoding"
 	ProxyAddressFlag              = "proxy-address"
 	ProxyPortFlag                 = "proxy-port"
+	Http2OriginFlag               = "http2-origin"
 )
 
 const (
 	socksProxy = "socks"
 )
 
+type WarpRoutingConfig struct {
+	Enabled        bool                  `yaml:"enabled" json:"enabled"`
+	ConnectTimeout config.CustomDuration `yaml:"connectTimeout" json:"connectTimeout,omitempty"`
+	TCPKeepAlive   config.CustomDuration `yaml:"tcpKeepAlive" json:"tcpKeepAlive,omitempty"`
+}
+
+func NewWarpRoutingConfig(raw *config.WarpRoutingConfig) WarpRoutingConfig {
+	cfg := WarpRoutingConfig{
+		Enabled:        raw.Enabled,
+		ConnectTimeout: defaultWarpRoutingConnectTimeout,
+		TCPKeepAlive:   defaultTCPKeepAlive,
+	}
+	if raw.ConnectTimeout != nil {
+		cfg.ConnectTimeout = *raw.ConnectTimeout
+	}
+	if raw.TCPKeepAlive != nil {
+		cfg.TCPKeepAlive = *raw.TCPKeepAlive
+	}
+	return cfg
+}
+
+func (c *WarpRoutingConfig) RawConfig() config.WarpRoutingConfig {
+	raw := config.WarpRoutingConfig{
+		Enabled: c.Enabled,
+	}
+	if c.ConnectTimeout.Duration != defaultWarpRoutingConnectTimeout.Duration {
+		raw.ConnectTimeout = &c.ConnectTimeout
+	}
+	if c.TCPKeepAlive.Duration != defaultTCPKeepAlive.Duration {
+		raw.TCPKeepAlive = &c.TCPKeepAlive
+	}
+	return raw
+}
+
 // RemoteConfig models ingress settings that can be managed remotely, for example through the dashboard.
 type RemoteConfig struct {
 	Ingress     Ingress
-	WarpRouting config.WarpRoutingConfig
+	WarpRouting WarpRoutingConfig
 }
 
 type RemoteConfigJSON struct {
@@ -72,18 +108,18 @@ func (rc *RemoteConfig) UnmarshalJSON(b []byte) error {
 	}
 
 	rc.Ingress = ingress
-	rc.WarpRouting = rawConfig.WarpRouting
+	rc.WarpRouting = NewWarpRoutingConfig(&rawConfig.WarpRouting)
 
 	return nil
 }
 
 func originRequestFromSingeRule(c *cli.Context) OriginRequestConfig {
-	var connectTimeout config.CustomDuration = defaultConnectTimeout
-	var tlsTimeout config.CustomDuration = defaultTLSTimeout
-	var tcpKeepAlive config.CustomDuration = defaultTCPKeepAlive
+	var connectTimeout = defaultHTTPConnectTimeout
+	var tlsTimeout = defaultTLSTimeout
+	var tcpKeepAlive = defaultTCPKeepAlive
 	var noHappyEyeballs bool
-	var keepAliveConnections int = defaultKeepAliveConnections
-	var keepAliveTimeout config.CustomDuration = defaultKeepAliveTimeout
+	var keepAliveConnections = defaultKeepAliveConnections
+	var keepAliveTimeout = defaultKeepAliveTimeout
 	var httpHostHeader string
 	var originServerName string
 	var caPool string
@@ -93,6 +129,7 @@ func originRequestFromSingeRule(c *cli.Context) OriginRequestConfig {
 	var proxyAddress = defaultProxyAddress
 	var proxyPort uint
 	var proxyType string
+	var http2Origin bool
 	if flag := ProxyConnectTimeoutFlag; c.IsSet(flag) {
 		connectTimeout = config.CustomDuration{Duration: c.Duration(flag)}
 	}
@@ -136,9 +173,13 @@ func originRequestFromSingeRule(c *cli.Context) OriginRequestConfig {
 		// Note TUN-3758 , we use Int because UInt is not supported with altsrc
 		proxyPort = uint(c.Int(flag))
 	}
+	if flag := Http2OriginFlag; c.IsSet(flag) {
+		http2Origin = c.Bool(flag)
+	}
 	if c.IsSet(Socks5Flag) {
 		proxyType = socksProxy
 	}
+
 	return OriginRequestConfig{
 		ConnectTimeout:         connectTimeout,
 		TLSTimeout:             tlsTimeout,
@@ -155,12 +196,13 @@ func originRequestFromSingeRule(c *cli.Context) OriginRequestConfig {
 		ProxyAddress:           proxyAddress,
 		ProxyPort:              proxyPort,
 		ProxyType:              proxyType,
+		Http2Origin:            http2Origin,
 	}
 }
 
 func originRequestFromConfig(c config.OriginRequestConfig) OriginRequestConfig {
 	out := OriginRequestConfig{
-		ConnectTimeout:       defaultConnectTimeout,
+		ConnectTimeout:       defaultHTTPConnectTimeout,
 		TLSTimeout:           defaultTLSTimeout,
 		TCPKeepAlive:         defaultTCPKeepAlive,
 		KeepAliveConnections: defaultKeepAliveConnections,
@@ -220,6 +262,9 @@ func originRequestFromConfig(c config.OriginRequestConfig) OriginRequestConfig {
 			}
 		}
 	}
+	if c.Http2Origin != nil {
+		out.Http2Origin = *c.Http2Origin
+	}
 	return out
 }
 
@@ -263,6 +308,8 @@ type OriginRequestConfig struct {
 	ProxyType string `yaml:"proxyType" json:"proxyType"`
 	// IP rules for the proxy service
 	IPRules []ipaccess.Rule `yaml:"ipRules" json:"ipRules"`
+	// Attempt to connect to origin with HTTP/2
+	Http2Origin bool `yaml:"http2Origin" json:"http2Origin"`
 }
 
 func (defaults *OriginRequestConfig) setConnectTimeout(overrides config.OriginRequestConfig) {
@@ -368,6 +415,12 @@ func (defaults *OriginRequestConfig) setIPRules(overrides config.OriginRequestCo
 	}
 }
 
+func (defaults *OriginRequestConfig) setHttp2Origin(overrides config.OriginRequestConfig) {
+	if val := overrides.Http2Origin; val != nil {
+		defaults.Http2Origin = *val
+	}
+}
+
 // SetConfig gets config for the requests that cloudflared sends to origins.
 // Each field has a setter method which sets a value for the field by trying to find:
 //   1. The user config for this rule
@@ -393,6 +446,7 @@ func setConfig(defaults OriginRequestConfig, overrides config.OriginRequestConfi
 	cfg.setProxyAddress(overrides)
 	cfg.setProxyType(overrides)
 	cfg.setIPRules(overrides)
+	cfg.setHttp2Origin(overrides)
 	return cfg
 }
 
@@ -404,7 +458,7 @@ func ConvertToRawOriginConfig(c OriginRequestConfig) config.OriginRequestConfig
 	var keepAliveTimeout *config.CustomDuration
 	var proxyAddress *string
 
-	if c.ConnectTimeout != defaultConnectTimeout {
+	if c.ConnectTimeout != defaultHTTPConnectTimeout {
 		connectTimeout = &c.ConnectTimeout
 	}
 	if c.TLSTimeout != defaultTLSTimeout {
@@ -440,6 +494,7 @@ func ConvertToRawOriginConfig(c OriginRequestConfig) config.OriginRequestConfig
 		ProxyPort:              zeroUIntToNil(c.ProxyPort),
 		ProxyType:              emptyStringToNil(c.ProxyType),
 		IPRules:                convertToRawIPRules(c.IPRules),
+		Http2Origin:            defaultBoolToNil(c.Http2Origin),
 	}
 }
 

ingress/config_test.go

@@ -8,7 +8,7 @@ import (
 
 	"github.com/stretchr/testify/require"
 	"github.com/urfave/cli/v2"
-	yaml "gopkg.in/yaml.v2"
+	yaml "gopkg.in/yaml.v3"
 
 	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/ipaccess"
@@ -274,7 +274,7 @@ func TestOriginRequestConfigDefaults(t *testing.T) {
 		// Rule 0 didn't override anything, so it inherits the cloudflared defaults
 		actual0 := ing.Rules[0].Config
 		expected0 := OriginRequestConfig{
-			ConnectTimeout:       defaultConnectTimeout,
+			ConnectTimeout:       defaultHTTPConnectTimeout,
 			TLSTimeout:           defaultTLSTimeout,
 			TCPKeepAlive:         defaultTCPKeepAlive,
 			KeepAliveConnections: defaultKeepAliveConnections,
@@ -404,7 +404,7 @@ func TestDefaultConfigFromCLI(t *testing.T) {
 	c := cli.NewContext(nil, set, nil)
 
 	expected := OriginRequestConfig{
-		ConnectTimeout:       defaultConnectTimeout,
+		ConnectTimeout:       defaultHTTPConnectTimeout,
 		TLSTimeout:           defaultTLSTimeout,
 		TCPKeepAlive:         defaultTCPKeepAlive,
 		KeepAliveConnections: defaultKeepAliveConnections,

ingress/ingress.go

@@ -97,8 +97,16 @@ type WarpRoutingService struct {
 	Proxy StreamBasedOriginProxy
 }
 
-func NewWarpRoutingService() *WarpRoutingService {
-	return &WarpRoutingService{Proxy: &rawTCPService{name: ServiceWarpRouting}}
+func NewWarpRoutingService(config WarpRoutingConfig) *WarpRoutingService {
+	svc := &rawTCPService{
+		name: ServiceWarpRouting,
+		dialer: net.Dialer{
+			Timeout:   config.ConnectTimeout.Duration,
+			KeepAlive: config.TCPKeepAlive.Duration,
+		},
+	}
+
+	return &WarpRoutingService{Proxy: svc}
 }
 
 // Get a single origin service from the CLI/config.

ingress/ingress_test.go

@@ -12,7 +12,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"github.com/urfave/cli/v2"
-	yaml "gopkg.in/yaml.v2"
+	yaml "gopkg.in/yaml.v3"
 
 	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/ipaccess"

ingress/origin_proxy.go

@@ -1,26 +1,20 @@
 package ingress
 
 import (
+	"context"
 	"fmt"
-	"net"
 	"net/http"
-
-	"github.com/pkg/errors"
-)
-
-var (
-	errUnsupportedConnectionType = errors.New("internal error: unsupported connection type")
 )
 
 // HTTPOriginProxy can be implemented by origin services that want to proxy http requests.
 type HTTPOriginProxy interface {
-	// RoundTrip is how cloudflared proxies eyeball requests to the actual origin services
+	// RoundTripper is how cloudflared proxies eyeball requests to the actual origin services
 	http.RoundTripper
 }
 
 // StreamBasedOriginProxy can be implemented by origin services that want to proxy ws/TCP.
 type StreamBasedOriginProxy interface {
-	EstablishConnection(dest string) (OriginConnection, error)
+	EstablishConnection(ctx context.Context, dest string) (OriginConnection, error)
 }
 
 func (o *unixSocketPath) RoundTrip(req *http.Request) (*http.Response, error) {
@@ -59,8 +53,8 @@ func (o *statusCode) RoundTrip(_ *http.Request) (*http.Response, error) {
 	return resp, nil
 }
 
-func (o *rawTCPService) EstablishConnection(dest string) (OriginConnection, error) {
-	conn, err := net.Dial("tcp", dest)
+func (o *rawTCPService) EstablishConnection(ctx context.Context, dest string) (OriginConnection, error) {
+	conn, err := o.dialer.DialContext(ctx, "tcp", dest)
 	if err != nil {
 		return nil, err
 	}
@@ -71,13 +65,13 @@ func (o *rawTCPService) EstablishConnection(dest string) (OriginConnection, erro
 	return originConn, nil
 }
 
-func (o *tcpOverWSService) EstablishConnection(dest string) (OriginConnection, error) {
+func (o *tcpOverWSService) EstablishConnection(ctx context.Context, dest string) (OriginConnection, error) {
 	var err error
 	if !o.isBastion {
 		dest = o.dest
 	}
 
-	conn, err := net.Dial("tcp", dest)
+	conn, err := o.dialer.DialContext(ctx, "tcp", dest)
 	if err != nil {
 		return nil, err
 	}
@@ -89,6 +83,6 @@ func (o *tcpOverWSService) EstablishConnection(dest string) (OriginConnection, e
 
 }
 
-func (o *socksProxyOverWSService) EstablishConnection(dest string) (OriginConnection, error) {
+func (o *socksProxyOverWSService) EstablishConnection(_ctx context.Context, _dest string) (OriginConnection, error) {
 	return o.conn, nil
 }

ingress/origin_proxy_test.go

@@ -36,7 +36,7 @@ func TestRawTCPServiceEstablishConnection(t *testing.T) {
 	require.NoError(t, err)
 
 	// Origin not listening for new connection, should return an error
-	_, err = rawTCPService.EstablishConnection(req.URL.String())
+	_, err = rawTCPService.EstablishConnection(context.Background(), req.URL.String())
 	require.Error(t, err)
 }
 
@@ -87,7 +87,7 @@ func TestTCPOverWSServiceEstablishConnection(t *testing.T) {
 		t.Run(test.testCase, func(t *testing.T) {
 			if test.expectErr {
 				bastionHost, _ := carrier.ResolveBastionDest(test.req)
-				_, err := test.service.EstablishConnection(bastionHost)
+				_, err := test.service.EstablishConnection(context.Background(), bastionHost)
 				assert.Error(t, err)
 			}
 		})
@@ -99,7 +99,7 @@ func TestTCPOverWSServiceEstablishConnection(t *testing.T) {
 	for _, service := range []*tcpOverWSService{newTCPOverWSService(originURL), newBastionService()} {
 		// Origin not listening for new connection, should return an error
 		bastionHost, _ := carrier.ResolveBastionDest(bastionReq)
-		_, err := service.EstablishConnection(bastionHost)
+		_, err := service.EstablishConnection(context.Background(), bastionHost)
 		assert.Error(t, err)
 	}
 }

ingress/origin_service.go

@@ -91,7 +91,8 @@ func (o httpService) MarshalJSON() ([]byte, error) {
 // rawTCPService dials TCP to the destination specified by the client
 // It's used by warp routing
 type rawTCPService struct {
-	name string
+	name   string
+	dialer net.Dialer
 }
 
 func (o *rawTCPService) String() string {
@@ -113,6 +114,7 @@ type tcpOverWSService struct {
 	dest          string
 	isBastion     bool
 	streamHandler streamHandlerFunc
+	dialer        net.Dialer
 }
 
 type socksProxyOverWSService struct {
@@ -176,6 +178,8 @@ func (o *tcpOverWSService) start(log *zerolog.Logger, _ <-chan struct{}, cfg Ori
 	} else {
 		o.streamHandler = DefaultStreamHandler
 	}
+	o.dialer.Timeout = cfg.ConnectTimeout.Duration
+	o.dialer.KeepAlive = cfg.TCPKeepAlive.Duration
 	return nil
 }
 
@@ -286,6 +290,7 @@ func newHTTPTransport(service OriginService, cfg OriginRequestConfig, log *zerol
 		TLSHandshakeTimeout:   cfg.TLSTimeout.Duration,
 		ExpectContinueTimeout: 1 * time.Second,
 		TLSClientConfig:       &tls.Config{RootCAs: originCertPool, InsecureSkipVerify: cfg.NoTLSVerify},
+		ForceAttemptHTTP2:     cfg.Http2Origin,
 	}
 	if _, isHelloWorld := service.(*helloWorld); !isHelloWorld && cfg.OriginServerName != "" {
 		httpTransport.TLSClientConfig.ServerName = cfg.OriginServerName

ingress/rule_test.go

@@ -182,25 +182,25 @@ func TestMarshalJSON(t *testing.T) {
 		{
 			name:     "Nil",
 			path:     nil,
-			expected: `{"hostname":"example.com","path":null,"service":"https://localhost:8000","originRequest":{"connectTimeout":30,"tlsTimeout":10,"tcpKeepAlive":30,"noHappyEyeballs":false,"keepAliveTimeout":90,"keepAliveConnections":100,"httpHostHeader":"","originServerName":"","caPool":"","noTLSVerify":false,"disableChunkedEncoding":false,"bastionMode":false,"proxyAddress":"127.0.0.1","proxyPort":0,"proxyType":"","ipRules":null}}`,
+			expected: `{"hostname":"example.com","path":null,"service":"https://localhost:8000","originRequest":{"connectTimeout":30,"tlsTimeout":10,"tcpKeepAlive":30,"noHappyEyeballs":false,"keepAliveTimeout":90,"keepAliveConnections":100,"httpHostHeader":"","originServerName":"","caPool":"","noTLSVerify":false,"disableChunkedEncoding":false,"bastionMode":false,"proxyAddress":"127.0.0.1","proxyPort":0,"proxyType":"","ipRules":null,"http2Origin":false}}`,
 			want:     true,
 		},
 		{
 			name:     "Nil regex",
 			path:     &Regexp{Regexp: nil},
-			expected: `{"hostname":"example.com","path":null,"service":"https://localhost:8000","originRequest":{"connectTimeout":30,"tlsTimeout":10,"tcpKeepAlive":30,"noHappyEyeballs":false,"keepAliveTimeout":90,"keepAliveConnections":100,"httpHostHeader":"","originServerName":"","caPool":"","noTLSVerify":false,"disableChunkedEncoding":false,"bastionMode":false,"proxyAddress":"127.0.0.1","proxyPort":0,"proxyType":"","ipRules":null}}`,
+			expected: `{"hostname":"example.com","path":null,"service":"https://localhost:8000","originRequest":{"connectTimeout":30,"tlsTimeout":10,"tcpKeepAlive":30,"noHappyEyeballs":false,"keepAliveTimeout":90,"keepAliveConnections":100,"httpHostHeader":"","originServerName":"","caPool":"","noTLSVerify":false,"disableChunkedEncoding":false,"bastionMode":false,"proxyAddress":"127.0.0.1","proxyPort":0,"proxyType":"","ipRules":null,"http2Origin":false}}`,
 			want:     true,
 		},
 		{
 			name:     "Empty",
 			path:     &Regexp{Regexp: regexp.MustCompile("")},
-			expected: `{"hostname":"example.com","path":"","service":"https://localhost:8000","originRequest":{"connectTimeout":30,"tlsTimeout":10,"tcpKeepAlive":30,"noHappyEyeballs":false,"keepAliveTimeout":90,"keepAliveConnections":100,"httpHostHeader":"","originServerName":"","caPool":"","noTLSVerify":false,"disableChunkedEncoding":false,"bastionMode":false,"proxyAddress":"127.0.0.1","proxyPort":0,"proxyType":"","ipRules":null}}`,
+			expected: `{"hostname":"example.com","path":"","service":"https://localhost:8000","originRequest":{"connectTimeout":30,"tlsTimeout":10,"tcpKeepAlive":30,"noHappyEyeballs":false,"keepAliveTimeout":90,"keepAliveConnections":100,"httpHostHeader":"","originServerName":"","caPool":"","noTLSVerify":false,"disableChunkedEncoding":false,"bastionMode":false,"proxyAddress":"127.0.0.1","proxyPort":0,"proxyType":"","ipRules":null,"http2Origin":false}}`,
 			want:     true,
 		},
 		{
 			name:     "Basic",
 			path:     &Regexp{Regexp: regexp.MustCompile("/echo")},
-			expected: `{"hostname":"example.com","path":"/echo","service":"https://localhost:8000","originRequest":{"connectTimeout":30,"tlsTimeout":10,"tcpKeepAlive":30,"noHappyEyeballs":false,"keepAliveTimeout":90,"keepAliveConnections":100,"httpHostHeader":"","originServerName":"","caPool":"","noTLSVerify":false,"disableChunkedEncoding":false,"bastionMode":false,"proxyAddress":"127.0.0.1","proxyPort":0,"proxyType":"","ipRules":null}}`,
+			expected: `{"hostname":"example.com","path":"/echo","service":"https://localhost:8000","originRequest":{"connectTimeout":30,"tlsTimeout":10,"tcpKeepAlive":30,"noHappyEyeballs":false,"keepAliveTimeout":90,"keepAliveConnections":100,"httpHostHeader":"","originServerName":"","caPool":"","noTLSVerify":false,"disableChunkedEncoding":false,"bastionMode":false,"proxyAddress":"127.0.0.1","proxyPort":0,"proxyType":"","ipRules":null,"http2Origin":false}}`,
 			want:     true,
 		},
 	}

metrics/readiness.go

@@ -5,21 +5,24 @@ import (
 	"fmt"
 	"net/http"
 
+	"github.com/google/uuid"
+	"github.com/rs/zerolog"
+
 	conn "github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/tunnelstate"
-
-	"github.com/rs/zerolog"
 )
 
 // ReadyServer serves HTTP 200 if the tunnel can serve traffic. Intended for k8s readiness checks.
 type ReadyServer struct {
-	tracker *tunnelstate.ConnTracker
+	clientID uuid.UUID
+	tracker  *tunnelstate.ConnTracker
 }
 
 // NewReadyServer initializes a ReadyServer and starts listening for dis/connection events.
-func NewReadyServer(log *zerolog.Logger) *ReadyServer {
+func NewReadyServer(log *zerolog.Logger, clientID uuid.UUID) *ReadyServer {
 	return &ReadyServer{
-		tracker: tunnelstate.NewConnTracker(log),
+		clientID: clientID,
+		tracker:  tunnelstate.NewConnTracker(log),
 	}
 }
 
@@ -28,8 +31,9 @@ func (rs *ReadyServer) OnTunnelEvent(c conn.Event) {
 }
 
 type body struct {
-	Status           int  `json:"status"`
-	ReadyConnections uint `json:"readyConnections"`
+	Status           int       `json:"status"`
+	ReadyConnections uint      `json:"readyConnections"`
+	ConnectorID      uuid.UUID `json:"connectorId"`
 }
 
 // ServeHTTP responds with HTTP 200 if the tunnel is connected to the edge.
@@ -39,6 +43,7 @@ func (rs *ReadyServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	body := body{
 		Status:           statusCode,
 		ReadyConnections: readyConnections,
+		ConnectorID:      rs.clientID,
 	}
 	msg, err := json.Marshal(body)
 	if err != nil {

metrics/readiness_test.go

@@ -4,12 +4,12 @@ import (
 	"net/http"
 	"testing"
 
+	"github.com/google/uuid"
 	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/assert"
 
-	"github.com/cloudflare/cloudflared/tunnelstate"
-
 	"github.com/cloudflare/cloudflared/connection"
+	"github.com/cloudflare/cloudflared/tunnelstate"
 )
 
 func TestReadyServer_makeResponse(t *testing.T) {
@@ -66,7 +66,7 @@ func TestReadyServer_makeResponse(t *testing.T) {
 
 func TestReadinessEventHandling(t *testing.T) {
 	nopLogger := zerolog.Nop()
-	rs := NewReadyServer(&nopLogger)
+	rs := NewReadyServer(&nopLogger, uuid.Nil)
 
 	// start not ok
 	code, ready := rs.makeResponse()

orchestration/config.go

@@ -19,8 +19,8 @@ type newLocalConfig struct {
 
 // Config is the original config as read and parsed by cloudflared.
 type Config struct {
-	Ingress            *ingress.Ingress
-	WarpRoutingEnabled bool
+	Ingress     *ingress.Ingress
+	WarpRouting ingress.WarpRoutingConfig
 
 	// Extra settings used to configure this instance but that are not eligible for remotely management
 	// ie. (--protocol, --loglevel, ...)
@@ -37,7 +37,7 @@ func (rc *newLocalConfig) MarshalJSON() ([]byte, error) {
 			// UI doesn't support top level configs, so we reconcile to individual ingress configs.
 			GlobalOriginRequest: nil,
 			IngressRules:        convertToUnvalidatedIngressRules(rc.RemoteConfig.Ingress),
-			WarpRouting:         rc.RemoteConfig.WarpRouting,
+			WarpRouting:         rc.RemoteConfig.WarpRouting.RawConfig(),
 		},
 	}
 

orchestration/config_test.go

@@ -3,16 +3,17 @@ package orchestration
 import (
 	"encoding/json"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/require"
 
+	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/ingress"
 )
 
 // TestNewLocalConfig_MarshalJSON tests that we are able to converte a compiled and validated config back
 // into an "unvalidated" format which is compatible with Remote Managed configurations.
 func TestNewLocalConfig_MarshalJSON(t *testing.T) {
-
 	rawConfig := []byte(`
 	{
 		"originRequest": {
@@ -57,7 +58,11 @@ func TestNewLocalConfig_MarshalJSON(t *testing.T) {
 					]
 				}
 			}
-		]
+		],
+        "warp-routing": {
+			"enabled": true,
+			"connectTimeout": 1
+        }
 	}
 	`)
 
@@ -73,10 +78,18 @@ func TestNewLocalConfig_MarshalJSON(t *testing.T) {
 	jsonSerde, err := json.Marshal(c)
 	require.NoError(t, err)
 
-	var config ingress.RemoteConfig
-	err = json.Unmarshal(jsonSerde, &config)
+	var remoteConfig ingress.RemoteConfig
+	err = json.Unmarshal(jsonSerde, &remoteConfig)
 	require.NoError(t, err)
 
-	require.Equal(t, config.WarpRouting.Enabled, false)
-	require.Equal(t, config.Ingress.Rules, expectedConfig.Ingress.Rules)
+	require.Equal(t, remoteConfig.WarpRouting, ingress.WarpRoutingConfig{
+		Enabled: true,
+		ConnectTimeout: config.CustomDuration{
+			Duration: time.Second,
+		},
+		TCPKeepAlive: config.CustomDuration{
+			Duration: 30 * time.Second, // default value is 30 seconds
+		},
+	})
+	require.Equal(t, remoteConfig.Ingress.Rules, expectedConfig.Ingress.Rules)
 }

orchestration/orchestrator.go

@@ -47,7 +47,7 @@ func NewOrchestrator(ctx context.Context, config *Config, tags []tunnelpogs.Tag,
 		log:            log,
 		shutdownC:      ctx.Done(),
 	}
-	if err := o.updateIngress(*config.Ingress, config.WarpRoutingEnabled); err != nil {
+	if err := o.updateIngress(*config.Ingress, config.WarpRouting); err != nil {
 		return nil, err
 	}
 	go o.waitToCloseLastProxy()
@@ -80,7 +80,7 @@ func (o *Orchestrator) UpdateConfig(version int32, config []byte) *tunnelpogs.Up
 		}
 	}
 
-	if err := o.updateIngress(newConf.Ingress, newConf.WarpRouting.Enabled); err != nil {
+	if err := o.updateIngress(newConf.Ingress, newConf.WarpRouting); err != nil {
 		o.log.Err(err).
 			Int32("version", version).
 			Str("config", string(config)).
@@ -103,7 +103,7 @@ func (o *Orchestrator) UpdateConfig(version int32, config []byte) *tunnelpogs.Up
 }
 
 // The caller is responsible to make sure there is no concurrent access
-func (o *Orchestrator) updateIngress(ingressRules ingress.Ingress, warpRoutingEnabled bool) error {
+func (o *Orchestrator) updateIngress(ingressRules ingress.Ingress, warpRouting ingress.WarpRoutingConfig) error {
 	select {
 	case <-o.shutdownC:
 		return fmt.Errorf("cloudflared already shutdown")
@@ -118,10 +118,10 @@ func (o *Orchestrator) updateIngress(ingressRules ingress.Ingress, warpRoutingEn
 	if err := ingressRules.StartOrigins(o.log, proxyShutdownC); err != nil {
 		return errors.Wrap(err, "failed to start origin")
 	}
-	newProxy := proxy.NewOriginProxy(ingressRules, warpRoutingEnabled, o.tags, o.log)
+	newProxy := proxy.NewOriginProxy(ingressRules, warpRouting, o.tags, o.log)
 	o.proxy.Store(newProxy)
 	o.config.Ingress = &ingressRules
-	o.config.WarpRoutingEnabled = warpRoutingEnabled
+	o.config.WarpRouting = warpRouting
 
 	// If proxyShutdownC is nil, there is no previous running proxy
 	if o.proxyShutdownC != nil {
@@ -139,7 +139,7 @@ func (o *Orchestrator) GetConfigJSON() ([]byte, error) {
 	c := &newLocalConfig{
 		RemoteConfig: ingress.RemoteConfig{
 			Ingress:     *o.config.Ingress,
-			WarpRouting: config.WarpRoutingConfig{Enabled: o.config.WarpRoutingEnabled},
+			WarpRouting: o.config.WarpRouting,
 		},
 		ConfigurationFlags: o.config.ConfigurationFlags,
 	}
@@ -166,7 +166,7 @@ func (o *Orchestrator) GetVersionedConfigJSON() ([]byte, error) {
 			OriginRequest ingress.OriginRequestConfig `json:"originRequest"`
 		}{
 			Ingress:       o.config.Ingress.Rules,
-			WarpRouting:   config.WarpRoutingConfig{Enabled: o.config.WarpRoutingEnabled},
+			WarpRouting:   o.config.WarpRouting.RawConfig(),
 			OriginRequest: o.config.Ingress.Defaults,
 		},
 	}

orchestration/orchestrator_test.go

@@ -48,8 +48,7 @@ var (
 // - receiving an old version is noop
 func TestUpdateConfiguration(t *testing.T) {
 	initConfig := &Config{
-		Ingress:            &ingress.Ingress{},
-		WarpRoutingEnabled: false,
+		Ingress: &ingress.Ingress{},
 	}
 	orchestrator, err := NewOrchestrator(context.Background(), initConfig, testTags, &testLogger)
 	require.NoError(t, err)
@@ -87,7 +86,8 @@ func TestUpdateConfiguration(t *testing.T) {
         }
     ],
     "warp-routing": {
-        "enabled": true
+        "enabled": true,
+        "connectTimeout": 10
     }
 }	
 `)
@@ -121,7 +121,8 @@ func TestUpdateConfiguration(t *testing.T) {
 	require.Equal(t, config.CustomDuration{Duration: time.Second * 90}, configV2.Ingress.Rules[2].Config.ConnectTimeout)
 	require.Equal(t, false, configV2.Ingress.Rules[2].Config.NoTLSVerify)
 	require.Equal(t, true, configV2.Ingress.Rules[2].Config.NoHappyEyeballs)
-	require.True(t, configV2.WarpRoutingEnabled)
+	require.True(t, configV2.WarpRouting.Enabled)
+	require.Equal(t, configV2.WarpRouting.ConnectTimeout.Duration, 10*time.Second)
 
 	originProxyV2, err := orchestrator.GetOriginProxy()
 	require.NoError(t, err)
@@ -164,7 +165,7 @@ func TestUpdateConfiguration(t *testing.T) {
 	require.Len(t, configV10.Ingress.Rules, 1)
 	require.True(t, configV10.Ingress.Rules[0].Matches("blogs.tunnel.io", "/2022/02/10"))
 	require.Equal(t, ingress.HelloWorldService, configV10.Ingress.Rules[0].Service.String())
-	require.False(t, configV10.WarpRoutingEnabled)
+	require.False(t, configV10.WarpRouting.Enabled)
 
 	originProxyV10, err := orchestrator.GetOriginProxy()
 	require.NoError(t, err)
@@ -246,8 +247,7 @@ func TestConcurrentUpdateAndRead(t *testing.T) {
 		appliedV2 = make(chan struct{})
 
 		initConfig = &Config{
-			Ingress:            &ingress.Ingress{},
-			WarpRoutingEnabled: false,
+			Ingress: &ingress.Ingress{},
 		}
 	)
 
@@ -355,7 +355,7 @@ func proxyHTTP(originProxy connection.OriginProxy, hostname string) (*http.Respo
 		return nil, err
 	}
 
-	err = originProxy.ProxyHTTP(respWriter, tracing.NewTracedRequest(req), false)
+	err = originProxy.ProxyHTTP(respWriter, tracing.NewTracedHTTPRequest(req, &log), false)
 	if err != nil {
 		return nil, err
 	}
@@ -476,8 +476,7 @@ func TestClosePreviousProxies(t *testing.T) {
 }
 `)
 		initConfig = &Config{
-			Ingress:            &ingress.Ingress{},
-			WarpRoutingEnabled: false,
+			Ingress: &ingress.Ingress{},
 		}
 	)
 
@@ -534,8 +533,7 @@ func TestPersistentConnection(t *testing.T) {
 	)
 	msg := t.Name()
 	initConfig := &Config{
-		Ingress:            &ingress.Ingress{},
-		WarpRoutingEnabled: false,
+		Ingress: &ingress.Ingress{},
 	}
 	orchestrator, err := NewOrchestrator(context.Background(), initConfig, testTags, &testLogger)
 	require.NoError(t, err)
@@ -606,7 +604,7 @@ func TestPersistentConnection(t *testing.T) {
 		respWriter, err := connection.NewHTTP2RespWriter(req, wsRespReadWriter, connection.TypeWebsocket, &log)
 		require.NoError(t, err)
 
-		err = originProxy.ProxyHTTP(respWriter, tracing.NewTracedRequest(req), true)
+		err = originProxy.ProxyHTTP(respWriter, tracing.NewTracedHTTPRequest(req, &log), true)
 		require.NoError(t, err)
 	}()
 
@@ -645,8 +643,7 @@ func TestPersistentConnection(t *testing.T) {
 func TestSerializeLocalConfig(t *testing.T) {
 	c := &newLocalConfig{
 		RemoteConfig: ingress.RemoteConfig{
-			Ingress:     ingress.Ingress{},
-			WarpRouting: config.WarpRoutingConfig{},
+			Ingress: ingress.Ingress{},
 		},
 		ConfigurationFlags: map[string]string{"a": "b"},
 	}

proxy/proxy.go

@@ -28,6 +28,7 @@ const (
 	LogFieldCFRay         = "cfRay"
 	LogFieldRule          = "ingressRule"
 	LogFieldOriginService = "originService"
+	LogFieldFlowID        = "flowID"
 )
 
 // Proxy represents a means to Proxy between cloudflared and the origin services.
@@ -41,7 +42,7 @@ type Proxy struct {
 // NewOriginProxy returns a new instance of the Proxy struct.
 func NewOriginProxy(
 	ingressRules ingress.Ingress,
-	warpRoutingEnabled bool,
+	warpRouting ingress.WarpRoutingConfig,
 	tags []tunnelpogs.Tag,
 	log *zerolog.Logger,
 ) *Proxy {
@@ -50,8 +51,8 @@ func NewOriginProxy(
 		tags:         tags,
 		log:          log,
 	}
-	if warpRoutingEnabled {
-		proxy.warpRouting = ingress.NewWarpRoutingService()
+	if warpRouting.Enabled {
+		proxy.warpRouting = ingress.NewWarpRoutingService(warpRouting)
 		log.Info().Msgf("Warp-routing is enabled")
 	}
 
@@ -62,7 +63,7 @@ func NewOriginProxy(
 // a simple roundtrip or a tcp/websocket dial depending on ingres rule setup.
 func (p *Proxy) ProxyHTTP(
 	w connection.ResponseWriter,
-	tr *tracing.TracedRequest,
+	tr *tracing.TracedHTTPRequest,
 	isWebsocket bool,
 ) error {
 	incrementRequests()
@@ -96,7 +97,7 @@ func (p *Proxy) ProxyHTTP(
 			logFields,
 		); err != nil {
 			rule, srv := ruleField(p.ingressRules, ruleNum)
-			p.logRequestError(err, cfRay, rule, srv)
+			p.logRequestError(err, cfRay, "", rule, srv)
 			return err
 		}
 		return nil
@@ -107,9 +108,9 @@ func (p *Proxy) ProxyHTTP(
 		}
 
 		rws := connection.NewHTTPResponseReadWriterAcker(w, req)
-		if err := p.proxyStream(req.Context(), rws, dest, originProxy, logFields); err != nil {
+		if err := p.proxyStream(tr.ToTracedContext(), rws, dest, originProxy); err != nil {
 			rule, srv := ruleField(p.ingressRules, ruleNum)
-			p.logRequestError(err, cfRay, rule, srv)
+			p.logRequestError(err, cfRay, "", rule, srv)
 			return err
 		}
 		return nil
@@ -136,17 +137,17 @@ func (p *Proxy) ProxyTCP(
 	serveCtx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
-	logFields := logFields{
-		cfRay:   req.CFRay,
-		lbProbe: req.LBProbe,
-		rule:    ingress.ServiceWarpRouting,
-	}
+	tracedCtx := tracing.NewTracedContext(serveCtx, req.CfTraceID, p.log)
 
-	if err := p.proxyStream(serveCtx, rwa, req.Dest, p.warpRouting.Proxy, logFields); err != nil {
-		p.logRequestError(err, req.CFRay, "", ingress.ServiceWarpRouting)
+	p.log.Debug().Str(LogFieldFlowID, req.FlowID).Msg("tcp proxy stream started")
+
+	if err := p.proxyStream(tracedCtx, rwa, req.Dest, p.warpRouting.Proxy); err != nil {
+		p.logRequestError(err, req.CFRay, req.FlowID, "", ingress.ServiceWarpRouting)
 		return err
 	}
 
+	p.log.Debug().Str(LogFieldFlowID, req.FlowID).Msg("tcp proxy stream finished successfully")
+
 	return nil
 }
 
@@ -161,7 +162,7 @@ func ruleField(ing ingress.Ingress, ruleNum int) (ruleID string, srv string) {
 // ProxyHTTPRequest proxies requests of underlying type http and websocket to the origin service.
 func (p *Proxy) proxyHTTPRequest(
 	w connection.ResponseWriter,
-	tr *tracing.TracedRequest,
+	tr *tracing.TracedHTTPRequest,
 	httpService ingress.HTTPOriginProxy,
 	isWebsocket bool,
 	disableChunkedEncoding bool,
@@ -197,6 +198,9 @@ func (p *Proxy) proxyHTTPRequest(
 	resp, err := httpService.RoundTrip(roundTripReq)
 	if err != nil {
 		tracing.EndWithErrorStatus(ttfbSpan, err)
+		if err := roundTripReq.Context().Err(); err != nil {
+			return errors.Wrap(err, "Incoming request ended abruptly")
+		}
 		return errors.Wrap(err, "Unable to reach the origin service. The service may be down or it may not be responding to traffic from cloudflared")
 	}
 
@@ -209,7 +213,7 @@ func (p *Proxy) proxyHTTPRequest(
 	}
 
 	// Add spans to response header (if available)
-	tr.AddSpans(resp.Header, p.log)
+	tr.AddSpans(resp.Header)
 
 	err = w.WriteRespHeaders(resp.StatusCode, resp.Header)
 	if err != nil {
@@ -246,18 +250,23 @@ func (p *Proxy) proxyHTTPRequest(
 // proxyStream proxies type TCP and other underlying types if the connection is defined as a stream oriented
 // ingress rule.
 func (p *Proxy) proxyStream(
-	ctx context.Context,
+	tr *tracing.TracedContext,
 	rwa connection.ReadWriteAcker,
 	dest string,
 	connectionProxy ingress.StreamBasedOriginProxy,
-	fields logFields,
 ) error {
-	originConn, err := connectionProxy.EstablishConnection(dest)
+	ctx := tr.Context
+	_, connectSpan := tr.Tracer().Start(ctx, "stream_connect")
+	originConn, err := connectionProxy.EstablishConnection(ctx, dest)
 	if err != nil {
+		tracing.EndWithErrorStatus(connectSpan, err)
 		return err
 	}
+	connectSpan.End()
 
-	if err := rwa.AckConnection(); err != nil {
+	encodedSpans := tr.GetSpans()
+
+	if err := rwa.AckConnection(encodedSpans); err != nil {
 		return err
 	}
 
@@ -317,6 +326,7 @@ type logFields struct {
 	cfRay   string
 	lbProbe bool
 	rule    interface{}
+	flowID  string
 }
 
 func (p *Proxy) logRequest(r *http.Request, fields logFields) {
@@ -360,12 +370,15 @@ func (p *Proxy) logOriginResponse(resp *http.Response, fields logFields) {
 	}
 }
 
-func (p *Proxy) logRequestError(err error, cfRay string, rule, service string) {
+func (p *Proxy) logRequestError(err error, cfRay string, flowID string, rule, service string) {
 	requestErrors.Inc()
 	log := p.log.Error().Err(err)
 	if cfRay != "" {
 		log = log.Str(LogFieldCFRay, cfRay)
 	}
+	if flowID != "" {
+		log = log.Str(LogFieldFlowID, flowID)
+	}
 	if rule != "" {
 		log = log.Str(LogFieldRule, rule)
 	}

proxy/proxy_test.go

@@ -32,7 +32,12 @@ import (
 )
 
 var (
-	testTags = []tunnelpogs.Tag{tunnelpogs.Tag{Name: "Name", Value: "value"}}
+	testTags        = []tunnelpogs.Tag{tunnelpogs.Tag{Name: "Name", Value: "value"}}
+	noWarpRouting   = ingress.WarpRoutingConfig{}
+	testWarpRouting = ingress.WarpRoutingConfig{
+		Enabled:        true,
+		ConnectTimeout: config.CustomDuration{Duration: time.Second},
+	}
 )
 
 type mockHTTPRespWriter struct {
@@ -138,7 +143,7 @@ func TestProxySingleOrigin(t *testing.T) {
 
 	require.NoError(t, ingressRule.StartOrigins(&log, ctx.Done()))
 
-	proxy := NewOriginProxy(ingressRule, false, testTags, &log)
+	proxy := NewOriginProxy(ingressRule, noWarpRouting, testTags, &log)
 	t.Run("testProxyHTTP", testProxyHTTP(proxy))
 	t.Run("testProxyWebsocket", testProxyWebsocket(proxy))
 	t.Run("testProxySSE", testProxySSE(proxy))
@@ -152,7 +157,8 @@ func testProxyHTTP(proxy connection.OriginProxy) func(t *testing.T) {
 		req, err := http.NewRequest(http.MethodGet, "http://localhost:8080", nil)
 		require.NoError(t, err)
 
-		err = proxy.ProxyHTTP(responseWriter, tracing.NewTracedRequest(req), false)
+		log := zerolog.Nop()
+		err = proxy.ProxyHTTP(responseWriter, tracing.NewTracedHTTPRequest(req, &log), false)
 		require.NoError(t, err)
 		for _, tag := range testTags {
 			assert.Equal(t, tag.Value, req.Header.Get(TagHeaderNamePrefix+tag.Name))
@@ -179,7 +185,8 @@ func testProxyWebsocket(proxy connection.OriginProxy) func(t *testing.T) {
 
 		errGroup, ctx := errgroup.WithContext(ctx)
 		errGroup.Go(func() error {
-			err = proxy.ProxyHTTP(responseWriter, tracing.NewTracedRequest(req), true)
+			log := zerolog.Nop()
+			err = proxy.ProxyHTTP(responseWriter, tracing.NewTracedHTTPRequest(req, &log), true)
 			require.NoError(t, err)
 
 			require.Equal(t, http.StatusSwitchingProtocols, responseWriter.Code)
@@ -240,7 +247,8 @@ func testProxySSE(proxy connection.OriginProxy) func(t *testing.T) {
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
-			err = proxy.ProxyHTTP(responseWriter, tracing.NewTracedRequest(req), false)
+			log := zerolog.Nop()
+			err = proxy.ProxyHTTP(responseWriter, tracing.NewTracedHTTPRequest(req, &log), false)
 			require.NoError(t, err)
 
 			require.Equal(t, http.StatusOK, responseWriter.Code)
@@ -345,14 +353,14 @@ func runIngressTestScenarios(t *testing.T, unvalidatedIngress []config.Unvalidat
 	ctx, cancel := context.WithCancel(context.Background())
 	require.NoError(t, ingress.StartOrigins(&log, ctx.Done()))
 
-	proxy := NewOriginProxy(ingress, false, testTags, &log)
+	proxy := NewOriginProxy(ingress, noWarpRouting, testTags, &log)
 
 	for _, test := range tests {
 		responseWriter := newMockHTTPRespWriter()
 		req, err := http.NewRequest(http.MethodGet, test.url, nil)
 		require.NoError(t, err)
 
-		err = proxy.ProxyHTTP(responseWriter, tracing.NewTracedRequest(req), false)
+		err = proxy.ProxyHTTP(responseWriter, tracing.NewTracedHTTPRequest(req, &log), false)
 		require.NoError(t, err)
 
 		assert.Equal(t, test.expectedStatus, responseWriter.Code)
@@ -393,13 +401,13 @@ func TestProxyError(t *testing.T) {
 
 	log := zerolog.Nop()
 
-	proxy := NewOriginProxy(ing, false, testTags, &log)
+	proxy := NewOriginProxy(ing, noWarpRouting, testTags, &log)
 
 	responseWriter := newMockHTTPRespWriter()
 	req, err := http.NewRequest(http.MethodGet, "http://127.0.0.1", nil)
 	assert.NoError(t, err)
 
-	assert.Error(t, proxy.ProxyHTTP(responseWriter, tracing.NewTracedRequest(req), false))
+	assert.Error(t, proxy.ProxyHTTP(responseWriter, tracing.NewTracedHTTPRequest(req, &log), false))
 }
 
 type replayer struct {
@@ -509,7 +517,7 @@ func TestConnections(t *testing.T) {
 				originService:         runEchoTCPService,
 				eyeballResponseWriter: newTCPRespWriter(replayer),
 				eyeballRequestBody:    newTCPRequestBody([]byte("test2")),
-				warpRoutingService:    ingress.NewWarpRoutingService(),
+				warpRoutingService:    ingress.NewWarpRoutingService(testWarpRouting),
 				connectionType:        connection.TypeTCP,
 				requestHeaders: map[string][]string{
 					"Cf-Cloudflared-Proxy-Src": {"non-blank-value"},
@@ -526,7 +534,7 @@ func TestConnections(t *testing.T) {
 				originService:        runEchoWSService,
 				// eyeballResponseWriter gets set after roundtrip dial.
 				eyeballRequestBody: newPipedWSRequestBody([]byte("test3")),
-				warpRoutingService: ingress.NewWarpRoutingService(),
+				warpRoutingService: ingress.NewWarpRoutingService(testWarpRouting),
 				requestHeaders: map[string][]string{
 					"Cf-Cloudflared-Proxy-Src": {"non-blank-value"},
 				},
@@ -652,7 +660,7 @@ func TestConnections(t *testing.T) {
 
 			ingressRule := createSingleIngressConfig(t, test.args.ingressServiceScheme+ln.Addr().String())
 			ingressRule.StartOrigins(logger, ctx.Done())
-			proxy := NewOriginProxy(ingressRule, true, testTags, logger)
+			proxy := NewOriginProxy(ingressRule, testWarpRouting, testTags, logger)
 			proxy.warpRouting = test.args.warpRoutingService
 
 			dest := ln.Addr().String()
@@ -677,7 +685,8 @@ func TestConnections(t *testing.T) {
 				rwa := connection.NewHTTPResponseReadWriterAcker(respWriter, req)
 				err = proxy.ProxyTCP(ctx, rwa, &connection.TCPRequest{Dest: dest})
 			} else {
-				err = proxy.ProxyHTTP(respWriter, tracing.NewTracedRequest(req), test.args.connectionType == connection.TypeWebsocket)
+				log := zerolog.Nop()
+				err = proxy.ProxyHTTP(respWriter, tracing.NewTracedHTTPRequest(req, &log), test.args.connectionType == connection.TypeWebsocket)
 			}
 
 			cancel()

quic/datagram.go

@@ -14,11 +14,11 @@ const (
 )
 
 type DatagramMuxer struct {
-	session quic.Session
+	session quic.Connection
 	logger  *zerolog.Logger
 }
 
-func NewDatagramMuxer(quicSession quic.Session, logger *zerolog.Logger) (*DatagramMuxer, error) {
+func NewDatagramMuxer(quicSession quic.Connection, logger *zerolog.Logger) (*DatagramMuxer, error) {
 	return &DatagramMuxer{
 		session: quicSession,
 		logger:  logger,
@@ -39,7 +39,6 @@ func (dm *DatagramMuxer) SendTo(sessionID uuid.UUID, payload []byte) error {
 	if err := dm.session.SendMessage(msgWithID); err != nil {
 		return errors.Wrap(err, "Failed to send datagram back to edge")
 	}
-	dm.logger.Debug().Str("sessionID", sessionID.String()).Int("bytes", len(payload)).Msg("Send datagram back to edge")
 	return nil
 }
 
@@ -55,7 +54,6 @@ func (dm *DatagramMuxer) ReceiveFrom() (uuid.UUID, []byte, error) {
 	if err != nil {
 		return uuid.Nil, nil, err
 	}
-	dm.logger.Debug().Str("sessionID", sessionID.String()).Int("bytes", len(payload)).Msg("Received datagram from edge")
 	return sessionID, payload, nil
 }
 

quic/datagram_test.go

@@ -56,7 +56,7 @@ func TestMaxDatagramPayload(t *testing.T) {
 	payload := make([]byte, maxDatagramPayloadSize)
 
 	quicConfig := &quic.Config{
-		KeepAlive:            true,
+		KeepAlivePeriod:      5 * time.Millisecond,
 		EnableDatagrams:      true,
 		MaxDatagramFrameSize: MaxDatagramFrameSize,
 	}

quic/pogs.go

@@ -1,6 +1,8 @@
 package quic
 
 import (
+	"fmt"
+
 	capnp "zombiezen.com/go/capnproto2"
 	"zombiezen.com/go/capnproto2/pogs"
 
@@ -16,6 +18,18 @@ const (
 	ConnectionTypeTCP
 )
 
+func (c ConnectionType) String() string {
+	switch c {
+	case ConnectionTypeHTTP:
+		return "http"
+	case ConnectionTypeWebsocket:
+		return "ws"
+	case ConnectionTypeTCP:
+		return "tcp"
+	}
+	panic(fmt.Sprintf("invalid ConnectionType: %d", c))
+}
+
 // ConnectRequest is the representation of metadata sent at the start of a QUIC application handshake.
 type ConnectRequest struct {
 	Dest     string         `capnp:"dest"`

quic/quic_protocol.go

@@ -37,7 +37,8 @@ const (
 	protocolVersionLength = 2
 
 	HandshakeIdleTimeout = 5 * time.Second
-	MaxIdleTimeout       = 15 * time.Second
+	MaxIdleTimeout       = 5 * time.Second
+	MaxIdlePingPeriod    = 1 * time.Second
 )
 
 // RequestServerStream is a stream to serve requests

quic/safe_stream.go

@@ -41,3 +41,14 @@ func (s *SafeStreamCloser) Close() error {
 	s.stream.CancelRead(0)
 	return s.stream.Close()
 }
+
+func (s *SafeStreamCloser) CloseWrite() error {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	// As documented by the quic-go library, this doesn't actually close the entire stream.
+	// It prevents further writes, which in turn will result in an EOF signal being sent the other side of stream when
+	// reading.
+	// We can still read from this stream.
+	return s.stream.Close()
+}

quic/safe_stream_test.go

@@ -7,6 +7,7 @@ import (
 	"net"
 	"sync"
 	"testing"
+	"time"
 
 	"github.com/lucas-clemente/quic-go"
 	"github.com/stretchr/testify/require"
@@ -15,7 +16,7 @@ import (
 var (
 	testTLSServerConfig = GenerateTLSConfig()
 	testQUICConfig      = &quic.Config{
-		KeepAlive:       true,
+		KeepAlivePeriod: 5 * time.Second,
 		EnableDatagrams: true,
 	}
 	exchanges       = 1000

release_pkgs.py

@@ -8,23 +8,24 @@
        them to be in an uploadable state.
     2. Upload these packages to a storage in a format that apt and yum expect.
 """
-from subprocess import Popen, PIPE
-import os
 import argparse
 import base64
-from pathlib import Path
 import logging
+import os
 import shutil
 from hashlib import sha256
+from pathlib import Path
+from subprocess import Popen, PIPE
 
-import gnupg
 import boto3
+import gnupg
 from botocore.client import Config
 from botocore.exceptions import ClientError
 
 # The front facing R2 URL to access assets from.
 R2_ASSET_URL = 'https://demo-r2-worker.cloudflare-tunnel.workers.dev/'
 
+
 class PkgUploader:
     def __init__(self, account_id, bucket_name, client_id, client_secret):
         self.account_id = account_id
@@ -35,14 +36,14 @@ class PkgUploader:
     def upload_pkg_to_r2(self, filename, upload_file_path):
         endpoint_url = f"https://{self.account_id}.r2.cloudflarestorage.com"
         token_secret_hash = sha256(self.client_secret.encode()).hexdigest()
-         
+
         config = Config(
-            region_name = 'auto',
+            region_name='auto',
             s3={
                 "addressing_style": "path",
             }
         )
-         
+
         r2 = boto3.client(
             "s3",
             endpoint_url=endpoint_url,
@@ -57,7 +58,7 @@ class PkgUploader:
         except ClientError as e:
             raise e
 
-                  
+
 class PkgCreator:
     """
         The distribution conf is what dictates to reprepro, the debian packaging building
@@ -72,15 +73,16 @@ class PkgCreator:
         description - (String)
         gpg_key_id - gpg key id of what you want to use to sign the packages.(String) 
     """
-    def create_distribution_conf(self, 
-            file_path,
-            origin,
-            label,
-            releases,
-            archs,
-            components, 
-            description,
-            gpg_key_id ):
+
+    def create_distribution_conf(self,
+                                 file_path,
+                                 origin,
+                                 label,
+                                 releases,
+                                 archs,
+                                 components,
+                                 description,
+                                 gpg_key_id):
         with open(file_path, "w+") as distributions_file:
             for release in releases:
                 distributions_file.write(f"Origin: {origin}\n")
@@ -102,6 +104,7 @@ class PkgCreator:
         db and pool contain information and metadata about builds. We can ignore these.
         dist: contains all the pkgs and signed releases that are necessary for an apt download.
     """
+
     def create_deb_pkgs(self, release, deb_file):
         print(f"creating deb pkgs: {release} : {deb_file}")
         p = Popen(["reprepro", "includedeb", release, deb_file], stdout=PIPE, stderr=PIPE)
@@ -120,20 +123,41 @@ class PkgCreator:
 
         self._sign_repomd()
 
+    """
+        creates a <binary>.repo file with details like so
+        [cloudflared-stable]
+        name=cloudflared-stable
+        baseurl=https://pkg.cloudflare.com/cloudflared/rpm
+        enabled=1
+        type=rpm
+        gpgcheck=1
+        gpgkey=https://pkg.cloudflare.com/cloudflare-main.gpg
+    """
+
+    def create_repo_file(self, file_path, binary_name, baseurl, gpgkey_url):
+        with open(os.path.join(file_path, binary_name + '.repo'), "w+") as repo_file:
+            repo_file.write(f"[{binary_name}-stable]")
+            repo_file.write(f"{binary_name}-stable")
+            repo_file.write(f"baseurl={baseurl}/rpm")
+            repo_file.write("enabled=1")
+            repo_file.write("type=rpm")
+            repo_file.write("gpgcheck=1")
+            repo_file.write(f"gpgkey={gpgkey_url}")
+
     def _sign_rpms(self, file_path):
-        p = Popen(["rpm" , "--define", f"_gpg_name {gpg_key_name}", "--addsign", file_path], stdout=PIPE, stderr=PIPE)
+        p = Popen(["rpm", "--define", f"_gpg_name {gpg_key_name}", "--addsign", file_path], stdout=PIPE, stderr=PIPE)
         out, err = p.communicate()
         if p.returncode != 0:
             print(f"rpm sign result result => {out}, {err}")
             raise
 
     def _sign_repomd(self):
-        p = Popen(["gpg", "--batch", "--detach-sign", "--armor", "./rpm/repodata/repomd.xml"], stdout=PIPE, stderr=PIPE) 
+        p = Popen(["gpg", "--batch", "--detach-sign", "--armor", "./rpm/repodata/repomd.xml"], stdout=PIPE, stderr=PIPE)
         out, err = p.communicate()
         if p.returncode != 0:
             print(f"sign repomd result => {out}, {err}")
             raise
-        
+
     """
         sets up and signs the RPM directories in the following format:
         - rpm 
@@ -143,9 +167,10 @@ class PkgCreator:
 
         this assumes the assets are in the format <prefix>-<aarch64/x86_64/386>.rpm
     """
+
     def _setup_rpm_pkg_directories(self, artifacts_path, gpg_key_name, archs=["aarch64", "x86_64", "386"]):
         for arch in archs:
-            for root, _ , files in os.walk(artifacts_path):
+            for root, _, files in os.walk(artifacts_path):
                 for file in files:
                     if file.endswith(f"{arch}.rpm"):
                         new_dir = f"./rpm/{arch}"
@@ -154,11 +179,12 @@ class PkgCreator:
                         new_path = os.path.join(new_dir, file)
                         shutil.copyfile(old_path, new_path)
                         self._sign_rpms(new_path)
-    
+
     """
         imports gpg keys into the system so reprepro and createrepo can use it to sign packages.
         it returns the GPG ID after a successful import
     """
+
     def import_gpg_keys(self, private_key, public_key):
         gpg = gnupg.GPG()
         private_key = base64.b64decode(private_key)
@@ -172,12 +198,13 @@ class PkgCreator:
         basically rpm --import <key_file>
         This enables us to sign rpms.
     """
+
     def import_rpm_key(self, public_key):
         file_name = "pb.key"
         with open(file_name, "wb") as f:
             public_key = base64.b64decode(public_key)
             f.write(public_key)
-        
+
         p = Popen(["rpm", "--import", file_name], stdout=PIPE, stderr=PIPE)
         out, err = p.communicate()
         if p.returncode != 0:
@@ -192,18 +219,21 @@ class PkgCreator:
               and the release will be uploaded to the default path. 
     binary: name of the binary to upload
 """
+
+
 def upload_from_directories(pkg_uploader, directory, release, binary):
-     for root, _ , files in os.walk(directory):
+    for root, _, files in os.walk(directory):
         for file in files:
             upload_file_name = os.path.join(binary, root, file)
             if release:
                 upload_file_name = os.path.join(release, upload_file_name)
-            filename = os.path.join(root,file)
-            try: 
+            filename = os.path.join(root, file)
+            try:
                 pkg_uploader.upload_pkg_to_r2(filename, upload_file_name)
             except ClientError as e:
                 logging.error(e)
-                return 
+                return
+
 
 """ 
     1. looks into a built_artifacts folder for cloudflared debs
@@ -217,19 +247,22 @@ def upload_from_directories(pkg_uploader, directory, release, binary):
 
     release_version: is the cloudflared release version. Only provide this if you want a permanent backup.
 """
-def create_deb_packaging(pkg_creator, pkg_uploader, releases, gpg_key_id, binary_name, archs, package_component, release_version):
+
+
+def create_deb_packaging(pkg_creator, pkg_uploader, releases, gpg_key_id, binary_name, archs, package_component,
+                         release_version):
     # set configuration for package creation.
     print(f"initialising configuration for {binary_name} , {archs}")
     Path("./conf").mkdir(parents=True, exist_ok=True)
     pkg_creator.create_distribution_conf(
-    "./conf/distributions",
-    binary_name,
-    binary_name,
-    releases,
-    archs,
-    package_component,
-    f"apt repository for {binary_name}",
-    gpg_key_id)
+        "./conf/distributions",
+        binary_name,
+        binary_name,
+        releases,
+        archs,
+        package_component,
+        f"apt repository for {binary_name}",
+        gpg_key_id)
 
     # create deb pkgs
     for release in releases:
@@ -246,9 +279,20 @@ def create_deb_packaging(pkg_creator, pkg_uploader, releases, gpg_key_id, binary
         upload_from_directories(pkg_uploader, "dists", release_version, binary_name)
         upload_from_directories(pkg_uploader, "pool", release_version, binary_name)
 
-def create_rpm_packaging(pkg_creator, pkg_uploader, artifacts_path, release_version, binary_name, gpg_key_name):
+
+def create_rpm_packaging(
+        pkg_creator,
+        pkg_uploader,
+        artifacts_path,
+        release_version,
+        binary_name,
+        gpg_key_name,
+        base_url,
+        gpg_key_url,
+):
     print(f"creating rpm pkgs...")
     pkg_creator.create_rpm_pkgs(artifacts_path, gpg_key_name)
+    pkg_creator.create_repo_file(artifacts_path, binary_name, base_url, gpg_key_url)
 
     print("uploading latest to r2...")
     upload_from_directories(pkg_uploader, "rpm", None, binary_name)
@@ -264,50 +308,61 @@ def parse_args():
     )
 
     parser.add_argument(
-            "--bucket", default=os.environ.get("R2_BUCKET"), help="R2 Bucket name"
+        "--bucket", default=os.environ.get("R2_BUCKET"), help="R2 Bucket name"
     )
     parser.add_argument(
-            "--id", default=os.environ.get("R2_CLIENT_ID"), help="R2 Client ID"
+        "--id", default=os.environ.get("R2_CLIENT_ID"), help="R2 Client ID"
     )
     parser.add_argument(
-            "--secret", default=os.environ.get("R2_CLIENT_SECRET"), help="R2 Client Secret"
+        "--secret", default=os.environ.get("R2_CLIENT_SECRET"), help="R2 Client Secret"
     )
     parser.add_argument(
-            "--account", default=os.environ.get("R2_ACCOUNT_ID"), help="R2 Account Tag"
+        "--account", default=os.environ.get("R2_ACCOUNT_ID"), help="R2 Account Tag"
     )
     parser.add_argument(
-            "--release-tag", default=os.environ.get("RELEASE_VERSION"), help="Release version you want your pkgs to be\
+        "--release-tag", default=os.environ.get("RELEASE_VERSION"), help="Release version you want your pkgs to be\
             prefixed with. Leave empty if you don't want tagged release versions backed up to R2."
     )
 
     parser.add_argument(
-           "--binary", default=os.environ.get("BINARY_NAME"), help="The name of the binary the packages are for"
+        "--binary", default=os.environ.get("BINARY_NAME"), help="The name of the binary the packages are for"
     )
 
     parser.add_argument(
-            "--gpg-private-key", default=os.environ.get("LINUX_SIGNING_PRIVATE_KEY"), help="GPG private key to sign the\
+        "--gpg-private-key", default=os.environ.get("LINUX_SIGNING_PRIVATE_KEY"), help="GPG private key to sign the\
             packages"
     )
 
     parser.add_argument(
-            "--gpg-public-key", default=os.environ.get("LINUX_SIGNING_PUBLIC_KEY"), help="GPG public key used for\
+        "--gpg-public-key", default=os.environ.get("LINUX_SIGNING_PUBLIC_KEY"), help="GPG public key used for\
             signing packages"
     )
 
     parser.add_argument(
-            "--deb-based-releases", default=["bookworm", "bullseye", "buster", "jammy", "impish", "focal", "bionic"],
-            help="list of debian based releases that need to be packaged for"
+        "--gpg-public-key-url", default=os.environ.get("GPG_PUBLIC_KEY_URL"), help="GPG public key url that\
+            downloaders can use to verify signing"
     )
 
     parser.add_argument(
-            "--archs", default=["amd64", "386", "arm64"], help="list of architectures we want to package for. Note that\
+        "--pkg-upload-url", default=os.environ.get("PKG_URL"), help="URL to be used by downloaders"
+    )
+
+    parser.add_argument(
+        "--deb-based-releases", default=["bookworm", "bullseye", "buster", "jammy", "impish", "focal", "bionic",
+                                         "xenial", "trusty"],
+        help="list of debian based releases that need to be packaged for"
+    )
+
+    parser.add_argument(
+        "--archs", default=["amd64", "386", "arm64", "arm", "armhf"], help="list of architectures we want to package for. Note that\
             it is the caller's responsiblity to ensure that these debs are already present in a directory. This script\
             will not build binaries or create their debs."
-            )
+    )
     args = parser.parse_args()
 
     return args
 
+
 if __name__ == "__main__":
     try:
         args = parse_args()
@@ -322,6 +377,15 @@ if __name__ == "__main__":
     pkg_uploader = PkgUploader(args.account, args.bucket, args.id, args.secret)
     print(f"signing with gpg_key: {gpg_key_id}")
     create_deb_packaging(pkg_creator, pkg_uploader, args.deb_based_releases, gpg_key_id, args.binary, args.archs,
-            "main", args.release_tag)
-    
-    create_rpm_packaging(pkg_creator, pkg_uploader, "./built_artifacts", args.release_tag, args.binary, gpg_key_name)
+                         "main", args.release_tag)
+
+    create_rpm_packaging(
+        pkg_creator,
+        pkg_uploader,
+        "./built_artifacts",
+        args.release_tag,
+        args.binary,
+        gpg_key_name,
+        args.gpg_public_key_url,
+        args.pkg_upload_url,
+    )

supervisor/supervisor.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"strings"
 	"time"
 
 	"github.com/google/uuid"
@@ -38,12 +39,14 @@ const (
 // Supervisor manages non-declarative tunnels. Establishes TCP connections with the edge, and
 // reconnects them if they disconnect.
 type Supervisor struct {
-	cloudflaredUUID   uuid.UUID
-	config            *TunnelConfig
-	orchestrator      *orchestration.Orchestrator
-	edgeIPs           *edgediscovery.Edge
-	tunnelErrors      chan tunnelError
-	tunnelsConnecting map[int]chan struct{}
+	cloudflaredUUID         uuid.UUID
+	config                  *TunnelConfig
+	orchestrator            *orchestration.Orchestrator
+	edgeIPs                 *edgediscovery.Edge
+	edgeTunnelServer        EdgeTunnelServer
+	tunnelErrors            chan tunnelError
+	tunnelsConnecting       map[int]chan struct{}
+	tunnelsProtocolFallback map[int]*protocolFallback
 	// nextConnectedIndex and nextConnectedSignal are used to wait for all
 	// currently-connecting tunnels to finish connecting so we can reset backoff timer
 	nextConnectedIndex  int
@@ -63,7 +66,6 @@ var errEarlyShutdown = errors.New("shutdown started")
 
 type tunnelError struct {
 	index int
-	addr  *allregions.EdgeAddr
 	err   error
 }
 
@@ -73,16 +75,42 @@ func NewSupervisor(config *TunnelConfig, orchestrator *orchestration.Orchestrato
 		return nil, fmt.Errorf("failed to generate cloudflared instance ID: %w", err)
 	}
 
+	isStaticEdge := len(config.EdgeAddrs) > 0
+
 	var edgeIPs *edgediscovery.Edge
-	if len(config.EdgeAddrs) > 0 {
+	if isStaticEdge { // static edge addresses
 		edgeIPs, err = edgediscovery.StaticEdge(config.Log, config.EdgeAddrs)
 	} else {
-		edgeIPs, err = edgediscovery.ResolveEdge(config.Log, config.Region)
+		edgeIPs, err = edgediscovery.ResolveEdge(config.Log, config.Region, config.EdgeIPVersion)
 	}
 	if err != nil {
 		return nil, err
 	}
 
+	reconnectCredentialManager := newReconnectCredentialManager(connection.MetricsNamespace, connection.TunnelSubsystem, config.HAConnections)
+	log := NewConnAwareLogger(config.Log, config.Observer)
+
+	var edgeAddrHandler EdgeAddrHandler
+	if isStaticEdge { // static edge addresses
+		edgeAddrHandler = &IPAddrFallback{}
+	} else if config.EdgeIPVersion == allregions.IPv6Only || config.EdgeIPVersion == allregions.Auto {
+		edgeAddrHandler = &IPAddrFallback{}
+	} else { // IPv4Only
+		edgeAddrHandler = &DefaultAddrFallback{}
+	}
+
+	edgeTunnelServer := EdgeTunnelServer{
+		config:            config,
+		cloudflaredUUID:   cloudflaredUUID,
+		orchestrator:      orchestrator,
+		credentialManager: reconnectCredentialManager,
+		edgeAddrs:         edgeIPs,
+		edgeAddrHandler:   edgeAddrHandler,
+		reconnectCh:       reconnectCh,
+		gracefulShutdownC: gracefulShutdownC,
+		connAwareLogger:   log,
+	}
+
 	useReconnectToken := false
 	if config.ClassicTunnel != nil {
 		useReconnectToken = config.ClassicTunnel.UseReconnectToken
@@ -93,11 +121,13 @@ func NewSupervisor(config *TunnelConfig, orchestrator *orchestration.Orchestrato
 		config:                     config,
 		orchestrator:               orchestrator,
 		edgeIPs:                    edgeIPs,
+		edgeTunnelServer:           edgeTunnelServer,
 		tunnelErrors:               make(chan tunnelError),
 		tunnelsConnecting:          map[int]chan struct{}{},
-		log:                        NewConnAwareLogger(config.Log, config.Observer),
+		tunnelsProtocolFallback:    map[int]*protocolFallback{},
+		log:                        log,
 		logTransport:               config.LogTransport,
-		reconnectCredentialManager: newReconnectCredentialManager(connection.MetricsNamespace, connection.TunnelSubsystem, config.HAConnections),
+		reconnectCredentialManager: reconnectCredentialManager,
 		useReconnectToken:          useReconnectToken,
 		reconnectCh:                reconnectCh,
 		gracefulShutdownC:          gracefulShutdownC,
@@ -144,11 +174,22 @@ func (s *Supervisor) Run(
 				tunnelsActive--
 			}
 			return nil
-		// startTunnel returned with error
+		// startTunnel completed with a response
 		// (note that this may also be caused by context cancellation)
 		case tunnelError := <-s.tunnelErrors:
 			tunnelsActive--
 			if tunnelError.err != nil && !shuttingDown {
+				switch tunnelError.err.(type) {
+				case ReconnectSignal:
+					// For tunnels that closed with reconnect signal, we reconnect immediately
+					go s.startTunnel(ctx, tunnelError.index, s.newConnectedTunnelSignal(tunnelError.index))
+					tunnelsActive++
+					continue
+				}
+				// Make sure we don't continue if there is no more fallback allowed
+				if _, retry := s.tunnelsProtocolFallback[tunnelError.index].GetMaxBackoffDuration(ctx); !retry {
+					continue
+				}
 				s.log.ConnAwareLogger().Err(tunnelError.err).Int(connection.LogFieldConnIndex, tunnelError.index).Msg("Connection terminated")
 				tunnelsWaiting = append(tunnelsWaiting, tunnelError.index)
 				s.waitForNextTunnel(tunnelError.index)
@@ -156,10 +197,9 @@ func (s *Supervisor) Run(
 				if backoffTimer == nil {
 					backoffTimer = backoff.BackoffTimer()
 				}
-
-				// Previously we'd mark the edge address as bad here, but now we'll just silently use another.
 			} else if tunnelsActive == 0 {
-				// all connected tunnels exited gracefully, no more work to do
+				s.log.ConnAwareLogger().Msg("no more connections active and exiting")
+				// All connected tunnels exited gracefully, no more work to do
 				return nil
 			}
 		// Backoff was set and its timer expired
@@ -193,6 +233,8 @@ func (s *Supervisor) Run(
 }
 
 // Returns nil if initialization succeeded, else the initialization error.
+// Attempts here will be made to connect one tunnel, if successful, it will
+// connect the available tunnels up to config.HAConnections.
 func (s *Supervisor) initialize(
 	ctx context.Context,
 	connectedSignal *signal.Signal,
@@ -202,8 +244,15 @@ func (s *Supervisor) initialize(
 		s.log.Logger().Info().Msgf("You requested %d HA connections but I can give you at most %d.", s.config.HAConnections, availableAddrs)
 		s.config.HAConnections = availableAddrs
 	}
+	s.tunnelsProtocolFallback[0] = &protocolFallback{
+		retry.BackoffHandler{MaxRetries: s.config.Retries},
+		s.config.ProtocolSelector.Current(),
+		false,
+	}
 
 	go s.startFirstTunnel(ctx, connectedSignal)
+
+	// Wait for response from first tunnel before proceeding to attempt other HA edge tunnels
 	select {
 	case <-ctx.Done():
 		<-s.tunnelErrors
@@ -214,8 +263,14 @@ func (s *Supervisor) initialize(
 		return errEarlyShutdown
 	case <-connectedSignal.Wait():
 	}
+
 	// At least one successful connection, so start the rest
 	for i := 1; i < s.config.HAConnections; i++ {
+		s.tunnelsProtocolFallback[i] = &protocolFallback{
+			retry.BackoffHandler{MaxRetries: s.config.Retries},
+			s.config.ProtocolSelector.Current(),
+			false,
+		}
 		ch := signal.New(make(chan struct{}))
 		go s.startTunnel(ctx, i, ch)
 		time.Sleep(registrationInterval)
@@ -230,102 +285,65 @@ func (s *Supervisor) startFirstTunnel(
 	connectedSignal *signal.Signal,
 ) {
 	var (
-		addr *allregions.EdgeAddr
-		err  error
+		err error
 	)
 	const firstConnIndex = 0
+	isStaticEdge := len(s.config.EdgeAddrs) > 0
 	defer func() {
-		s.tunnelErrors <- tunnelError{index: firstConnIndex, addr: addr, err: err}
+		s.tunnelErrors <- tunnelError{index: firstConnIndex, err: err}
 	}()
 
-	addr, err = s.edgeIPs.GetAddr(firstConnIndex)
-	if err != nil {
-		return
-	}
-
-	err = ServeTunnelLoop(
-		ctx,
-		s.reconnectCredentialManager,
-		s.config,
-		s.orchestrator,
-		addr,
-		s.log,
-		firstConnIndex,
-		connectedSignal,
-		s.cloudflaredUUID,
-		s.reconnectCh,
-		s.gracefulShutdownC,
-	)
 	// If the first tunnel disconnects, keep restarting it.
-	edgeErrors := 0
-	for s.unusedIPs() {
+	for {
+		err = s.edgeTunnelServer.Serve(ctx, firstConnIndex, s.tunnelsProtocolFallback[firstConnIndex], connectedSignal)
 		if ctx.Err() != nil {
 			return
 		}
-		switch err.(type) {
-		case nil:
-			return
-		// try the next address if it was a quic.IdleTimeoutError, dialError(network problem) or
-		// dupConnRegisterTunnelError
-		case *quic.IdleTimeoutError, edgediscovery.DialError, connection.DupConnRegisterTunnelError:
-			edgeErrors++
-		default:
+		if err == nil {
 			return
 		}
-		if edgeErrors >= 2 {
-			addr, err = s.edgeIPs.GetDifferentAddr(firstConnIndex)
-			if err != nil {
+		// Make sure we don't continue if there is no more fallback allowed
+		if _, retry := s.tunnelsProtocolFallback[firstConnIndex].GetMaxBackoffDuration(ctx); !retry {
+			return
+		}
+		// Try again for Unauthorized errors because we hope them to be
+		// transient due to edge propagation lag on new Tunnels.
+		if strings.Contains(err.Error(), "Unauthorized") {
+			continue
+		}
+		switch err.(type) {
+		case edgediscovery.ErrNoAddressesLeft:
+			// If your provided addresses are not available, we will keep trying regardless.
+			if !isStaticEdge {
 				return
 			}
+		case connection.DupConnRegisterTunnelError,
+			*quic.IdleTimeoutError,
+			edgediscovery.DialError,
+			*connection.EdgeQuicDialError:
+			// Try again for these types of errors
+		default:
+			// Uncaught errors should bail startup
+			return
 		}
-		err = ServeTunnelLoop(
-			ctx,
-			s.reconnectCredentialManager,
-			s.config,
-			s.orchestrator,
-			addr,
-			s.log,
-			firstConnIndex,
-			connectedSignal,
-			s.cloudflaredUUID,
-			s.reconnectCh,
-			s.gracefulShutdownC,
-		)
 	}
 }
 
 // startTunnel starts a new tunnel connection. The resulting error will be sent on
-// s.tunnelErrors.
+// s.tunnelError as this is expected to run in a goroutine.
 func (s *Supervisor) startTunnel(
 	ctx context.Context,
 	index int,
 	connectedSignal *signal.Signal,
 ) {
 	var (
-		addr *allregions.EdgeAddr
-		err  error
+		err error
 	)
 	defer func() {
-		s.tunnelErrors <- tunnelError{index: index, addr: addr, err: err}
+		s.tunnelErrors <- tunnelError{index: index, err: err}
 	}()
 
-	addr, err = s.edgeIPs.GetDifferentAddr(index)
-	if err != nil {
-		return
-	}
-	err = ServeTunnelLoop(
-		ctx,
-		s.reconnectCredentialManager,
-		s.config,
-		s.orchestrator,
-		addr,
-		s.log,
-		uint8(index),
-		connectedSignal,
-		s.cloudflaredUUID,
-		s.reconnectCh,
-		s.gracefulShutdownC,
-	)
+	err = s.edgeTunnelServer.Serve(ctx, uint8(index), s.tunnelsProtocolFallback[index], connectedSignal)
 }
 
 func (s *Supervisor) newConnectedTunnelSignal(index int) *signal.Signal {

supervisor/tunnel.go

@@ -43,6 +43,7 @@ type TunnelConfig struct {
 	CloseConnOnce   *sync.Once // Used to close connectedSignal no more than once
 	EdgeAddrs       []string
 	Region          string
+	EdgeIPVersion   allregions.ConfigIPVersion
 	HAConnections   int
 	IncidentLookup  IncidentLookup
 	IsAutoupdated   bool
@@ -121,30 +122,82 @@ func StartTunnelDaemon(
 	return s.Run(ctx, connectedSignal)
 }
 
-func ServeTunnelLoop(
-	ctx context.Context,
-	credentialManager *reconnectCredentialManager,
-	config *TunnelConfig,
-	orchestrator *orchestration.Orchestrator,
-	addr *allregions.EdgeAddr,
-	connAwareLogger *ConnAwareLogger,
-	connIndex uint8,
-	connectedSignal *signal.Signal,
-	cloudflaredUUID uuid.UUID,
-	reconnectCh chan ReconnectSignal,
-	gracefulShutdownC <-chan struct{},
-) error {
+// EdgeAddrHandler provides a mechanism switch between behaviors in ServeTunnel
+// for handling the errors when attempting to make edge connections.
+type EdgeAddrHandler interface {
+	// ShouldGetNewAddress will check the edge connection error and determine if
+	// the edge address should be replaced with a new one. Also, will return if the
+	// error should be recognized as a connectivity error, or otherwise, a general
+	// application error.
+	ShouldGetNewAddress(err error) (needsNewAddress bool, isConnectivityError bool)
+}
+
+// DefaultAddrFallback will always return false for isConnectivityError since this
+// handler is a way to provide the legacy behavior in the new edge discovery algorithm.
+type DefaultAddrFallback struct {
+	edgeErrors int
+}
+
+func (f DefaultAddrFallback) ShouldGetNewAddress(err error) (needsNewAddress bool, isConnectivityError bool) {
+	switch err.(type) {
+	case nil: // maintain current IP address
+	// DupConnRegisterTunnelError should indicate to get a new address immediately
+	case connection.DupConnRegisterTunnelError:
+		return true, false
+	// Try the next address if it was a quic.IdleTimeoutError
+	case *quic.IdleTimeoutError,
+		edgediscovery.DialError,
+		*connection.EdgeQuicDialError:
+		// Wait for two failures before falling back to a new address
+		f.edgeErrors++
+		if f.edgeErrors >= 2 {
+			f.edgeErrors = 0
+			return true, false
+		}
+	default: // maintain current IP address
+	}
+	return false, false
+}
+
+// IPAddrFallback will have more conditions to fall back to a new address for certain
+// edge connection errors. This means that this handler will return true for isConnectivityError
+// for more cases like duplicate connection register and edge quic dial errors.
+type IPAddrFallback struct{}
+
+func (f IPAddrFallback) ShouldGetNewAddress(err error) (needsNewAddress bool, isConnectivityError bool) {
+	switch err.(type) {
+	case nil: // maintain current IP address
+	// Try the next address if it was a quic.IdleTimeoutError
+	// DupConnRegisterTunnelError needs to also receive a new ip address
+	case connection.DupConnRegisterTunnelError,
+		*quic.IdleTimeoutError:
+		return true, false
+	// Network problems should be retried with new address immediately and report
+	// as connectivity error
+	case edgediscovery.DialError, *connection.EdgeQuicDialError:
+		return true, true
+	default: // maintain current IP address
+	}
+	return false, false
+}
+
+type EdgeTunnelServer struct {
+	config            *TunnelConfig
+	cloudflaredUUID   uuid.UUID
+	orchestrator      *orchestration.Orchestrator
+	credentialManager *reconnectCredentialManager
+	edgeAddrHandler   EdgeAddrHandler
+	edgeAddrs         *edgediscovery.Edge
+	reconnectCh       chan ReconnectSignal
+	gracefulShutdownC <-chan struct{}
+
+	connAwareLogger *ConnAwareLogger
+}
+
+func (e EdgeTunnelServer) Serve(ctx context.Context, connIndex uint8, protocolFallback *protocolFallback, connectedSignal *signal.Signal) error {
 	haConnections.Inc()
 	defer haConnections.Dec()
 
-	logger := config.Log.With().Uint8(connection.LogFieldConnIndex, connIndex).Logger()
-	connLog := connAwareLogger.ReplaceLogger(&logger)
-
-	protocolFallback := &protocolFallback{
-		retry.BackoffHandler{MaxRetries: config.Retries},
-		config.ProtocolSelector.Current(),
-		false,
-	}
 	connectedFuse := h2mux.NewBooleanFuse()
 	go func() {
 		if connectedFuse.Await() {
@@ -153,54 +206,83 @@ func ServeTunnelLoop(
 	}()
 	// Ensure the above goroutine will terminate if we return without connecting
 	defer connectedFuse.Fuse(false)
+
+	// Fetch IP address to associated connection index
+	addr, err := e.edgeAddrs.GetAddr(int(connIndex))
+	switch err.(type) {
+	case nil: // no error
+	case edgediscovery.ErrNoAddressesLeft:
+		return err
+	default:
+		return err
+	}
+
+	logger := e.config.Log.With().
+		IPAddr(connection.LogFieldIPAddress, addr.UDP.IP).
+		Uint8(connection.LogFieldConnIndex, connIndex).
+		Logger()
+	connLog := e.connAwareLogger.ReplaceLogger(&logger)
 	// Each connection to keep its own copy of protocol, because individual connections might fallback
 	// to another protocol when a particular metal doesn't support new protocol
-	for {
-		err, recoverable := ServeTunnel(
-			ctx,
-			connLog,
-			credentialManager,
-			config,
-			orchestrator,
-			addr,
-			connIndex,
-			connectedFuse,
-			protocolFallback,
-			cloudflaredUUID,
-			reconnectCh,
-			protocolFallback.protocol,
-			gracefulShutdownC,
-		)
+	// Each connection can also have it's own IP version because individual connections might fallback
+	// to another IP version.
+	err, recoverable := ServeTunnel(
+		ctx,
+		connLog,
+		e.credentialManager,
+		e.config,
+		e.orchestrator,
+		addr,
+		connIndex,
+		connectedFuse,
+		protocolFallback,
+		e.cloudflaredUUID,
+		e.reconnectCh,
+		protocolFallback.protocol,
+		e.gracefulShutdownC,
+	)
 
-		if recoverable {
-			duration, ok := protocolFallback.GetMaxBackoffDuration(ctx)
-			if !ok {
-				return err
-			}
-			config.Observer.SendReconnect(connIndex)
-			connLog.Logger().Info().Msgf("Retrying connection in up to %s seconds", duration)
+	// If the connection is recoverable, we want to maintain the same IP
+	// but backoff a reconnect with some duration.
+	if recoverable {
+		duration, ok := protocolFallback.GetMaxBackoffDuration(ctx)
+		if !ok {
+			return err
 		}
+		e.config.Observer.SendReconnect(connIndex)
+		connLog.Logger().Info().Msgf("Retrying connection in up to %s seconds", duration)
+	}
 
-		select {
-		case <-ctx.Done():
-			return ctx.Err()
-		case <-gracefulShutdownC:
-			return nil
-		case <-protocolFallback.BackoffTimer():
-			if !recoverable {
-				return err
-			}
-
-			if !selectNextProtocol(
-				connLog.Logger(),
-				protocolFallback,
-				config.ProtocolSelector,
-				err,
-			) {
-				return err
-			}
+	// Check if the connection error was from an IP issue with the host or
+	// establishing a connection to the edge and if so, rotate the IP address.
+	yes, hasConnectivityError := e.edgeAddrHandler.ShouldGetNewAddress(err)
+	if yes {
+		if _, err := e.edgeAddrs.GetDifferentAddr(int(connIndex), hasConnectivityError); err != nil {
+			return err
 		}
 	}
+
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case <-e.gracefulShutdownC:
+		return nil
+	case <-protocolFallback.BackoffTimer():
+		if !recoverable {
+			return err
+		}
+
+		if !selectNextProtocol(
+			connLog.Logger(),
+			protocolFallback,
+			e.config.ProtocolSelector,
+			err,
+		) {
+			return err
+		}
+	}
+
+	return err
 }
 
 // protocolFallback is a wrapper around backoffHandler that will try fallback option when backoff reaches
@@ -232,6 +314,10 @@ func selectNextProtocol(
 ) bool {
 	var idleTimeoutError *quic.IdleTimeoutError
 	isNetworkActivityTimeout := errors.As(cause, &idleTimeoutError)
+	edgeQuicDialError, ok := cause.(*connection.EdgeQuicDialError)
+	if !isNetworkActivityTimeout && ok {
+		isNetworkActivityTimeout = errors.As(edgeQuicDialError.Cause, &idleTimeoutError)
+	}
 	_, hasFallback := selector.Fallback()
 
 	if protocolBackoff.ReachedMaxRetries() || (hasFallback && isNetworkActivityTimeout) {
@@ -240,7 +326,7 @@ func selectNextProtocol(
 				"Cloudflare Network with `quic` protocol, then most likely your machine/network is getting its egress " +
 				"UDP to port 7844 (or others) blocked or dropped. Make sure to allow egress connectivity as per " +
 				"https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/configuration/ports-and-ips/\n" +
-				"If you are using private routing to this Tunnel, then UDP (and Private DNS Resolution) will not work" +
+				"If you are using private routing to this Tunnel, then UDP (and Private DNS Resolution) will not work " +
 				"unless your cloudflared can connect with Cloudflare Network with `quic`.")
 		}
 
@@ -325,8 +411,11 @@ func ServeTunnel(
 				connLog.ConnAwareLogger().Msg(activeIncidentsMsg(incidents))
 			}
 			return err.Cause, !err.Permanent
+		case *connection.EdgeQuicDialError:
+			return err, true
 		case ReconnectSignal:
 			connLog.Logger().Info().
+				IPAddr(connection.LogFieldIPAddress, addr.UDP.IP).
 				Uint8(connection.LogFieldConnIndex, connIndex).
 				Msgf("Restarting connection due to reconnect signal in %s", err.Delay)
 			err.DelayBeforeReconnect()
@@ -368,6 +457,7 @@ func serveTunnel(
 		connectedFuse,
 		config.NamedTunnel,
 		connIndex,
+		addr.UDP.IP,
 		nil,
 		gracefulShutdownC,
 		config.GracePeriod,
@@ -467,6 +557,7 @@ func ServeH2mux(
 		connIndex,
 		config.Observer,
 		gracefulShutdownC,
+		config.Log,
 	)
 	if err != nil {
 		if !recoverable {
@@ -525,6 +616,7 @@ func ServeHTTP2(
 		err := listenReconnect(serveCtx, reconnectCh, gracefulShutdownC)
 		if err != nil {
 			// forcefully break the connection (this is only used for testing)
+			connLog.Logger().Debug().Msg("Forcefully breaking http2 connection")
 			_ = tlsServerConn.Close()
 		}
 		return err
@@ -549,9 +641,9 @@ func ServeQUIC(
 	quicConfig := &quic.Config{
 		HandshakeIdleTimeout:  quicpogs.HandshakeIdleTimeout,
 		MaxIdleTimeout:        quicpogs.MaxIdleTimeout,
+		KeepAlivePeriod:       quicpogs.MaxIdlePingPeriod,
 		MaxIncomingStreams:    connection.MaxConcurrentStreams,
 		MaxIncomingUniStreams: connection.MaxConcurrentStreams,
-		KeepAlive:             true,
 		EnableDatagrams:       true,
 		MaxDatagramFrameSize:  quicpogs.MaxDatagramFrameSize,
 		Tracer:                quicpogs.NewClientTracer(connLogger.Logger(), connIndex),
@@ -583,6 +675,7 @@ func ServeQUIC(
 		err := listenReconnect(serveCtx, reconnectCh, gracefulShutdownC)
 		if err != nil {
 			// forcefully break the connection (this is only used for testing)
+			connLogger.Logger().Debug().Msg("Forcefully breaking quic connection")
 			quicConn.Close()
 		}
 		return err

token/token.go

@@ -29,6 +29,10 @@ const (
 	AccessLoginWorkerPath = "/cdn-cgi/access/login"
 )
 
+var (
+	userAgent = "DEV"
+)
+
 type AppInfo struct {
 	AuthDomain string
 	AppAUD     string
@@ -144,6 +148,10 @@ func isTokenLocked(lockFilePath string) bool {
 	return exists && err == nil
 }
 
+func Init(version string) {
+	userAgent = fmt.Sprintf("cloudflared/%s", version)
+}
+
 // FetchTokenWithRedirect will either load a stored token or generate a new one
 // it appends the full url as the redirect URL to the access cli request if opening the browser
 func FetchTokenWithRedirect(appURL *url.URL, appInfo *AppInfo, log *zerolog.Logger) (string, error) {
@@ -261,6 +269,7 @@ func GetAppInfo(reqURL *url.URL) (*AppInfo, error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to create app info request")
 	}
+	appInfoReq.Header.Add("User-Agent", userAgent)
 	resp, err := client.Do(appInfoReq)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to get app info")
@@ -311,6 +320,7 @@ func exchangeOrgToken(appURL *url.URL, orgToken string) (string, error) {
 	if err != nil {
 		return "", errors.Wrap(err, "failed to create app token request")
 	}
+	appTokenRequest.Header.Add("User-Agent", userAgent)
 	resp, err := client.Do(appTokenRequest)
 	if err != nil {
 		return "", errors.Wrap(err, "failed to get app token")

token/transfer.go

@@ -113,7 +113,12 @@ func transferRequest(requestURL string, log *zerolog.Logger) ([]byte, string, er
 
 // poll the endpoint for the request resource, waiting for the user interaction
 func poll(client *http.Client, requestURL string, log *zerolog.Logger) ([]byte, string, error) {
-	resp, err := client.Get(requestURL)
+	req, err := http.NewRequest(http.MethodGet, requestURL, nil)
+	if err != nil {
+		return nil, "", err
+	}
+	req.Header.Set("User-Agent", userAgent)
+	resp, err := client.Do(req)
 	if err != nil {
 		return nil, "", err
 	}

tracing/tracing.go

@@ -8,6 +8,7 @@ import (
 	"net/http"
 	"os"
 	"runtime"
+	"strings"
 
 	"github.com/rs/zerolog"
 	otelContrib "go.opentelemetry.io/contrib/propagators/jaeger"
@@ -33,6 +34,9 @@ const (
 
 	MaxErrorDescriptionLen = 100
 	traceHttpStatusCodeKey = "upstreamStatusCode"
+
+	traceID128bitsWidth = 128 / 4
+	separator           = ":"
 )
 
 var (
@@ -66,22 +70,50 @@ func Init(version string) {
 	cloudflaredVersionAttribute = semconv.ProcessRuntimeVersionKey.String(version)
 }
 
-type TracedRequest struct {
+type TracedHTTPRequest struct {
 	*http.Request
-	trace.TracerProvider
-	exporter InMemoryClient
+	*cfdTracer
 }
 
-// NewTracedRequest creates a new tracer for the current request context.
-func NewTracedRequest(req *http.Request) *TracedRequest {
+// NewTracedHTTPRequest creates a new tracer for the current HTTP request context.
+func NewTracedHTTPRequest(req *http.Request, log *zerolog.Logger) *TracedHTTPRequest {
 	ctx, exists := extractTrace(req)
 	if !exists {
-		return &TracedRequest{req, trace.NewNoopTracerProvider(), &NoopOtlpClient{}}
+		return &TracedHTTPRequest{req, &cfdTracer{trace.NewNoopTracerProvider(), &NoopOtlpClient{}, log}}
 	}
+	return &TracedHTTPRequest{req.WithContext(ctx), newCfdTracer(ctx, log)}
+}
+
+func (tr *TracedHTTPRequest) ToTracedContext() *TracedContext {
+	return &TracedContext{tr.Context(), tr.cfdTracer}
+}
+
+type TracedContext struct {
+	context.Context
+	*cfdTracer
+}
+
+// NewTracedHTTPRequest creates a new tracer for the current HTTP request context.
+func NewTracedContext(ctx context.Context, traceContext string, log *zerolog.Logger) *TracedContext {
+	ctx, exists := extractTraceFromString(ctx, traceContext)
+	if !exists {
+		return &TracedContext{ctx, &cfdTracer{trace.NewNoopTracerProvider(), &NoopOtlpClient{}, log}}
+	}
+	return &TracedContext{ctx, newCfdTracer(ctx, log)}
+}
+
+type cfdTracer struct {
+	trace.TracerProvider
+	exporter InMemoryClient
+	log      *zerolog.Logger
+}
+
+// NewCfdTracer creates a new tracer for the current request context.
+func newCfdTracer(ctx context.Context, log *zerolog.Logger) *cfdTracer {
 	mc := new(InMemoryOtlpClient)
-	exp, err := otlptrace.New(req.Context(), mc)
+	exp, err := otlptrace.New(ctx, mc)
 	if err != nil {
-		return &TracedRequest{req, trace.NewNoopTracerProvider(), &NoopOtlpClient{}}
+		return &cfdTracer{trace.NewNoopTracerProvider(), &NoopOtlpClient{}, log}
 	}
 	tp := tracesdk.NewTracerProvider(
 		// We want to dump to in-memory exporter immediately
@@ -98,36 +130,41 @@ func NewTracedRequest(req *http.Request) *TracedRequest {
 		)),
 	)
 
-	return &TracedRequest{req.WithContext(ctx), tp, mc}
+	return &cfdTracer{tp, mc, log}
 }
 
-func (cft *TracedRequest) Tracer() trace.Tracer {
+func (cft *cfdTracer) Tracer() trace.Tracer {
 	return cft.TracerProvider.Tracer(tracerInstrumentName)
 }
 
-// Spans returns the spans as base64 encoded protobuf otlp traces.
-func (cft *TracedRequest) AddSpans(headers http.Header, log *zerolog.Logger) {
-	if headers == nil {
-		log.Error().Msgf("provided headers map is nil")
-		return
-	}
-
+// GetSpans returns the spans as base64 encoded string of protobuf otlp traces.
+func (cft *cfdTracer) GetSpans() (enc string) {
 	enc, err := cft.exporter.Spans()
 	switch err {
 	case nil:
 		break
 	case errNoTraces:
-		log.Error().Err(err).Msgf("expected traces to be available")
+		cft.log.Trace().Err(err).Msgf("expected traces to be available")
 		return
 	case errNoopTracer:
 		return // noop tracer has no traces
 	default:
-		log.Error().Err(err)
+		cft.log.Debug().Err(err)
 		return
 	}
+	return
+}
+
+// AddSpans assigns spans as base64 encoded protobuf otlp traces to provided
+// HTTP headers.
+func (cft *cfdTracer) AddSpans(headers http.Header) {
+	if headers == nil {
+		return
+	}
+
+	enc := cft.GetSpans()
 	// No need to add header if no traces
 	if enc == "" {
-		log.Error().Msgf("no traces provided and no error from exporter")
 		return
 	}
 
@@ -166,6 +203,33 @@ func endSpan(span trace.Span, upstreamStatusCode int, spanStatusCode codes.Code,
 	span.End()
 }
 
+// extractTraceFromString will extract the trace information from the provided
+// propagated trace string context.
+func extractTraceFromString(ctx context.Context, trace string) (context.Context, bool) {
+	if trace == "" {
+		return ctx, false
+	}
+	// Jaeger specific separator
+	parts := strings.Split(trace, separator)
+	if len(parts) != 4 {
+		return ctx, false
+	}
+	if parts[0] == "" {
+		return ctx, false
+	}
+	// Correctly left pad the trace to a length of 32
+	if len(parts[0]) < traceID128bitsWidth {
+		left := traceID128bitsWidth - len(parts[0])
+		parts[0] = strings.Repeat("0", left) + parts[0]
+		trace = strings.Join(parts, separator)
+	}
+
+	// Override the 'cf-trace-id' as 'uber-trace-id' so the jaeger propagator can extract it.
+	traceHeader := map[string]string{TracerContextNameOverride: trace}
+	remoteCtx := otel.GetTextMapPropagator().Extract(ctx, propagation.MapCarrier(traceHeader))
+	return remoteCtx, true
+}
+
 // extractTrace attempts to check for a cf-trace-id from a request and return the
 // trace context with the provided http.Request.
 func extractTrace(req *http.Request) (context.Context, bool) {

tracing/tracing_test.go

@@ -14,38 +14,42 @@ import (
 )
 
 func TestNewCfTracer(t *testing.T) {
+	log := zerolog.Nop()
 	req := httptest.NewRequest("GET", "http://localhost", nil)
 	req.Header.Add(TracerContextName, "14cb070dde8e51fc5ae8514e69ba42ca:b38f1bf5eae406f3:0:1")
-	tr := NewTracedRequest(req)
+	tr := NewTracedHTTPRequest(req, &log)
 	assert.NotNil(t, tr)
 	assert.IsType(t, tracesdk.NewTracerProvider(), tr.TracerProvider)
 	assert.IsType(t, &InMemoryOtlpClient{}, tr.exporter)
 }
 
 func TestNewCfTracerMultiple(t *testing.T) {
+	log := zerolog.Nop()
 	req := httptest.NewRequest("GET", "http://localhost", nil)
 	req.Header.Add(TracerContextName, "1241ce3ecdefc68854e8514e69ba42ca:b38f1bf5eae406f3:0:1")
 	req.Header.Add(TracerContextName, "14cb070dde8e51fc5ae8514e69ba42ca:b38f1bf5eae406f3:0:1")
-	tr := NewTracedRequest(req)
+	tr := NewTracedHTTPRequest(req, &log)
 	assert.NotNil(t, tr)
 	assert.IsType(t, tracesdk.NewTracerProvider(), tr.TracerProvider)
 	assert.IsType(t, &InMemoryOtlpClient{}, tr.exporter)
 }
 
 func TestNewCfTracerNilHeader(t *testing.T) {
+	log := zerolog.Nop()
 	req := httptest.NewRequest("GET", "http://localhost", nil)
 	req.Header[http.CanonicalHeaderKey(TracerContextName)] = nil
-	tr := NewTracedRequest(req)
+	tr := NewTracedHTTPRequest(req, &log)
 	assert.NotNil(t, tr)
 	assert.IsType(t, trace.NewNoopTracerProvider(), tr.TracerProvider)
 	assert.IsType(t, &NoopOtlpClient{}, tr.exporter)
 }
 
 func TestNewCfTracerInvalidHeaders(t *testing.T) {
+	log := zerolog.Nop()
 	req := httptest.NewRequest("GET", "http://localhost", nil)
 	for _, test := range [][]string{nil, {""}} {
 		req.Header[http.CanonicalHeaderKey(TracerContextName)] = test
-		tr := NewTracedRequest(req)
+		tr := NewTracedHTTPRequest(req, &log)
 		assert.NotNil(t, tr)
 		assert.IsType(t, trace.NewNoopTracerProvider(), tr.TracerProvider)
 		assert.IsType(t, &NoopOtlpClient{}, tr.exporter)
@@ -53,9 +57,10 @@ func TestNewCfTracerInvalidHeaders(t *testing.T) {
 }
 
 func TestAddingSpansWithNilMap(t *testing.T) {
+	log := zerolog.Nop()
 	req := httptest.NewRequest("GET", "http://localhost", nil)
 	req.Header.Add(TracerContextName, "14cb070dde8e51fc5ae8514e69ba42ca:b38f1bf5eae406f3:0:1")
-	tr := NewTracedRequest(req)
+	tr := NewTracedHTTPRequest(req, &log)
 
 	exporter := tr.exporter.(*InMemoryOtlpClient)
 
@@ -65,5 +70,5 @@ func TestAddingSpansWithNilMap(t *testing.T) {
 	assert.NoError(t, err)
 
 	// a panic shouldn't occur
-	tr.AddSpans(nil, &zerolog.Logger{})
+	tr.AddSpans(nil)
 }

vendor/github.com/francoispqt/gojay/.gitignore

@@ -1,5 +0,0 @@
-vendor
-*.out
-*.log
-*.test
-.vscode

vendor/github.com/francoispqt/gojay/.travis.yml

@@ -1,15 +0,0 @@
-language: go
-
-go:
-  - "1.10.x"
-  - "1.11.x"
-  - "1.12.x"
-
-script:
-  - go get github.com/golang/dep/cmd/dep github.com/stretchr/testify
-  - dep ensure -v -vendor-only
-  - go test ./gojay/codegen/test/... -race
-  - go test -race -coverprofile=coverage.txt -covermode=atomic
-
-after_success:
-  - bash <(curl -s https://codecov.io/bash)

vendor/github.com/francoispqt/gojay/Gopkg.lock

@@ -1,163 +0,0 @@
-# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
-
-
-[[projects]]
-  digest = "1:1a37f9f2ae10d161d9688fb6008ffa14e1631e5068cc3e9698008b9e8d40d575"
-  name = "cloud.google.com/go"
-  packages = ["compute/metadata"]
-  pruneopts = ""
-  revision = "457ea5c15ccf3b87db582c450e80101989da35f7"
-  version = "v0.40.0"
-
-[[projects]]
-  digest = "1:968d8903d598e3fae738325d3410f33f07ea6a2b9ee5591e9c262ee37df6845a"
-  name = "github.com/go-errors/errors"
-  packages = ["."]
-  pruneopts = ""
-  revision = "a6af135bd4e28680facf08a3d206b454abc877a4"
-  version = "v1.0.1"
-
-[[projects]]
-  digest = "1:529d738b7976c3848cae5cf3a8036440166835e389c1f617af701eeb12a0518d"
-  name = "github.com/golang/protobuf"
-  packages = ["proto"]
-  pruneopts = ""
-  revision = "b5d812f8a3706043e23a9cd5babf2e5423744d30"
-  version = "v1.3.1"
-
-[[projects]]
-  branch = "master"
-  digest = "1:cae59d7b8243c671c9f544965522ba35c0fec48ee80adb9f1400cd2f33abbbec"
-  name = "github.com/mailru/easyjson"
-  packages = [
-    ".",
-    "buffer",
-    "jlexer",
-    "jwriter",
-  ]
-  pruneopts = ""
-  revision = "1ea4449da9834f4d333f1cc461c374aea217d249"
-
-[[projects]]
-  digest = "1:1d7e1867c49a6dd9856598ef7c3123604ea3daabf5b83f303ff457bcbc410b1d"
-  name = "github.com/pkg/errors"
-  packages = ["."]
-  pruneopts = ""
-  revision = "ba968bfe8b2f7e042a574c888954fccecfa385b4"
-  version = "v0.8.1"
-
-[[projects]]
-  digest = "1:8d4bbd8ab012efc77ab6b97286f2aff262bcdeac9803bb57d75cf7d0a5e6a877"
-  name = "github.com/viant/assertly"
-  packages = ["."]
-  pruneopts = ""
-  revision = "04f45e0aeb6f3455884877b047a97bcc95dc9493"
-  version = "v0.4.8"
-
-[[projects]]
-  digest = "1:5913451bc2d274673c0716efe226a137625740cd9380641f4d8300ff4f2d82a0"
-  name = "github.com/viant/toolbox"
-  packages = [
-    ".",
-    "cred",
-    "data",
-    "storage",
-    "url",
-  ]
-  pruneopts = ""
-  revision = "1be8e4d172138324f40d55ea61a2aeab0c5ce864"
-  version = "v0.24.0"
-
-[[projects]]
-  branch = "master"
-  digest = "1:9d150270ca2c3356f2224a0878daa1652e4d0b25b345f18b4f6e156cc4b8ec5e"
-  name = "golang.org/x/crypto"
-  packages = [
-    "blowfish",
-    "curve25519",
-    "ed25519",
-    "ed25519/internal/edwards25519",
-    "internal/chacha20",
-    "internal/subtle",
-    "poly1305",
-    "ssh",
-  ]
-  pruneopts = ""
-  revision = "f99c8df09eb5bff426315721bfa5f16a99cad32c"
-
-[[projects]]
-  branch = "master"
-  digest = "1:5a56f211e7c12a65c5585c629457a2fb91d8719844ee8fab92727ea8adb5721c"
-  name = "golang.org/x/net"
-  packages = [
-    "context",
-    "context/ctxhttp",
-    "websocket",
-  ]
-  pruneopts = ""
-  revision = "461777fb6f67e8cb9d70cda16573678d085a74cf"
-
-[[projects]]
-  branch = "master"
-  digest = "1:01bdbbc604dcd5afb6f66a717f69ad45e9643c72d5bc11678d44ffa5c50f9e42"
-  name = "golang.org/x/oauth2"
-  packages = [
-    ".",
-    "google",
-    "internal",
-    "jws",
-    "jwt",
-  ]
-  pruneopts = ""
-  revision = "0f29369cfe4552d0e4bcddc57cc75f4d7e672a33"
-
-[[projects]]
-  branch = "master"
-  digest = "1:8ddb956f67d4c176abbbc42b7514aaeaf9ea30daa24e27d2cf30ad82f9334a2c"
-  name = "golang.org/x/sys"
-  packages = ["cpu"]
-  pruneopts = ""
-  revision = "1e42afee0f762ed3d76e6dd942e4181855fd1849"
-
-[[projects]]
-  digest = "1:47f391ee443f578f01168347818cb234ed819521e49e4d2c8dd2fb80d48ee41a"
-  name = "google.golang.org/appengine"
-  packages = [
-    ".",
-    "internal",
-    "internal/app_identity",
-    "internal/base",
-    "internal/datastore",
-    "internal/log",
-    "internal/modules",
-    "internal/remote_api",
-    "internal/urlfetch",
-    "urlfetch",
-  ]
-  pruneopts = ""
-  revision = "b2f4a3cf3c67576a2ee09e1fe62656a5086ce880"
-  version = "v1.6.1"
-
-[[projects]]
-  digest = "1:cedccf16b71e86db87a24f8d4c70b0a855872eb967cb906a66b95de56aefbd0d"
-  name = "gopkg.in/yaml.v2"
-  packages = ["."]
-  pruneopts = ""
-  revision = "51d6538a90f86fe93ac480b35f37b2be17fef232"
-  version = "v2.2.2"
-
-[solve-meta]
-  analyzer-name = "dep"
-  analyzer-version = 1
-  input-imports = [
-    "github.com/go-errors/errors",
-    "github.com/mailru/easyjson",
-    "github.com/mailru/easyjson/jlexer",
-    "github.com/mailru/easyjson/jwriter",
-    "github.com/viant/assertly",
-    "github.com/viant/toolbox",
-    "github.com/viant/toolbox/url",
-    "golang.org/x/net/websocket",
-  ]
-  solver-name = "gps-cdcl"
-  solver-version = 1

vendor/github.com/francoispqt/gojay/Gopkg.toml

@@ -1,23 +0,0 @@
-# Gopkg.toml example
-#
-# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md
-# for detailed Gopkg.toml documentation.
-#
-# required = ["github.com/user/thing/cmd/thing"]
-# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"]
-#
-# [[constraint]]
-#   name = "github.com/user/project"
-#   version = "1.0.0"
-#
-# [[constraint]]
-#   name = "github.com/user/project2"
-#   branch = "dev"
-#   source = "github.com/myfork/project2"
-#
-# [[override]]
-#  name = "github.com/x/y"
-#  version = "2.4.0"
-
-
-ignored = ["github.com/francoispqt/benchmarks*","github.com/stretchr/testify*","github.com/stretchr/testify","github.com/json-iterator/go","github.com/buger/jsonparser"]

vendor/github.com/francoispqt/gojay/LICENSE

@@ -1,21 +0,0 @@
-MIT License
-
-Copyright (c) 2016 gojay
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

vendor/github.com/francoispqt/gojay/Makefile

@@ -1,11 +0,0 @@
-.PHONY: test
-test:
-	go test -race -run=^Test -v
-
-.PHONY: cover
-cover: 
-	go test -coverprofile=coverage.out -covermode=atomic
-
-.PHONY: coverhtml
-coverhtml: 
-	go tool cover -html=coverage.out

vendor/github.com/francoispqt/gojay/README.md

@@ -1,855 +0,0 @@
-[![Build Status](https://travis-ci.org/francoispqt/gojay.svg?branch=master)](https://travis-ci.org/francoispqt/gojay)
-[![codecov](https://codecov.io/gh/francoispqt/gojay/branch/master/graph/badge.svg)](https://codecov.io/gh/francoispqt/gojay)
-[![Go Report Card](https://goreportcard.com/badge/github.com/francoispqt/gojay)](https://goreportcard.com/report/github.com/francoispqt/gojay)
-[![Go doc](http://img.shields.io/badge/go-documentation-blue.svg?style=flat-square
-)](https://godoc.org/github.com/francoispqt/gojay)
-![MIT License](https://img.shields.io/badge/license-mit-blue.svg?style=flat-square)
-[![Sourcegraph](https://sourcegraph.com/github.com/francoispqt/gojay/-/badge.svg)](https://sourcegraph.com/github.com/francoispqt/gojay)
-![stability-stable](https://img.shields.io/badge/stability-stable-green.svg)
-
-# GoJay
-
-<img src="https://github.com/francoispqt/gojay/raw/master/gojay.png" width="200px">
-
-GoJay is a performant JSON encoder/decoder for Golang (currently the most performant, [see benchmarks](#benchmark-results)).
-
-It has a simple API and doesn't use reflection. It relies on small interfaces to decode/encode structures and slices.
-
-Gojay also comes with powerful stream decoding features and an even faster [Unsafe](#unsafe-api) API.
-
-There is also a [code generation tool](https://github.com/francoispqt/gojay/tree/master/gojay) to make usage easier and faster.
-
-# Why another JSON parser?
-
-I looked at other fast decoder/encoder and realised it was mostly hardly readable static code generation or a lot of reflection, poor streaming features, and not so fast in the end.
-
-Also, I wanted to build a decoder that could consume an io.Reader of line or comma delimited JSON, in a JIT way. To consume a flow of JSON objects from a TCP connection for example or from a standard output. Same way I wanted to build an encoder that could encode a flow of data to a io.Writer.
-
-This is how GoJay aims to be a very fast, JIT stream parser with 0 reflection, low allocation with a friendly API.
-
-# Get started
-
-```bash
-go get github.com/francoispqt/gojay
-```
-
-* [Encoder](#encoding)
-* [Decoder](#decoding)
-* [Stream API](#stream-api)
-* [Code Generation](https://github.com/francoispqt/gojay/tree/master/gojay)
-
-## Decoding
-
-Decoding is done through two different API similar to standard `encoding/json`:
-* [Unmarshal](#unmarshal-api)
-* [Decode](#decode-api)
-
-
-Example of basic stucture decoding with Unmarshal:
-```go
-import "github.com/francoispqt/gojay"
-
-type user struct {
-    id int
-    name string
-    email string
-}
-// implement gojay.UnmarshalerJSONObject
-func (u *user) UnmarshalJSONObject(dec *gojay.Decoder, key string) error {
-    switch key {
-    case "id":
-        return dec.Int(&u.id)
-    case "name":
-        return dec.String(&u.name)
-    case "email":
-        return dec.String(&u.email)
-    }
-    return nil
-}
-func (u *user) NKeys() int {
-    return 3
-}
-
-func main() {
-    u := &user{}
-    d := []byte(`{"id":1,"name":"gojay","email":"gojay@email.com"}`)
-    err := gojay.UnmarshalJSONObject(d, u)
-    if err != nil {
-        log.Fatal(err)
-    }
-}
-```
-
-with Decode:
-```go
-func main() {
-    u := &user{}
-    dec := gojay.NewDecoder(bytes.NewReader([]byte(`{"id":1,"name":"gojay","email":"gojay@email.com"}`)))
-    err := dec.DecodeObject(d, u)
-    if err != nil {
-        log.Fatal(err)
-    }
-}
-```
-
-### Unmarshal API
-
-Unmarshal API decodes a `[]byte` to a given pointer with a single function.
-
-Behind the doors, Unmarshal API borrows a `*gojay.Decoder` resets its settings and decodes the data to the given pointer and releases the `*gojay.Decoder` to the pool when it finishes, whether it encounters an error or not.
-
-If it cannot find the right Decoding strategy for the type of the given pointer, it returns an `InvalidUnmarshalError`. You can test the error returned by doing `if ok := err.(InvalidUnmarshalError); ok {}`.
-
-Unmarshal API comes with three functions:
-* Unmarshal
-```go
-func Unmarshal(data []byte, v interface{}) error
-```
-
-* UnmarshalJSONObject
-```go
-func UnmarshalJSONObject(data []byte, v gojay.UnmarshalerJSONObject) error
-```
-
-* UnmarshalJSONArray
-```go
-func UnmarshalJSONArray(data []byte, v gojay.UnmarshalerJSONArray) error
-```
-
-
-### Decode API
-
-Decode API decodes a `[]byte` to a given pointer by creating or borrowing a `*gojay.Decoder` with an `io.Reader` and calling `Decode` methods.
-
-__Getting a *gojay.Decoder or Borrowing__
-
-You can either get a fresh `*gojay.Decoder` calling `dec := gojay.NewDecoder(io.Reader)` or borrow one from the pool by calling `dec := gojay.BorrowDecoder(io.Reader)`.
-
-After using a decoder, you can release it by calling `dec.Release()`. Beware, if you reuse the decoder after releasing it, it will panic with an error of type `InvalidUsagePooledDecoderError`. If you want to fully benefit from the pooling, you must release your decoders after using.
-
-Example getting a fresh an releasing:
-```go
-str := ""
-dec := gojay.NewDecoder(strings.NewReader(`"test"`))
-defer dec.Release()
-if err := dec.Decode(&str); err != nil {
-    log.Fatal(err)
-}
-```
-Example borrowing a decoder and releasing:
-```go
-str := ""
-dec := gojay.BorrowDecoder(strings.NewReader(`"test"`))
-defer dec.Release()
-if err := dec.Decode(&str); err != nil {
-    log.Fatal(err)
-}
-```
-
-`*gojay.Decoder` has multiple methods to decode to specific types:
-* Decode
-```go
-func (dec *gojay.Decoder) Decode(v interface{}) error
-```
-* DecodeObject
-```go
-func (dec *gojay.Decoder) DecodeObject(v gojay.UnmarshalerJSONObject) error
-```
-* DecodeArray
-```go
-func (dec *gojay.Decoder) DecodeArray(v gojay.UnmarshalerJSONArray) error
-```
-* DecodeInt
-```go
-func (dec *gojay.Decoder) DecodeInt(v *int) error
-```
-* DecodeBool
-```go
-func (dec *gojay.Decoder) DecodeBool(v *bool) error
-```
-* DecodeString
-```go
-func (dec *gojay.Decoder) DecodeString(v *string) error
-```
-
-All DecodeXxx methods are used to decode top level JSON values. If you are decoding keys or items of a JSON object or array, don't use the Decode methods.
-
-Example:
-```go
-reader := strings.NewReader(`"John Doe"`)
-dec := NewDecoder(reader)
-
-var str string
-err := dec.DecodeString(&str)
-if err != nil {
-    log.Fatal(err)
-}
-
-fmt.Println(str) // John Doe
-```
-
-### Structs and Maps
-#### UnmarshalerJSONObject Interface
-
-To unmarshal a JSON object to a structure, the structure must implement the `UnmarshalerJSONObject` interface:
-```go
-type UnmarshalerJSONObject interface {
-	UnmarshalJSONObject(*gojay.Decoder, string) error
-	NKeys() int
-}
-```
-`UnmarshalJSONObject` method takes two arguments, the first one is a pointer to the Decoder (*gojay.Decoder) and the second one is the string value of the current key being parsed. If the JSON data is not an object, the UnmarshalJSONObject method will never be called.
-
-`NKeys` method must return the number of keys to Unmarshal in the JSON object or 0. If zero is returned, all keys will be parsed.
-
-Example of implementation for a struct:
-```go
-type user struct {
-    id int
-    name string
-    email string
-}
-// implement UnmarshalerJSONObject
-func (u *user) UnmarshalJSONObject(dec *gojay.Decoder, key string) error {
-    switch key {
-    case "id":
-        return dec.Int(&u.id)
-    case "name":
-        return dec.String(&u.name)
-    case "email":
-        return dec.String(&u.email)
-    }
-    return nil
-}
-func (u *user) NKeys() int {
-    return 3
-}
-```
-
-Example of implementation for a `map[string]string`:
-```go
-// define our custom map type implementing UnmarshalerJSONObject
-type message map[string]string
-
-// Implementing Unmarshaler
-func (m message) UnmarshalJSONObject(dec *gojay.Decoder, k string) error {
-	str := ""
-	err := dec.String(&str)
-	if err != nil {
-		return err
-	}
-	m[k] = str
-	return nil
-}
-
-// we return 0, it tells the Decoder to decode all keys
-func (m message) NKeys() int {
-	return 0
-}
-```
-
-### Arrays, Slices and Channels
-
-To unmarshal a JSON object to a slice an array or a channel, it must implement the UnmarshalerJSONArray interface:
-```go
-type UnmarshalerJSONArray interface {
-	UnmarshalJSONArray(*gojay.Decoder) error
-}
-```
-UnmarshalJSONArray method takes one argument, a pointer to the Decoder (*gojay.Decoder). If the JSON data is not an array, the Unmarshal method will never be called.
-
-Example of implementation with a slice:
-```go
-type testSlice []string
-// implement UnmarshalerJSONArray
-func (t *testSlice) UnmarshalJSONArray(dec *gojay.Decoder) error {
-	str := ""
-	if err := dec.String(&str); err != nil {
-		return err
-	}
-	*t = append(*t, str)
-	return nil
-}
-
-func main() {
-	dec := gojay.BorrowDecoder(strings.NewReader(`["Tom", "Jim"]`))
-	var slice testSlice
-	err := dec.DecodeArray(&slice)
-	if err != nil {
-		log.Fatal(err)
-	}
-	fmt.Println(slice) // [Tom Jim]
-	dec.Release()
-}
-```
-
-Example of implementation with a channel:
-```go
-type testChannel chan string
-// implement UnmarshalerJSONArray
-func (c testChannel) UnmarshalJSONArray(dec *gojay.Decoder) error {
-	str := ""
-	if err := dec.String(&str); err != nil {
-		return err
-	}
-	c <- str
-	return nil
-}
-
-func main() {
-	dec := gojay.BorrowDecoder(strings.NewReader(`["Tom", "Jim"]`))
-	c := make(testChannel, 2)
-	err := dec.DecodeArray(c)
-	if err != nil {
-		log.Fatal(err)
-	}
-	for i := 0; i < 2; i++ {
-		fmt.Println(<-c)
-	}
-	close(c)
-	dec.Release()
-}
-```
-
-Example of implementation with an array:
-```go
-type testArray [3]string
-// implement UnmarshalerJSONArray
-func (a *testArray) UnmarshalJSONArray(dec *Decoder) error {
-	var str string
-	if err := dec.String(&str); err != nil {
-		return err
-	}
-	a[dec.Index()] = str
-	return nil
-}
-
-func main() {
-	dec := gojay.BorrowDecoder(strings.NewReader(`["Tom", "Jim", "Bob"]`))
-	var a testArray
-	err := dec.DecodeArray(&a)
-	fmt.Println(a) // [Tom Jim Bob]
-	dec.Release()
-}
-```
-
-### Other types
-To decode other types (string, int, int32, int64, uint32, uint64, float, booleans), you don't need to implement any interface.
-
-Example of encoding strings:
-```go
-func main() {
-    json := []byte(`"Jay"`)
-    var v string
-    err := gojay.Unmarshal(json, &v)
-    if err != nil {
-        log.Fatal(err)
-    }
-    fmt.Println(v) // Jay
-}
-```
-
-### Decode values methods
-When decoding a JSON object of a JSON array using `UnmarshalerJSONObject` or `UnmarshalerJSONArray` interface, the `gojay.Decoder` provides dozens of methods to Decode multiple types.
-
-Non exhaustive list of methods available (to see all methods, check the godoc):
-```go
-dec.Int
-dec.Int8
-dec.Int16
-dec.Int32
-dec.Int64
-dec.Uint8
-dec.Uint16
-dec.Uint32
-dec.Uint64
-dec.String
-dec.Time
-dec.Bool
-dec.SQLNullString
-dec.SQLNullInt64
-```
-
-
-## Encoding
-
-Encoding is done through two different API similar to standard `encoding/json`:
-* [Marshal](#marshal-api)
-* [Encode](#encode-api)
-
-Example of basic structure encoding with Marshal:
-```go
-import "github.com/francoispqt/gojay"
-
-type user struct {
-	id    int
-	name  string
-	email string
-}
-
-// implement MarshalerJSONObject
-func (u *user) MarshalJSONObject(enc *gojay.Encoder) {
-	enc.IntKey("id", u.id)
-	enc.StringKey("name", u.name)
-	enc.StringKey("email", u.email)
-}
-func (u *user) IsNil() bool {
-	return u == nil
-}
-
-func main() {
-	u := &user{1, "gojay", "gojay@email.com"}
-	b, err := gojay.MarshalJSONObject(u)
-	if err != nil {
-		log.Fatal(err)
-	}
-	fmt.Println(string(b)) // {"id":1,"name":"gojay","email":"gojay@email.com"}
-}
-```
-
-with Encode:
-```go
-func main() {
-	u := &user{1, "gojay", "gojay@email.com"}
-	b := strings.Builder{}
-	enc := gojay.NewEncoder(&b)
-	if err := enc.Encode(u); err != nil {
-		log.Fatal(err)
-	}
-	fmt.Println(b.String()) // {"id":1,"name":"gojay","email":"gojay@email.com"}
-}
-```
-
-### Marshal API
-
-Marshal API encodes a value to a JSON `[]byte` with a single function.
-
-Behind the doors, Marshal API borrows a `*gojay.Encoder` resets its settings and encodes the data to an internal byte buffer and releases the `*gojay.Encoder` to the pool when it finishes, whether it encounters an error or not.
-
-If it cannot find the right Encoding strategy for the type of the given value, it returns an `InvalidMarshalError`. You can test the error returned by doing `if ok := err.(InvalidMarshalError); ok {}`.
-
-Marshal API comes with three functions:
-* Marshal
-```go
-func Marshal(v interface{}) ([]byte, error)
-```
-
-* MarshalJSONObject
-```go
-func MarshalJSONObject(v gojay.MarshalerJSONObject) ([]byte, error)
-```
-
-* MarshalJSONArray
-```go
-func MarshalJSONArray(v gojay.MarshalerJSONArray) ([]byte, error)
-```
-
-### Encode API
-
-Encode API decodes a value to JSON by creating or borrowing a `*gojay.Encoder` sending it to an `io.Writer` and calling `Encode` methods.
-
-__Getting a *gojay.Encoder or Borrowing__
-
-You can either get a fresh `*gojay.Encoder` calling `enc := gojay.NewEncoder(io.Writer)` or borrow one from the pool by calling `enc := gojay.BorrowEncoder(io.Writer)`.
-
-After using an encoder, you can release it by calling `enc.Release()`. Beware, if you reuse the encoder after releasing it, it will panic with an error of type `InvalidUsagePooledEncoderError`. If you want to fully benefit from the pooling, you must release your encoders after using.
-
-Example getting a fresh encoder an releasing:
-```go
-str := "test"
-b := strings.Builder{}
-enc := gojay.NewEncoder(&b)
-defer enc.Release()
-if err := enc.Encode(str); err != nil {
-    log.Fatal(err)
-}
-```
-Example borrowing an encoder and releasing:
-```go
-str := "test"
-b := strings.Builder{}
-enc := gojay.BorrowEncoder(b)
-defer enc.Release()
-if err := enc.Encode(str); err != nil {
-    log.Fatal(err)
-}
-```
-
-`*gojay.Encoder` has multiple methods to encoder specific types to JSON:
-* Encode
-```go
-func (enc *gojay.Encoder) Encode(v interface{}) error
-```
-* EncodeObject
-```go
-func (enc *gojay.Encoder) EncodeObject(v gojay.MarshalerJSONObject) error
-```
-* EncodeArray
-```go
-func (enc *gojay.Encoder) EncodeArray(v gojay.MarshalerJSONArray) error
-```
-* EncodeInt
-```go
-func (enc *gojay.Encoder) EncodeInt(n int) error
-```
-* EncodeInt64
-```go
-func (enc *gojay.Encoder) EncodeInt64(n int64) error
-```
-* EncodeFloat
-```go
-func (enc *gojay.Encoder) EncodeFloat(n float64) error
-```
-* EncodeBool
-```go
-func (enc *gojay.Encoder) EncodeBool(v bool) error
-```
-* EncodeString
-```go
-func (enc *gojay.Encoder) EncodeString(s string) error
-```
-
-### Structs and Maps
-
-To encode a structure, the structure must implement the MarshalerJSONObject interface:
-```go
-type MarshalerJSONObject interface {
-	MarshalJSONObject(enc *gojay.Encoder)
-	IsNil() bool
-}
-```
-`MarshalJSONObject` method takes one argument, a pointer to the Encoder (*gojay.Encoder). The method must add all the keys in the JSON Object by calling Decoder's methods.
-
-IsNil method returns a boolean indicating if the interface underlying value is nil or not. It is used to safely ensure that the underlying value is not nil without using Reflection.
-
-Example of implementation for a struct:
-```go
-type user struct {
-	id    int
-	name  string
-	email string
-}
-
-// implement MarshalerJSONObject
-func (u *user) MarshalJSONObject(enc *gojay.Encoder) {
-	enc.IntKey("id", u.id)
-	enc.StringKey("name", u.name)
-	enc.StringKey("email", u.email)
-}
-func (u *user) IsNil() bool {
-	return u == nil
-}
-```
-
-Example of implementation for a `map[string]string`:
-```go
-// define our custom map type implementing MarshalerJSONObject
-type message map[string]string
-
-// Implementing Marshaler
-func (m message) MarshalJSONObject(enc *gojay.Encoder) {
-	for k, v := range m {
-		enc.StringKey(k, v)
-	}
-}
-
-func (m message) IsNil() bool {
-	return m == nil
-}
-```
-
-### Arrays and Slices
-To encode an array or a slice, the slice/array must implement the MarshalerJSONArray interface:
-```go
-type MarshalerJSONArray interface {
-	MarshalJSONArray(enc *gojay.Encoder)
-	IsNil() bool
-}
-```
-`MarshalJSONArray` method takes one argument, a pointer to the Encoder (*gojay.Encoder). The method must add all element in the JSON Array by calling Decoder's methods.
-
-`IsNil` method returns a boolean indicating if the interface underlying value is nil(empty) or not. It is used to safely ensure that the underlying value is not nil without using Reflection and also to in `OmitEmpty` feature.
-
-Example of implementation:
-```go
-type users []*user
-// implement MarshalerJSONArray
-func (u *users) MarshalJSONArray(enc *gojay.Encoder) {
-	for _, e := range u {
-		enc.Object(e)
-	}
-}
-func (u *users) IsNil() bool {
-	return len(u) == 0
-}
-```
-
-### Other types
-To encode other types (string, int, float, booleans), you don't need to implement any interface.
-
-Example of encoding strings:
-```go
-func main() {
-	name := "Jay"
-	b, err := gojay.Marshal(name)
-	if err != nil {
-		log.Fatal(err)
-	}
-	fmt.Println(string(b)) // "Jay"
-}
-```
-
-# Stream API
-
-### Stream Decoding
-GoJay ships with a powerful stream decoder.
-
-It allows to read continuously from an io.Reader stream and do JIT decoding writing unmarshalled JSON to a channel to allow async consuming.
-
-When using the Stream API, the Decoder implements context.Context to provide graceful cancellation.
-
-To decode a stream of JSON, you must call `gojay.Stream.DecodeStream` and pass it a `UnmarshalerStream` implementation.
-
-```go
-type UnmarshalerStream interface {
-	UnmarshalStream(*StreamDecoder) error
-}
-```
-
-Example of implementation of stream reading from a WebSocket connection:
-```go
-// implement UnmarshalerStream
-type ChannelStream chan *user
-
-func (c ChannelStream) UnmarshalStream(dec *gojay.StreamDecoder) error {
-	u := &user{}
-	if err := dec.Object(u); err != nil {
-		return err
-	}
-	c <- u
-	return nil
-}
-
-func main() {
-	// get our websocket connection
-	origin := "http://localhost/"
-	url := "ws://localhost:12345/ws"
-	ws, err := websocket.Dial(url, "", origin)
-	if err != nil {
-		log.Fatal(err)
-	}
-	// create our channel which will receive our objects
-	streamChan := ChannelStream(make(chan *user))
-	// borrow a decoder
-	dec := gojay.Stream.BorrowDecoder(ws)
-	// start decoding, it will block until a JSON message is decoded from the WebSocket
-	// or until Done channel is closed
-	go dec.DecodeStream(streamChan)
-	for {
-		select {
-		case v := <-streamChan:
-			// Got something from my websocket!
-			log.Println(v)
-		case <-dec.Done():
-			log.Println("finished reading from WebSocket")
-			os.Exit(0)
-		}
-	}
-}
-```
-
-### Stream Encoding
-GoJay ships with a powerful stream encoder part of the Stream API.
-
-It allows to write continuously to an io.Writer and do JIT encoding of data fed to a channel to allow async consuming. You can set multiple consumers on the channel to be as performant as possible. Consumers are non blocking and are scheduled individually in their own go routine.
-
-When using the Stream API, the Encoder implements context.Context to provide graceful cancellation.
-
-To encode a stream of data, you must call `EncodeStream` and pass it a `MarshalerStream` implementation.
-
-```go
-type MarshalerStream interface {
-	MarshalStream(enc *gojay.StreamEncoder)
-}
-```
-
-Example of implementation of stream writing to a WebSocket:
-```go
-// Our structure which will be pushed to our stream
-type user struct {
-	id    int
-	name  string
-	email string
-}
-
-func (u *user) MarshalJSONObject(enc *gojay.Encoder) {
-	enc.IntKey("id", u.id)
-	enc.StringKey("name", u.name)
-	enc.StringKey("email", u.email)
-}
-func (u *user) IsNil() bool {
-	return u == nil
-}
-
-// Our MarshalerStream implementation
-type StreamChan chan *user
-
-func (s StreamChan) MarshalStream(enc *gojay.StreamEncoder) {
-	select {
-	case <-enc.Done():
-		return
-	case o := <-s:
-		enc.Object(o)
-	}
-}
-
-// Our main function
-func main() {
-	// get our websocket connection
-	origin := "http://localhost/"
-	url := "ws://localhost:12345/ws"
-	ws, err := websocket.Dial(url, "", origin)
-	if err != nil {
-		log.Fatal(err)
-	}
-	// we borrow an encoder set stdout as the writer,
-	// set the number of consumer to 10
-	// and tell the encoder to separate each encoded element
-	// added to the channel by a new line character
-	enc := gojay.Stream.BorrowEncoder(ws).NConsumer(10).LineDelimited()
-	// instantiate our MarshalerStream
-	s := StreamChan(make(chan *user))
-	// start the stream encoder
-	// will block its goroutine until enc.Cancel(error) is called
-	// or until something is written to the channel
-	go enc.EncodeStream(s)
-	// write to our MarshalerStream
-	for i := 0; i < 1000; i++ {
-		s <- &user{i, "username", "user@email.com"}
-	}
-	// Wait
-	<-enc.Done()
-}
-```
-
-# Unsafe API
-
-Unsafe API has the same functions than the regular API, it only has `Unmarshal API` for now. It is unsafe because it makes assumptions on the quality of the given JSON.
-
-If you are not sure if your JSON is valid, don't use the Unsafe API.
-
-Also, the `Unsafe` API does not copy the buffer when using Unmarshal API, which, in case of string decoding, can lead to data corruption if a byte buffer is reused. Using the `Decode` API makes `Unsafe` API safer as the io.Reader relies on `copy` builtin method and `Decoder` will have its own internal buffer :)
-
-Access the `Unsafe` API this way:
-```go
-gojay.Unsafe.Unmarshal(b, v)
-```
-
-
-# Benchmarks
-
-Benchmarks encode and decode three different data based on size (small, medium, large).
-
-To run benchmark for decoder:
-```bash
-cd $GOPATH/src/github.com/francoispqt/gojay/benchmarks/decoder && make bench
-```
-
-To run benchmark for encoder:
-```bash
-cd $GOPATH/src/github.com/francoispqt/gojay/benchmarks/encoder && make bench
-```
-
-# Benchmark Results
-## Decode
-
-<img src="https://images2.imgbox.com/78/01/49OExcPh_o.png" width="500px">
-
-### Small Payload
-[benchmark code is here](https://github.com/francoispqt/gojay/blob/master/benchmarks/decoder/decoder_bench_small_test.go)
-
-[benchmark data is here](https://github.com/francoispqt/gojay/blob/master/benchmarks/benchmarks_small.go)
-
-|                 | ns/op     | bytes/op     | allocs/op |
-|-----------------|-----------|--------------|-----------|
-| Std Library     | 2547      | 496          | 4         |
-| JsonIter        | 2046      | 312          | 12        |
-| JsonParser      | 1408      | 0            | 0         |
-| EasyJson        | 929       | 240          | 2         |
-| **GoJay**       | **807**   | **256**      | **2**     |
-| **GoJay-unsafe**| **712**   | **112**      | **1**     |
-
-### Medium Payload
-[benchmark code is here](https://github.com/francoispqt/gojay/blob/master/benchmarks/decoder/decoder_bench_medium_test.go)
-
-[benchmark data is here](https://github.com/francoispqt/gojay/blob/master/benchmarks/benchmarks_medium.go)
-
-|                 | ns/op     | bytes/op | allocs/op |
-|-----------------|-----------|----------|-----------|
-| Std Library     | 30148     | 2152     | 496       |
-| JsonIter        | 16309     | 2976     | 80        |
-| JsonParser      | 7793      | 0        | 0         |
-| EasyJson        | 7957      | 232      | 6         |
-| **GoJay**       | **4984**  | **2448** | **8**     |
-| **GoJay-unsafe**| **4809**  | **144**  | **7**     |
-
-### Large Payload
-[benchmark code is here](https://github.com/francoispqt/gojay/blob/master/benchmarks/decoder/decoder_bench_large_test.go)
-
-[benchmark data is here](https://github.com/francoispqt/gojay/blob/master/benchmarks/benchmarks_large.go)
-
-|                 | ns/op     | bytes/op    | allocs/op |
-|-----------------|-----------|-------------|-----------|
-| JsonIter        | 210078    | 41712       | 1136      |
-| EasyJson        | 106626    | 160         | 2         |
-| JsonParser      | 66813     | 0           | 0         |
-| **GoJay**       | **52153** | **31241**   | **77**    |
-| **GoJay-unsafe**| **48277** | **2561**    | **76**    |
-
-## Encode
-
-<img src="https://images2.imgbox.com/e9/cc/pnM8c7Gf_o.png" width="500px">
-
-### Small Struct
-[benchmark code is here](https://github.com/francoispqt/gojay/blob/master/benchmarks/encoder/encoder_bench_small_test.go)
-
-[benchmark data is here](https://github.com/francoispqt/gojay/blob/master/benchmarks/benchmarks_small.go)
-
-|                | ns/op    | bytes/op     | allocs/op |
-|----------------|----------|--------------|-----------|
-| Std Library    | 1280     | 464          | 3         |
-| EasyJson       | 871      | 944          | 6         |
-| JsonIter       | 866      | 272          | 3         |
-| **GoJay**      | **543**  | **112**      | **1**     |
-| **GoJay-func** | **347**  | **0**        | **0**     |
-
-### Medium Struct
-[benchmark code is here](https://github.com/francoispqt/gojay/blob/master/benchmarks/encoder/encoder_bench_medium_test.go)
-
-[benchmark data is here](https://github.com/francoispqt/gojay/blob/master/benchmarks/benchmarks_medium.go)
-
-|             | ns/op    | bytes/op     | allocs/op |
-|-------------|----------|--------------|-----------|
-| Std Library | 5006     | 1496         | 25        |
-| JsonIter    | 2232     | 1544         | 20        |
-| EasyJson    | 1997     | 1544         | 19        |
-| **GoJay**   | **1522** | **312**      | **14**    |
-
-### Large Struct
-[benchmark code is here](https://github.com/francoispqt/gojay/blob/master/benchmarks/encoder/encoder_bench_large_test.go)
-
-[benchmark data is here](https://github.com/francoispqt/gojay/blob/master/benchmarks/benchmarks_large.go)
-
-|             | ns/op     | bytes/op     | allocs/op |
-|-------------|-----------|--------------|-----------|
-| Std Library | 66441     | 20576        | 332       |
-| JsonIter    | 35247     | 20255        | 328       |
-| EasyJson    | 32053     | 15474        | 327       |
-| **GoJay**   | **27847** | **9802**     | **318**   |
-
-# Contributing
-
-Contributions are welcome :)
-
-If you encounter issues please report it in Github and/or send an email at [francois@parquet.ninja](mailto:francois@parquet.ninja)
-

vendor/github.com/francoispqt/gojay/decode.go

@@ -1,386 +0,0 @@
-package gojay
-
-import (
-	"fmt"
-	"io"
-)
-
-// UnmarshalJSONArray parses the JSON-encoded data and stores the result in the value pointed to by v.
-//
-// v must implement UnmarshalerJSONArray.
-//
-// If a JSON value is not appropriate for a given target type, or if a JSON number
-// overflows the target type, UnmarshalJSONArray skips that field and completes the unmarshaling as best it can.
-func UnmarshalJSONArray(data []byte, v UnmarshalerJSONArray) error {
-	dec := borrowDecoder(nil, 0)
-	defer dec.Release()
-	dec.data = make([]byte, len(data))
-	copy(dec.data, data)
-	dec.length = len(data)
-	_, err := dec.decodeArray(v)
-	if err != nil {
-		return err
-	}
-	if dec.err != nil {
-		return dec.err
-	}
-	return nil
-}
-
-// UnmarshalJSONObject parses the JSON-encoded data and stores the result in the value pointed to by v.
-//
-// v must implement UnmarshalerJSONObject.
-//
-// If a JSON value is not appropriate for a given target type, or if a JSON number
-// overflows the target type, UnmarshalJSONObject skips that field and completes the unmarshaling as best it can.
-func UnmarshalJSONObject(data []byte, v UnmarshalerJSONObject) error {
-	dec := borrowDecoder(nil, 0)
-	defer dec.Release()
-	dec.data = make([]byte, len(data))
-	copy(dec.data, data)
-	dec.length = len(data)
-	_, err := dec.decodeObject(v)
-	if err != nil {
-		return err
-	}
-	if dec.err != nil {
-		return dec.err
-	}
-	return nil
-}
-
-// Unmarshal parses the JSON-encoded data and stores the result in the value pointed to by v.
-// If v is nil, not an implementation of UnmarshalerJSONObject or UnmarshalerJSONArray or not one of the following types:
-// 	*string, **string, *int, **int, *int8, **int8, *int16, **int16, *int32, **int32, *int64, **int64, *uint8, **uint8, *uint16, **uint16,
-// 	*uint32, **uint32, *uint64, **uint64, *float64, **float64, *float32, **float32, *bool, **bool
-// Unmarshal returns an InvalidUnmarshalError.
-//
-//
-// If a JSON value is not appropriate for a given target type, or if a JSON number
-// overflows the target type, Unmarshal skips that field and completes the unmarshaling as best it can.
-// If no more serious errors are encountered, Unmarshal returns an UnmarshalTypeError describing the earliest such error.
-// In any case, it's not guaranteed that all the remaining fields following the problematic one will be unmarshaled into the target object.
-func Unmarshal(data []byte, v interface{}) error {
-	var err error
-	var dec *Decoder
-	switch vt := v.(type) {
-	case *string:
-		dec = borrowDecoder(nil, 0)
-		dec.length = len(data)
-		dec.data = data
-		err = dec.decodeString(vt)
-	case **string:
-		dec = borrowDecoder(nil, 0)
-		dec.length = len(data)
-		dec.data = data
-		err = dec.decodeStringNull(vt)
-	case *int:
-		dec = borrowDecoder(nil, 0)
-		dec.length = len(data)
-		dec.data = data
-		err = dec.decodeInt(vt)
-	case **int:
-		dec = borrowDecoder(nil, 0)
-		dec.length = len(data)
-		dec.data = data
-		err = dec.decodeIntNull(vt)
-	case *int8:
-		dec = borrowDecoder(nil, 0)
-		dec.length = len(data)
-		dec.data = data
-		err = dec.decodeInt8(vt)
-	case **int8:
-		dec = borrowDecoder(nil, 0)
-		dec.length = len(data)
-		dec.data = data
-		err = dec.decodeInt8Null(vt)
-	case *int16:
-		dec = borrowDecoder(nil, 0)
-		dec.length = len(data)
-		dec.data = data
-		err = dec.decodeInt16(vt)
-	case **int16:
-		dec = borrowDecoder(nil, 0)
-		dec.length = len(data)
-		dec.data = data
-		err = dec.decodeInt16Null(vt)
-	case *int32:
-		dec = borrowDecoder(nil, 0)
-		dec.length = len(data)
-		dec.data = data
-		err = dec.decodeInt32(vt)
-	case **int32:
-		dec = borrowDecoder(nil, 0)
-		dec.length = len(data)
-		dec.data = data
-		err = dec.decodeInt32Null(vt)
-	case *int64:
-		dec = borrowDecoder(nil, 0)
-		dec.length = len(data)
-		dec.data = data
-		err = dec.decodeInt64(vt)
-	case **int64:
-		dec = borrowDecoder(nil, 0)
-		dec.length = len(data)
-		dec.data = data
-		err = dec.decodeInt64Null(vt)
-	case *uint8:
-		dec = borrowDecoder(nil, 0)
-		dec.length = len(data)
-		dec.data = data
-		err = dec.decodeUint8(vt)
-	case **uint8:
-		dec = borrowDecoder(nil, 0)
-		dec.length = len(data)
-		dec.data = data
-		err = dec.decodeUint8Null(vt)
-	case *uint16:
-		dec = borrowDecoder(nil, 0)
-		dec.length = len(data)
-		dec.data = data
-		err = dec.decodeUint16(vt)
-	case **uint16:
-		dec = borrowDecoder(nil, 0)
-		dec.length = len(data)
-		dec.data = data
-		err = dec.decodeUint16Null(vt)
-	case *uint32:
-		dec = borrowDecoder(nil, 0)
-		dec.length = len(data)
-		dec.data = data
-		err = dec.decodeUint32(vt)
-	case **uint32:
-		dec = borrowDecoder(nil, 0)
-		dec.length = len(data)
-		dec.data = data
-		err = dec.decodeUint32Null(vt)
-	case *uint64:
-		dec = borrowDecoder(nil, 0)
-		dec.length = len(data)
-		dec.data = data
-		err = dec.decodeUint64(vt)
-	case **uint64:
-		dec = borrowDecoder(nil, 0)
-		dec.length = len(data)
-		dec.data = data
-		err = dec.decodeUint64Null(vt)
-	case *float64:
-		dec = borrowDecoder(nil, 0)
-		dec.length = len(data)
-		dec.data = data
-		err = dec.decodeFloat64(vt)
-	case **float64:
-		dec = borrowDecoder(nil, 0)
-		dec.length = len(data)
-		dec.data = data
-		err = dec.decodeFloat64Null(vt)
-	case *float32:
-		dec = borrowDecoder(nil, 0)
-		dec.length = len(data)
-		dec.data = data
-		err = dec.decodeFloat32(vt)
-	case **float32:
-		dec = borrowDecoder(nil, 0)
-		dec.length = len(data)
-		dec.data = data
-		err = dec.decodeFloat32Null(vt)
-	case *bool:
-		dec = borrowDecoder(nil, 0)
-		dec.length = len(data)
-		dec.data = data
-		err = dec.decodeBool(vt)
-	case **bool:
-		dec = borrowDecoder(nil, 0)
-		dec.length = len(data)
-		dec.data = data
-		err = dec.decodeBoolNull(vt)
-	case UnmarshalerJSONObject:
-		dec = borrowDecoder(nil, 0)
-		dec.length = len(data)
-		dec.data = make([]byte, len(data))
-		copy(dec.data, data)
-		_, err = dec.decodeObject(vt)
-	case UnmarshalerJSONArray:
-		dec = borrowDecoder(nil, 0)
-		dec.length = len(data)
-		dec.data = make([]byte, len(data))
-		copy(dec.data, data)
-		_, err = dec.decodeArray(vt)
-	case *interface{}:
-		dec = borrowDecoder(nil, 0)
-		dec.length = len(data)
-		dec.data = make([]byte, len(data))
-		copy(dec.data, data)
-		err = dec.decodeInterface(vt)
-	default:
-		return InvalidUnmarshalError(fmt.Sprintf(invalidUnmarshalErrorMsg, vt))
-	}
-	defer dec.Release()
-	if err != nil {
-		return err
-	}
-	return dec.err
-}
-
-// UnmarshalerJSONObject is the interface to implement to decode a JSON Object.
-type UnmarshalerJSONObject interface {
-	UnmarshalJSONObject(*Decoder, string) error
-	NKeys() int
-}
-
-// UnmarshalerJSONArray is the interface to implement to decode a JSON Array.
-type UnmarshalerJSONArray interface {
-	UnmarshalJSONArray(*Decoder) error
-}
-
-// A Decoder reads and decodes JSON values from an input stream.
-type Decoder struct {
-	r          io.Reader
-	data       []byte
-	err        error
-	isPooled   byte
-	called     byte
-	child      byte
-	cursor     int
-	length     int
-	keysDone   int
-	arrayIndex int
-}
-
-// Decode reads the next JSON-encoded value from the decoder's input (io.Reader) and stores it in the value pointed to by v.
-//
-// See the documentation for Unmarshal for details about the conversion of JSON into a Go value.
-// The differences between Decode and Unmarshal are:
-// 	- Decode reads from an io.Reader in the Decoder, whereas Unmarshal reads from a []byte
-// 	- Decode leaves to the user the option of borrowing and releasing a Decoder, whereas Unmarshal internally always borrows a Decoder and releases it when the unmarshaling is completed
-func (dec *Decoder) Decode(v interface{}) error {
-	if dec.isPooled == 1 {
-		panic(InvalidUsagePooledDecoderError("Invalid usage of pooled decoder"))
-	}
-	var err error
-	switch vt := v.(type) {
-	case *string:
-		err = dec.decodeString(vt)
-	case **string:
-		err = dec.decodeStringNull(vt)
-	case *int:
-		err = dec.decodeInt(vt)
-	case **int:
-		err = dec.decodeIntNull(vt)
-	case *int8:
-		err = dec.decodeInt8(vt)
-	case **int8:
-		err = dec.decodeInt8Null(vt)
-	case *int16:
-		err = dec.decodeInt16(vt)
-	case **int16:
-		err = dec.decodeInt16Null(vt)
-	case *int32:
-		err = dec.decodeInt32(vt)
-	case **int32:
-		err = dec.decodeInt32Null(vt)
-	case *int64:
-		err = dec.decodeInt64(vt)
-	case **int64:
-		err = dec.decodeInt64Null(vt)
-	case *uint8:
-		err = dec.decodeUint8(vt)
-	case **uint8:
-		err = dec.decodeUint8Null(vt)
-	case *uint16:
-		err = dec.decodeUint16(vt)
-	case **uint16:
-		err = dec.decodeUint16Null(vt)
-	case *uint32:
-		err = dec.decodeUint32(vt)
-	case **uint32:
-		err = dec.decodeUint32Null(vt)
-	case *uint64:
-		err = dec.decodeUint64(vt)
-	case **uint64:
-		err = dec.decodeUint64Null(vt)
-	case *float64:
-		err = dec.decodeFloat64(vt)
-	case **float64:
-		err = dec.decodeFloat64Null(vt)
-	case *float32:
-		err = dec.decodeFloat32(vt)
-	case **float32:
-		err = dec.decodeFloat32Null(vt)
-	case *bool:
-		err = dec.decodeBool(vt)
-	case **bool:
-		err = dec.decodeBoolNull(vt)
-	case UnmarshalerJSONObject:
-		_, err = dec.decodeObject(vt)
-	case UnmarshalerJSONArray:
-		_, err = dec.decodeArray(vt)
-	case *EmbeddedJSON:
-		err = dec.decodeEmbeddedJSON(vt)
-	case *interface{}:
-		err = dec.decodeInterface(vt)
-	default:
-		return InvalidUnmarshalError(fmt.Sprintf(invalidUnmarshalErrorMsg, vt))
-	}
-	if err != nil {
-		return err
-	}
-	return dec.err
-}
-
-// Non exported
-
-func isDigit(b byte) bool {
-	switch b {
-	case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
-		return true
-	default:
-		return false
-	}
-}
-
-func (dec *Decoder) read() bool {
-	if dec.r != nil {
-		// if we reach the end, double the buffer to ensure there's always more space
-		if len(dec.data) == dec.length {
-			nLen := dec.length * 2
-			if nLen == 0 {
-				nLen = 512
-			}
-			Buf := make([]byte, nLen, nLen)
-			copy(Buf, dec.data)
-			dec.data = Buf
-		}
-		var n int
-		var err error
-		for n == 0 {
-			n, err = dec.r.Read(dec.data[dec.length:])
-			if err != nil {
-				if err != io.EOF {
-					dec.err = err
-					return false
-				}
-				if n == 0 {
-					return false
-				}
-				dec.length = dec.length + n
-				return true
-			}
-		}
-		dec.length = dec.length + n
-		return true
-	}
-	return false
-}
-
-func (dec *Decoder) nextChar() byte {
-	for ; dec.cursor < dec.length || dec.read(); dec.cursor++ {
-		switch dec.data[dec.cursor] {
-		case ' ', '\n', '\t', '\r', ',':
-			continue
-		}
-		d := dec.data[dec.cursor]
-		return d
-	}
-	return 0
-}

vendor/github.com/francoispqt/gojay/decode_array.go

@@ -1,247 +0,0 @@
-package gojay
-
-import "reflect"
-
-// DecodeArray reads the next JSON-encoded value from the decoder's input (io.Reader)
-// and stores it in the value pointed to by v.
-//
-// v must implement UnmarshalerJSONArray.
-//
-// See the documentation for Unmarshal for details about the conversion of JSON into a Go value.
-func (dec *Decoder) DecodeArray(v UnmarshalerJSONArray) error {
-	if dec.isPooled == 1 {
-		panic(InvalidUsagePooledDecoderError("Invalid usage of pooled decoder"))
-	}
-	_, err := dec.decodeArray(v)
-	return err
-}
-func (dec *Decoder) decodeArray(arr UnmarshalerJSONArray) (int, error) {
-	// remember last array index in case of nested arrays
-	lastArrayIndex := dec.arrayIndex
-	dec.arrayIndex = 0
-	defer func() {
-		dec.arrayIndex = lastArrayIndex
-	}()
-	for ; dec.cursor < dec.length || dec.read(); dec.cursor++ {
-		switch dec.data[dec.cursor] {
-		case ' ', '\n', '\t', '\r', ',':
-			continue
-		case '[':
-			dec.cursor = dec.cursor + 1
-			// array is open, char is not space start readings
-			for dec.nextChar() != 0 {
-				// closing array
-				if dec.data[dec.cursor] == ']' {
-					dec.cursor = dec.cursor + 1
-					return dec.cursor, nil
-				}
-				// calling unmarshall function for each element of the slice
-				err := arr.UnmarshalJSONArray(dec)
-				if err != nil {
-					return 0, err
-				}
-				dec.arrayIndex++
-			}
-			return 0, dec.raiseInvalidJSONErr(dec.cursor)
-		case 'n':
-			// is null
-			dec.cursor++
-			err := dec.assertNull()
-			if err != nil {
-				return 0, err
-			}
-			return dec.cursor, nil
-		case '{', '"', 'f', 't', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
-			// can't unmarshall to struct
-			// we skip array and set Error
-			dec.err = dec.makeInvalidUnmarshalErr(arr)
-			err := dec.skipData()
-			if err != nil {
-				return 0, err
-			}
-			return dec.cursor, nil
-		default:
-			return 0, dec.raiseInvalidJSONErr(dec.cursor)
-		}
-	}
-	return 0, dec.raiseInvalidJSONErr(dec.cursor)
-}
-func (dec *Decoder) decodeArrayNull(v interface{}) (int, error) {
-	// remember last array index in case of nested arrays
-	lastArrayIndex := dec.arrayIndex
-	dec.arrayIndex = 0
-	defer func() {
-		dec.arrayIndex = lastArrayIndex
-	}()
-	vv := reflect.ValueOf(v)
-	vvt := vv.Type()
-	if vvt.Kind() != reflect.Ptr || vvt.Elem().Kind() != reflect.Ptr {
-		dec.err = ErrUnmarshalPtrExpected
-		return 0, dec.err
-	}
-	// not an array not an error, but do not know what to do
-	// do not check syntax
-	for ; dec.cursor < dec.length || dec.read(); dec.cursor++ {
-		switch dec.data[dec.cursor] {
-		case ' ', '\n', '\t', '\r', ',':
-			continue
-		case '[':
-			dec.cursor = dec.cursor + 1
-			// create our new type
-			elt := vv.Elem()
-			n := reflect.New(elt.Type().Elem())
-			var arr UnmarshalerJSONArray
-			var ok bool
-			if arr, ok = n.Interface().(UnmarshalerJSONArray); !ok {
-				dec.err = dec.makeInvalidUnmarshalErr((UnmarshalerJSONArray)(nil))
-				return 0, dec.err
-			}
-			// array is open, char is not space start readings
-			for dec.nextChar() != 0 {
-				// closing array
-				if dec.data[dec.cursor] == ']' {
-					elt.Set(n)
-					dec.cursor = dec.cursor + 1
-					return dec.cursor, nil
-				}
-				// calling unmarshall function for each element of the slice
-				err := arr.UnmarshalJSONArray(dec)
-				if err != nil {
-					return 0, err
-				}
-				dec.arrayIndex++
-			}
-			return 0, dec.raiseInvalidJSONErr(dec.cursor)
-		case 'n':
-			// is null
-			dec.cursor++
-			err := dec.assertNull()
-			if err != nil {
-				return 0, err
-			}
-			return dec.cursor, nil
-		case '{', '"', 'f', 't', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
-			// can't unmarshall to struct
-			// we skip array and set Error
-			dec.err = dec.makeInvalidUnmarshalErr((UnmarshalerJSONArray)(nil))
-			err := dec.skipData()
-			if err != nil {
-				return 0, err
-			}
-			return dec.cursor, nil
-		default:
-			return 0, dec.raiseInvalidJSONErr(dec.cursor)
-		}
-	}
-	return 0, dec.raiseInvalidJSONErr(dec.cursor)
-}
-
-func (dec *Decoder) skipArray() (int, error) {
-	var arraysOpen = 1
-	var arraysClosed = 0
-	// var stringOpen byte = 0
-	for j := dec.cursor; j < dec.length || dec.read(); j++ {
-		switch dec.data[j] {
-		case ']':
-			arraysClosed++
-			// everything is closed return
-			if arraysOpen == arraysClosed {
-				// add char to object data
-				return j + 1, nil
-			}
-		case '[':
-			arraysOpen++
-		case '"':
-			j++
-			var isInEscapeSeq bool
-			var isFirstQuote = true
-			for ; j < dec.length || dec.read(); j++ {
-				if dec.data[j] != '"' {
-					continue
-				}
-				if dec.data[j-1] != '\\' || (!isInEscapeSeq && !isFirstQuote) {
-					break
-				} else {
-					isInEscapeSeq = false
-				}
-				if isFirstQuote {
-					isFirstQuote = false
-				}
-				// loop backward and count how many anti slash found
-				// to see if string is effectively escaped
-				ct := 0
-				for i := j - 1; i > 0; i-- {
-					if dec.data[i] != '\\' {
-						break
-					}
-					ct++
-				}
-				// is pair number of slashes, quote is not escaped
-				if ct&1 == 0 {
-					break
-				}
-				isInEscapeSeq = true
-			}
-		default:
-			continue
-		}
-	}
-	return 0, dec.raiseInvalidJSONErr(dec.cursor)
-}
-
-// DecodeArrayFunc is a func type implementing UnmarshalerJSONArray.
-// Use it to cast a `func(*Decoder) error` to Unmarshal an array on the fly.
-
-type DecodeArrayFunc func(*Decoder) error
-
-// UnmarshalJSONArray implements UnmarshalerJSONArray.
-func (f DecodeArrayFunc) UnmarshalJSONArray(dec *Decoder) error {
-	return f(dec)
-}
-
-// IsNil implements UnmarshalerJSONArray.
-func (f DecodeArrayFunc) IsNil() bool {
-	return f == nil
-}
-
-// Add Values functions
-
-// AddArray decodes the JSON value within an object or an array to a UnmarshalerJSONArray.
-func (dec *Decoder) AddArray(v UnmarshalerJSONArray) error {
-	return dec.Array(v)
-}
-
-// AddArrayNull decodes the JSON value within an object or an array to a UnmarshalerJSONArray.
-func (dec *Decoder) AddArrayNull(v interface{}) error {
-	return dec.ArrayNull(v)
-}
-
-// Array decodes the JSON value within an object or an array to a UnmarshalerJSONArray.
-func (dec *Decoder) Array(v UnmarshalerJSONArray) error {
-	newCursor, err := dec.decodeArray(v)
-	if err != nil {
-		return err
-	}
-	dec.cursor = newCursor
-	dec.called |= 1
-	return nil
-}
-
-// ArrayNull decodes the JSON value within an object or an array to a UnmarshalerJSONArray.
-// v should be a pointer to an UnmarshalerJSONArray,
-// if `null` value is encountered in JSON, it will leave the value v untouched,
-// else it will create a new instance of the UnmarshalerJSONArray behind v.
-func (dec *Decoder) ArrayNull(v interface{}) error {
-	newCursor, err := dec.decodeArrayNull(v)
-	if err != nil {
-		return err
-	}
-	dec.cursor = newCursor
-	dec.called |= 1
-	return nil
-}
-
-// Index returns the index of an array being decoded.
-func (dec *Decoder) Index() int {
-	return dec.arrayIndex
-}

vendor/github.com/francoispqt/gojay/decode_bool.go

@@ -1,241 +0,0 @@
-package gojay
-
-// DecodeBool reads the next JSON-encoded value from the decoder's input (io.Reader)
-// and stores it in the boolean pointed to by v.
-//
-// See the documentation for Unmarshal for details about the conversion of JSON into a Go value.
-func (dec *Decoder) DecodeBool(v *bool) error {
-	if dec.isPooled == 1 {
-		panic(InvalidUsagePooledDecoderError("Invalid usage of pooled decoder"))
-	}
-	return dec.decodeBool(v)
-}
-func (dec *Decoder) decodeBool(v *bool) error {
-	for ; dec.cursor < dec.length || dec.read(); dec.cursor++ {
-		switch dec.data[dec.cursor] {
-		case ' ', '\n', '\t', '\r', ',':
-			continue
-		case 't':
-			dec.cursor++
-			err := dec.assertTrue()
-			if err != nil {
-				return err
-			}
-			*v = true
-			return nil
-		case 'f':
-			dec.cursor++
-			err := dec.assertFalse()
-			if err != nil {
-				return err
-			}
-			*v = false
-			return nil
-		case 'n':
-			dec.cursor++
-			err := dec.assertNull()
-			if err != nil {
-				return err
-			}
-			*v = false
-			return nil
-		default:
-			dec.err = dec.makeInvalidUnmarshalErr(v)
-			err := dec.skipData()
-			if err != nil {
-				return err
-			}
-			return nil
-		}
-	}
-	return nil
-}
-func (dec *Decoder) decodeBoolNull(v **bool) error {
-	for ; dec.cursor < dec.length || dec.read(); dec.cursor++ {
-		switch dec.data[dec.cursor] {
-		case ' ', '\n', '\t', '\r', ',':
-			continue
-		case 't':
-			dec.cursor++
-			err := dec.assertTrue()
-			if err != nil {
-				return err
-			}
-			if *v == nil {
-				*v = new(bool)
-			}
-			**v = true
-			return nil
-		case 'f':
-			dec.cursor++
-			err := dec.assertFalse()
-			if err != nil {
-				return err
-			}
-			if *v == nil {
-				*v = new(bool)
-			}
-			**v = false
-			return nil
-		case 'n':
-			dec.cursor++
-			err := dec.assertNull()
-			if err != nil {
-				return err
-			}
-			return nil
-		default:
-			dec.err = dec.makeInvalidUnmarshalErr(v)
-			err := dec.skipData()
-			if err != nil {
-				return err
-			}
-			return nil
-		}
-	}
-	return nil
-}
-
-func (dec *Decoder) assertTrue() error {
-	i := 0
-	for ; dec.cursor < dec.length || dec.read(); dec.cursor++ {
-		switch i {
-		case 0:
-			if dec.data[dec.cursor] != 'r' {
-				return dec.raiseInvalidJSONErr(dec.cursor)
-			}
-		case 1:
-			if dec.data[dec.cursor] != 'u' {
-				return dec.raiseInvalidJSONErr(dec.cursor)
-			}
-		case 2:
-			if dec.data[dec.cursor] != 'e' {
-				return dec.raiseInvalidJSONErr(dec.cursor)
-			}
-		case 3:
-			switch dec.data[dec.cursor] {
-			case ' ', '\b', '\t', '\n', ',', ']', '}':
-				// dec.cursor--
-				return nil
-			default:
-				return dec.raiseInvalidJSONErr(dec.cursor)
-			}
-		}
-		i++
-	}
-	if i == 3 {
-		return nil
-	}
-	return dec.raiseInvalidJSONErr(dec.cursor)
-}
-
-func (dec *Decoder) assertNull() error {
-	i := 0
-	for ; dec.cursor < dec.length || dec.read(); dec.cursor++ {
-		switch i {
-		case 0:
-			if dec.data[dec.cursor] != 'u' {
-				return dec.raiseInvalidJSONErr(dec.cursor)
-			}
-		case 1:
-			if dec.data[dec.cursor] != 'l' {
-				return dec.raiseInvalidJSONErr(dec.cursor)
-			}
-		case 2:
-			if dec.data[dec.cursor] != 'l' {
-				return dec.raiseInvalidJSONErr(dec.cursor)
-			}
-		case 3:
-			switch dec.data[dec.cursor] {
-			case ' ', '\t', '\n', ',', ']', '}':
-				// dec.cursor--
-				return nil
-			default:
-				return dec.raiseInvalidJSONErr(dec.cursor)
-			}
-		}
-		i++
-	}
-	if i == 3 {
-		return nil
-	}
-	return dec.raiseInvalidJSONErr(dec.cursor)
-}
-
-func (dec *Decoder) assertFalse() error {
-	i := 0
-	for ; dec.cursor < dec.length || dec.read(); dec.cursor++ {
-		switch i {
-		case 0:
-			if dec.data[dec.cursor] != 'a' {
-				return dec.raiseInvalidJSONErr(dec.cursor)
-			}
-		case 1:
-			if dec.data[dec.cursor] != 'l' {
-				return dec.raiseInvalidJSONErr(dec.cursor)
-			}
-		case 2:
-			if dec.data[dec.cursor] != 's' {
-				return dec.raiseInvalidJSONErr(dec.cursor)
-			}
-		case 3:
-			if dec.data[dec.cursor] != 'e' {
-				return dec.raiseInvalidJSONErr(dec.cursor)
-			}
-		case 4:
-			switch dec.data[dec.cursor] {
-			case ' ', '\t', '\n', ',', ']', '}':
-				// dec.cursor--
-				return nil
-			default:
-				return dec.raiseInvalidJSONErr(dec.cursor)
-			}
-		}
-		i++
-	}
-	if i == 4 {
-		return nil
-	}
-	return dec.raiseInvalidJSONErr(dec.cursor)
-}
-
-// Add Values functions
-
-// AddBool decodes the JSON value within an object or an array to a *bool.
-// If next key is neither null nor a JSON boolean, an InvalidUnmarshalError will be returned.
-// If next key is null, bool will be false.
-func (dec *Decoder) AddBool(v *bool) error {
-	return dec.Bool(v)
-}
-
-// AddBoolNull decodes the JSON value within an object or an array to a *bool.
-// If next key is neither null nor a JSON boolean, an InvalidUnmarshalError will be returned.
-// If next key is null, bool will be false.
-// If a `null` is encountered, gojay does not change the value of the pointer.
-func (dec *Decoder) AddBoolNull(v **bool) error {
-	return dec.BoolNull(v)
-}
-
-// Bool decodes the JSON value within an object or an array to a *bool.
-// If next key is neither null nor a JSON boolean, an InvalidUnmarshalError will be returned.
-// If next key is null, bool will be false.
-func (dec *Decoder) Bool(v *bool) error {
-	err := dec.decodeBool(v)
-	if err != nil {
-		return err
-	}
-	dec.called |= 1
-	return nil
-}
-
-// BoolNull decodes the JSON value within an object or an array to a *bool.
-// If next key is neither null nor a JSON boolean, an InvalidUnmarshalError will be returned.
-// If next key is null, bool will be false.
-func (dec *Decoder) BoolNull(v **bool) error {
-	err := dec.decodeBoolNull(v)
-	if err != nil {
-		return err
-	}
-	dec.called |= 1
-	return nil
-}

vendor/github.com/francoispqt/gojay/decode_embedded_json.go

@@ -1,85 +0,0 @@
-package gojay
-
-// EmbeddedJSON is a raw encoded JSON value.
-// It can be used to delay JSON decoding or precompute a JSON encoding.
-type EmbeddedJSON []byte
-
-func (dec *Decoder) decodeEmbeddedJSON(ej *EmbeddedJSON) error {
-	var err error
-	if ej == nil {
-		return InvalidUnmarshalError("Invalid nil pointer given")
-	}
-	var beginOfEmbeddedJSON int
-	for ; dec.cursor < dec.length || dec.read(); dec.cursor++ {
-		switch dec.data[dec.cursor] {
-		case ' ', '\n', '\t', '\r', ',':
-			continue
-		// is null
-		case 'n':
-			beginOfEmbeddedJSON = dec.cursor
-			dec.cursor++
-			err := dec.assertNull()
-			if err != nil {
-				return err
-			}
-		case 't':
-			beginOfEmbeddedJSON = dec.cursor
-			dec.cursor++
-			err := dec.assertTrue()
-			if err != nil {
-				return err
-			}
-		// is false
-		case 'f':
-			beginOfEmbeddedJSON = dec.cursor
-			dec.cursor++
-			err := dec.assertFalse()
-			if err != nil {
-				return err
-			}
-		// is an object
-		case '{':
-			beginOfEmbeddedJSON = dec.cursor
-			dec.cursor = dec.cursor + 1
-			dec.cursor, err = dec.skipObject()
-		// is string
-		case '"':
-			beginOfEmbeddedJSON = dec.cursor
-			dec.cursor = dec.cursor + 1
-			err = dec.skipString() // why no new dec.cursor in result?
-		// is array
-		case '[':
-			beginOfEmbeddedJSON = dec.cursor
-			dec.cursor = dec.cursor + 1
-			dec.cursor, err = dec.skipArray()
-		case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-':
-			beginOfEmbeddedJSON = dec.cursor
-			dec.cursor, err = dec.skipNumber()
-		}
-		break
-	}
-	if err == nil {
-		if dec.cursor-1 >= beginOfEmbeddedJSON {
-			*ej = append(*ej, dec.data[beginOfEmbeddedJSON:dec.cursor]...)
-		}
-		dec.called |= 1
-	}
-	return err
-}
-
-// AddEmbeddedJSON adds an EmbeddedsJSON to the value pointed by v.
-// It can be used to delay JSON decoding or precompute a JSON encoding.
-func (dec *Decoder) AddEmbeddedJSON(v *EmbeddedJSON) error {
-	return dec.EmbeddedJSON(v)
-}
-
-// EmbeddedJSON adds an EmbeddedsJSON to the value pointed by v.
-// It can be used to delay JSON decoding or precompute a JSON encoding.
-func (dec *Decoder) EmbeddedJSON(v *EmbeddedJSON) error {
-	err := dec.decodeEmbeddedJSON(v)
-	if err != nil {
-		return err
-	}
-	dec.called |= 1
-	return nil
-}

vendor/github.com/francoispqt/gojay/decode_interface.go

@@ -1,130 +0,0 @@
-package gojay
-
-// TODO @afiune for now we are using the standard json unmarshaling but in
-// the future it would be great to implement one here inside this repo
-import "encoding/json"
-
-// DecodeInterface reads the next JSON-encoded value from the decoder's input (io.Reader) and stores it in the value pointed to by i.
-//
-// i must be an interface poiter
-func (dec *Decoder) DecodeInterface(i *interface{}) error {
-	if dec.isPooled == 1 {
-		panic(InvalidUsagePooledDecoderError("Invalid usage of pooled decoder"))
-	}
-	err := dec.decodeInterface(i)
-	return err
-}
-
-func (dec *Decoder) decodeInterface(i *interface{}) error {
-	start, end, err := dec.getObject()
-	if err != nil {
-		dec.cursor = start
-		return err
-	}
-
-	// if start & end are equal the object is a null, don't unmarshal
-	if start == end {
-		return nil
-	}
-
-	object := dec.data[start:end]
-	if err = json.Unmarshal(object, i); err != nil {
-		return err
-	}
-
-	dec.cursor = end
-	return nil
-}
-
-// @afiune Maybe return the type as well?
-func (dec *Decoder) getObject() (start int, end int, err error) {
-	// start cursor
-	for ; dec.cursor < dec.length || dec.read(); dec.cursor++ {
-		switch dec.data[dec.cursor] {
-		case ' ', '\n', '\t', '\r', ',':
-			continue
-		// is null
-		case 'n':
-			dec.cursor++
-			err = dec.assertNull()
-			if err != nil {
-				return
-			}
-			// Set start & end to the same cursor to indicate the object
-			// is a null and should not be unmarshal
-			start = dec.cursor
-			end = dec.cursor
-			return
-		case 't':
-			start = dec.cursor
-			dec.cursor++
-			err = dec.assertTrue()
-			if err != nil {
-				return
-			}
-			end = dec.cursor
-			dec.cursor++
-			return
-		// is false
-		case 'f':
-			start = dec.cursor
-			dec.cursor++
-			err = dec.assertFalse()
-			if err != nil {
-				return
-			}
-			end = dec.cursor
-			dec.cursor++
-			return
-		// is an object
-		case '{':
-			start = dec.cursor
-			dec.cursor++
-			end, err = dec.skipObject()
-			dec.cursor = end
-			return
-		// is string
-		case '"':
-			start = dec.cursor
-			dec.cursor++
-			start, end, err = dec.getString()
-			start--
-			dec.cursor = end
-			return
-		// is array
-		case '[':
-			start = dec.cursor
-			dec.cursor++
-			end, err = dec.skipArray()
-			dec.cursor = end
-			return
-		case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-':
-			start = dec.cursor
-			end, err = dec.skipNumber()
-			dec.cursor = end
-			return
-		default:
-			err = dec.raiseInvalidJSONErr(dec.cursor)
-			return
-		}
-	}
-	err = dec.raiseInvalidJSONErr(dec.cursor)
-	return
-}
-
-// Add Values functions
-
-// AddInterface decodes the JSON value within an object or an array to a interface{}.
-func (dec *Decoder) AddInterface(v *interface{}) error {
-	return dec.Interface(v)
-}
-
-// Interface decodes the JSON value within an object or an array to an interface{}.
-func (dec *Decoder) Interface(value *interface{}) error {
-	err := dec.decodeInterface(value)
-	if err != nil {
-		return err
-	}
-	dec.called |= 1
-	return nil
-}

vendor/github.com/francoispqt/gojay/decode_number.go

@@ -1,118 +0,0 @@
-package gojay
-
-import (
-	"math"
-)
-
-var digits []int8
-
-const maxInt64toMultiply = math.MaxInt64 / 10
-const maxInt32toMultiply = math.MaxInt32 / 10
-const maxInt16toMultiply = math.MaxInt16 / 10
-const maxInt8toMultiply = math.MaxInt8 / 10
-const maxUint8toMultiply = math.MaxUint8 / 10
-const maxUint16toMultiply = math.MaxUint16 / 10
-const maxUint32toMultiply = math.MaxUint32 / 10
-const maxUint64toMultiply = math.MaxUint64 / 10
-const maxUint32Length = 10
-const maxUint64Length = 20
-const maxUint16Length = 5
-const maxUint8Length = 3
-const maxInt32Length = 10
-const maxInt64Length = 19
-const maxInt16Length = 5
-const maxInt8Length = 3
-const invalidNumber = int8(-1)
-
-var pow10uint64 = [21]uint64{
-	0,
-	1,
-	10,
-	100,
-	1000,
-	10000,
-	100000,
-	1000000,
-	10000000,
-	100000000,
-	1000000000,
-	10000000000,
-	100000000000,
-	1000000000000,
-	10000000000000,
-	100000000000000,
-	1000000000000000,
-	10000000000000000,
-	100000000000000000,
-	1000000000000000000,
-	10000000000000000000,
-}
-
-var skipNumberEndCursorIncrement [256]int
-
-func init() {
-	digits = make([]int8, 256)
-	for i := 0; i < len(digits); i++ {
-		digits[i] = invalidNumber
-	}
-	for i := int8('0'); i <= int8('9'); i++ {
-		digits[i] = i - int8('0')
-	}
-
-	for i := 0; i < 256; i++ {
-		switch i {
-		case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '.', 'e', 'E', '+', '-':
-			skipNumberEndCursorIncrement[i] = 1
-		}
-	}
-}
-
-func (dec *Decoder) skipNumber() (int, error) {
-	end := dec.cursor + 1
-	// look for following numbers
-	for j := dec.cursor + 1; j < dec.length || dec.read(); j++ {
-		end += skipNumberEndCursorIncrement[dec.data[j]]
-
-		switch dec.data[j] {
-		case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '.', 'e', 'E', '+', '-', ' ', '\n', '\t', '\r':
-			continue
-		case ',', '}', ']':
-			return end, nil
-		default:
-			// invalid json we expect numbers, dot (single one), comma, or spaces
-			return end, dec.raiseInvalidJSONErr(dec.cursor)
-		}
-	}
-
-	return end, nil
-}
-
-func (dec *Decoder) getExponent() (int64, error) {
-	start := dec.cursor
-	end := dec.cursor
-	for ; dec.cursor < dec.length || dec.read(); dec.cursor++ {
-		switch dec.data[dec.cursor] { // is positive
-		case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
-			end = dec.cursor + 1
-		case '-':
-			dec.cursor++
-			exp, err := dec.getExponent()
-			return -exp, err
-		case '+':
-			dec.cursor++
-			return dec.getExponent()
-		default:
-			// if nothing return 0
-			// could raise error
-			if start == end {
-				return 0, dec.raiseInvalidJSONErr(dec.cursor)
-			}
-			return dec.atoi64(start, end-1), nil
-		}
-	}
-	if start == end {
-
-		return 0, dec.raiseInvalidJSONErr(dec.cursor)
-	}
-	return dec.atoi64(start, end-1), nil
-}

vendor/github.com/francoispqt/gojay/decode_number_float.go

@@ -1,516 +0,0 @@
-package gojay
-
-// DecodeFloat64 reads the next JSON-encoded value from the decoder's input (io.Reader) and stores it in the float64 pointed to by v.
-//
-// See the documentation for Unmarshal for details about the conversion of JSON into a Go value.
-func (dec *Decoder) DecodeFloat64(v *float64) error {
-	if dec.isPooled == 1 {
-		panic(InvalidUsagePooledDecoderError("Invalid usage of pooled decoder"))
-	}
-	return dec.decodeFloat64(v)
-}
-func (dec *Decoder) decodeFloat64(v *float64) error {
-	for ; dec.cursor < dec.length || dec.read(); dec.cursor++ {
-		switch c := dec.data[dec.cursor]; c {
-		case ' ', '\n', '\t', '\r', ',':
-			continue
-		case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
-			val, err := dec.getFloat()
-			if err != nil {
-				return err
-			}
-			*v = val
-			return nil
-		case '-':
-			dec.cursor = dec.cursor + 1
-			val, err := dec.getFloatNegative()
-			if err != nil {
-				return err
-			}
-			*v = -val
-			return nil
-		case 'n':
-			dec.cursor++
-			err := dec.assertNull()
-			if err != nil {
-				return err
-			}
-			return nil
-		default:
-			dec.err = dec.makeInvalidUnmarshalErr(v)
-			err := dec.skipData()
-			if err != nil {
-				return err
-			}
-			return nil
-		}
-	}
-	return dec.raiseInvalidJSONErr(dec.cursor)
-}
-func (dec *Decoder) decodeFloat64Null(v **float64) error {
-	for ; dec.cursor < dec.length || dec.read(); dec.cursor++ {
-		switch c := dec.data[dec.cursor]; c {
-		case ' ', '\n', '\t', '\r', ',':
-			continue
-		case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
-			val, err := dec.getFloat()
-			if err != nil {
-				return err
-			}
-			if *v == nil {
-				*v = new(float64)
-			}
-			**v = val
-			return nil
-		case '-':
-			dec.cursor = dec.cursor + 1
-			val, err := dec.getFloatNegative()
-			if err != nil {
-				return err
-			}
-			if *v == nil {
-				*v = new(float64)
-			}
-			**v = -val
-			return nil
-		case 'n':
-			dec.cursor++
-			err := dec.assertNull()
-			if err != nil {
-				return err
-			}
-			return nil
-		default:
-			dec.err = dec.makeInvalidUnmarshalErr(v)
-			err := dec.skipData()
-			if err != nil {
-				return err
-			}
-			return nil
-		}
-	}
-	return dec.raiseInvalidJSONErr(dec.cursor)
-}
-
-func (dec *Decoder) getFloatNegative() (float64, error) {
-	// look for following numbers
-	for ; dec.cursor < dec.length || dec.read(); dec.cursor++ {
-		switch dec.data[dec.cursor] {
-		case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
-			return dec.getFloat()
-		default:
-			return 0, dec.raiseInvalidJSONErr(dec.cursor)
-		}
-	}
-	return 0, dec.raiseInvalidJSONErr(dec.cursor)
-}
-
-func (dec *Decoder) getFloat() (float64, error) {
-	var end = dec.cursor
-	var start = dec.cursor
-	// look for following numbers
-	for j := dec.cursor + 1; j < dec.length || dec.read(); j++ {
-		switch dec.data[j] {
-		case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
-			end = j
-			continue
-		case '.':
-			// we get part before decimal as integer
-			beforeDecimal := dec.atoi64(start, end)
-			// then we get part after decimal as integer
-			start = j + 1
-			// get number after the decimal point
-			for i := j + 1; i < dec.length || dec.read(); i++ {
-				c := dec.data[i]
-				if isDigit(c) {
-					end = i
-					// multiply the before decimal point portion by 10 using bitwise
-					// make sure it doesn't overflow
-					if end-start < 18 {
-						beforeDecimal = (beforeDecimal << 3) + (beforeDecimal << 1)
-					}
-					continue
-				} else if (c == 'e' || c == 'E') && j < i-1 {
-					// we have an exponent, convert first the value we got before the exponent
-					var afterDecimal int64
-					expI := end - start + 2
-					// if exp is too long, it means number is too long, just truncate the number
-					if expI >= len(pow10uint64) || expI < 0 {
-						expI = len(pow10uint64) - 2
-						afterDecimal = dec.atoi64(start, start+expI-2)
-					} else {
-						// then we add both integers
-						// then we divide the number by the power found
-						afterDecimal = dec.atoi64(start, end)
-					}
-					dec.cursor = i + 1
-					pow := pow10uint64[expI]
-					floatVal := float64(beforeDecimal+afterDecimal) / float64(pow)
-					exp, err := dec.getExponent()
-					if err != nil {
-						return 0, err
-					}
-					pExp := (exp + (exp >> 31)) ^ (exp >> 31) + 1 // absolute exponent
-					if pExp >= int64(len(pow10uint64)) || pExp < 0 {
-						return 0, dec.raiseInvalidJSONErr(dec.cursor)
-					}
-					// if exponent is negative
-					if exp < 0 {
-						return float64(floatVal) * (1 / float64(pow10uint64[pExp])), nil
-					}
-					return float64(floatVal) * float64(pow10uint64[pExp]), nil
-				}
-				dec.cursor = i
-				break
-			}
-			if end >= dec.length || end < start {
-				return 0, dec.raiseInvalidJSONErr(dec.cursor)
-			}
-			var afterDecimal int64
-			expI := end - start + 2
-			// if exp is too long, it means number is too long, just truncate the number
-			if expI >= len(pow10uint64) || expI < 0 {
-				expI = 19
-				afterDecimal = dec.atoi64(start, start+expI-2)
-			} else {
-				afterDecimal = dec.atoi64(start, end)
-			}
-
-			pow := pow10uint64[expI]
-			// then we add both integers
-			// then we divide the number by the power found
-			return float64(beforeDecimal+afterDecimal) / float64(pow), nil
-		case 'e', 'E':
-			dec.cursor = j + 1
-			// we get part before decimal as integer
-			beforeDecimal := uint64(dec.atoi64(start, end))
-			// get exponent
-			exp, err := dec.getExponent()
-			if err != nil {
-				return 0, err
-			}
-			pExp := (exp + (exp >> 31)) ^ (exp >> 31) + 1 // abs
-			if pExp >= int64(len(pow10uint64)) || pExp < 0 {
-				return 0, dec.raiseInvalidJSONErr(dec.cursor)
-			}
-			// if exponent is negative
-			if exp < 0 {
-				return float64(beforeDecimal) * (1 / float64(pow10uint64[pExp])), nil
-			}
-			return float64(beforeDecimal) * float64(pow10uint64[pExp]), nil
-		case ' ', '\n', '\t', '\r', ',', '}', ']': // does not have decimal
-			dec.cursor = j
-			return float64(dec.atoi64(start, end)), nil
-		}
-		// invalid json we expect numbers, dot (single one), comma, or spaces
-		return 0, dec.raiseInvalidJSONErr(dec.cursor)
-	}
-	return float64(dec.atoi64(start, end)), nil
-}
-
-// DecodeFloat32 reads the next JSON-encoded value from the decoder's input (io.Reader) and stores it in the float32 pointed to by v.
-//
-// See the documentation for Unmarshal for details about the conversion of JSON into a Go value.
-func (dec *Decoder) DecodeFloat32(v *float32) error {
-	if dec.isPooled == 1 {
-		panic(InvalidUsagePooledDecoderError("Invalid usage of pooled decoder"))
-	}
-	return dec.decodeFloat32(v)
-}
-func (dec *Decoder) decodeFloat32(v *float32) error {
-	for ; dec.cursor < dec.length || dec.read(); dec.cursor++ {
-		switch c := dec.data[dec.cursor]; c {
-		case ' ', '\n', '\t', '\r', ',':
-			continue
-		case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
-			val, err := dec.getFloat32()
-			if err != nil {
-				return err
-			}
-			*v = val
-			return nil
-		case '-':
-			dec.cursor = dec.cursor + 1
-			val, err := dec.getFloat32Negative()
-			if err != nil {
-				return err
-			}
-			*v = -val
-			return nil
-		case 'n':
-			dec.cursor++
-			err := dec.assertNull()
-			if err != nil {
-				return err
-			}
-			return nil
-		default:
-			dec.err = dec.makeInvalidUnmarshalErr(v)
-			err := dec.skipData()
-			if err != nil {
-				return err
-			}
-			return nil
-		}
-	}
-	return dec.raiseInvalidJSONErr(dec.cursor)
-}
-func (dec *Decoder) decodeFloat32Null(v **float32) error {
-	for ; dec.cursor < dec.length || dec.read(); dec.cursor++ {
-		switch c := dec.data[dec.cursor]; c {
-		case ' ', '\n', '\t', '\r', ',':
-			continue
-		case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
-			val, err := dec.getFloat32()
-			if err != nil {
-				return err
-			}
-			if *v == nil {
-				*v = new(float32)
-			}
-			**v = val
-			return nil
-		case '-':
-			dec.cursor = dec.cursor + 1
-			val, err := dec.getFloat32Negative()
-			if err != nil {
-				return err
-			}
-			if *v == nil {
-				*v = new(float32)
-			}
-			**v = -val
-			return nil
-		case 'n':
-			dec.cursor++
-			err := dec.assertNull()
-			if err != nil {
-				return err
-			}
-			return nil
-		default:
-			dec.err = dec.makeInvalidUnmarshalErr(v)
-			err := dec.skipData()
-			if err != nil {
-				return err
-			}
-			return nil
-		}
-	}
-	return dec.raiseInvalidJSONErr(dec.cursor)
-}
-
-func (dec *Decoder) getFloat32Negative() (float32, error) {
-	// look for following numbers
-	for ; dec.cursor < dec.length || dec.read(); dec.cursor++ {
-		switch dec.data[dec.cursor] {
-		case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
-			return dec.getFloat32()
-		default:
-			return 0, dec.raiseInvalidJSONErr(dec.cursor)
-		}
-	}
-	return 0, dec.raiseInvalidJSONErr(dec.cursor)
-}
-
-func (dec *Decoder) getFloat32() (float32, error) {
-	var end = dec.cursor
-	var start = dec.cursor
-	// look for following numbers
-	for j := dec.cursor + 1; j < dec.length || dec.read(); j++ {
-		switch dec.data[j] {
-		case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
-			end = j
-			continue
-		case '.':
-			// we get part before decimal as integer
-			beforeDecimal := dec.atoi64(start, end)
-			// then we get part after decimal as integer
-			start = j + 1
-			// get number after the decimal point
-			// multiple the before decimal point portion by 10 using bitwise
-			for i := j + 1; i < dec.length || dec.read(); i++ {
-				c := dec.data[i]
-				if isDigit(c) {
-					end = i
-					// multiply the before decimal point portion by 10 using bitwise
-					// make sure it desn't overflow
-					if end-start < 9 {
-						beforeDecimal = (beforeDecimal << 3) + (beforeDecimal << 1)
-					}
-					continue
-				} else if (c == 'e' || c == 'E') && j < i-1 {
-					// we get the number before decimal
-					var afterDecimal int64
-					expI := end - start + 2
-					// if exp is too long, it means number is too long, just truncate the number
-					if expI >= 12 || expI < 0 {
-						expI = 10
-						afterDecimal = dec.atoi64(start, start+expI-2)
-					} else {
-						afterDecimal = dec.atoi64(start, end)
-					}
-					dec.cursor = i + 1
-					pow := pow10uint64[expI]
-					// then we add both integers
-					// then we divide the number by the power found
-					floatVal := float32(beforeDecimal+afterDecimal) / float32(pow)
-					exp, err := dec.getExponent()
-					if err != nil {
-						return 0, err
-					}
-					pExp := (exp + (exp >> 31)) ^ (exp >> 31) + 1 // abs
-					if pExp >= int64(len(pow10uint64)) || pExp < 0 {
-						return 0, dec.raiseInvalidJSONErr(dec.cursor)
-					}
-					// if exponent is negative
-					if exp < 0 {
-						return float32(floatVal) * (1 / float32(pow10uint64[pExp])), nil
-					}
-					return float32(floatVal) * float32(pow10uint64[pExp]), nil
-				}
-				dec.cursor = i
-				break
-			}
-			if end >= dec.length || end < start {
-				return 0, dec.raiseInvalidJSONErr(dec.cursor)
-			}
-			// then we add both integers
-			// then we divide the number by the power found
-			var afterDecimal int64
-			expI := end - start + 2
-			// if exp is too long, it means number is too long, just truncate the number
-			if expI >= 12 || expI < 0 {
-				expI = 10
-				afterDecimal = dec.atoi64(start, start+expI-2)
-			} else {
-				// then we add both integers
-				// then we divide the number by the power found
-				afterDecimal = dec.atoi64(start, end)
-			}
-			pow := pow10uint64[expI]
-			return float32(beforeDecimal+afterDecimal) / float32(pow), nil
-		case 'e', 'E':
-			dec.cursor = j + 1
-			// we get part before decimal as integer
-			beforeDecimal := dec.atoi64(start, end)
-			// get exponent
-			exp, err := dec.getExponent()
-			if err != nil {
-				return 0, err
-			}
-			pExp := (exp + (exp >> 31)) ^ (exp >> 31) + 1
-			if pExp >= int64(len(pow10uint64)) || pExp < 0 {
-				return 0, dec.raiseInvalidJSONErr(dec.cursor)
-			}
-			// if exponent is negative
-			if exp < 0 {
-				return float32(beforeDecimal) * (1 / float32(pow10uint64[pExp])), nil
-			}
-			return float32(beforeDecimal) * float32(pow10uint64[pExp]), nil
-		case ' ', '\n', '\t', '\r', ',', '}', ']': // does not have decimal
-			dec.cursor = j
-			return float32(dec.atoi64(start, end)), nil
-		}
-		// invalid json we expect numbers, dot (single one), comma, or spaces
-		return 0, dec.raiseInvalidJSONErr(dec.cursor)
-	}
-	return float32(dec.atoi64(start, end)), nil
-}
-
-// Add Values functions
-
-// AddFloat decodes the JSON value within an object or an array to a *float64.
-// If next key value overflows float64, an InvalidUnmarshalError error will be returned.
-func (dec *Decoder) AddFloat(v *float64) error {
-	return dec.Float64(v)
-}
-
-// AddFloatNull decodes the JSON value within an object or an array to a *float64.
-// If next key value overflows float64, an InvalidUnmarshalError error will be returned.
-// If a `null` is encountered, gojay does not change the value of the pointer.
-func (dec *Decoder) AddFloatNull(v **float64) error {
-	return dec.Float64Null(v)
-}
-
-// AddFloat64 decodes the JSON value within an object or an array to a *float64.
-// If next key value overflows float64, an InvalidUnmarshalError error will be returned.
-func (dec *Decoder) AddFloat64(v *float64) error {
-	return dec.Float64(v)
-}
-
-// AddFloat64Null decodes the JSON value within an object or an array to a *float64.
-// If next key value overflows float64, an InvalidUnmarshalError error will be returned.
-// If a `null` is encountered, gojay does not change the value of the pointer.
-func (dec *Decoder) AddFloat64Null(v **float64) error {
-	return dec.Float64Null(v)
-}
-
-// AddFloat32 decodes the JSON value within an object or an array to a *float64.
-// If next key value overflows float64, an InvalidUnmarshalError error will be returned.
-func (dec *Decoder) AddFloat32(v *float32) error {
-	return dec.Float32(v)
-}
-
-// AddFloat32Null decodes the JSON value within an object or an array to a *float64.
-// If next key value overflows float64, an InvalidUnmarshalError error will be returned.
-// If a `null` is encountered, gojay does not change the value of the pointer.
-func (dec *Decoder) AddFloat32Null(v **float32) error {
-	return dec.Float32Null(v)
-}
-
-// Float decodes the JSON value within an object or an array to a *float64.
-// If next key value overflows float64, an InvalidUnmarshalError error will be returned.
-func (dec *Decoder) Float(v *float64) error {
-	return dec.Float64(v)
-}
-
-// FloatNull decodes the JSON value within an object or an array to a *float64.
-// If next key value overflows float64, an InvalidUnmarshalError error will be returned.
-func (dec *Decoder) FloatNull(v **float64) error {
-	return dec.Float64Null(v)
-}
-
-// Float64 decodes the JSON value within an object or an array to a *float64.
-// If next key value overflows float64, an InvalidUnmarshalError error will be returned.
-func (dec *Decoder) Float64(v *float64) error {
-	err := dec.decodeFloat64(v)
-	if err != nil {
-		return err
-	}
-	dec.called |= 1
-	return nil
-}
-
-// Float64Null decodes the JSON value within an object or an array to a *float64.
-// If next key value overflows float64, an InvalidUnmarshalError error will be returned.
-func (dec *Decoder) Float64Null(v **float64) error {
-	err := dec.decodeFloat64Null(v)
-	if err != nil {
-		return err
-	}
-	dec.called |= 1
-	return nil
-}
-
-// Float32 decodes the JSON value within an object or an array to a *float64.
-// If next key value overflows float64, an InvalidUnmarshalError error will be returned.
-func (dec *Decoder) Float32(v *float32) error {
-	err := dec.decodeFloat32(v)
-	if err != nil {
-		return err
-	}
-	dec.called |= 1
-	return nil
-}
-
-// Float32Null decodes the JSON value within an object or an array to a *float64.
-// If next key value overflows float64, an InvalidUnmarshalError error will be returned.
-func (dec *Decoder) Float32Null(v **float32) error {
-	err := dec.decodeFloat32Null(v)
-	if err != nil {
-		return err
-	}
-	dec.called |= 1
-	return nil
-}

vendor/github.com/francoispqt/gojay/decode_number_uint.go

@@ -1,715 +0,0 @@
-package gojay
-
-import (
-	"math"
-)
-
-// DecodeUint8 reads the next JSON-encoded value from the decoder's input (io.Reader) and stores it in the uint8 pointed to by v.
-//
-// See the documentation for Unmarshal for details about the conversion of JSON into a Go value.
-func (dec *Decoder) DecodeUint8(v *uint8) error {
-	if dec.isPooled == 1 {
-		panic(InvalidUsagePooledDecoderError("Invalid usage of pooled decoder"))
-	}
-	return dec.decodeUint8(v)
-}
-
-func (dec *Decoder) decodeUint8(v *uint8) error {
-	for ; dec.cursor < dec.length || dec.read(); dec.cursor++ {
-		switch c := dec.data[dec.cursor]; c {
-		case ' ', '\n', '\t', '\r', ',':
-			continue
-		case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
-			val, err := dec.getUint8()
-			if err != nil {
-				return err
-			}
-			*v = val
-			return nil
-		case '-': // if negative, we just set it to 0 and set error
-			dec.err = dec.makeInvalidUnmarshalErr(v)
-			err := dec.skipData()
-			if err != nil {
-				return err
-			}
-			return nil
-		case 'n':
-			dec.cursor++
-			err := dec.assertNull()
-			if err != nil {
-				return err
-			}
-			return nil
-		default:
-			dec.err = dec.makeInvalidUnmarshalErr(v)
-			err := dec.skipData()
-			if err != nil {
-				return err
-			}
-			return nil
-		}
-	}
-	return dec.raiseInvalidJSONErr(dec.cursor)
-}
-func (dec *Decoder) decodeUint8Null(v **uint8) error {
-	for ; dec.cursor < dec.length || dec.read(); dec.cursor++ {
-		switch c := dec.data[dec.cursor]; c {
-		case ' ', '\n', '\t', '\r', ',':
-			continue
-		case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
-			val, err := dec.getUint8()
-			if err != nil {
-				return err
-			}
-			if *v == nil {
-				*v = new(uint8)
-			}
-			**v = val
-			return nil
-		case '-': // if negative, we just set it to 0 and set error
-			dec.err = dec.makeInvalidUnmarshalErr(v)
-			err := dec.skipData()
-			if err != nil {
-				return err
-			}
-			if *v == nil {
-				*v = new(uint8)
-			}
-			return nil
-		case 'n':
-			dec.cursor++
-			err := dec.assertNull()
-			if err != nil {
-				return err
-			}
-			return nil
-		default:
-			dec.err = dec.makeInvalidUnmarshalErr(v)
-			err := dec.skipData()
-			if err != nil {
-				return err
-			}
-			return nil
-		}
-	}
-	return dec.raiseInvalidJSONErr(dec.cursor)
-}
-
-func (dec *Decoder) getUint8() (uint8, error) {
-	var end = dec.cursor
-	var start = dec.cursor
-	// look for following numbers
-	for j := dec.cursor + 1; j < dec.length || dec.read(); j++ {
-		switch dec.data[j] {
-		case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
-			end = j
-			continue
-		case ' ', '\n', '\t', '\r':
-			continue
-		case '.', ',', '}', ']':
-			dec.cursor = j
-			return dec.atoui8(start, end), nil
-		}
-		// invalid json we expect numbers, dot (single one), comma, or spaces
-		return 0, dec.raiseInvalidJSONErr(dec.cursor)
-	}
-	return dec.atoui8(start, end), nil
-}
-
-// DecodeUint16 reads the next JSON-encoded value from the decoder's input (io.Reader) and stores it in the uint16 pointed to by v.
-//
-// See the documentation for Unmarshal for details about the conversion of JSON into a Go value.
-func (dec *Decoder) DecodeUint16(v *uint16) error {
-	if dec.isPooled == 1 {
-		panic(InvalidUsagePooledDecoderError("Invalid usage of pooled decoder"))
-	}
-	return dec.decodeUint16(v)
-}
-
-func (dec *Decoder) decodeUint16(v *uint16) error {
-	for ; dec.cursor < dec.length || dec.read(); dec.cursor++ {
-		switch c := dec.data[dec.cursor]; c {
-		case ' ', '\n', '\t', '\r', ',':
-			continue
-		case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
-			val, err := dec.getUint16()
-			if err != nil {
-				return err
-			}
-			*v = val
-			return nil
-		case '-':
-			dec.err = dec.makeInvalidUnmarshalErr(v)
-			err := dec.skipData()
-			if err != nil {
-				return err
-			}
-			return nil
-		case 'n':
-			dec.cursor++
-			err := dec.assertNull()
-			if err != nil {
-				return err
-			}
-			return nil
-		default:
-			dec.err = dec.makeInvalidUnmarshalErr(v)
-			err := dec.skipData()
-			if err != nil {
-				return err
-			}
-			return nil
-		}
-	}
-	return dec.raiseInvalidJSONErr(dec.cursor)
-}
-func (dec *Decoder) decodeUint16Null(v **uint16) error {
-	for ; dec.cursor < dec.length || dec.read(); dec.cursor++ {
-		switch c := dec.data[dec.cursor]; c {
-		case ' ', '\n', '\t', '\r', ',':
-			continue
-		case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
-			val, err := dec.getUint16()
-			if err != nil {
-				return err
-			}
-			if *v == nil {
-				*v = new(uint16)
-			}
-			**v = val
-			return nil
-		case '-':
-			dec.err = dec.makeInvalidUnmarshalErr(v)
-			err := dec.skipData()
-			if err != nil {
-				return err
-			}
-			if *v == nil {
-				*v = new(uint16)
-			}
-			return nil
-		case 'n':
-			dec.cursor++
-			err := dec.assertNull()
-			if err != nil {
-				return err
-			}
-			return nil
-		default:
-			dec.err = dec.makeInvalidUnmarshalErr(v)
-			err := dec.skipData()
-			if err != nil {
-				return err
-			}
-			return nil
-		}
-	}
-	return dec.raiseInvalidJSONErr(dec.cursor)
-}
-
-func (dec *Decoder) getUint16() (uint16, error) {
-	var end = dec.cursor
-	var start = dec.cursor
-	// look for following numbers
-	for j := dec.cursor + 1; j < dec.length || dec.read(); j++ {
-		switch dec.data[j] {
-		case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
-			end = j
-			continue
-		case ' ', '\n', '\t', '\r':
-			continue
-		case '.', ',', '}', ']':
-			dec.cursor = j
-			return dec.atoui16(start, end), nil
-		}
-		// invalid json we expect numbers, dot (single one), comma, or spaces
-		return 0, dec.raiseInvalidJSONErr(dec.cursor)
-	}
-	return dec.atoui16(start, end), nil
-}
-
-// DecodeUint32 reads the next JSON-encoded value from the decoder's input (io.Reader) and stores it in the uint32 pointed to by v.
-//
-// See the documentation for Unmarshal for details about the conversion of JSON into a Go value.
-func (dec *Decoder) DecodeUint32(v *uint32) error {
-	if dec.isPooled == 1 {
-		panic(InvalidUsagePooledDecoderError("Invalid usage of pooled decoder"))
-	}
-	return dec.decodeUint32(v)
-}
-
-func (dec *Decoder) decodeUint32(v *uint32) error {
-	for ; dec.cursor < dec.length || dec.read(); dec.cursor++ {
-		switch c := dec.data[dec.cursor]; c {
-		case ' ', '\n', '\t', '\r', ',':
-			continue
-		case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
-			val, err := dec.getUint32()
-			if err != nil {
-				return err
-			}
-			*v = val
-			return nil
-		case '-':
-			dec.err = dec.makeInvalidUnmarshalErr(v)
-			err := dec.skipData()
-			if err != nil {
-				return err
-			}
-			return nil
-		case 'n':
-			dec.cursor++
-			err := dec.assertNull()
-			if err != nil {
-				return err
-			}
-			return nil
-		default:
-			dec.err = dec.makeInvalidUnmarshalErr(v)
-			err := dec.skipData()
-			if err != nil {
-				return err
-			}
-			return nil
-		}
-	}
-	return dec.raiseInvalidJSONErr(dec.cursor)
-}
-func (dec *Decoder) decodeUint32Null(v **uint32) error {
-	for ; dec.cursor < dec.length || dec.read(); dec.cursor++ {
-		switch c := dec.data[dec.cursor]; c {
-		case ' ', '\n', '\t', '\r', ',':
-			continue
-		case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
-			val, err := dec.getUint32()
-			if err != nil {
-				return err
-			}
-			if *v == nil {
-				*v = new(uint32)
-			}
-			**v = val
-			return nil
-		case '-':
-			dec.err = dec.makeInvalidUnmarshalErr(v)
-			err := dec.skipData()
-			if err != nil {
-				return err
-			}
-			if *v == nil {
-				*v = new(uint32)
-			}
-			return nil
-		case 'n':
-			dec.cursor++
-			err := dec.assertNull()
-			if err != nil {
-				return err
-			}
-			return nil
-		default:
-			dec.err = dec.makeInvalidUnmarshalErr(v)
-			err := dec.skipData()
-			if err != nil {
-				return err
-			}
-			return nil
-		}
-	}
-	return dec.raiseInvalidJSONErr(dec.cursor)
-}
-
-func (dec *Decoder) getUint32() (uint32, error) {
-	var end = dec.cursor
-	var start = dec.cursor
-	// look for following numbers
-	for j := dec.cursor + 1; j < dec.length || dec.read(); j++ {
-		switch dec.data[j] {
-		case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
-			end = j
-			continue
-		case ' ', '\n', '\t', '\r':
-			continue
-		case '.', ',', '}', ']':
-			dec.cursor = j
-			return dec.atoui32(start, end), nil
-		}
-		// invalid json we expect numbers, dot (single one), comma, or spaces
-		return 0, dec.raiseInvalidJSONErr(dec.cursor)
-	}
-	return dec.atoui32(start, end), nil
-}
-
-// DecodeUint64 reads the next JSON-encoded value from the decoder's input (io.Reader) and stores it in the uint64 pointed to by v.
-//
-// See the documentation for Unmarshal for details about the conversion of JSON into a Go value.
-func (dec *Decoder) DecodeUint64(v *uint64) error {
-	if dec.isPooled == 1 {
-		panic(InvalidUsagePooledDecoderError("Invalid usage of pooled decoder"))
-	}
-	return dec.decodeUint64(v)
-}
-func (dec *Decoder) decodeUint64(v *uint64) error {
-	for ; dec.cursor < dec.length || dec.read(); dec.cursor++ {
-		switch c := dec.data[dec.cursor]; c {
-		case ' ', '\n', '\t', '\r', ',':
-			continue
-		case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
-			val, err := dec.getUint64()
-			if err != nil {
-				return err
-			}
-			*v = val
-			return nil
-		case '-':
-			dec.err = dec.makeInvalidUnmarshalErr(v)
-			err := dec.skipData()
-			if err != nil {
-				return err
-			}
-			return nil
-		case 'n':
-			dec.cursor++
-			err := dec.assertNull()
-			if err != nil {
-				return err
-			}
-			return nil
-		default:
-			dec.err = dec.makeInvalidUnmarshalErr(v)
-			err := dec.skipData()
-			if err != nil {
-				return err
-			}
-			return nil
-		}
-	}
-	return dec.raiseInvalidJSONErr(dec.cursor)
-}
-func (dec *Decoder) decodeUint64Null(v **uint64) error {
-	for ; dec.cursor < dec.length || dec.read(); dec.cursor++ {
-		switch c := dec.data[dec.cursor]; c {
-		case ' ', '\n', '\t', '\r', ',':
-			continue
-		case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
-			val, err := dec.getUint64()
-			if err != nil {
-				return err
-			}
-			if *v == nil {
-				*v = new(uint64)
-			}
-			**v = val
-			return nil
-		case '-':
-			dec.err = dec.makeInvalidUnmarshalErr(v)
-			err := dec.skipData()
-			if err != nil {
-				return err
-			}
-			if *v == nil {
-				*v = new(uint64)
-			}
-			return nil
-		case 'n':
-			dec.cursor++
-			err := dec.assertNull()
-			if err != nil {
-				return err
-			}
-			return nil
-		default:
-			dec.err = dec.makeInvalidUnmarshalErr(v)
-			err := dec.skipData()
-			if err != nil {
-				return err
-			}
-			return nil
-		}
-	}
-	return dec.raiseInvalidJSONErr(dec.cursor)
-}
-
-func (dec *Decoder) getUint64() (uint64, error) {
-	var end = dec.cursor
-	var start = dec.cursor
-	// look for following numbers
-	for j := dec.cursor + 1; j < dec.length || dec.read(); j++ {
-		switch dec.data[j] {
-		case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
-			end = j
-			continue
-		case ' ', '\n', '\t', '\r', '.', ',', '}', ']':
-			dec.cursor = j
-			return dec.atoui64(start, end), nil
-		}
-		// invalid json we expect numbers, dot (single one), comma, or spaces
-		return 0, dec.raiseInvalidJSONErr(dec.cursor)
-	}
-	return dec.atoui64(start, end), nil
-}
-
-func (dec *Decoder) atoui64(start, end int) uint64 {
-	var ll = end + 1 - start
-	var val = uint64(digits[dec.data[start]])
-	end = end + 1
-	if ll < maxUint64Length {
-		for i := start + 1; i < end; i++ {
-			uintv := uint64(digits[dec.data[i]])
-			val = (val << 3) + (val << 1) + uintv
-		}
-	} else if ll == maxUint64Length {
-		for i := start + 1; i < end; i++ {
-			uintv := uint64(digits[dec.data[i]])
-			if val > maxUint64toMultiply {
-				dec.err = dec.makeInvalidUnmarshalErr(val)
-				return 0
-			}
-			val = (val << 3) + (val << 1)
-			if math.MaxUint64-val < uintv {
-				dec.err = dec.makeInvalidUnmarshalErr(val)
-				return 0
-			}
-			val += uintv
-		}
-	} else {
-		dec.err = dec.makeInvalidUnmarshalErr(val)
-		return 0
-	}
-	return val
-}
-
-func (dec *Decoder) atoui32(start, end int) uint32 {
-	var ll = end + 1 - start
-	var val uint32
-	val = uint32(digits[dec.data[start]])
-	end = end + 1
-	if ll < maxUint32Length {
-		for i := start + 1; i < end; i++ {
-			uintv := uint32(digits[dec.data[i]])
-			val = (val << 3) + (val << 1) + uintv
-		}
-	} else if ll == maxUint32Length {
-		for i := start + 1; i < end; i++ {
-			uintv := uint32(digits[dec.data[i]])
-			if val > maxUint32toMultiply {
-				dec.err = dec.makeInvalidUnmarshalErr(val)
-				return 0
-			}
-			val = (val << 3) + (val << 1)
-			if math.MaxUint32-val < uintv {
-				dec.err = dec.makeInvalidUnmarshalErr(val)
-				return 0
-			}
-			val += uintv
-		}
-	} else if ll > maxUint32Length {
-		dec.err = dec.makeInvalidUnmarshalErr(val)
-		val = 0
-	}
-	return val
-}
-
-func (dec *Decoder) atoui16(start, end int) uint16 {
-	var ll = end + 1 - start
-	var val uint16
-	val = uint16(digits[dec.data[start]])
-	end = end + 1
-	if ll < maxUint16Length {
-		for i := start + 1; i < end; i++ {
-			uintv := uint16(digits[dec.data[i]])
-			val = (val << 3) + (val << 1) + uintv
-		}
-	} else if ll == maxUint16Length {
-		for i := start + 1; i < end; i++ {
-			uintv := uint16(digits[dec.data[i]])
-			if val > maxUint16toMultiply {
-				dec.err = dec.makeInvalidUnmarshalErr(val)
-				return 0
-			}
-			val = (val << 3) + (val << 1)
-			if math.MaxUint16-val < uintv {
-				dec.err = dec.makeInvalidUnmarshalErr(val)
-				return 0
-			}
-			val += uintv
-		}
-	} else if ll > maxUint16Length {
-		dec.err = dec.makeInvalidUnmarshalErr(val)
-		val = 0
-	}
-	return val
-}
-
-func (dec *Decoder) atoui8(start, end int) uint8 {
-	var ll = end + 1 - start
-	var val uint8
-	val = uint8(digits[dec.data[start]])
-	end = end + 1
-	if ll < maxUint8Length {
-		for i := start + 1; i < end; i++ {
-			uintv := uint8(digits[dec.data[i]])
-			val = (val << 3) + (val << 1) + uintv
-		}
-	} else if ll == maxUint8Length {
-		for i := start + 1; i < end; i++ {
-			uintv := uint8(digits[dec.data[i]])
-			if val > maxUint8toMultiply {
-				dec.err = dec.makeInvalidUnmarshalErr(val)
-				return 0
-			}
-			val = (val << 3) + (val << 1)
-			if math.MaxUint8-val < uintv {
-				dec.err = dec.makeInvalidUnmarshalErr(val)
-				return 0
-			}
-			val += uintv
-		}
-	} else if ll > maxUint8Length {
-		dec.err = dec.makeInvalidUnmarshalErr(val)
-		val = 0
-	}
-	return val
-}
-
-// Add Values functions
-
-// AddUint8 decodes the JSON value within an object or an array to an *int.
-// If next key value overflows uint8, an InvalidUnmarshalError error will be returned.
-func (dec *Decoder) AddUint8(v *uint8) error {
-	return dec.Uint8(v)
-}
-
-// AddUint8Null decodes the JSON value within an object or an array to an *int.
-// If next key value overflows uint8, an InvalidUnmarshalError error will be returned.
-// If a `null` is encountered, gojay does not change the value of the pointer.
-func (dec *Decoder) AddUint8Null(v **uint8) error {
-	return dec.Uint8Null(v)
-}
-
-// AddUint16 decodes the JSON value within an object or an array to an *int.
-// If next key value overflows uint16, an InvalidUnmarshalError error will be returned.
-func (dec *Decoder) AddUint16(v *uint16) error {
-	return dec.Uint16(v)
-}
-
-// AddUint16Null decodes the JSON value within an object or an array to an *int.
-// If next key value overflows uint16, an InvalidUnmarshalError error will be returned.
-// If a `null` is encountered, gojay does not change the value of the pointer.
-func (dec *Decoder) AddUint16Null(v **uint16) error {
-	return dec.Uint16Null(v)
-}
-
-// AddUint32 decodes the JSON value within an object or an array to an *int.
-// If next key value overflows uint32, an InvalidUnmarshalError error will be returned.
-func (dec *Decoder) AddUint32(v *uint32) error {
-	return dec.Uint32(v)
-}
-
-// AddUint32Null decodes the JSON value within an object or an array to an *int.
-// If next key value overflows uint32, an InvalidUnmarshalError error will be returned.
-// If a `null` is encountered, gojay does not change the value of the pointer.
-func (dec *Decoder) AddUint32Null(v **uint32) error {
-	return dec.Uint32Null(v)
-}
-
-// AddUint64 decodes the JSON value within an object or an array to an *int.
-// If next key value overflows uint64, an InvalidUnmarshalError error will be returned.
-func (dec *Decoder) AddUint64(v *uint64) error {
-	return dec.Uint64(v)
-}
-
-// AddUint64Null decodes the JSON value within an object or an array to an *int.
-// If next key value overflows uint64, an InvalidUnmarshalError error will be returned.
-// If a `null` is encountered, gojay does not change the value of the pointer.
-func (dec *Decoder) AddUint64Null(v **uint64) error {
-	return dec.Uint64Null(v)
-}
-
-// Uint8 decodes the JSON value within an object or an array to an *int.
-// If next key value overflows uint8, an InvalidUnmarshalError error will be returned.
-func (dec *Decoder) Uint8(v *uint8) error {
-	err := dec.decodeUint8(v)
-	if err != nil {
-		return err
-	}
-	dec.called |= 1
-	return nil
-}
-
-// Uint8Null decodes the JSON value within an object or an array to an *int.
-// If next key value overflows uint8, an InvalidUnmarshalError error will be returned.
-func (dec *Decoder) Uint8Null(v **uint8) error {
-	err := dec.decodeUint8Null(v)
-	if err != nil {
-		return err
-	}
-	dec.called |= 1
-	return nil
-}
-
-// Uint16 decodes the JSON value within an object or an array to an *int.
-// If next key value overflows uint16, an InvalidUnmarshalError error will be returned.
-func (dec *Decoder) Uint16(v *uint16) error {
-	err := dec.decodeUint16(v)
-	if err != nil {
-		return err
-	}
-	dec.called |= 1
-	return nil
-}
-
-// Uint16Null decodes the JSON value within an object or an array to an *int.
-// If next key value overflows uint16, an InvalidUnmarshalError error will be returned.
-func (dec *Decoder) Uint16Null(v **uint16) error {
-	err := dec.decodeUint16Null(v)
-	if err != nil {
-		return err
-	}
-	dec.called |= 1
-	return nil
-}
-
-// Uint32 decodes the JSON value within an object or an array to an *int.
-// If next key value overflows uint32, an InvalidUnmarshalError error will be returned.
-func (dec *Decoder) Uint32(v *uint32) error {
-	err := dec.decodeUint32(v)
-	if err != nil {
-		return err
-	}
-	dec.called |= 1
-	return nil
-}
-
-// Uint32Null decodes the JSON value within an object or an array to an *int.
-// If next key value overflows uint32, an InvalidUnmarshalError error will be returned.
-func (dec *Decoder) Uint32Null(v **uint32) error {
-	err := dec.decodeUint32Null(v)
-	if err != nil {
-		return err
-	}
-	dec.called |= 1
-	return nil
-}
-
-// Uint64 decodes the JSON value within an object or an array to an *int.
-// If next key value overflows uint64, an InvalidUnmarshalError error will be returned.
-func (dec *Decoder) Uint64(v *uint64) error {
-	err := dec.decodeUint64(v)
-	if err != nil {
-		return err
-	}
-	dec.called |= 1
-	return nil
-}
-
-// Uint64Null decodes the JSON value within an object or an array to an *int.
-// If next key value overflows uint64, an InvalidUnmarshalError error will be returned.
-func (dec *Decoder) Uint64Null(v **uint64) error {
-	err := dec.decodeUint64Null(v)
-	if err != nil {
-		return err
-	}
-	dec.called |= 1
-	return nil
-}