Release 2022.2.0

Right now the proxying of cloudflared -> unix socket is a bit of
a no man's land, where we do not have the ability to specify the
actual protocol since the user just configures "unix:/path/"

In practice, we proxy using an HTTP client.
But it could be that the origin expects HTTP or HTTPS. However,
we have no way of knowing.

So how are we proxying to it? We are configuring the http.Request
in ways that depend on the transport and edge implementation, and
it so happens that for h2mux and http2 we are using a http.Request
whose Scheme is HTTP, whereas for quic we are generating a http.Request
whose scheme is HTTPS.

Since it does not make sense to have different behaviours depending
on the transport, we are making a (hopefully temporary) change so
that proxied requests to Unix sockets are systematically HTTP.

In practice we should do https://github.com/cloudflare/cloudflared/issues/502
to make this configurable.

CHANGES.md

@@ -1,8 +1,16 @@
 **Experimental**: This is a new format for release notes. The format and availability is subject to change.
 
+## 2021.12.2
+### Bug Fixes
+- Fix logging when `quic` transport is used and UDP traffic is proxied.
+- FIPS compliant cloudflared binaries will now be released as separate artifacts. Recall that these are only for linux
+and amd64.
+
 ## 2021.12.1
-### Bug Fixe
- - Fixes Github issue #530 where cloudflared 2021.12.0 could not reach origins that were HTTPS and using certain encryption methods forbidden by FIPS compliance (such as Let's Encrypt certificates). To address this fix we have temporarily reverted FIPS compliance from amd64 linux binaries that was recently introduced (or fixed actually as it was never working before).
+### Bug Fixes
+ - Fixes Github issue #530 where cloudflared 2021.12.0 could not reach origins that were HTTPS and using certain encryption
+methods forbidden by FIPS compliance (such as Let's Encrypt certificates). To address this fix we have temporarily reverted
+FIPS compliance from amd64 linux binaries that was recently introduced (or fixed actually as it was never working before).
 
 ## 2021.12.0
 ### New Features

Makefile

@@ -3,23 +3,39 @@ MSI_VERSION   := $(shell git tag -l --sort=v:refname | grep "w" | tail -1 | cut
 #MSI_VERSION expects the format of the tag to be: (wX.X.X). Starts with the w character to not break cfsetup.
 #e.g. w3.0.1 or w4.2.10. It trims off the w character when creating the MSI.
 
-ifeq ($(FIPS), true)
-	GO_BUILD_TAGS := $(GO_BUILD_TAGS) fips
-endif
-
-ifneq ($(GO_BUILD_TAGS),)
-	GO_BUILD_TAGS := -tags $(GO_BUILD_TAGS)
+ifeq ($(ORIGINAL_NAME), true)
+	# Used for builds that want FIPS compilation but want the artifacts generated to still have the original name.
+	BINARY_NAME := cloudflared
+else ifeq ($(FIPS), true)
+	# Used for FIPS compliant builds that do not match the case above.
+	BINARY_NAME := cloudflared-fips
+else
+	# Used for all other (non-FIPS) builds.
+	BINARY_NAME := cloudflared
 endif
 
 ifeq ($(NIGHTLY), true)
-	DEB_PACKAGE_NAME := cloudflared-nightly
+	DEB_PACKAGE_NAME := $(BINARY_NAME)-nightly
 	NIGHTLY_FLAGS := --conflicts cloudflared --replaces cloudflared
 else
-	DEB_PACKAGE_NAME := cloudflared
+	DEB_PACKAGE_NAME := $(BINARY_NAME)
 endif
 
 DATE          := $(shell date -u '+%Y-%m-%d-%H%M UTC')
-VERSION_FLAGS := -ldflags='-X "main.Version=$(VERSION)" -X "main.BuildTime=$(DATE)"'
+VERSION_FLAGS := -X "main.Version=$(VERSION)" -X "main.BuildTime=$(DATE)"
+
+LINK_FLAGS :=
+ifeq ($(FIPS), true)
+	LINK_FLAGS := -linkmode=external -extldflags=-static $(LINK_FLAGS)
+	# Prevent linking with libc regardless of CGO enabled or not.
+	GO_BUILD_TAGS := $(GO_BUILD_TAGS) osusergo netgo fips
+	VERSION_FLAGS := $(VERSION_FLAGS) -X "main.BuildType=FIPS"
+endif
+
+LDFLAGS := -ldflags='$(VERSION_FLAGS) $(LINK_FLAGS)'
+ifneq ($(GO_BUILD_TAGS),)
+	GO_BUILD_TAGS := -tags "$(GO_BUILD_TAGS)"
+endif
 
 IMPORT_PATH   := github.com/cloudflare/cloudflared
 PACKAGE_DIR   := $(CURDIR)/packaging
@@ -61,9 +77,9 @@ else
 endif
 
 ifeq ($(TARGET_OS), windows)
-	EXECUTABLE_PATH=./cloudflared.exe
+	EXECUTABLE_PATH=./$(BINARY_NAME).exe
 else
-	EXECUTABLE_PATH=./cloudflared
+	EXECUTABLE_PATH=./$(BINARY_NAME)
 endif
 
 ifeq ($(FLAVOR), centos-7)
@@ -80,17 +96,15 @@ clean:
 	go clean
 
 .PHONY: cloudflared
-cloudflared: 
+cloudflared:
 ifeq ($(FIPS), true)
 	$(info Building cloudflared with go-fips)
-	-test -f fips/fips.go && mv fips/fips.go fips/fips.go.linux-amd64
-	mv fips/fips.go.linux-amd64 fips/fips.go
+	cp -f fips/fips.go.linux-amd64 cmd/cloudflared/fips.go
 endif
-
-	GOOS=$(TARGET_OS) GOARCH=$(TARGET_ARCH) go build -v -mod=vendor $(GO_BUILD_TAGS) $(VERSION_FLAGS) $(IMPORT_PATH)/cmd/cloudflared
-
+	GOOS=$(TARGET_OS) GOARCH=$(TARGET_ARCH) go build -v -mod=vendor $(GO_BUILD_TAGS) $(LDFLAGS) $(IMPORT_PATH)/cmd/cloudflared
 ifeq ($(FIPS), true)
-	mv fips/fips.go fips/fips.go.linux-amd64
+	rm -f cmd/cloudflared/fips.go
+	./check-fips.sh cloudflared
 endif
 
 .PHONY: container
@@ -100,10 +114,10 @@ container:
 .PHONY: test
 test: vet
 ifndef CI
-	go test -v -mod=vendor -race $(VERSION_FLAGS) ./...
+	go test -v -mod=vendor -race $(LDFLAGS) ./...
 else
 	@mkdir -p .cover
-	go test -v -mod=vendor -race $(VERSION_FLAGS) -coverprofile=".cover/c.out" ./...
+	go test -v -mod=vendor -race $(LDFLAGS) -coverprofile=".cover/c.out" ./...
 	go tool cover -html ".cover/c.out" -o .cover/all.html
 endif
 
@@ -112,10 +126,10 @@ test-ssh-server:
 	docker-compose -f ssh_server_tests/docker-compose.yml up
 
 define publish_package
-	chmod 664 cloudflared*.$(1); \
+	chmod 664 $(BINARY_NAME)*.$(1); \
 	for HOST in $(CF_PKG_HOSTS); do \
 		ssh-keyscan -t ecdsa $$HOST >> ~/.ssh/known_hosts; \
-		scp -p -4 cloudflared*.$(1) cfsync@$$HOST:/state/cf-pkg/staging/$(2)/$(TARGET_PUBLIC_REPO)/cloudflared/; \
+		scp -p -4 $(BINARY_NAME)*.$(1) cfsync@$$HOST:/state/cf-pkg/staging/$(2)/$(TARGET_PUBLIC_REPO)/$(BINARY_NAME)/; \
 	done
 endef
 
@@ -127,6 +141,8 @@ publish-deb: cloudflared-deb
 publish-rpm: cloudflared-rpm
 	$(call publish_package,rpm,yum)
 
+# When we build packages, the package name will be FIPS-aware.
+# But we keep the binary installed by it to be named "cloudflared" regardless.
 define build_package
 	mkdir -p $(PACKAGE_DIR)
 	cp cloudflared $(PACKAGE_DIR)/cloudflared
@@ -247,8 +263,8 @@ tunnelrpc-deps:
 	capnp compile -ogo tunnelrpc/tunnelrpc.capnp
 
 .PHONY: quic-deps
-quic-deps: 
-	which capnp 
+quic-deps:
+	which capnp
 	which capnpc-go
 	capnp compile -ogo quic/schema/quic_metadata_protocol.capnp
 
@@ -258,9 +274,9 @@ vet:
 	# go get github.com/sudarshan-reddy/go-sumtype (don't do this in build directory or this will cause vendor issues)
 	# Note: If you have github.com/BurntSushi/go-sumtype then you might have to use the repo above instead
 	# for now because it uses an older version of golang.org/x/tools.
-	which go-sumtype  
+	which go-sumtype
 	go-sumtype $$(go list -mod=vendor ./...)
 
 .PHONY: goimports
 goimports:
-	for d in $$(go list -mod=readonly -f '{{.Dir}}' -a ./... | fgrep -v tunnelrpc) ; do goimports -format-only -local github.com/cloudflare/cloudflared -w $$d ; done
+	for d in $$(go list -mod=readonly -f '{{.Dir}}' -a ./... | fgrep -v tunnelrpc) ; do goimports -format-only -local github.com/cloudflare/cloudflared -w $$d ; done

RELEASE_NOTES

@@ -1,3 +1,52 @@
+2022.2.0
+- 2022-02-02 TUN-4947: Use http when talking to Unix sockets origins
+- 2022-02-02 TUN-5695: Define RPC method to update configuration
+- 2022-01-27 TUN-5621: Correctly manage QUIC stream closing
+- 2022-01-28 TUN-5702: Allow to deserialize config from JSON
+
+2022.1.3
+- 2022-01-21 TUN-5477: Unhide vnet commands
+- 2022-01-24 TUN-5669: Change network command to vnet
+- 2022-01-25 TUN-5675: Remove github.com/dgrijalva/jwt-go dependency by upgrading coredns version
+- 2022-01-27 TUN-5719: Re-attempt connection to edge with QUIC despite network error when there is no fallback
+- 2022-01-28 TUN-5724: Fix SSE streaming by guaranteeing we write everything we read
+- 2022-01-17 TUN-5547: Bump golang x/net package to fix http2 transport bugs
+- 2022-01-19 TUN-5659: Proxy UDP with zero-byte payload
+- 2021-10-22 Add X-Forwarded-Host for http proxy
+
+2022.1.2
+- 2022-01-13 TUN-5650: Fix pynacl version to 1.4.0 and pygithub version to 1.55 so release doesn't break unexpectedly
+
+2022.1.1
+- 2022-01-10 TUN-5631: Build everything with go 1.17.5
+- 2022-01-06 TUN-5623: Configure quic max datagram frame size to 1350 bytes for none Windows platforms
+
+2022.1.0
+- 2022-01-03 TUN-5612: Add support for specifying TLS min/max version
+- 2022-01-03 TUN-5612: Make tls min/max version public visible
+- 2022-01-03 TUN-5551: Internally published debian artifacts are now named just cloudflared even though they are FIPS compliant
+- 2022-01-04 TUN-5600: Close QUIC transports as soon as possible while respecting graceful shutdown
+- 2022-01-05 TUN-5616: Never fallback transport if user chooses it on purpose
+- 2022-01-05 TUN-5204: Unregister QUIC transports on disconnect
+- 2022-01-04 TUN-5600: Add coverage to component tests for various transports
+
+2021.12.4
+- 2021-12-27 TUN-5482: Refactor tunnelstore client related packages for more coherent package
+- 2021-12-27 TUN-5551: Change internally published debian package to be FIPS compliant
+- 2021-12-27 TUN-5551: Show whether the binary was built for FIPS compliance
+
+2021.12.3
+- 2021-12-22 TUN-5584: Changes for release 2021.12.2
+- 2021-12-22 TUN-5590: QUIC datagram max user payload is 1217 bytes
+- 2021-12-22 TUN-5593: Read full packet from UDP connection, even if it exceeds MTU of the transport. When packet length is greater than the MTU of the transport, we will silently drop packets (for now).
+- 2021-12-23 TUN-5597: Log session ID when session is terminated by edge
+
+2021.12.2
+- 2021-12-20 TUN-5571: Remove redundant session manager log, it's already logged in origin/tunnel.ServeQUIC
+- 2021-12-20 TUN-5570: Only log RPC server events at error level to reduce noise
+- 2021-12-14 TUN-5494: Send a RPC with terminate reason to edge if the session is closed locally
+- 2021-11-09 TUN-5551: Reintroduce FIPS compliance for linux amd64 now as separate binaries
+
 2021.12.1
 - 2021-12-16 TUN-5549: Revert "TUN-5277: Ensure cloudflared binary is FIPS compliant on linux amd64"
 

build-packages-fips.sh

@@ -0,0 +1,25 @@
+VERSION=$(git describe --tags --always --match "[0-9][0-9][0-9][0-9].*.*")
+echo $VERSION
+
+# This controls the directory the built artifacts go into
+export ARTIFACT_DIR=built_artifacts/
+mkdir -p $ARTIFACT_DIR
+
+arch=("amd64")
+export TARGET_ARCH=$arch
+export TARGET_OS=linux
+export FIPS=true
+# For BoringCrypto to link, we need CGO enabled. Otherwise compilation fails.
+export CGO_ENABLED=1
+
+make cloudflared-deb
+mv cloudflared-fips\_$VERSION\_$arch.deb $ARTIFACT_DIR/cloudflared-fips-linux-$arch.deb
+
+# rpm packages invert the - and _ and use x86_64 instead of amd64.
+RPMVERSION=$(echo $VERSION|sed -r 's/-/_/g')
+RPMARCH="x86_64"
+make cloudflared-rpm
+mv cloudflared-fips-$RPMVERSION-1.$RPMARCH.rpm $ARTIFACT_DIR/cloudflared-fips-linux-$RPMARCH.rpm
+
+# finally move the linux binary as well.
+mv ./cloudflared $ARTIFACT_DIR/cloudflared-fips-linux-$arch

build-packages.sh

@@ -1,12 +1,15 @@
-VERSION=$(git describe --tags --always --dirty="-dev" --match "[0-9][0-9][0-9][0-9].*.*")
+VERSION=$(git describe --tags --always --match "[0-9][0-9][0-9][0-9].*.*")
 echo $VERSION
+
+# Avoid depending on C code since we don't need it.
 export CGO_ENABLED=0
+
 # This controls the directory the built artifacts go into
 export ARTIFACT_DIR=built_artifacts/
 mkdir -p $ARTIFACT_DIR
 windowsArchs=("amd64" "386")
 export TARGET_OS=windows
-for arch in ${windowsArchs[@]}; do 
+for arch in ${windowsArchs[@]}; do
     export TARGET_ARCH=$arch
     make cloudflared-msi
     mv ./cloudflared.exe $ARTIFACT_DIR/cloudflared-windows-$arch.exe
@@ -14,15 +17,14 @@ for arch in ${windowsArchs[@]}; do
 done
 
 
-export FIPS=true
-linuxArchs=("amd64" "386" "arm" "arm64")
+linuxArchs=("386" "amd64" "arm" "arm64")
 export TARGET_OS=linux
-for arch in ${linuxArchs[@]}; do 
+for arch in ${linuxArchs[@]}; do
     export TARGET_ARCH=$arch
     make cloudflared-deb
     mv cloudflared\_$VERSION\_$arch.deb $ARTIFACT_DIR/cloudflared-linux-$arch.deb
 
-    # rpm packages invert the - and _ and use x86_64 instead of amd64. 
+    # rpm packages invert the - and _ and use x86_64 instead of amd64.
     RPMVERSION=$(echo $VERSION|sed -r 's/-/_/g')
     RPMARCH=$arch
     if [ $arch == "amd64" ];then
@@ -37,4 +39,3 @@ for arch in ${linuxArchs[@]}; do
     # finally move the linux binary as well.
     mv ./cloudflared $ARTIFACT_DIR/cloudflared-linux-$arch
 done
-

cfapi/base_client.go

@@ -0,0 +1,186 @@
+package cfapi
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog"
+	"golang.org/x/net/http2"
+)
+
+const (
+	defaultTimeout  = 15 * time.Second
+	jsonContentType = "application/json"
+)
+
+var (
+	ErrUnauthorized = errors.New("unauthorized")
+	ErrBadRequest   = errors.New("incorrect request parameters")
+	ErrNotFound     = errors.New("not found")
+	ErrAPINoSuccess = errors.New("API call failed")
+)
+
+type RESTClient struct {
+	baseEndpoints *baseEndpoints
+	authToken     string
+	userAgent     string
+	client        http.Client
+	log           *zerolog.Logger
+}
+
+type baseEndpoints struct {
+	accountLevel  url.URL
+	zoneLevel     url.URL
+	accountRoutes url.URL
+	accountVnets  url.URL
+}
+
+var _ Client = (*RESTClient)(nil)
+
+func NewRESTClient(baseURL, accountTag, zoneTag, authToken, userAgent string, log *zerolog.Logger) (*RESTClient, error) {
+	if strings.HasSuffix(baseURL, "/") {
+		baseURL = baseURL[:len(baseURL)-1]
+	}
+	accountLevelEndpoint, err := url.Parse(fmt.Sprintf("%s/accounts/%s/tunnels", baseURL, accountTag))
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to create account level endpoint")
+	}
+	accountRoutesEndpoint, err := url.Parse(fmt.Sprintf("%s/accounts/%s/teamnet/routes", baseURL, accountTag))
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to create route account-level endpoint")
+	}
+	accountVnetsEndpoint, err := url.Parse(fmt.Sprintf("%s/accounts/%s/teamnet/virtual_networks", baseURL, accountTag))
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to create virtual network account-level endpoint")
+	}
+	zoneLevelEndpoint, err := url.Parse(fmt.Sprintf("%s/zones/%s/tunnels", baseURL, zoneTag))
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to create account level endpoint")
+	}
+	httpTransport := http.Transport{
+		TLSHandshakeTimeout:   defaultTimeout,
+		ResponseHeaderTimeout: defaultTimeout,
+	}
+	http2.ConfigureTransport(&httpTransport)
+	return &RESTClient{
+		baseEndpoints: &baseEndpoints{
+			accountLevel:  *accountLevelEndpoint,
+			zoneLevel:     *zoneLevelEndpoint,
+			accountRoutes: *accountRoutesEndpoint,
+			accountVnets:  *accountVnetsEndpoint,
+		},
+		authToken: authToken,
+		userAgent: userAgent,
+		client: http.Client{
+			Transport: &httpTransport,
+			Timeout:   defaultTimeout,
+		},
+		log: log,
+	}, nil
+}
+
+func (r *RESTClient) sendRequest(method string, url url.URL, body interface{}) (*http.Response, error) {
+	var bodyReader io.Reader
+	if body != nil {
+		if bodyBytes, err := json.Marshal(body); err != nil {
+			return nil, errors.Wrap(err, "failed to serialize json body")
+		} else {
+			bodyReader = bytes.NewBuffer(bodyBytes)
+		}
+	}
+
+	req, err := http.NewRequest(method, url.String(), bodyReader)
+	if err != nil {
+		return nil, errors.Wrapf(err, "can't create %s request", method)
+	}
+	req.Header.Set("User-Agent", r.userAgent)
+	if bodyReader != nil {
+		req.Header.Set("Content-Type", jsonContentType)
+	}
+	req.Header.Add("X-Auth-User-Service-Key", r.authToken)
+	req.Header.Add("Accept", "application/json;version=1")
+	return r.client.Do(req)
+}
+
+func parseResponse(reader io.Reader, data interface{}) error {
+	// Schema for Tunnelstore responses in the v1 API.
+	// Roughly, it's a wrapper around a particular result that adds failures/errors/etc
+	var result response
+	// First, parse the wrapper and check the API call succeeded
+	if err := json.NewDecoder(reader).Decode(&result); err != nil {
+		return errors.Wrap(err, "failed to decode response")
+	}
+	if err := result.checkErrors(); err != nil {
+		return err
+	}
+	if !result.Success {
+		return ErrAPINoSuccess
+	}
+	// At this point we know the API call succeeded, so, parse out the inner
+	// result into the datatype provided as a parameter.
+	if err := json.Unmarshal(result.Result, &data); err != nil {
+		return errors.Wrap(err, "the Cloudflare API response was an unexpected type")
+	}
+	return nil
+}
+
+type response struct {
+	Success  bool            `json:"success,omitempty"`
+	Errors   []apiErr        `json:"errors,omitempty"`
+	Messages []string        `json:"messages,omitempty"`
+	Result   json.RawMessage `json:"result,omitempty"`
+}
+
+func (r *response) checkErrors() error {
+	if len(r.Errors) == 0 {
+		return nil
+	}
+	if len(r.Errors) == 1 {
+		return r.Errors[0]
+	}
+	var messages string
+	for _, e := range r.Errors {
+		messages += fmt.Sprintf("%s; ", e)
+	}
+	return fmt.Errorf("API errors: %s", messages)
+}
+
+type apiErr struct {
+	Code    json.Number `json:"code,omitempty"`
+	Message string      `json:"message,omitempty"`
+}
+
+func (e apiErr) Error() string {
+	return fmt.Sprintf("code: %v, reason: %s", e.Code, e.Message)
+}
+
+func (r *RESTClient) statusCodeToError(op string, resp *http.Response) error {
+	if resp.Header.Get("Content-Type") == "application/json" {
+		var errorsResp response
+		if json.NewDecoder(resp.Body).Decode(&errorsResp) == nil {
+			if err := errorsResp.checkErrors(); err != nil {
+				return errors.Errorf("Failed to %s: %s", op, err)
+			}
+		}
+	}
+
+	switch resp.StatusCode {
+	case http.StatusOK:
+		return nil
+	case http.StatusBadRequest:
+		return ErrBadRequest
+	case http.StatusUnauthorized, http.StatusForbidden:
+		return ErrUnauthorized
+	case http.StatusNotFound:
+		return ErrNotFound
+	}
+	return errors.Errorf("API call to %s failed with status %d: %s", op,
+		resp.StatusCode, http.StatusText(resp.StatusCode))
+}

cfapi/client.go

@@ -0,0 +1,39 @@
+package cfapi
+
+import (
+	"github.com/google/uuid"
+)
+
+type TunnelClient interface {
+	CreateTunnel(name string, tunnelSecret []byte) (*Tunnel, error)
+	GetTunnel(tunnelID uuid.UUID) (*Tunnel, error)
+	DeleteTunnel(tunnelID uuid.UUID) error
+	ListTunnels(filter *TunnelFilter) ([]*Tunnel, error)
+	ListActiveClients(tunnelID uuid.UUID) ([]*ActiveClient, error)
+	CleanupConnections(tunnelID uuid.UUID, params *CleanupParams) error
+}
+
+type HostnameClient interface {
+	RouteTunnel(tunnelID uuid.UUID, route HostnameRoute) (HostnameRouteResult, error)
+}
+
+type IPRouteClient interface {
+	ListRoutes(filter *IpRouteFilter) ([]*DetailedRoute, error)
+	AddRoute(newRoute NewRoute) (Route, error)
+	DeleteRoute(params DeleteRouteParams) error
+	GetByIP(params GetRouteByIpParams) (DetailedRoute, error)
+}
+
+type VnetClient interface {
+	CreateVirtualNetwork(newVnet NewVirtualNetwork) (VirtualNetwork, error)
+	ListVirtualNetworks(filter *VnetFilter) ([]*VirtualNetwork, error)
+	DeleteVirtualNetwork(id uuid.UUID) error
+	UpdateVirtualNetwork(id uuid.UUID, updates UpdateVirtualNetwork) error
+}
+
+type Client interface {
+	TunnelClient
+	HostnameClient
+	IPRouteClient
+	VnetClient
+}

cfapi/hostname.go

@@ -0,0 +1,192 @@
+package cfapi
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"path"
+
+	"github.com/google/uuid"
+	"github.com/pkg/errors"
+)
+
+type Change = string
+
+const (
+	ChangeNew       = "new"
+	ChangeUpdated   = "updated"
+	ChangeUnchanged = "unchanged"
+)
+
+// HostnameRoute represents a record type that can route to a tunnel
+type HostnameRoute interface {
+	json.Marshaler
+	RecordType() string
+	UnmarshalResult(body io.Reader) (HostnameRouteResult, error)
+	String() string
+}
+
+type HostnameRouteResult interface {
+	// SuccessSummary explains what will route to this tunnel when it's provisioned successfully
+	SuccessSummary() string
+}
+
+type DNSRoute struct {
+	userHostname      string
+	overwriteExisting bool
+}
+
+type DNSRouteResult struct {
+	route *DNSRoute
+	CName Change `json:"cname"`
+	Name  string `json:"name"`
+}
+
+func NewDNSRoute(userHostname string, overwriteExisting bool) HostnameRoute {
+	return &DNSRoute{
+		userHostname:      userHostname,
+		overwriteExisting: overwriteExisting,
+	}
+}
+
+func (dr *DNSRoute) MarshalJSON() ([]byte, error) {
+	s := struct {
+		Type              string `json:"type"`
+		UserHostname      string `json:"user_hostname"`
+		OverwriteExisting bool   `json:"overwrite_existing"`
+	}{
+		Type:              dr.RecordType(),
+		UserHostname:      dr.userHostname,
+		OverwriteExisting: dr.overwriteExisting,
+	}
+	return json.Marshal(&s)
+}
+
+func (dr *DNSRoute) UnmarshalResult(body io.Reader) (HostnameRouteResult, error) {
+	var result DNSRouteResult
+	err := parseResponse(body, &result)
+	result.route = dr
+	return &result, err
+}
+
+func (dr *DNSRoute) RecordType() string {
+	return "dns"
+}
+
+func (dr *DNSRoute) String() string {
+	return fmt.Sprintf("%s %s", dr.RecordType(), dr.userHostname)
+}
+
+func (res *DNSRouteResult) SuccessSummary() string {
+	var msgFmt string
+	switch res.CName {
+	case ChangeNew:
+		msgFmt = "Added CNAME %s which will route to this tunnel"
+	case ChangeUpdated: // this is not currently returned by tunnelsore
+		msgFmt = "%s updated to route to your tunnel"
+	case ChangeUnchanged:
+		msgFmt = "%s is already configured to route to your tunnel"
+	}
+	return fmt.Sprintf(msgFmt, res.hostname())
+}
+
+// hostname yields the resulting name for the DNS route; if that is not available from Cloudflare API, then the
+// requested name is returned instead (should not be the common path, it is just a fall-back).
+func (res *DNSRouteResult) hostname() string {
+	if res.Name != "" {
+		return res.Name
+	}
+	return res.route.userHostname
+}
+
+type LBRoute struct {
+	lbName string
+	lbPool string
+}
+
+type LBRouteResult struct {
+	route        *LBRoute
+	LoadBalancer Change `json:"load_balancer"`
+	Pool         Change `json:"pool"`
+}
+
+func NewLBRoute(lbName, lbPool string) HostnameRoute {
+	return &LBRoute{
+		lbName: lbName,
+		lbPool: lbPool,
+	}
+}
+
+func (lr *LBRoute) MarshalJSON() ([]byte, error) {
+	s := struct {
+		Type   string `json:"type"`
+		LBName string `json:"lb_name"`
+		LBPool string `json:"lb_pool"`
+	}{
+		Type:   lr.RecordType(),
+		LBName: lr.lbName,
+		LBPool: lr.lbPool,
+	}
+	return json.Marshal(&s)
+}
+
+func (lr *LBRoute) RecordType() string {
+	return "lb"
+}
+
+func (lb *LBRoute) String() string {
+	return fmt.Sprintf("%s %s %s", lb.RecordType(), lb.lbName, lb.lbPool)
+}
+
+func (lr *LBRoute) UnmarshalResult(body io.Reader) (HostnameRouteResult, error) {
+	var result LBRouteResult
+	err := parseResponse(body, &result)
+	result.route = lr
+	return &result, err
+}
+
+func (res *LBRouteResult) SuccessSummary() string {
+	var msg string
+	switch res.LoadBalancer + "," + res.Pool {
+	case "new,new":
+		msg = "Created load balancer %s and added a new pool %s with this tunnel as an origin"
+	case "new,updated":
+		msg = "Created load balancer %s with an existing pool %s which was updated to use this tunnel as an origin"
+	case "new,unchanged":
+		msg = "Created load balancer %s with an existing pool %s which already has this tunnel as an origin"
+	case "updated,new":
+		msg = "Added new pool %[2]s with this tunnel as an origin to load balancer %[1]s"
+	case "updated,updated":
+		msg = "Updated pool %[2]s to use this tunnel as an origin and added it to load balancer %[1]s"
+	case "updated,unchanged":
+		msg = "Added pool %[2]s, which already has this tunnel as an origin, to load balancer %[1]s"
+	case "unchanged,updated":
+		msg = "Added this tunnel as an origin in pool %[2]s which is already used by load balancer %[1]s"
+	case "unchanged,unchanged":
+		msg = "Load balancer %s already uses pool %s which has this tunnel as an origin"
+	case "unchanged,new":
+		// this state is not possible
+		fallthrough
+	default:
+		msg = "Something went wrong: failed to modify load balancer %s with pool %s; please check traffic manager configuration in the dashboard"
+	}
+
+	return fmt.Sprintf(msg, res.route.lbName, res.route.lbPool)
+}
+
+func (r *RESTClient) RouteTunnel(tunnelID uuid.UUID, route HostnameRoute) (HostnameRouteResult, error) {
+	endpoint := r.baseEndpoints.zoneLevel
+	endpoint.Path = path.Join(endpoint.Path, fmt.Sprintf("%v/routes", tunnelID))
+	resp, err := r.sendRequest("PUT", endpoint, route)
+	if err != nil {
+		return nil, errors.Wrap(err, "REST request failed")
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode == http.StatusOK {
+		return route.UnmarshalResult(resp.Body)
+	}
+
+	return nil, r.statusCodeToError("add route", resp)
+}

cfapi/hostname_test.go

@@ -1,17 +1,9 @@
-package tunnelstore
+package cfapi
 
 import (
-	"bytes"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"net"
-	"reflect"
 	"strings"
 	"testing"
-	"time"
 
-	"github.com/google/uuid"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -105,125 +97,3 @@ func TestLBRouteResultSuccessSummary(t *testing.T) {
 		assert.Equal(t, tt.expected, actual, "case %d", i+1)
 	}
 }
-
-func Test_parseListTunnels(t *testing.T) {
-	type args struct {
-		body string
-	}
-	tests := []struct {
-		name    string
-		args    args
-		want    []*Tunnel
-		wantErr bool
-	}{
-		{
-			name: "empty list",
-			args: args{body: `{"success": true, "result": []}`},
-			want: []*Tunnel{},
-		},
-		{
-			name:    "success is false",
-			args:    args{body: `{"success": false, "result": []}`},
-			wantErr: true,
-		},
-		{
-			name:    "errors are present",
-			args:    args{body: `{"errors": [{"code": 1003, "message":"An A, AAAA or CNAME record already exists with that host"}], "result": []}`},
-			wantErr: true,
-		},
-		{
-			name:    "invalid response",
-			args:    args{body: `abc`},
-			wantErr: true,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			body := ioutil.NopCloser(bytes.NewReader([]byte(tt.args.body)))
-			got, err := parseListTunnels(body)
-			if (err != nil) != tt.wantErr {
-				t.Errorf("parseListTunnels() error = %v, wantErr %v", err, tt.wantErr)
-				return
-			}
-			if !reflect.DeepEqual(got, tt.want) {
-				t.Errorf("parseListTunnels() = %v, want %v", got, tt.want)
-			}
-		})
-	}
-}
-
-func Test_unmarshalTunnel(t *testing.T) {
-	type args struct {
-		reader io.Reader
-	}
-	tests := []struct {
-		name    string
-		args    args
-		want    *Tunnel
-		wantErr bool
-	}{
-		// TODO: Add test cases.
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			got, err := unmarshalTunnel(tt.args.reader)
-			if (err != nil) != tt.wantErr {
-				t.Errorf("unmarshalTunnel() error = %v, wantErr %v", err, tt.wantErr)
-				return
-			}
-			if !reflect.DeepEqual(got, tt.want) {
-				t.Errorf("unmarshalTunnel() = %v, want %v", got, tt.want)
-			}
-		})
-	}
-}
-
-func TestUnmarshalTunnelOk(t *testing.T) {
-
-	jsonBody := `{"success": true, "result": {"id": "00000000-0000-0000-0000-000000000000","name":"test","created_at":"0001-01-01T00:00:00Z","connections":[]}}`
-	expected := Tunnel{
-		ID:          uuid.Nil,
-		Name:        "test",
-		CreatedAt:   time.Time{},
-		Connections: []Connection{},
-	}
-	actual, err := unmarshalTunnel(bytes.NewReader([]byte(jsonBody)))
-	assert.NoError(t, err)
-	assert.Equal(t, &expected, actual)
-}
-
-func TestUnmarshalTunnelErr(t *testing.T) {
-
-	tests := []string{
-		`abc`,
-		`{"success": true, "result": abc}`,
-		`{"success": false, "result": {"id": "00000000-0000-0000-0000-000000000000","name":"test","created_at":"0001-01-01T00:00:00Z","connections":[]}}}`,
-		`{"errors": [{"code": 1003, "message":"An A, AAAA or CNAME record already exists with that host"}], "result": {"id": "00000000-0000-0000-0000-000000000000","name":"test","created_at":"0001-01-01T00:00:00Z","connections":[]}}}`,
-	}
-
-	for i, test := range tests {
-		_, err := unmarshalTunnel(bytes.NewReader([]byte(test)))
-		assert.Error(t, err, fmt.Sprintf("Test #%v failed", i))
-	}
-}
-
-func TestUnmarshalConnections(t *testing.T) {
-	jsonBody := `{"success":true,"messages":[],"errors":[],"result":[{"id":"d4041254-91e3-4deb-bd94-b46e11680b1e","features":["ha-origin"],"version":"2021.2.5","arch":"darwin_amd64","conns":[{"colo_name":"LIS","id":"ac2286e5-c708-4588-a6a0-ba6b51940019","is_pending_reconnect":false,"origin_ip":"148.38.28.2","opened_at":"0001-01-01T00:00:00Z"}],"run_at":"0001-01-01T00:00:00Z"}]}`
-	expected := ActiveClient{
-		ID:       uuid.MustParse("d4041254-91e3-4deb-bd94-b46e11680b1e"),
-		Features: []string{"ha-origin"},
-		Version:  "2021.2.5",
-		Arch:     "darwin_amd64",
-		RunAt:    time.Time{},
-		Connections: []Connection{{
-			ID:                 uuid.MustParse("ac2286e5-c708-4588-a6a0-ba6b51940019"),
-			ColoName:           "LIS",
-			IsPendingReconnect: false,
-			OriginIP:           net.ParseIP("148.38.28.2"),
-			OpenedAt:           time.Time{},
-		}},
-	}
-	actual, err := parseConnectionsDetails(bytes.NewReader([]byte(jsonBody)))
-	assert.NoError(t, err)
-	assert.Equal(t, []*ActiveClient{&expected}, actual)
-}

cfapi/ip_route.go

@@ -1,9 +1,13 @@
-package teamnet
+package cfapi
 
 import (
 	"encoding/json"
 	"fmt"
+	"io"
 	"net"
+	"net/http"
+	"net/url"
+	"path"
 	"time"
 
 	"github.com/google/uuid"
@@ -133,3 +137,104 @@ type GetRouteByIpParams struct {
 	// Optional field. If unset, backend will assume the default vnet for the account.
 	VNetID *uuid.UUID
 }
+
+// ListRoutes calls the Tunnelstore GET endpoint for all routes under an account.
+func (r *RESTClient) ListRoutes(filter *IpRouteFilter) ([]*DetailedRoute, error) {
+	endpoint := r.baseEndpoints.accountRoutes
+	endpoint.RawQuery = filter.Encode()
+	resp, err := r.sendRequest("GET", endpoint, nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "REST request failed")
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode == http.StatusOK {
+		return parseListDetailedRoutes(resp.Body)
+	}
+
+	return nil, r.statusCodeToError("list routes", resp)
+}
+
+// AddRoute calls the Tunnelstore POST endpoint for a given route.
+func (r *RESTClient) AddRoute(newRoute NewRoute) (Route, error) {
+	endpoint := r.baseEndpoints.accountRoutes
+	endpoint.Path = path.Join(endpoint.Path, "network", url.PathEscape(newRoute.Network.String()))
+	resp, err := r.sendRequest("POST", endpoint, newRoute)
+	if err != nil {
+		return Route{}, errors.Wrap(err, "REST request failed")
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode == http.StatusOK {
+		return parseRoute(resp.Body)
+	}
+
+	return Route{}, r.statusCodeToError("add route", resp)
+}
+
+// DeleteRoute calls the Tunnelstore DELETE endpoint for a given route.
+func (r *RESTClient) DeleteRoute(params DeleteRouteParams) error {
+	endpoint := r.baseEndpoints.accountRoutes
+	endpoint.Path = path.Join(endpoint.Path, "network", url.PathEscape(params.Network.String()))
+	setVnetParam(&endpoint, params.VNetID)
+
+	resp, err := r.sendRequest("DELETE", endpoint, nil)
+	if err != nil {
+		return errors.Wrap(err, "REST request failed")
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode == http.StatusOK {
+		_, err := parseRoute(resp.Body)
+		return err
+	}
+
+	return r.statusCodeToError("delete route", resp)
+}
+
+// GetByIP checks which route will proxy a given IP.
+func (r *RESTClient) GetByIP(params GetRouteByIpParams) (DetailedRoute, error) {
+	endpoint := r.baseEndpoints.accountRoutes
+	endpoint.Path = path.Join(endpoint.Path, "ip", url.PathEscape(params.Ip.String()))
+	setVnetParam(&endpoint, params.VNetID)
+
+	resp, err := r.sendRequest("GET", endpoint, nil)
+	if err != nil {
+		return DetailedRoute{}, errors.Wrap(err, "REST request failed")
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode == http.StatusOK {
+		return parseDetailedRoute(resp.Body)
+	}
+
+	return DetailedRoute{}, r.statusCodeToError("get route by IP", resp)
+}
+
+func parseListDetailedRoutes(body io.ReadCloser) ([]*DetailedRoute, error) {
+	var routes []*DetailedRoute
+	err := parseResponse(body, &routes)
+	return routes, err
+}
+
+func parseRoute(body io.ReadCloser) (Route, error) {
+	var route Route
+	err := parseResponse(body, &route)
+	return route, err
+}
+
+func parseDetailedRoute(body io.ReadCloser) (DetailedRoute, error) {
+	var route DetailedRoute
+	err := parseResponse(body, &route)
+	return route, err
+}
+
+// setVnetParam overwrites the URL's query parameters with a query param to scope the HostnameRoute action to a certain
+// virtual network (if one is provided).
+func setVnetParam(endpoint *url.URL, vnetID *uuid.UUID) {
+	queryParams := url.Values{}
+	if vnetID != nil {
+		queryParams.Set("virtual_network_id", vnetID.String())
+	}
+	endpoint.RawQuery = queryParams.Encode()
+}

cfapi/ip_route_filter.go

@@ -1,4 +1,4 @@
-package teamnet
+package cfapi
 
 import (
 	"fmt"
@@ -13,90 +13,90 @@ import (
 )
 
 var (
-	filterDeleted = cli.BoolFlag{
+	filterIpRouteDeleted = cli.BoolFlag{
 		Name:  "filter-is-deleted",
 		Usage: "If false (default), only show non-deleted routes. If true, only show deleted routes.",
 	}
-	filterTunnelID = cli.StringFlag{
+	filterIpRouteTunnelID = cli.StringFlag{
 		Name:  "filter-tunnel-id",
 		Usage: "Show only routes with the given tunnel ID.",
 	}
-	filterSubset = cli.StringFlag{
+	filterSubsetIpRoute = cli.StringFlag{
 		Name:    "filter-network-is-subset-of",
 		Aliases: []string{"nsub"},
 		Usage:   "Show only routes whose network is a subset of the given network.",
 	}
-	filterSuperset = cli.StringFlag{
+	filterSupersetIpRoute = cli.StringFlag{
 		Name:    "filter-network-is-superset-of",
 		Aliases: []string{"nsup"},
 		Usage:   "Show only routes whose network is a superset of the given network.",
 	}
-	filterComment = cli.StringFlag{
+	filterIpRouteComment = cli.StringFlag{
 		Name:  "filter-comment-is",
 		Usage: "Show only routes with this comment.",
 	}
-	filterVnet = cli.StringFlag{
-		Name:  "filter-virtual-network-id",
+	filterIpRouteByVnet = cli.StringFlag{
+		Name:  "filter-vnet-id",
 		Usage: "Show only routes that are attached to the given virtual network ID.",
 	}
 
 	// Flags contains all filter flags.
-	FilterFlags = []cli.Flag{
-		&filterDeleted,
-		&filterTunnelID,
-		&filterSubset,
-		&filterSuperset,
-		&filterComment,
-		&filterVnet,
+	IpRouteFilterFlags = []cli.Flag{
+		&filterIpRouteDeleted,
+		&filterIpRouteTunnelID,
+		&filterSubsetIpRoute,
+		&filterSupersetIpRoute,
+		&filterIpRouteComment,
+		&filterIpRouteByVnet,
 	}
 )
 
-// Filter which routes get queried.
-type Filter struct {
+// IpRouteFilter which routes get queried.
+type IpRouteFilter struct {
 	queryParams url.Values
 }
 
-// NewFromCLI parses CLI flags to discover which filters should get applied.
-func NewFromCLI(c *cli.Context) (*Filter, error) {
-	f := &Filter{
+// NewIpRouteFilterFromCLI parses CLI flags to discover which filters should get applied.
+func NewIpRouteFilterFromCLI(c *cli.Context) (*IpRouteFilter, error) {
+	f := &IpRouteFilter{
 		queryParams: url.Values{},
 	}
 
 	// Set deletion filter
-	if flag := filterDeleted.Name; c.IsSet(flag) && c.Bool(flag) {
+	if flag := filterIpRouteDeleted.Name; c.IsSet(flag) && c.Bool(flag) {
 		f.deleted()
 	} else {
 		f.notDeleted()
 	}
 
-	if subset, err := cidrFromFlag(c, filterSubset); err != nil {
+	if subset, err := cidrFromFlag(c, filterSubsetIpRoute); err != nil {
 		return nil, err
 	} else if subset != nil {
 		f.networkIsSupersetOf(*subset)
 	}
 
-	if superset, err := cidrFromFlag(c, filterSuperset); err != nil {
+	if superset, err := cidrFromFlag(c, filterSupersetIpRoute); err != nil {
 		return nil, err
 	} else if superset != nil {
 		f.networkIsSupersetOf(*superset)
 	}
 
-	if comment := c.String(filterComment.Name); comment != "" {
+	if comment := c.String(filterIpRouteComment.Name); comment != "" {
 		f.commentIs(comment)
 	}
 
-	if tunnelID := c.String(filterTunnelID.Name); tunnelID != "" {
+	if tunnelID := c.String(filterIpRouteTunnelID.Name); tunnelID != "" {
 		u, err := uuid.Parse(tunnelID)
 		if err != nil {
-			return nil, errors.Wrapf(err, "Couldn't parse UUID from %s", filterTunnelID.Name)
+			return nil, errors.Wrapf(err, "Couldn't parse UUID from %s", filterIpRouteTunnelID.Name)
 		}
 		f.tunnelID(u)
 	}
 
-	if vnetId := c.String(filterVnet.Name); vnetId != "" {
+	if vnetId := c.String(filterIpRouteByVnet.Name); vnetId != "" {
 		u, err := uuid.Parse(vnetId)
 		if err != nil {
-			return nil, errors.Wrapf(err, "Couldn't parse UUID from %s", filterVnet.Name)
+			return nil, errors.Wrapf(err, "Couldn't parse UUID from %s", filterIpRouteByVnet.Name)
 		}
 		f.vnetID(u)
 	}
@@ -124,42 +124,42 @@ func cidrFromFlag(c *cli.Context, flag cli.StringFlag) (*net.IPNet, error) {
 	return subset, nil
 }
 
-func (f *Filter) commentIs(comment string) {
+func (f *IpRouteFilter) commentIs(comment string) {
 	f.queryParams.Set("comment", comment)
 }
 
-func (f *Filter) notDeleted() {
+func (f *IpRouteFilter) notDeleted() {
 	f.queryParams.Set("is_deleted", "false")
 }
 
-func (f *Filter) deleted() {
+func (f *IpRouteFilter) deleted() {
 	f.queryParams.Set("is_deleted", "true")
 }
 
-func (f *Filter) networkIsSubsetOf(superset net.IPNet) {
+func (f *IpRouteFilter) networkIsSubsetOf(superset net.IPNet) {
 	f.queryParams.Set("network_subset", superset.String())
 }
 
-func (f *Filter) networkIsSupersetOf(subset net.IPNet) {
+func (f *IpRouteFilter) networkIsSupersetOf(subset net.IPNet) {
 	f.queryParams.Set("network_superset", subset.String())
 }
 
-func (f *Filter) existedAt(existedAt time.Time) {
+func (f *IpRouteFilter) existedAt(existedAt time.Time) {
 	f.queryParams.Set("existed_at", existedAt.Format(time.RFC3339))
 }
 
-func (f *Filter) tunnelID(id uuid.UUID) {
+func (f *IpRouteFilter) tunnelID(id uuid.UUID) {
 	f.queryParams.Set("tunnel_id", id.String())
 }
 
-func (f *Filter) vnetID(id uuid.UUID) {
+func (f *IpRouteFilter) vnetID(id uuid.UUID) {
 	f.queryParams.Set("virtual_network_id", id.String())
 }
 
-func (f *Filter) MaxFetchSize(max uint) {
+func (f *IpRouteFilter) MaxFetchSize(max uint) {
 	f.queryParams.Set("per_page", strconv.Itoa(int(max)))
 }
 
-func (f Filter) Encode() string {
+func (f IpRouteFilter) Encode() string {
 	return f.queryParams.Encode()
 }

cfapi/ip_route_test.go

@@ -1,4 +1,4 @@
-package teamnet
+package cfapi
 
 import (
 	"encoding/json"

cfapi/tunnel.go

@@ -0,0 +1,183 @@
+package cfapi
+
+import (
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"net/url"
+	"path"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/pkg/errors"
+)
+
+var ErrTunnelNameConflict = errors.New("tunnel with name already exists")
+
+type Tunnel struct {
+	ID          uuid.UUID    `json:"id"`
+	Name        string       `json:"name"`
+	CreatedAt   time.Time    `json:"created_at"`
+	DeletedAt   time.Time    `json:"deleted_at"`
+	Connections []Connection `json:"connections"`
+}
+
+type Connection struct {
+	ColoName           string    `json:"colo_name"`
+	ID                 uuid.UUID `json:"id"`
+	IsPendingReconnect bool      `json:"is_pending_reconnect"`
+	OriginIP           net.IP    `json:"origin_ip"`
+	OpenedAt           time.Time `json:"opened_at"`
+}
+
+type ActiveClient struct {
+	ID          uuid.UUID    `json:"id"`
+	Features    []string     `json:"features"`
+	Version     string       `json:"version"`
+	Arch        string       `json:"arch"`
+	RunAt       time.Time    `json:"run_at"`
+	Connections []Connection `json:"conns"`
+}
+
+type newTunnel struct {
+	Name         string `json:"name"`
+	TunnelSecret []byte `json:"tunnel_secret"`
+}
+
+type CleanupParams struct {
+	queryParams url.Values
+}
+
+func NewCleanupParams() *CleanupParams {
+	return &CleanupParams{
+		queryParams: url.Values{},
+	}
+}
+
+func (cp *CleanupParams) ForClient(clientID uuid.UUID) {
+	cp.queryParams.Set("client_id", clientID.String())
+}
+
+func (cp CleanupParams) encode() string {
+	return cp.queryParams.Encode()
+}
+
+func (r *RESTClient) CreateTunnel(name string, tunnelSecret []byte) (*Tunnel, error) {
+	if name == "" {
+		return nil, errors.New("tunnel name required")
+	}
+	if _, err := uuid.Parse(name); err == nil {
+		return nil, errors.New("you cannot use UUIDs as tunnel names")
+	}
+	body := &newTunnel{
+		Name:         name,
+		TunnelSecret: tunnelSecret,
+	}
+
+	resp, err := r.sendRequest("POST", r.baseEndpoints.accountLevel, body)
+	if err != nil {
+		return nil, errors.Wrap(err, "REST request failed")
+	}
+	defer resp.Body.Close()
+
+	switch resp.StatusCode {
+	case http.StatusOK:
+		return unmarshalTunnel(resp.Body)
+	case http.StatusConflict:
+		return nil, ErrTunnelNameConflict
+	}
+
+	return nil, r.statusCodeToError("create tunnel", resp)
+}
+
+func (r *RESTClient) GetTunnel(tunnelID uuid.UUID) (*Tunnel, error) {
+	endpoint := r.baseEndpoints.accountLevel
+	endpoint.Path = path.Join(endpoint.Path, fmt.Sprintf("%v", tunnelID))
+	resp, err := r.sendRequest("GET", endpoint, nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "REST request failed")
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode == http.StatusOK {
+		return unmarshalTunnel(resp.Body)
+	}
+
+	return nil, r.statusCodeToError("get tunnel", resp)
+}
+
+func (r *RESTClient) DeleteTunnel(tunnelID uuid.UUID) error {
+	endpoint := r.baseEndpoints.accountLevel
+	endpoint.Path = path.Join(endpoint.Path, fmt.Sprintf("%v", tunnelID))
+	resp, err := r.sendRequest("DELETE", endpoint, nil)
+	if err != nil {
+		return errors.Wrap(err, "REST request failed")
+	}
+	defer resp.Body.Close()
+
+	return r.statusCodeToError("delete tunnel", resp)
+}
+
+func (r *RESTClient) ListTunnels(filter *TunnelFilter) ([]*Tunnel, error) {
+	endpoint := r.baseEndpoints.accountLevel
+	endpoint.RawQuery = filter.encode()
+	resp, err := r.sendRequest("GET", endpoint, nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "REST request failed")
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode == http.StatusOK {
+		return parseListTunnels(resp.Body)
+	}
+
+	return nil, r.statusCodeToError("list tunnels", resp)
+}
+
+func parseListTunnels(body io.ReadCloser) ([]*Tunnel, error) {
+	var tunnels []*Tunnel
+	err := parseResponse(body, &tunnels)
+	return tunnels, err
+}
+
+func (r *RESTClient) ListActiveClients(tunnelID uuid.UUID) ([]*ActiveClient, error) {
+	endpoint := r.baseEndpoints.accountLevel
+	endpoint.Path = path.Join(endpoint.Path, fmt.Sprintf("%v/connections", tunnelID))
+	resp, err := r.sendRequest("GET", endpoint, nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "REST request failed")
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode == http.StatusOK {
+		return parseConnectionsDetails(resp.Body)
+	}
+
+	return nil, r.statusCodeToError("list connection details", resp)
+}
+
+func parseConnectionsDetails(reader io.Reader) ([]*ActiveClient, error) {
+	var clients []*ActiveClient
+	err := parseResponse(reader, &clients)
+	return clients, err
+}
+
+func (r *RESTClient) CleanupConnections(tunnelID uuid.UUID, params *CleanupParams) error {
+	endpoint := r.baseEndpoints.accountLevel
+	endpoint.RawQuery = params.encode()
+	endpoint.Path = path.Join(endpoint.Path, fmt.Sprintf("%v/connections", tunnelID))
+	resp, err := r.sendRequest("DELETE", endpoint, nil)
+	if err != nil {
+		return errors.Wrap(err, "REST request failed")
+	}
+	defer resp.Body.Close()
+
+	return r.statusCodeToError("cleanup connections", resp)
+}
+
+func unmarshalTunnel(reader io.Reader) (*Tunnel, error) {
+	var tunnel Tunnel
+	err := parseResponse(reader, &tunnel)
+	return &tunnel, err
+}

cfapi/tunnel_filter.go

@@ -1,4 +1,4 @@
-package tunnelstore
+package cfapi
 
 import (
 	"net/url"
@@ -12,44 +12,44 @@ const (
 	TimeLayout = time.RFC3339
 )
 
-type Filter struct {
+type TunnelFilter struct {
 	queryParams url.Values
 }
 
-func NewFilter() *Filter {
-	return &Filter{
+func NewTunnelFilter() *TunnelFilter {
+	return &TunnelFilter{
 		queryParams: url.Values{},
 	}
 }
 
-func (f *Filter) ByName(name string) {
+func (f *TunnelFilter) ByName(name string) {
 	f.queryParams.Set("name", name)
 }
 
-func (f *Filter) ByNamePrefix(namePrefix string) {
+func (f *TunnelFilter) ByNamePrefix(namePrefix string) {
 	f.queryParams.Set("name_prefix", namePrefix)
 }
 
-func (f *Filter) ExcludeNameWithPrefix(excludePrefix string) {
+func (f *TunnelFilter) ExcludeNameWithPrefix(excludePrefix string) {
 	f.queryParams.Set("exclude_prefix", excludePrefix)
 }
 
-func (f *Filter) NoDeleted() {
+func (f *TunnelFilter) NoDeleted() {
 	f.queryParams.Set("is_deleted", "false")
 }
 
-func (f *Filter) ByExistedAt(existedAt time.Time) {
+func (f *TunnelFilter) ByExistedAt(existedAt time.Time) {
 	f.queryParams.Set("existed_at", existedAt.Format(TimeLayout))
 }
 
-func (f *Filter) ByTunnelID(tunnelID uuid.UUID) {
+func (f *TunnelFilter) ByTunnelID(tunnelID uuid.UUID) {
 	f.queryParams.Set("uuid", tunnelID.String())
 }
 
-func (f *Filter) MaxFetchSize(max uint) {
+func (f *TunnelFilter) MaxFetchSize(max uint) {
 	f.queryParams.Set("per_page", strconv.Itoa(int(max)))
 }
 
-func (f Filter) encode() string {
+func (f TunnelFilter) encode() string {
 	return f.queryParams.Encode()
 }

cfapi/tunnel_test.go

@@ -0,0 +1,149 @@
+package cfapi
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"reflect"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/assert"
+)
+
+var loc, _ = time.LoadLocation("UTC")
+
+func Test_parseListTunnels(t *testing.T) {
+	type args struct {
+		body string
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    []*Tunnel
+		wantErr bool
+	}{
+		{
+			name: "empty list",
+			args: args{body: `{"success": true, "result": []}`},
+			want: []*Tunnel{},
+		},
+		{
+			name:    "success is false",
+			args:    args{body: `{"success": false, "result": []}`},
+			wantErr: true,
+		},
+		{
+			name:    "errors are present",
+			args:    args{body: `{"errors": [{"code": 1003, "message":"An A, AAAA or CNAME record already exists with that host"}], "result": []}`},
+			wantErr: true,
+		},
+		{
+			name:    "invalid response",
+			args:    args{body: `abc`},
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			body := ioutil.NopCloser(bytes.NewReader([]byte(tt.args.body)))
+			got, err := parseListTunnels(body)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("parseListTunnels() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("parseListTunnels() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func Test_unmarshalTunnel(t *testing.T) {
+	type args struct {
+		body string
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    *Tunnel
+		wantErr bool
+	}{
+		{
+			name: "empty list",
+			args: args{body: `{"success": true, "result": {"id":"b34cc7ce-925b-46ee-bc23-4cb5c18d8292","created_at":"2021-07-29T13:46:14.090955Z","deleted_at":"2021-07-29T14:07:27.559047Z","name":"qt-bIWWN7D662ogh61pCPfu5s2XgqFY1OyV","account_id":6946212,"account_tag":"5ab4e9dfbd435d24068829fda0077963","conns_active_at":null,"conns_inactive_at":"2021-07-29T13:47:22.548482Z","tun_type":"cfd_tunnel","metadata":{"qtid":"a6fJROgkXutNruBGaJjD"}}}`},
+			want: &Tunnel{
+				ID:          uuid.MustParse("b34cc7ce-925b-46ee-bc23-4cb5c18d8292"),
+				Name:        "qt-bIWWN7D662ogh61pCPfu5s2XgqFY1OyV",
+				CreatedAt:   time.Date(2021, 07, 29, 13, 46, 14, 90955000, loc),
+				DeletedAt:   time.Date(2021, 07, 29, 14, 7, 27, 559047000, loc),
+				Connections: nil,
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := unmarshalTunnel(strings.NewReader(tt.args.body))
+			if (err != nil) != tt.wantErr {
+				t.Errorf("unmarshalTunnel() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("unmarshalTunnel() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestUnmarshalTunnelOk(t *testing.T) {
+
+	jsonBody := `{"success": true, "result": {"id": "00000000-0000-0000-0000-000000000000","name":"test","created_at":"0001-01-01T00:00:00Z","connections":[]}}`
+	expected := Tunnel{
+		ID:          uuid.Nil,
+		Name:        "test",
+		CreatedAt:   time.Time{},
+		Connections: []Connection{},
+	}
+	actual, err := unmarshalTunnel(bytes.NewReader([]byte(jsonBody)))
+	assert.NoError(t, err)
+	assert.Equal(t, &expected, actual)
+}
+
+func TestUnmarshalTunnelErr(t *testing.T) {
+
+	tests := []string{
+		`abc`,
+		`{"success": true, "result": abc}`,
+		`{"success": false, "result": {"id": "00000000-0000-0000-0000-000000000000","name":"test","created_at":"0001-01-01T00:00:00Z","connections":[]}}}`,
+		`{"errors": [{"code": 1003, "message":"An A, AAAA or CNAME record already exists with that host"}], "result": {"id": "00000000-0000-0000-0000-000000000000","name":"test","created_at":"0001-01-01T00:00:00Z","connections":[]}}}`,
+	}
+
+	for i, test := range tests {
+		_, err := unmarshalTunnel(bytes.NewReader([]byte(test)))
+		assert.Error(t, err, fmt.Sprintf("Test #%v failed", i))
+	}
+}
+
+func TestUnmarshalConnections(t *testing.T) {
+	jsonBody := `{"success":true,"messages":[],"errors":[],"result":[{"id":"d4041254-91e3-4deb-bd94-b46e11680b1e","features":["ha-origin"],"version":"2021.2.5","arch":"darwin_amd64","conns":[{"colo_name":"LIS","id":"ac2286e5-c708-4588-a6a0-ba6b51940019","is_pending_reconnect":false,"origin_ip":"148.38.28.2","opened_at":"0001-01-01T00:00:00Z"}],"run_at":"0001-01-01T00:00:00Z"}]}`
+	expected := ActiveClient{
+		ID:       uuid.MustParse("d4041254-91e3-4deb-bd94-b46e11680b1e"),
+		Features: []string{"ha-origin"},
+		Version:  "2021.2.5",
+		Arch:     "darwin_amd64",
+		RunAt:    time.Time{},
+		Connections: []Connection{{
+			ID:                 uuid.MustParse("ac2286e5-c708-4588-a6a0-ba6b51940019"),
+			ColoName:           "LIS",
+			IsPendingReconnect: false,
+			OriginIP:           net.ParseIP("148.38.28.2"),
+			OpenedAt:           time.Time{},
+		}},
+	}
+	actual, err := parseConnectionsDetails(bytes.NewReader([]byte(jsonBody)))
+	assert.NoError(t, err)
+	assert.Equal(t, []*ActiveClient{&expected}, actual)
+}

cfapi/virtual_network.go

@@ -1,21 +1,59 @@
-package tunnelstore
+package cfapi
 
 import (
+	"fmt"
 	"io"
 	"net/http"
 	"net/url"
 	"path"
+	"strconv"
+	"time"
 
 	"github.com/google/uuid"
 	"github.com/pkg/errors"
-
-	"github.com/cloudflare/cloudflared/vnet"
 )
 
-func (r *RESTClient) CreateVirtualNetwork(newVnet vnet.NewVirtualNetwork) (vnet.VirtualNetwork, error) {
+type NewVirtualNetwork struct {
+	Name      string `json:"name"`
+	Comment   string `json:"comment"`
+	IsDefault bool   `json:"is_default"`
+}
+
+type VirtualNetwork struct {
+	ID        uuid.UUID `json:"id"`
+	Comment   string    `json:"comment"`
+	Name      string    `json:"name"`
+	IsDefault bool      `json:"is_default_network"`
+	CreatedAt time.Time `json:"created_at"`
+	DeletedAt time.Time `json:"deleted_at"`
+}
+
+type UpdateVirtualNetwork struct {
+	Name      *string `json:"name,omitempty"`
+	Comment   *string `json:"comment,omitempty"`
+	IsDefault *bool   `json:"is_default_network,omitempty"`
+}
+
+func (virtualNetwork VirtualNetwork) TableString() string {
+	deletedColumn := "-"
+	if !virtualNetwork.DeletedAt.IsZero() {
+		deletedColumn = virtualNetwork.DeletedAt.Format(time.RFC3339)
+	}
+	return fmt.Sprintf(
+		"%s\t%s\t%s\t%s\t%s\t%s\t",
+		virtualNetwork.ID,
+		virtualNetwork.Name,
+		strconv.FormatBool(virtualNetwork.IsDefault),
+		virtualNetwork.Comment,
+		virtualNetwork.CreatedAt.Format(time.RFC3339),
+		deletedColumn,
+	)
+}
+
+func (r *RESTClient) CreateVirtualNetwork(newVnet NewVirtualNetwork) (VirtualNetwork, error) {
 	resp, err := r.sendRequest("POST", r.baseEndpoints.accountVnets, newVnet)
 	if err != nil {
-		return vnet.VirtualNetwork{}, errors.Wrap(err, "REST request failed")
+		return VirtualNetwork{}, errors.Wrap(err, "REST request failed")
 	}
 	defer resp.Body.Close()
 
@@ -23,10 +61,10 @@ func (r *RESTClient) CreateVirtualNetwork(newVnet vnet.NewVirtualNetwork) (vnet.
 		return parseVnet(resp.Body)
 	}
 
-	return vnet.VirtualNetwork{}, r.statusCodeToError("add virtual network", resp)
+	return VirtualNetwork{}, r.statusCodeToError("add virtual network", resp)
 }
 
-func (r *RESTClient) ListVirtualNetworks(filter *vnet.Filter) ([]*vnet.VirtualNetwork, error) {
+func (r *RESTClient) ListVirtualNetworks(filter *VnetFilter) ([]*VirtualNetwork, error) {
 	endpoint := r.baseEndpoints.accountVnets
 	endpoint.RawQuery = filter.Encode()
 	resp, err := r.sendRequest("GET", endpoint, nil)
@@ -59,7 +97,7 @@ func (r *RESTClient) DeleteVirtualNetwork(id uuid.UUID) error {
 	return r.statusCodeToError("delete virtual network", resp)
 }
 
-func (r *RESTClient) UpdateVirtualNetwork(id uuid.UUID, updates vnet.UpdateVirtualNetwork) error {
+func (r *RESTClient) UpdateVirtualNetwork(id uuid.UUID, updates UpdateVirtualNetwork) error {
 	endpoint := r.baseEndpoints.accountVnets
 	endpoint.Path = path.Join(endpoint.Path, url.PathEscape(id.String()))
 	resp, err := r.sendRequest("PATCH", endpoint, updates)
@@ -76,14 +114,14 @@ func (r *RESTClient) UpdateVirtualNetwork(id uuid.UUID, updates vnet.UpdateVirtu
 	return r.statusCodeToError("update virtual network", resp)
 }
 
-func parseListVnets(body io.ReadCloser) ([]*vnet.VirtualNetwork, error) {
-	var vnets []*vnet.VirtualNetwork
+func parseListVnets(body io.ReadCloser) ([]*VirtualNetwork, error) {
+	var vnets []*VirtualNetwork
 	err := parseResponse(body, &vnets)
 	return vnets, err
 }
 
-func parseVnet(body io.ReadCloser) (vnet.VirtualNetwork, error) {
-	var vnet vnet.VirtualNetwork
+func parseVnet(body io.ReadCloser) (VirtualNetwork, error) {
+	var vnet VirtualNetwork
 	err := parseResponse(body, &vnet)
 	return vnet, err
 }

cfapi/virtual_network_filter.go

@@ -1,4 +1,4 @@
-package vnet
+package cfapi
 
 import (
 	"net/url"
@@ -10,68 +10,68 @@ import (
 )
 
 var (
-	filterId = cli.StringFlag{
+	filterVnetId = cli.StringFlag{
 		Name:  "id",
 		Usage: "List virtual networks with the given `ID`",
 	}
-	filterName = cli.StringFlag{
+	filterVnetByName = cli.StringFlag{
 		Name:  "name",
 		Usage: "List virtual networks with the given `NAME`",
 	}
-	filterDefault = cli.BoolFlag{
+	filterDefaultVnet = cli.BoolFlag{
 		Name:  "is-default",
 		Usage: "If true, lists the virtual network that is the default one. If false, lists all non-default virtual networks for the account. If absent, all are included in the results regardless of their default status.",
 	}
-	filterDeleted = cli.BoolFlag{
+	filterDeletedVnet = cli.BoolFlag{
 		Name:  "show-deleted",
 		Usage: "If false (default), only show non-deleted virtual networks. If true, only show deleted virtual networks.",
 	}
-	FilterFlags = []cli.Flag{
-		&filterId,
-		&filterName,
-		&filterDefault,
-		&filterDeleted,
+	VnetFilterFlags = []cli.Flag{
+		&filterVnetId,
+		&filterVnetByName,
+		&filterDefaultVnet,
+		&filterDeletedVnet,
 	}
 )
 
-// Filter which virtual networks get queried.
-type Filter struct {
+// VnetFilter which virtual networks get queried.
+type VnetFilter struct {
 	queryParams url.Values
 }
 
-func NewFilter() *Filter {
-	return &Filter{
+func NewVnetFilter() *VnetFilter {
+	return &VnetFilter{
 		queryParams: url.Values{},
 	}
 }
 
-func (f *Filter) ById(vnetId uuid.UUID) {
+func (f *VnetFilter) ById(vnetId uuid.UUID) {
 	f.queryParams.Set("id", vnetId.String())
 }
 
-func (f *Filter) ByName(name string) {
+func (f *VnetFilter) ByName(name string) {
 	f.queryParams.Set("name", name)
 }
 
-func (f *Filter) ByDefaultStatus(isDefault bool) {
+func (f *VnetFilter) ByDefaultStatus(isDefault bool) {
 	f.queryParams.Set("is_default", strconv.FormatBool(isDefault))
 }
 
-func (f *Filter) WithDeleted(isDeleted bool) {
+func (f *VnetFilter) WithDeleted(isDeleted bool) {
 	f.queryParams.Set("is_deleted", strconv.FormatBool(isDeleted))
 }
 
-func (f *Filter) MaxFetchSize(max uint) {
+func (f *VnetFilter) MaxFetchSize(max uint) {
 	f.queryParams.Set("per_page", strconv.Itoa(int(max)))
 }
 
-func (f Filter) Encode() string {
+func (f VnetFilter) Encode() string {
 	return f.queryParams.Encode()
 }
 
 // NewFromCLI parses CLI flags to discover which filters should get applied to list virtual networks.
-func NewFromCLI(c *cli.Context) (*Filter, error) {
-	f := NewFilter()
+func NewFromCLI(c *cli.Context) (*VnetFilter, error) {
+	f := NewVnetFilter()
 
 	if id := c.String("id"); id != "" {
 		vnetId, err := uuid.Parse(id)

cfapi/virtual_network_test.go

@@ -1,4 +1,4 @@
-package vnet
+package cfapi
 
 import (
 	"encoding/json"

cfsetup.yaml

@@ -1,20 +1,10 @@
-pinned_go: &pinned_go go=1.17-1
-pinned_go_fips: &pinned_go_fips go-boring=1.16.6-6
+pinned_go: &pinned_go go=1.17.5-1
+pinned_go_fips: &pinned_go_fips go-boring=1.17.5-1
 
 build_dir: &build_dir /cfsetup_build
 default-flavor: buster
 stretch: &stretch
   build:
-    build_dir: *build_dir
-    builddeps:
-      - *pinned_go_fips
-      - build-essential
-    post-cache:
-      - export GOOS=linux
-      - export GOARCH=amd64
-      - export FIPS=true
-      - make cloudflared
-  build-non-fips:  # helpful to catch problems with non-fips (only used for releasing non-linux artifacts) before releases
     build_dir: *build_dir
     builddeps:
       - *pinned_go
@@ -23,11 +13,22 @@ stretch: &stretch
       - export GOOS=linux
       - export GOARCH=amd64
       - make cloudflared
-  build-all-packages: #except osxpkg
+  build-fips:
     build_dir: *build_dir
     builddeps:
       - *pinned_go_fips
       - build-essential
+    post-cache:
+      - export GOOS=linux
+      - export GOARCH=amd64
+      - export FIPS=true
+      - make cloudflared
+  # except FIPS (handled in github-fips-release-pkgs) and macos (handled in github-release-macos-amd64)
+  github-release-pkgs:
+    build_dir: *build_dir
+    builddeps:
+      - *pinned_go
+      - build-essential
       - fakeroot
       - rubygem-fpm
       - rpm
@@ -35,15 +36,22 @@ stretch: &stretch
       # libmsi and libgcab are libraries the wixl binary depends on.
       - libmsi-dev
       - libgcab-dev
-    pre-cache:
-      # TODO: https://jira.cfops.it/browse/TUN-4792 Replace this wixl with the official one once msitools supports
-      # environment.
+      - python3-dev
+      - libffi-dev
+      - python3-setuptools
+      - python3-pip
+    pre-cache: &github_release_pkgs_pre_cache
       - wget https://github.com/sudarshan-reddy/msitools/releases/download/v0.101b/wixl -P /usr/local/bin
       - chmod a+x /usr/local/bin/wixl
+      - pip3 install pynacl==1.4.0
+      - pip3 install pygithub==1.55
     post-cache:
-      - export FIPS=true
+      # build all packages (except macos and FIPS) and move them to /cfsetup/built_artifacts
       - ./build-packages.sh
-  github-release-pkgs:
+      # release the packages built and moved to /cfsetup/built_artifacts
+      - make github-release-built-pkgs
+  # handle FIPS separately so that we built with gofips compiler
+  github-fips-release-pkgs:
     build_dir: *build_dir
     builddeps:
       - *pinned_go_fips
@@ -59,18 +67,25 @@ stretch: &stretch
       - libffi-dev
       - python3-setuptools
       - python3-pip
-    pre-cache:
-      - wget https://github.com/sudarshan-reddy/msitools/releases/download/v0.101b/wixl -P /usr/local/bin
-      - chmod a+x /usr/local/bin/wixl
-      - pip3 install pygithub
+    pre-cache: *github_release_pkgs_pre_cache
     post-cache:
-      # build all packages and move them to /cfsetup/built_artifacts
-      - ./build-packages.sh
-      # release the packages built and moved to /cfsetup/built_artifacts
+      # same logic as above, but for FIPS packages only
+      - ./build-packages-fips.sh
       - make github-release-built-pkgs
   build-deb:
     build_dir: *build_dir
     builddeps: &build_deb_deps
+      - *pinned_go
+      - build-essential
+      - fakeroot
+      - rubygem-fpm
+    post-cache:
+      - export GOOS=linux
+      - export GOARCH=amd64
+      - make cloudflared-deb
+  build-fips-internal-deb:
+    build_dir: *build_dir
+    builddeps: &build_fips_deb_deps
       - *pinned_go_fips
       - build-essential
       - fakeroot
@@ -79,15 +94,17 @@ stretch: &stretch
       - export GOOS=linux
       - export GOARCH=amd64
       - export FIPS=true
+      - export ORIGINAL_NAME=true
       - make cloudflared-deb
-  build-deb-nightly:
+  build-fips-internal-deb-nightly:
     build_dir: *build_dir
-    builddeps: *build_deb_deps
+    builddeps: *build_fips_deb_deps
     post-cache:
       - export GOOS=linux
       - export GOARCH=amd64
-      - export FIPS=true
       - export NIGHTLY=true
+      - export FIPS=true
+      - export ORIGINAL_NAME=true
       - make cloudflared-deb
   build-deb-arm64:
     build_dir: *build_dir
@@ -99,7 +116,7 @@ stretch: &stretch
   publish-deb:
     build_dir: *build_dir
     builddeps:
-      - *pinned_go_fips
+      - *pinned_go
       - build-essential
       - fakeroot
       - rubygem-fpm
@@ -107,7 +124,6 @@ stretch: &stretch
     post-cache:
       - export GOOS=linux
       - export GOARCH=amd64
-      - export FIPS=true
       - make publish-deb
   github-release-macos-amd64:
     build_dir: *build_dir
@@ -119,18 +135,32 @@ stretch: &stretch
       - python3-setuptools
       - python3-pip
     pre-cache: &install_pygithub
-      - pip3 install pygithub
+      - pip3 install pynacl==1.4.0
+      - pip3 install pygithub==1.55
     post-cache:
       - make github-mac-upload
   test:
+    build_dir: *build_dir
+    builddeps:
+      - *pinned_go
+      - build-essential
+      - gotest-to-teamcity
+    pre-cache: &test_pre_cache
+      - go get golang.org/x/tools/cmd/goimports
+      - go get github.com/sudarshan-reddy/go-sumtype@v0.0.0-20210827105221-82eca7e5abb1
+    post-cache:
+      - export GOOS=linux
+      - export GOARCH=amd64
+      - export PATH="$HOME/go/bin:$PATH"
+      - ./fmt-check.sh
+      - make test | gotest-to-teamcity
+  test-fips:
     build_dir: *build_dir
     builddeps:
       - *pinned_go_fips
       - build-essential
       - gotest-to-teamcity
-    pre-cache:
-      - go get golang.org/x/tools/cmd/goimports
-      - go get github.com/sudarshan-reddy/go-sumtype@v0.0.0-20210827105221-82eca7e5abb1
+    pre-cache: *test_pre_cache
     post-cache:
       - export GOOS=linux
       - export GOARCH=amd64
@@ -216,23 +246,10 @@ centos-7:
     pre-cache:
       - yum install -y fakeroot
       - yum upgrade -y binutils-2.27-44.base.el7.x86_64
-      - wget https://golang.org/dl/go1.16.3.linux-amd64.tar.gz -P /tmp/
-      - tar -C /usr/local -xzf /tmp/go1.16.3.linux-amd64.tar.gz
+      - wget https://go.dev/dl/go1.17.5.linux-amd64.tar.gz -P /tmp/
+      - tar -C /usr/local -xzf /tmp/go1.17.5.linux-amd64.tar.gz
     post-cache:
       - export PATH=$PATH:/usr/local/go/bin
       - export GOOS=linux
       - export GOARCH=amd64
-      - make publish-rpm
-  build-rpm:
-    build_dir: *build_dir
-    builddeps: *el7_builddeps
-    pre-cache:
-      - yum install -y fakeroot
-      - yum upgrade -y binutils-2.27-44.base.el7.x86_64
-      - wget https://golang.org/dl/go1.16.3.linux-amd64.tar.gz -P /tmp/
-      - tar -C /usr/local -xzf /tmp/go1.16.3.linux-amd64.tar.gz
-    post-cache:
-      - export PATH=$PATH:/usr/local/go/bin
-      - export GOOS=linux
-      - export GOARCH=amd64
-      - make cloudflared-rpm
+      - make publish-rpm

check-fips.sh

@@ -0,0 +1,15 @@
+# Pass the path to the executable to check for FIPS compliance
+exe=$1
+
+if [ "$(go tool nm "${exe}" | grep -c '_Cfunc__goboringcrypto_')" -eq 0 ]; then
+    # Asserts that executable is using FIPS-compliant boringcrypto
+    echo "${exe}: missing goboring symbols" >&2
+    exit 1
+fi
+if [ "$(go tool nm "${exe}" | grep -c 'crypto/internal/boring/sig.FIPSOnly')" -eq 0 ]; then
+    # Asserts that executable is using FIPS-only schemes
+    echo "${exe}: missing fipsonly symbols" >&2
+    exit 1
+fi
+
+echo "${exe} is FIPS-compliant"

cmd/cloudflared/cliutil/build_info.go

@@ -1,4 +1,4 @@
-package buildinfo
+package cliutil
 
 import (
 	"fmt"
@@ -11,23 +11,39 @@ type BuildInfo struct {
 	GoOS               string `json:"go_os"`
 	GoVersion          string `json:"go_version"`
 	GoArch             string `json:"go_arch"`
+	BuildType          string `json:"build_type"`
 	CloudflaredVersion string `json:"cloudflared_version"`
 }
 
-func GetBuildInfo(cloudflaredVersion string) *BuildInfo {
+func GetBuildInfo(buildType, version string) *BuildInfo {
 	return &BuildInfo{
 		GoOS:               runtime.GOOS,
 		GoVersion:          runtime.Version(),
 		GoArch:             runtime.GOARCH,
-		CloudflaredVersion: cloudflaredVersion,
+		BuildType:          buildType,
+		CloudflaredVersion: version,
 	}
 }
 
 func (bi *BuildInfo) Log(log *zerolog.Logger) {
 	log.Info().Msgf("Version %s", bi.CloudflaredVersion)
+	if bi.BuildType != "" {
+		log.Info().Msgf("Built%s", bi.GetBuildTypeMsg())
+	}
 	log.Info().Msgf("GOOS: %s, GOVersion: %s, GoArch: %s", bi.GoOS, bi.GoVersion, bi.GoArch)
 }
 
 func (bi *BuildInfo) OSArch() string {
 	return fmt.Sprintf("%s_%s", bi.GoOS, bi.GoArch)
 }
+
+func (bi *BuildInfo) Version() string {
+	return bi.CloudflaredVersion
+}
+
+func (bi *BuildInfo) GetBuildTypeMsg() string {
+	if bi.BuildType == "" {
+		return ""
+	}
+	return fmt.Sprintf(" with %s", bi.BuildType)
+}

cmd/cloudflared/generic_service.go

@@ -1,3 +1,4 @@
+//go:build !windows && !darwin && !linux
 // +build !windows,!darwin,!linux
 
 package main

cmd/cloudflared/linux_service.go

@@ -1,3 +1,4 @@
+//go:build linux
 // +build linux
 
 package main

cmd/cloudflared/macos_service.go

@@ -1,3 +1,4 @@
+//go:build darwin
 // +build darwin
 
 package main

cmd/cloudflared/main.go

@@ -31,6 +31,7 @@ const (
 var (
 	Version   = "DEV"
 	BuildTime = "unknown"
+	BuildType = ""
 	// Mostly network errors that we don't want reported back to Sentry, this is done by substring match.
 	ignoredErrors = []string{
 		"connection reset by peer",
@@ -46,9 +47,10 @@ var (
 
 func main() {
 	rand.Seed(time.Now().UnixNano())
-	metrics.RegisterBuildInfo(BuildTime, Version)
+	metrics.RegisterBuildInfo(BuildType, BuildTime, Version)
 	raven.SetRelease(Version)
 	maxprocs.Set()
+	bInfo := cliutil.GetBuildInfo(BuildType, Version)
 
 	// Graceful shutdown channel used by the app. When closed, app must terminate gracefully.
 	// Windows service manager closes this channel when it receives stop command.
@@ -71,7 +73,7 @@ func main() {
    Terms (https://www.cloudflare.com/terms/) and Privacy Policy (https://www.cloudflare.com/privacypolicy/).`,
 		time.Now().Year(),
 	)
-	app.Version = fmt.Sprintf("%s (built %s)", Version, BuildTime)
+	app.Version = fmt.Sprintf("%s (built %s%s)", Version, BuildTime, bInfo.GetBuildTypeMsg())
 	app.Description = `cloudflared connects your machine or user identity to Cloudflare's global network.
 	You can use it to authenticate a session to reach an API behind Access, route web traffic to this machine,
 	and configure access control.
@@ -81,7 +83,7 @@ func main() {
 	app.Action = action(graceShutdownC)
 	app.Commands = commands(cli.ShowVersion)
 
-	tunnel.Init(Version, graceShutdownC) // we need this to support the tunnel sub command...
+	tunnel.Init(bInfo, graceShutdownC) // we need this to support the tunnel sub command...
 	access.Init(graceShutdownC)
 	updater.Init(Version)
 	runApp(app, graceShutdownC)

cmd/cloudflared/tunnel/cmd.go

@@ -21,7 +21,7 @@ import (
 	"github.com/urfave/cli/v2"
 	"github.com/urfave/cli/v2/altsrc"
 
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
+	"github.com/cloudflare/cloudflared/cfapi"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/proxydns"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/ui"
@@ -35,7 +35,6 @@ import (
 	"github.com/cloudflare/cloudflared/signal"
 	"github.com/cloudflare/cloudflared/tlsconfig"
 	"github.com/cloudflare/cloudflared/tunneldns"
-	"github.com/cloudflare/cloudflared/tunnelstore"
 )
 
 const (
@@ -86,7 +85,7 @@ const (
 
 var (
 	graceShutdownC chan struct{}
-	version        string
+	buildInfo      *cliutil.BuildInfo
 
 	routeFailMsg = fmt.Sprintf("failed to provision routing, please create it manually via Cloudflare dashboard or UI; "+
 		"most likely you already have a conflicting record there. You can also rerun this command with --%s to overwrite "+
@@ -102,8 +101,7 @@ func Commands() []*cli.Command {
 		buildLoginSubcommand(false),
 		buildCreateCommand(),
 		buildRouteCommand(),
-		// TODO TUN-5477 this should not be hidden
-		buildVirtualNetworkSubcommand(true),
+		buildVirtualNetworkSubcommand(false),
 		buildRunCommand(),
 		buildListCommand(),
 		buildInfoCommand(),
@@ -175,8 +173,8 @@ func TunnelCommand(c *cli.Context) error {
 	return runClassicTunnel(sc)
 }
 
-func Init(ver string, gracefulShutdown chan struct{}) {
-	version, graceShutdownC = ver, gracefulShutdown
+func Init(info *cliutil.BuildInfo, gracefulShutdown chan struct{}) {
+	buildInfo, graceShutdownC = info, gracefulShutdown
 }
 
 // runAdhocNamedTunnel create, route and run a named tunnel in one command
@@ -209,22 +207,22 @@ func runAdhocNamedTunnel(sc *subcommandContext, name, credentialsOutputPath stri
 
 // runClassicTunnel creates a "classic" non-named tunnel
 func runClassicTunnel(sc *subcommandContext) error {
-	return StartServer(sc.c, version, nil, sc.log, sc.isUIEnabled)
+	return StartServer(sc.c, buildInfo, nil, sc.log, sc.isUIEnabled)
 }
 
-func routeFromFlag(c *cli.Context) (route tunnelstore.Route, ok bool) {
+func routeFromFlag(c *cli.Context) (route cfapi.HostnameRoute, ok bool) {
 	if hostname := c.String("hostname"); hostname != "" {
 		if lbPool := c.String("lb-pool"); lbPool != "" {
-			return tunnelstore.NewLBRoute(hostname, lbPool), true
+			return cfapi.NewLBRoute(hostname, lbPool), true
 		}
-		return tunnelstore.NewDNSRoute(hostname, c.Bool(overwriteDNSFlagName)), true
+		return cfapi.NewDNSRoute(hostname, c.Bool(overwriteDNSFlagName)), true
 	}
 	return nil, false
 }
 
 func StartServer(
 	c *cli.Context,
-	version string,
+	info *cliutil.BuildInfo,
 	namedTunnel *connection.NamedTunnelConfig,
 	log *zerolog.Logger,
 	isUIEnabled bool,
@@ -271,8 +269,7 @@ func StartServer(
 		defer trace.Stop()
 	}
 
-	buildInfo := buildinfo.GetBuildInfo(version)
-	buildInfo.Log(log)
+	info.Log(log)
 	logClientOptions(c, log)
 
 	// this context drives the server, when it's cancelled tunnel and all other components (origins, dns, etc...) should stop
@@ -336,7 +333,7 @@ func StartServer(
 		observer.SendURL(quickTunnelURL)
 	}
 
-	tunnelConfig, ingressRules, err := prepareTunnelConfig(c, buildInfo, version, log, logTransport, observer, namedTunnel)
+	tunnelConfig, ingressRules, err := prepareTunnelConfig(c, info, log, logTransport, observer, namedTunnel)
 	if err != nil {
 		log.Err(err).Msg("Couldn't start tunnel")
 		return err
@@ -377,7 +374,7 @@ func StartServer(
 
 	if isUIEnabled {
 		tunnelUI := ui.NewUIModel(
-			version,
+			info.Version(),
 			hostname,
 			metricsListener.Addr().String(),
 			&ingressRules,

cmd/cloudflared/tunnel/configuration.go

@@ -16,7 +16,8 @@ import (
 	"github.com/urfave/cli/v2"
 	"golang.org/x/crypto/ssh/terminal"
 
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
+
 	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/edgediscovery"
@@ -148,8 +149,7 @@ func getOriginCert(originCertPath string, log *zerolog.Logger) ([]byte, error) {
 
 func prepareTunnelConfig(
 	c *cli.Context,
-	buildInfo *buildinfo.BuildInfo,
-	version string,
+	info *cliutil.BuildInfo,
 	log, logTransport *zerolog.Logger,
 	observer *connection.Observer,
 	namedTunnel *connection.NamedTunnelConfig,
@@ -193,8 +193,8 @@ func prepareTunnelConfig(
 		namedTunnel.Client = tunnelpogs.ClientInfo{
 			ClientID: clientUUID[:],
 			Features: dedup(features),
-			Version:  version,
-			Arch:     buildInfo.OSArch(),
+			Version:  info.Version(),
+			Arch:     info.OSArch(),
 		}
 		ingressRules, err = ingress.ParseIngress(cfg)
 		if err != nil && err != ingress.ErrNoIngressRules {
@@ -281,7 +281,7 @@ func prepareTunnelConfig(
 
 	return &origin.TunnelConfig{
 		ConnectionConfig: connectionConfig,
-		OSArch:           buildInfo.OSArch(),
+		OSArch:           info.OSArch(),
 		ClientID:         clientID,
 		EdgeAddrs:        c.StringSlice("edge"),
 		Region:           c.String("region"),
@@ -293,7 +293,7 @@ func prepareTunnelConfig(
 		Log:              log,
 		LogTransport:     logTransport,
 		Observer:         observer,
-		ReportedVersion:  version,
+		ReportedVersion:  info.Version(),
 		// Note TUN-3758 , we use Int because UInt is not supported with altsrc
 		Retries:          uint(c.Int("retries")),
 		RunFromTerminal:  isRunningFromTerminal(),

cmd/cloudflared/tunnel/configuration_test.go

@@ -1,4 +1,6 @@
+//go:build ignore
 // +build ignore
+
 // TODO: Remove the above build tag and include this test when we start compiling with Golang 1.10.0+
 
 package tunnel

cmd/cloudflared/tunnel/info.go

@@ -5,12 +5,12 @@ import (
 
 	"github.com/google/uuid"
 
-	"github.com/cloudflare/cloudflared/tunnelstore"
+	"github.com/cloudflare/cloudflared/cfapi"
 )
 
 type Info struct {
-	ID         uuid.UUID                   `json:"id"`
-	Name       string                      `json:"name"`
-	CreatedAt  time.Time                   `json:"createdAt"`
-	Connectors []*tunnelstore.ActiveClient `json:"conns"`
+	ID         uuid.UUID             `json:"id"`
+	Name       string                `json:"name"`
+	CreatedAt  time.Time             `json:"createdAt"`
+	Connectors []*cfapi.ActiveClient `json:"conns"`
 }

cmd/cloudflared/tunnel/quick_tunnel.go

@@ -76,7 +76,7 @@ func RunQuickTunnel(sc *subcommandContext) error {
 
 	return StartServer(
 		sc.c,
-		version,
+		buildInfo,
 		&connection.NamedTunnelConfig{Credentials: credentials, QuickTunnelUrl: data.Result.Hostname},
 		sc.log,
 		sc.isUIEnabled,

cmd/cloudflared/tunnel/signal_test.go

@@ -1,3 +1,4 @@
+//go:build !windows
 // +build !windows
 
 package tunnel

cmd/cloudflared/tunnel/subcommand_context.go

@@ -14,9 +14,9 @@ import (
 	"github.com/urfave/cli/v2"
 
 	"github.com/cloudflare/cloudflared/certutil"
+	"github.com/cloudflare/cloudflared/cfapi"
 	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/logger"
-	"github.com/cloudflare/cloudflared/tunnelstore"
 )
 
 type errInvalidJSONCredential struct {
@@ -37,7 +37,7 @@ type subcommandContext struct {
 	fs          fileSystem
 
 	// These fields should be accessed using their respective Getter
-	tunnelstoreClient tunnelstore.Client
+	tunnelstoreClient cfapi.Client
 	userCredential    *userCredential
 }
 
@@ -68,7 +68,7 @@ type userCredential struct {
 	certPath string
 }
 
-func (sc *subcommandContext) client() (tunnelstore.Client, error) {
+func (sc *subcommandContext) client() (cfapi.Client, error) {
 	if sc.tunnelstoreClient != nil {
 		return sc.tunnelstoreClient, nil
 	}
@@ -76,8 +76,8 @@ func (sc *subcommandContext) client() (tunnelstore.Client, error) {
 	if err != nil {
 		return nil, err
 	}
-	userAgent := fmt.Sprintf("cloudflared/%s", version)
-	client, err := tunnelstore.NewRESTClient(
+	userAgent := fmt.Sprintf("cloudflared/%s", buildInfo.Version())
+	client, err := cfapi.NewRESTClient(
 		sc.c.String("api-url"),
 		credential.cert.AccountID,
 		credential.cert.ZoneID,
@@ -149,7 +149,7 @@ func (sc *subcommandContext) readTunnelCredentials(credFinder CredFinder) (conne
 	return credentials, nil
 }
 
-func (sc *subcommandContext) create(name string, credentialsFilePath string, secret string) (*tunnelstore.Tunnel, error) {
+func (sc *subcommandContext) create(name string, credentialsFilePath string, secret string) (*cfapi.Tunnel, error) {
 	client, err := sc.client()
 	if err != nil {
 		return nil, errors.Wrap(err, "couldn't create client to talk to Cloudflare Tunnel backend")
@@ -224,7 +224,7 @@ func (sc *subcommandContext) create(name string, credentialsFilePath string, sec
 	return tunnel, nil
 }
 
-func (sc *subcommandContext) list(filter *tunnelstore.Filter) ([]*tunnelstore.Tunnel, error) {
+func (sc *subcommandContext) list(filter *cfapi.TunnelFilter) ([]*cfapi.Tunnel, error) {
 	client, err := sc.client()
 	if err != nil {
 		return nil, err
@@ -251,7 +251,7 @@ func (sc *subcommandContext) delete(tunnelIDs []uuid.UUID) error {
 			return fmt.Errorf("Tunnel %s has already been deleted", tunnel.ID)
 		}
 		if forceFlagSet {
-			if err := client.CleanupConnections(tunnel.ID, tunnelstore.NewCleanupParams()); err != nil {
+			if err := client.CleanupConnections(tunnel.ID, cfapi.NewCleanupParams()); err != nil {
 				return errors.Wrapf(err, "Error cleaning up connections for tunnel %s", tunnel.ID)
 			}
 		}
@@ -303,7 +303,7 @@ func (sc *subcommandContext) run(tunnelID uuid.UUID) error {
 
 	return StartServer(
 		sc.c,
-		version,
+		buildInfo,
 		&connection.NamedTunnelConfig{Credentials: credentials},
 		sc.log,
 		sc.isUIEnabled,
@@ -311,7 +311,7 @@ func (sc *subcommandContext) run(tunnelID uuid.UUID) error {
 }
 
 func (sc *subcommandContext) cleanupConnections(tunnelIDs []uuid.UUID) error {
-	params := tunnelstore.NewCleanupParams()
+	params := cfapi.NewCleanupParams()
 	extraLog := ""
 	if connector := sc.c.String("connector-id"); connector != "" {
 		connectorID, err := uuid.Parse(connector)
@@ -335,7 +335,7 @@ func (sc *subcommandContext) cleanupConnections(tunnelIDs []uuid.UUID) error {
 	return nil
 }
 
-func (sc *subcommandContext) route(tunnelID uuid.UUID, r tunnelstore.Route) (tunnelstore.RouteResult, error) {
+func (sc *subcommandContext) route(tunnelID uuid.UUID, r cfapi.HostnameRoute) (cfapi.HostnameRouteResult, error) {
 	client, err := sc.client()
 	if err != nil {
 		return nil, err
@@ -345,8 +345,8 @@ func (sc *subcommandContext) route(tunnelID uuid.UUID, r tunnelstore.Route) (tun
 }
 
 // Query Tunnelstore to find the active tunnel with the given name.
-func (sc *subcommandContext) tunnelActive(name string) (*tunnelstore.Tunnel, bool, error) {
-	filter := tunnelstore.NewFilter()
+func (sc *subcommandContext) tunnelActive(name string) (*cfapi.Tunnel, bool, error) {
+	filter := cfapi.NewTunnelFilter()
 	filter.NoDeleted()
 	filter.ByName(name)
 	tunnels, err := sc.list(filter)
@@ -391,7 +391,7 @@ func (sc *subcommandContext) findIDs(inputs []string) ([]uuid.UUID, error) {
 	uuids, names := splitUuids(inputs)
 
 	for _, name := range names {
-		filter := tunnelstore.NewFilter()
+		filter := cfapi.NewTunnelFilter()
 		filter.NoDeleted()
 		filter.ByName(name)
 

cmd/cloudflared/tunnel/subcommand_context_teamnet.go

@@ -3,12 +3,12 @@ package tunnel
 import (
 	"github.com/pkg/errors"
 
-	"github.com/cloudflare/cloudflared/teamnet"
+	"github.com/cloudflare/cloudflared/cfapi"
 )
 
 const noClientMsg = "error while creating backend client"
 
-func (sc *subcommandContext) listRoutes(filter *teamnet.Filter) ([]*teamnet.DetailedRoute, error) {
+func (sc *subcommandContext) listRoutes(filter *cfapi.IpRouteFilter) ([]*cfapi.DetailedRoute, error) {
 	client, err := sc.client()
 	if err != nil {
 		return nil, errors.Wrap(err, noClientMsg)
@@ -16,15 +16,15 @@ func (sc *subcommandContext) listRoutes(filter *teamnet.Filter) ([]*teamnet.Deta
 	return client.ListRoutes(filter)
 }
 
-func (sc *subcommandContext) addRoute(newRoute teamnet.NewRoute) (teamnet.Route, error) {
+func (sc *subcommandContext) addRoute(newRoute cfapi.NewRoute) (cfapi.Route, error) {
 	client, err := sc.client()
 	if err != nil {
-		return teamnet.Route{}, errors.Wrap(err, noClientMsg)
+		return cfapi.Route{}, errors.Wrap(err, noClientMsg)
 	}
 	return client.AddRoute(newRoute)
 }
 
-func (sc *subcommandContext) deleteRoute(params teamnet.DeleteRouteParams) error {
+func (sc *subcommandContext) deleteRoute(params cfapi.DeleteRouteParams) error {
 	client, err := sc.client()
 	if err != nil {
 		return errors.Wrap(err, noClientMsg)
@@ -32,10 +32,10 @@ func (sc *subcommandContext) deleteRoute(params teamnet.DeleteRouteParams) error
 	return client.DeleteRoute(params)
 }
 
-func (sc *subcommandContext) getRouteByIP(params teamnet.GetRouteByIpParams) (teamnet.DetailedRoute, error) {
+func (sc *subcommandContext) getRouteByIP(params cfapi.GetRouteByIpParams) (cfapi.DetailedRoute, error) {
 	client, err := sc.client()
 	if err != nil {
-		return teamnet.DetailedRoute{}, errors.Wrap(err, noClientMsg)
+		return cfapi.DetailedRoute{}, errors.Wrap(err, noClientMsg)
 	}
 	return client.GetByIP(params)
 }

cmd/cloudflared/tunnel/subcommand_context_test.go

@@ -13,8 +13,8 @@ import (
 	"github.com/rs/zerolog"
 	"github.com/urfave/cli/v2"
 
+	"github.com/cloudflare/cloudflared/cfapi"
 	"github.com/cloudflare/cloudflared/connection"
-	"github.com/cloudflare/cloudflared/tunnelstore"
 )
 
 type mockFileSystem struct {
@@ -36,7 +36,7 @@ func Test_subcommandContext_findCredentials(t *testing.T) {
 		log               *zerolog.Logger
 		isUIEnabled       bool
 		fs                fileSystem
-		tunnelstoreClient tunnelstore.Client
+		tunnelstoreClient cfapi.Client
 		userCredential    *userCredential
 	}
 	type args struct {
@@ -187,13 +187,13 @@ func Test_subcommandContext_findCredentials(t *testing.T) {
 }
 
 type deleteMockTunnelStore struct {
-	tunnelstore.Client
+	cfapi.Client
 	mockTunnels      map[uuid.UUID]mockTunnelBehaviour
 	deletedTunnelIDs []uuid.UUID
 }
 
 type mockTunnelBehaviour struct {
-	tunnel     tunnelstore.Tunnel
+	tunnel     cfapi.Tunnel
 	deleteErr  error
 	cleanupErr error
 }
@@ -209,7 +209,7 @@ func newDeleteMockTunnelStore(tunnels ...mockTunnelBehaviour) *deleteMockTunnelS
 	}
 }
 
-func (d *deleteMockTunnelStore) GetTunnel(tunnelID uuid.UUID) (*tunnelstore.Tunnel, error) {
+func (d *deleteMockTunnelStore) GetTunnel(tunnelID uuid.UUID) (*cfapi.Tunnel, error) {
 	tunnel, ok := d.mockTunnels[tunnelID]
 	if !ok {
 		return nil, fmt.Errorf("Couldn't find tunnel: %v", tunnelID)
@@ -233,7 +233,7 @@ func (d *deleteMockTunnelStore) DeleteTunnel(tunnelID uuid.UUID) error {
 	return nil
 }
 
-func (d *deleteMockTunnelStore) CleanupConnections(tunnelID uuid.UUID, _ *tunnelstore.CleanupParams) error {
+func (d *deleteMockTunnelStore) CleanupConnections(tunnelID uuid.UUID, _ *cfapi.CleanupParams) error {
 	tunnel, ok := d.mockTunnels[tunnelID]
 	if !ok {
 		return fmt.Errorf("Couldn't find tunnel: %v", tunnelID)
@@ -284,10 +284,10 @@ func Test_subcommandContext_Delete(t *testing.T) {
 				}(),
 				tunnelstoreClient: newDeleteMockTunnelStore(
 					mockTunnelBehaviour{
-						tunnel: tunnelstore.Tunnel{ID: tunnelID1},
+						tunnel: cfapi.Tunnel{ID: tunnelID1},
 					},
 					mockTunnelBehaviour{
-						tunnel: tunnelstore.Tunnel{ID: tunnelID2},
+						tunnel: cfapi.Tunnel{ID: tunnelID2},
 					},
 				),
 			},

cmd/cloudflared/tunnel/subcommand_context_vnets.go

@@ -4,18 +4,18 @@ import (
 	"github.com/google/uuid"
 	"github.com/pkg/errors"
 
-	"github.com/cloudflare/cloudflared/vnet"
+	"github.com/cloudflare/cloudflared/cfapi"
 )
 
-func (sc *subcommandContext) addVirtualNetwork(newVnet vnet.NewVirtualNetwork) (vnet.VirtualNetwork, error) {
+func (sc *subcommandContext) addVirtualNetwork(newVnet cfapi.NewVirtualNetwork) (cfapi.VirtualNetwork, error) {
 	client, err := sc.client()
 	if err != nil {
-		return vnet.VirtualNetwork{}, errors.Wrap(err, noClientMsg)
+		return cfapi.VirtualNetwork{}, errors.Wrap(err, noClientMsg)
 	}
 	return client.CreateVirtualNetwork(newVnet)
 }
 
-func (sc *subcommandContext) listVirtualNetworks(filter *vnet.Filter) ([]*vnet.VirtualNetwork, error) {
+func (sc *subcommandContext) listVirtualNetworks(filter *cfapi.VnetFilter) ([]*cfapi.VirtualNetwork, error) {
 	client, err := sc.client()
 	if err != nil {
 		return nil, errors.Wrap(err, noClientMsg)
@@ -31,7 +31,7 @@ func (sc *subcommandContext) deleteVirtualNetwork(vnetId uuid.UUID) error {
 	return client.DeleteVirtualNetwork(vnetId)
 }
 
-func (sc *subcommandContext) updateVirtualNetwork(vnetId uuid.UUID, updates vnet.UpdateVirtualNetwork) error {
+func (sc *subcommandContext) updateVirtualNetwork(vnetId uuid.UUID, updates cfapi.UpdateVirtualNetwork) error {
 	client, err := sc.client()
 	if err != nil {
 		return errors.Wrap(err, noClientMsg)

cmd/cloudflared/tunnel/subcommands.go

@@ -21,11 +21,11 @@ import (
 	"golang.org/x/net/idna"
 	yaml "gopkg.in/yaml.v2"
 
+	"github.com/cloudflare/cloudflared/cfapi"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/updater"
 	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/connection"
-	"github.com/cloudflare/cloudflared/tunnelstore"
 )
 
 const (
@@ -64,8 +64,8 @@ var (
 		Name:        "when",
 		Aliases:     []string{"w"},
 		Usage:       "List tunnels that are active at the given `TIME` in RFC3339 format",
-		Layout:      tunnelstore.TimeLayout,
-		DefaultText: fmt.Sprintf("current time, %s", time.Now().Format(tunnelstore.TimeLayout)),
+		Layout:      cfapi.TimeLayout,
+		DefaultText: fmt.Sprintf("current time, %s", time.Now().Format(cfapi.TimeLayout)),
 	}
 	listIDFlag = &cli.StringFlag{
 		Name:    "id",
@@ -260,7 +260,7 @@ func listCommand(c *cli.Context) error {
 	warningChecker := updater.StartWarningCheck(c)
 	defer warningChecker.LogWarningIfAny(sc.log)
 
-	filter := tunnelstore.NewFilter()
+	filter := cfapi.NewTunnelFilter()
 	if !c.Bool("show-deleted") {
 		filter.NoDeleted()
 	}
@@ -335,7 +335,7 @@ func listCommand(c *cli.Context) error {
 	return nil
 }
 
-func formatAndPrintTunnelList(tunnels []*tunnelstore.Tunnel, showRecentlyDisconnected bool) {
+func formatAndPrintTunnelList(tunnels []*cfapi.Tunnel, showRecentlyDisconnected bool) {
 	writer := tabWriter()
 	defer writer.Flush()
 
@@ -357,7 +357,7 @@ func formatAndPrintTunnelList(tunnels []*tunnelstore.Tunnel, showRecentlyDisconn
 	}
 }
 
-func fmtConnections(connections []tunnelstore.Connection, showRecentlyDisconnected bool) string {
+func fmtConnections(connections []cfapi.Connection, showRecentlyDisconnected bool) string {
 
 	// Count connections per colo
 	numConnsPerColo := make(map[string]uint, len(connections))
@@ -477,8 +477,8 @@ func tunnelInfo(c *cli.Context) error {
 	return nil
 }
 
-func getTunnel(sc *subcommandContext, tunnelID uuid.UUID) (*tunnelstore.Tunnel, error) {
-	filter := tunnelstore.NewFilter()
+func getTunnel(sc *subcommandContext, tunnelID uuid.UUID) (*cfapi.Tunnel, error) {
+	filter := cfapi.NewTunnelFilter()
 	filter.ByTunnelID(tunnelID)
 	tunnels, err := sc.list(filter)
 	if err != nil {
@@ -711,7 +711,7 @@ Further information about managing Cloudflare WARP traffic to your tunnel is ava
 			{
 				Name:        "dns",
 				Action:      cliutil.ConfiguredAction(routeDnsCommand),
-				Usage:       "Route a hostname by creating a DNS CNAME record to a tunnel",
+				Usage:       "HostnameRoute a hostname by creating a DNS CNAME record to a tunnel",
 				UsageText:   "cloudflared tunnel route dns [TUNNEL] [HOSTNAME]",
 				Description: `Creates a DNS CNAME record hostname that points to the tunnel.`,
 				Flags:       []cli.Flag{overwriteDNSFlag},
@@ -728,7 +728,7 @@ Further information about managing Cloudflare WARP traffic to your tunnel is ava
 	}
 }
 
-func dnsRouteFromArg(c *cli.Context, overwriteExisting bool) (tunnelstore.Route, error) {
+func dnsRouteFromArg(c *cli.Context, overwriteExisting bool) (cfapi.HostnameRoute, error) {
 	const (
 		userHostnameIndex = 1
 		expectedNArgs     = 2
@@ -742,10 +742,10 @@ func dnsRouteFromArg(c *cli.Context, overwriteExisting bool) (tunnelstore.Route,
 	} else if !validateHostname(userHostname, true) {
 		return nil, errors.Errorf("%s is not a valid hostname", userHostname)
 	}
-	return tunnelstore.NewDNSRoute(userHostname, overwriteExisting), nil
+	return cfapi.NewDNSRoute(userHostname, overwriteExisting), nil
 }
 
-func lbRouteFromArg(c *cli.Context) (tunnelstore.Route, error) {
+func lbRouteFromArg(c *cli.Context) (cfapi.HostnameRoute, error) {
 	const (
 		lbNameIndex   = 1
 		lbPoolIndex   = 2
@@ -768,7 +768,7 @@ func lbRouteFromArg(c *cli.Context) (tunnelstore.Route, error) {
 		return nil, errors.Errorf("%s is not a valid pool name", lbPool)
 	}
 
-	return tunnelstore.NewLBRoute(lbName, lbPool), nil
+	return cfapi.NewLBRoute(lbName, lbPool), nil
 }
 
 var nameRegex = regexp.MustCompile("^[_a-zA-Z0-9][-_.a-zA-Z0-9]*$")
@@ -815,7 +815,7 @@ func routeCommand(c *cli.Context, routeType string) error {
 	if err != nil {
 		return err
 	}
-	var route tunnelstore.Route
+	var route cfapi.HostnameRoute
 	switch routeType {
 	case "dns":
 		route, err = dnsRouteFromArg(c, c.Bool(overwriteDNSFlagName))

cmd/cloudflared/tunnel/subcommands_test.go

@@ -8,12 +8,12 @@ import (
 	homedir "github.com/mitchellh/go-homedir"
 	"github.com/stretchr/testify/assert"
 
-	"github.com/cloudflare/cloudflared/tunnelstore"
+	"github.com/cloudflare/cloudflared/cfapi"
 )
 
 func Test_fmtConnections(t *testing.T) {
 	type args struct {
-		connections []tunnelstore.Connection
+		connections []cfapi.Connection
 	}
 	tests := []struct {
 		name string
@@ -23,14 +23,14 @@ func Test_fmtConnections(t *testing.T) {
 		{
 			name: "empty",
 			args: args{
-				connections: []tunnelstore.Connection{},
+				connections: []cfapi.Connection{},
 			},
 			want: "",
 		},
 		{
 			name: "trivial",
 			args: args{
-				connections: []tunnelstore.Connection{
+				connections: []cfapi.Connection{
 					{
 						ColoName: "DFW",
 						ID:       uuid.MustParse("ea550130-57fd-4463-aab1-752822231ddd"),
@@ -42,7 +42,7 @@ func Test_fmtConnections(t *testing.T) {
 		{
 			name: "with a pending reconnect",
 			args: args{
-				connections: []tunnelstore.Connection{
+				connections: []cfapi.Connection{
 					{
 						ColoName:           "DFW",
 						ID:                 uuid.MustParse("ea550130-57fd-4463-aab1-752822231ddd"),
@@ -55,7 +55,7 @@ func Test_fmtConnections(t *testing.T) {
 		{
 			name: "many colos",
 			args: args{
-				connections: []tunnelstore.Connection{
+				connections: []cfapi.Connection{
 					{
 						ColoName: "YRV",
 						ID:       uuid.MustParse("ea550130-57fd-4463-aab1-752822231ddd"),

cmd/cloudflared/tunnel/teamnet_subcommands.go

@@ -8,17 +8,16 @@ import (
 
 	"github.com/google/uuid"
 	"github.com/pkg/errors"
+	"github.com/urfave/cli/v2"
 
+	"github.com/cloudflare/cloudflared/cfapi"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/updater"
-	"github.com/cloudflare/cloudflared/teamnet"
-
-	"github.com/urfave/cli/v2"
 )
 
 var (
 	vnetFlag = &cli.StringFlag{
-		Name:    "virtual-network",
+		Name:    "vnet",
 		Aliases: []string{"vn"},
 		Usage:   "The ID or name of the virtual network to which the route is associated to.",
 	}
@@ -36,7 +35,7 @@ determine who can reach certain routes.
 By default IP routes all exist within a single virtual network. If you use the same IP
 space(s) in different physical private networks, all meant to be reachable via IP routes,
 then you have to manage the ambiguous IP routes by associating them to virtual networks.
-See "cloudflared tunnel network --help" for more information.`,
+See "cloudflared tunnel vnet --help" for more information.`,
 		Subcommands: []*cli.Command{
 			{
 				Name:      "add",
@@ -51,9 +50,9 @@ the specified Tunnel, and reach an IP in the given CIDR, as long as that IP is
 reachable from cloudflared.
 If the CIDR exists in more than one private network, to be connected with Cloudflare
 Tunnels, then you have to manage those IP routes with virtual networks (see
-"cloudflared tunnel network --help)". In those cases, you then have to tell
+"cloudflared tunnel vnet --help)". In those cases, you then have to tell
 which virtual network's routing table you want to add the route to with:
-"cloudflared tunnel route ip add --virtual-network [ID/name] [CIDR] [TUNNEL]".`,
+"cloudflared tunnel route ip add --vnet [ID/name] [CIDR] [TUNNEL]".`,
 				Flags: []cli.Flag{vnetFlag},
 			},
 			{
@@ -92,7 +91,7 @@ to tell which virtual network whose routing table you want to use.`,
 
 func showRoutesFlags() []cli.Flag {
 	flags := make([]cli.Flag, 0)
-	flags = append(flags, teamnet.FilterFlags...)
+	flags = append(flags, cfapi.IpRouteFilterFlags...)
 	flags = append(flags, outputFormatFlag)
 	return flags
 }
@@ -103,7 +102,7 @@ func showRoutesCommand(c *cli.Context) error {
 		return err
 	}
 
-	filter, err := teamnet.NewFromCLI(c)
+	filter, err := cfapi.NewIpRouteFilterFromCLI(c)
 	if err != nil {
 		return errors.Wrap(err, "invalid config for routing filters")
 	}
@@ -168,7 +167,7 @@ func addRouteCommand(c *cli.Context) error {
 		vnetId = &id
 	}
 
-	_, err = sc.addRoute(teamnet.NewRoute{
+	_, err = sc.addRoute(cfapi.NewRoute{
 		Comment:  comment,
 		Network:  *network,
 		TunnelID: tunnelID,
@@ -199,7 +198,7 @@ func deleteRouteCommand(c *cli.Context) error {
 		return errors.New("Invalid network CIDR")
 	}
 
-	params := teamnet.DeleteRouteParams{
+	params := cfapi.DeleteRouteParams{
 		Network: *network,
 	}
 
@@ -233,7 +232,7 @@ func getRouteByIPCommand(c *cli.Context) error {
 		return fmt.Errorf("Invalid IP %s", ipInput)
 	}
 
-	params := teamnet.GetRouteByIpParams{
+	params := cfapi.GetRouteByIpParams{
 		Ip: ip,
 	}
 
@@ -252,12 +251,12 @@ func getRouteByIPCommand(c *cli.Context) error {
 	if route.IsZero() {
 		fmt.Printf("No route matches the IP %s\n", ip)
 	} else {
-		formatAndPrintRouteList([]*teamnet.DetailedRoute{&route})
+		formatAndPrintRouteList([]*cfapi.DetailedRoute{&route})
 	}
 	return nil
 }
 
-func formatAndPrintRouteList(routes []*teamnet.DetailedRoute) {
+func formatAndPrintRouteList(routes []*cfapi.DetailedRoute) {
 	const (
 		minWidth = 0
 		tabWidth = 8

cmd/cloudflared/tunnel/vnets_subcommands.go

@@ -9,9 +9,9 @@ import (
 	"github.com/pkg/errors"
 	"github.com/urfave/cli/v2"
 
+	"github.com/cloudflare/cloudflared/cfapi"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/updater"
-	"github.com/cloudflare/cloudflared/vnet"
 )
 
 var (
@@ -37,7 +37,7 @@ var (
 
 func buildVirtualNetworkSubcommand(hidden bool) *cli.Command {
 	return &cli.Command{
-		Name:      "network",
+		Name:      "vnet",
 		Usage:     "Configure and query virtual networks to manage private IP routes with overlapping IPs.",
 		UsageText: "cloudflared tunnel [--config FILEPATH] network COMMAND [arguments...]",
 		Description: `cloudflared allows to manage IP routes that expose origins in your private network space via their IP directly
@@ -102,7 +102,7 @@ default or update an existing one to become the default.`,
 
 func listVirtualNetworksFlags() []cli.Flag {
 	flags := make([]cli.Flag, 0)
-	flags = append(flags, vnet.FilterFlags...)
+	flags = append(flags, cfapi.VnetFilterFlags...)
 	flags = append(flags, outputFormatFlag)
 	return flags
 }
@@ -128,7 +128,7 @@ func addVirtualNetworkCommand(c *cli.Context) error {
 		comment = args.Get(1)
 	}
 
-	newVnet := vnet.NewVirtualNetwork{
+	newVnet := cfapi.NewVirtualNetwork{
 		Name:      name,
 		Comment:   comment,
 		IsDefault: c.Bool(makeDefaultFlag.Name),
@@ -160,7 +160,7 @@ func listVirtualNetworksCommand(c *cli.Context) error {
 	warningChecker := updater.StartWarningCheck(c)
 	defer warningChecker.LogWarningIfAny(sc.log)
 
-	filter, err := vnet.NewFromCLI(c)
+	filter, err := cfapi.NewFromCLI(c)
 	if err != nil {
 		return errors.Wrap(err, "invalid flags for filtering virtual networks")
 	}
@@ -177,7 +177,7 @@ func listVirtualNetworksCommand(c *cli.Context) error {
 	if len(vnets) > 0 {
 		formatAndPrintVnetsList(vnets)
 	} else {
-		fmt.Println("No virtual networks were found for the given filter flags. You can use 'cloudflared tunnel network add' to add a virtual network.")
+		fmt.Println("No virtual networks were found for the given filter flags. You can use 'cloudflared tunnel vnet add' to add a virtual network.")
 	}
 
 	return nil
@@ -220,7 +220,7 @@ func updateVirtualNetworkCommand(c *cli.Context) error {
 		return err
 	}
 
-	updates := vnet.UpdateVirtualNetwork{}
+	updates := cfapi.UpdateVirtualNetwork{}
 
 	if c.IsSet(newNameFlag.Name) {
 		newName := c.String(newNameFlag.Name)
@@ -248,7 +248,7 @@ func getVnetId(sc *subcommandContext, input string) (uuid.UUID, error) {
 		return val, nil
 	}
 
-	filter := vnet.NewFilter()
+	filter := cfapi.NewVnetFilter()
 	filter.WithDeleted(false)
 	filter.ByName(input)
 
@@ -264,7 +264,7 @@ func getVnetId(sc *subcommandContext, input string) (uuid.UUID, error) {
 	return vnets[0].ID, nil
 }
 
-func formatAndPrintVnetsList(vnets []*vnet.VirtualNetwork) {
+func formatAndPrintVnetsList(vnets []*cfapi.VirtualNetwork) {
 	const (
 		minWidth = 0
 		tabWidth = 8

cmd/cloudflared/updater/workers_service_test.go

@@ -1,3 +1,4 @@
+//go:build !windows
 // +build !windows
 
 package updater

cmd/cloudflared/windows_service.go

@@ -1,3 +1,4 @@
+//go:build windows
 // +build windows
 
 package main

component-tests/constants.py

@@ -3,3 +3,7 @@ MAX_RETRIES = 5
 BACKOFF_SECS = 7
 
 PROXY_DNS_PORT = 9053
+
+
+def protocols():
+    return ["h2mux", "http2", "quic"]

component-tests/test_reconnect.py

@@ -7,6 +7,7 @@ import pytest
 from flaky import flaky
 
 from conftest import CfdModes
+from constants import protocols
 from util import start_cloudflared, wait_tunnel_ready, check_tunnel_not_connected
 
 
@@ -18,9 +19,16 @@ class TestReconnect:
         "stdin-control": True,
     }
 
+    def _extra_config(self, protocol):
+        return {
+            "stdin-control": True,
+            "protocol": protocol,
+        }
+
     @pytest.mark.skipif(platform.system() == "Windows", reason=f"Currently buggy on Windows TUN-4584")
-    def test_named_reconnect(self, tmp_path, component_tests_config):
-        config = component_tests_config(self.extra_config)
+    @pytest.mark.parametrize("protocol", protocols())
+    def test_named_reconnect(self, tmp_path, component_tests_config, protocol):
+        config = component_tests_config(self._extra_config(protocol))
         with start_cloudflared(tmp_path, config, new_process=True, allow_input=True, capture_output=False) as cloudflared:
             # Repeat the test multiple times because some issues only occur after multiple reconnects
             self.assert_reconnect(config, cloudflared, 5)

component-tests/test_termination.py

@@ -8,6 +8,7 @@ import time
 import pytest
 import requests
 
+from constants import protocols
 from util import start_cloudflared, wait_tunnel_ready, check_tunnel_not_connected
 
 
@@ -17,17 +18,21 @@ def supported_signals():
     return [signal.SIGTERM, signal.SIGINT]
 
 
-class TestTermination():
+class TestTermination:
     grace_period = 5
     timeout = 10
-    extra_config = {
-        "grace-period": f"{grace_period}s",
-    }
     sse_endpoint = "/sse?freq=1s"
 
+    def _extra_config(self, protocol):
+        return {
+            "grace-period": f"{self.grace_period}s",
+            "protocol": protocol,
+        }
+
     @pytest.mark.parametrize("signal", supported_signals())
-    def test_graceful_shutdown(self, tmp_path, component_tests_config, signal):
-        config = component_tests_config(self.extra_config)
+    @pytest.mark.parametrize("protocol", protocols())
+    def test_graceful_shutdown(self, tmp_path, component_tests_config, signal, protocol):
+        config = component_tests_config(self._extra_config(protocol))
         with start_cloudflared(
                 tmp_path, config, new_process=True, capture_output=False) as cloudflared:
             wait_tunnel_ready(tunnel_url=config.get_url())
@@ -47,8 +52,9 @@ class TestTermination():
     # test cloudflared terminates before grace period expires when all eyeball
     # connections are drained
     @pytest.mark.parametrize("signal", supported_signals())
-    def test_shutdown_once_no_connection(self, tmp_path, component_tests_config, signal):
-        config = component_tests_config(self.extra_config)
+    @pytest.mark.parametrize("protocol", protocols())
+    def test_shutdown_once_no_connection(self, tmp_path, component_tests_config, signal, protocol):
+        config = component_tests_config(self._extra_config(protocol))
         with start_cloudflared(
                 tmp_path, config, new_process=True, capture_output=False) as cloudflared:
             wait_tunnel_ready(tunnel_url=config.get_url())
@@ -66,8 +72,9 @@ class TestTermination():
                 self.wait_eyeball_thread(in_flight_req, self.grace_period)
 
     @pytest.mark.parametrize("signal", supported_signals())
-    def test_no_connection_shutdown(self, tmp_path, component_tests_config, signal):
-        config = component_tests_config(self.extra_config)
+    @pytest.mark.parametrize("protocol", protocols())
+    def test_no_connection_shutdown(self, tmp_path, component_tests_config, signal, protocol):
+        config = component_tests_config(self._extra_config(protocol))
         with start_cloudflared(
                 tmp_path, config, new_process=True, capture_output=False) as cloudflared:
             wait_tunnel_ready(tunnel_url=config.get_url())

config/configuration.go

@@ -175,60 +175,62 @@ func ValidateUrl(c *cli.Context, allowURLFromArgs bool) (*url.URL, error) {
 }
 
 type UnvalidatedIngressRule struct {
-	Hostname      string
-	Path          string
-	Service       string
-	OriginRequest OriginRequestConfig `yaml:"originRequest"`
+	Hostname      string              `json:"hostname"`
+	Path          string              `json:"path"`
+	Service       string              `json:"service"`
+	OriginRequest OriginRequestConfig `yaml:"originRequest" json:"originRequest"`
 }
 
 // OriginRequestConfig is a set of optional fields that users may set to
 // customize how cloudflared sends requests to origin services. It is used to set
 // up general config that apply to all rules, and also, specific per-rule
 // config.
-// Note: To specify a time.Duration in go-yaml, use e.g. "3s" or "24h".
+// Note:
+// - To specify a time.Duration in go-yaml, use e.g. "3s" or "24h".
+// - To specify a time.Duration in json, use int64 of the nanoseconds
 type OriginRequestConfig struct {
 	// HTTP proxy timeout for establishing a new connection
-	ConnectTimeout *time.Duration `yaml:"connectTimeout"`
+	ConnectTimeout *time.Duration `yaml:"connectTimeout" json:"connectTimeout"`
 	// HTTP proxy timeout for completing a TLS handshake
-	TLSTimeout *time.Duration `yaml:"tlsTimeout"`
+	TLSTimeout *time.Duration `yaml:"tlsTimeout" json:"tlsTimeout"`
 	// HTTP proxy TCP keepalive duration
-	TCPKeepAlive *time.Duration `yaml:"tcpKeepAlive"`
+	TCPKeepAlive *time.Duration `yaml:"tcpKeepAlive" json:"tcpKeepAlive"`
 	// HTTP proxy should disable "happy eyeballs" for IPv4/v6 fallback
-	NoHappyEyeballs *bool `yaml:"noHappyEyeballs"`
+	NoHappyEyeballs *bool `yaml:"noHappyEyeballs" json:"noHappyEyeballs"`
 	// HTTP proxy maximum keepalive connection pool size
-	KeepAliveConnections *int `yaml:"keepAliveConnections"`
+	KeepAliveConnections *int `yaml:"keepAliveConnections" json:"keepAliveConnections"`
 	// HTTP proxy timeout for closing an idle connection
-	KeepAliveTimeout *time.Duration `yaml:"keepAliveTimeout"`
+	KeepAliveTimeout *time.Duration `yaml:"keepAliveTimeout" json:"keepAliveTimeout"`
 	// Sets the HTTP Host header for the local webserver.
-	HTTPHostHeader *string `yaml:"httpHostHeader"`
+	HTTPHostHeader *string `yaml:"httpHostHeader" json:"httpHostHeader"`
 	// Hostname on the origin server certificate.
-	OriginServerName *string `yaml:"originServerName"`
+	OriginServerName *string `yaml:"originServerName" json:"originServerName"`
 	// Path to the CA for the certificate of your origin.
 	// This option should be used only if your certificate is not signed by Cloudflare.
-	CAPool *string `yaml:"caPool"`
+	CAPool *string `yaml:"caPool" json:"caPool"`
 	// Disables TLS verification of the certificate presented by your origin.
 	// Will allow any certificate from the origin to be accepted.
 	// Note: The connection from your machine to Cloudflare's Edge is still encrypted.
-	NoTLSVerify *bool `yaml:"noTLSVerify"`
+	NoTLSVerify *bool `yaml:"noTLSVerify" json:"noTLSVerify"`
 	// Disables chunked transfer encoding.
 	// Useful if you are running a WSGI server.
-	DisableChunkedEncoding *bool `yaml:"disableChunkedEncoding"`
+	DisableChunkedEncoding *bool `yaml:"disableChunkedEncoding" json:"disableChunkedEncoding"`
 	// Runs as jump host
-	BastionMode *bool `yaml:"bastionMode"`
+	BastionMode *bool `yaml:"bastionMode" json:"bastionMode"`
 	// Listen address for the proxy.
-	ProxyAddress *string `yaml:"proxyAddress"`
+	ProxyAddress *string `yaml:"proxyAddress" json:"proxyAddress"`
 	// Listen port for the proxy.
-	ProxyPort *uint `yaml:"proxyPort"`
+	ProxyPort *uint `yaml:"proxyPort" json:"proxyPort"`
 	// Valid options are 'socks' or empty.
-	ProxyType *string `yaml:"proxyType"`
+	ProxyType *string `yaml:"proxyType" json:"proxyType"`
 	// IP rules for the proxy service
-	IPRules []IngressIPRule `yaml:"ipRules"`
+	IPRules []IngressIPRule `yaml:"ipRules" json:"ipRules"`
 }
 
 type IngressIPRule struct {
-	Prefix *string `yaml:"prefix"`
-	Ports  []int   `yaml:"ports"`
-	Allow  bool    `yaml:"allow"`
+	Prefix *string `yaml:"prefix" json:"prefix"`
+	Ports  []int   `yaml:"ports" json:"ports"`
+	Allow  bool    `yaml:"allow" json:"allow"`
 }
 
 type Configuration struct {
@@ -240,7 +242,7 @@ type Configuration struct {
 }
 
 type WarpRoutingConfig struct {
-	Enabled bool `yaml:"enabled"`
+	Enabled bool `yaml:"enabled" json:"enabled"`
 }
 
 type configFileSettings struct {

config/configuration_test.go

@@ -1,6 +1,7 @@
 package config
 
 import (
+	"encoding/json"
 	"testing"
 	"time"
 
@@ -26,6 +27,18 @@ func TestConfigFileSettings(t *testing.T) {
 	)
 	rawYAML := `
 tunnel: config-file-test
+originRequest:
+  ipRules:
+  - prefix: "10.0.0.0/8"
+    ports:
+    - 80
+    - 8080
+    allow: false
+  - prefix: "fc00::/7"
+    ports:
+    - 443
+    - 4443
+    allow: true
 ingress:
  - hostname: tunnel1.example.com
    path: /id
@@ -53,6 +66,21 @@ counters:
 	assert.Equal(t, firstIngress, config.Ingress[0])
 	assert.Equal(t, secondIngress, config.Ingress[1])
 	assert.Equal(t, warpRouting, config.WarpRouting)
+	privateV4 := "10.0.0.0/8"
+	privateV6 := "fc00::/7"
+	ipRules := []IngressIPRule{
+		{
+			Prefix: &privateV4,
+			Ports:  []int{80, 8080},
+			Allow:  false,
+		},
+		{
+			Prefix: &privateV6,
+			Ports:  []int{443, 4443},
+			Allow:  true,
+		},
+	}
+	assert.Equal(t, ipRules, config.OriginRequest.IPRules)
 
 	retries, err := config.Int("retries")
 	assert.NoError(t, err)
@@ -81,3 +109,71 @@ counters:
 	assert.Equal(t, 456, counters[1])
 
 }
+
+func TestUnmarshalOriginRequestConfig(t *testing.T) {
+	raw := []byte(`
+{
+	"connectTimeout": 10000000000,
+	"tlsTimeout": 30000000000,
+	"tcpKeepAlive": 30000000000,
+	"noHappyEyeballs": true,
+	"keepAliveTimeout": 60000000000,
+	"keepAliveConnections": 10,
+	"httpHostHeader": "app.tunnel.com",
+	"originServerName": "app.tunnel.com",
+	"caPool": "/etc/capool",
+	"noTLSVerify": true,
+	"disableChunkedEncoding": true,
+	"bastionMode": true,
+	"proxyAddress": "127.0.0.3",
+	"proxyPort": 9000,
+	"proxyType": "socks",
+	"ipRules": [
+		{
+			"prefix": "10.0.0.0/8",
+			"ports": [80, 8080],
+			"allow": false
+		},
+		{
+			"prefix": "fc00::/7",
+			"ports": [443, 4443],
+			"allow": true
+		}
+	]
+}
+`)
+	var config OriginRequestConfig
+	assert.NoError(t, json.Unmarshal(raw, &config))
+	assert.Equal(t, time.Second*10, *config.ConnectTimeout)
+	assert.Equal(t, time.Second*30, *config.TLSTimeout)
+	assert.Equal(t, time.Second*30, *config.TCPKeepAlive)
+	assert.Equal(t, true, *config.NoHappyEyeballs)
+	assert.Equal(t, time.Second*60, *config.KeepAliveTimeout)
+	assert.Equal(t, 10, *config.KeepAliveConnections)
+	assert.Equal(t, "app.tunnel.com", *config.HTTPHostHeader)
+	assert.Equal(t, "app.tunnel.com", *config.OriginServerName)
+	assert.Equal(t, "/etc/capool", *config.CAPool)
+	assert.Equal(t, true, *config.NoTLSVerify)
+	assert.Equal(t, true, *config.DisableChunkedEncoding)
+	assert.Equal(t, true, *config.BastionMode)
+	assert.Equal(t, "127.0.0.3", *config.ProxyAddress)
+	assert.Equal(t, true, *config.NoTLSVerify)
+	assert.Equal(t, uint(9000), *config.ProxyPort)
+	assert.Equal(t, "socks", *config.ProxyType)
+
+	privateV4 := "10.0.0.0/8"
+	privateV6 := "fc00::/7"
+	ipRules := []IngressIPRule{
+		{
+			Prefix: &privateV4,
+			Ports:  []int{80, 8080},
+			Allow:  false,
+		},
+		{
+			Prefix: &privateV6,
+			Ports:  []int{443, 4443},
+			Allow:  true,
+		},
+	}
+	assert.Equal(t, ipRules, config.IPRules)
+}

connection/control.go

@@ -29,7 +29,9 @@ type controlStream struct {
 
 // ControlStreamHandler registers connections with origintunneld and initiates graceful shutdown.
 type ControlStreamHandler interface {
-	ServeControlStream(ctx context.Context, rw io.ReadWriteCloser, connOptions *tunnelpogs.ConnectionOptions, shouldWaitForUnregister bool) error
+	// ServeControlStream handles the control plane of the transport in the current goroutine calling this
+	ServeControlStream(ctx context.Context, rw io.ReadWriteCloser, connOptions *tunnelpogs.ConnectionOptions) error
+	// IsStopped tells whether the method above has finished
 	IsStopped() bool
 }
 
@@ -61,7 +63,6 @@ func (c *controlStream) ServeControlStream(
 	ctx context.Context,
 	rw io.ReadWriteCloser,
 	connOptions *tunnelpogs.ConnectionOptions,
-	shouldWaitForUnregister bool,
 ) error {
 	rpcClient := c.newRPCClientFunc(ctx, rw, c.observer.log)
 
@@ -71,12 +72,7 @@ func (c *controlStream) ServeControlStream(
 	}
 	c.connectedFuse.Connected()
 
-	if shouldWaitForUnregister {
-		c.waitForUnregister(ctx, rpcClient)
-	} else {
-		go c.waitForUnregister(ctx, rpcClient)
-	}
-
+	c.waitForUnregister(ctx, rpcClient)
 	return nil
 }
 

connection/http2.go

@@ -108,7 +108,7 @@ func (c *HTTP2Connection) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
 	switch connType {
 	case TypeControlStream:
-		if err := c.controlStreamHandler.ServeControlStream(r.Context(), respWriter, c.connOptions, true); err != nil {
+		if err := c.controlStreamHandler.ServeControlStream(r.Context(), respWriter, c.connOptions); err != nil {
 			c.controlStreamErr = err
 			c.log.Error().Err(err)
 			respWriter.WriteErrorResponse()

connection/protocol.go

@@ -19,9 +19,7 @@ const (
 	edgeH2TLSServerName = "h2.cftunnel.com"
 	// edgeQUICServerName is the server name to establish quic connection with edge.
 	edgeQUICServerName = "quic.cftunnel.com"
-	// threshold to switch back to h2mux when the user intentionally pick --protocol http2
-	explicitHTTP2FallbackThreshold = -1
-	autoSelectFlag                 = "auto"
+	autoSelectFlag     = "auto"
 )
 
 var (
@@ -237,25 +235,23 @@ func selectNamedTunnelProtocols(
 	threshold int32,
 	protocol Protocol,
 ) (ProtocolSelector, error) {
-	if protocolFlag == H2mux.String() {
-		return &staticProtocolSelector{
-			current: H2mux,
-		}, nil
+	// If the user picks a protocol, then we stick to it no matter what.
+	switch protocolFlag {
+	case H2mux.String():
+		return &staticProtocolSelector{current: H2mux}, nil
+	case QUIC.String():
+		return &staticProtocolSelector{current: QUIC}, nil
+	case HTTP2.String():
+		return &staticProtocolSelector{current: HTTP2}, nil
 	}
 
-	if protocolFlag == QUIC.String() {
-		return newAutoProtocolSelector(QUIC, []Protocol{QUIC, HTTP2, H2mux}, explicitHTTP2FallbackThreshold, fetchFunc, ttl, log), nil
+	// If the user does not pick (hopefully the majority) then we use the one derived from the TXT DNS record and
+	// fallback on failures.
+	if protocolFlag == autoSelectFlag {
+		return newAutoProtocolSelector(protocol, []Protocol{QUIC, HTTP2, H2mux}, threshold, fetchFunc, ttl, log), nil
 	}
 
-	if protocolFlag == HTTP2.String() {
-		return newAutoProtocolSelector(HTTP2, []Protocol{HTTP2, H2mux}, explicitHTTP2FallbackThreshold, fetchFunc, ttl, log), nil
-	}
-
-	if protocolFlag != autoSelectFlag {
-		return nil, fmt.Errorf("Unknown protocol %s, %s", protocolFlag, AvailableProtocolFlagMessage)
-	}
-
-	return newAutoProtocolSelector(protocol, []Protocol{QUIC, HTTP2, H2mux}, threshold, fetchFunc, ttl, log), nil
+	return nil, fmt.Errorf("Unknown protocol %s, %s", protocolFlag, AvailableProtocolFlagMessage)
 }
 
 func selectWarpRoutingProtocols(
@@ -266,19 +262,21 @@ func selectWarpRoutingProtocols(
 	threshold int32,
 	protocol Protocol,
 ) (ProtocolSelector, error) {
-	if protocolFlag == QUIC.String() {
-		return newAutoProtocolSelector(QUICWarp, []Protocol{QUICWarp, HTTP2Warp}, explicitHTTP2FallbackThreshold, fetchFunc, ttl, log), nil
+	// If the user picks a protocol, then we stick to it no matter what.
+	switch protocolFlag {
+	case QUIC.String():
+		return &staticProtocolSelector{current: QUICWarp}, nil
+	case HTTP2.String():
+		return &staticProtocolSelector{current: HTTP2Warp}, nil
 	}
 
-	if protocolFlag == HTTP2.String() {
-		return newAutoProtocolSelector(HTTP2Warp, []Protocol{HTTP2Warp}, explicitHTTP2FallbackThreshold, fetchFunc, ttl, log), nil
+	// If the user does not pick (hopefully the majority) then we use the one derived from the TXT DNS record and
+	// fallback on failures.
+	if protocolFlag == autoSelectFlag {
+		return newAutoProtocolSelector(protocol, []Protocol{QUICWarp, HTTP2Warp}, threshold, fetchFunc, ttl, log), nil
 	}
 
-	if protocolFlag != autoSelectFlag {
-		return nil, fmt.Errorf("Unknown protocol %s, %s", protocolFlag, AvailableProtocolFlagMessage)
-	}
-
-	return newAutoProtocolSelector(protocol, []Protocol{QUICWarp, HTTP2Warp}, threshold, fetchFunc, ttl, log), nil
+	return nil, fmt.Errorf("Unknown protocol %s, %s", protocolFlag, AvailableProtocolFlagMessage)
 }
 
 func switchThreshold(accountTag string) int32 {

connection/protocol_test.go

@@ -72,21 +72,33 @@ func TestNewProtocolSelector(t *testing.T) {
 			name:              "named tunnel over http2",
 			protocol:          "http2",
 			expectedProtocol:  HTTP2,
-			hasFallback:       true,
-			expectedFallback:  H2mux,
 			fetchFunc:         mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 0}),
 			namedTunnelConfig: testNamedTunnelConfig,
 		},
 		{
-			name:              "named tunnel http2 disabled",
+			name:              "named tunnel http2 disabled still gets http2 because it is manually picked",
 			protocol:          "http2",
-			expectedProtocol:  H2mux,
+			expectedProtocol:  HTTP2,
 			fetchFunc:         mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: -1}),
 			namedTunnelConfig: testNamedTunnelConfig,
 		},
+		{
+			name:              "named tunnel quic disabled still gets quic because it is manually picked",
+			protocol:          "quic",
+			expectedProtocol:  QUIC,
+			fetchFunc:         mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 100}, edgediscovery.ProtocolPercent{Protocol: "quic", Percentage: -1}),
+			namedTunnelConfig: testNamedTunnelConfig,
+		},
+		{
+			name:              "named tunnel quic and http2 disabled",
+			protocol:          "auto",
+			expectedProtocol:  H2mux,
+			fetchFunc:         mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: -1}, edgediscovery.ProtocolPercent{Protocol: "quic", Percentage: -1}),
+			namedTunnelConfig: testNamedTunnelConfig,
+		},
 		{
 			name:             "named tunnel quic disabled",
-			protocol:         "quic",
+			protocol:         "auto",
 			expectedProtocol: HTTP2,
 			// Hasfallback true is because if http2 fails, then we further fallback to h2mux.
 			hasFallback:       true,
@@ -155,7 +167,7 @@ func TestNewProtocolSelector(t *testing.T) {
 		},
 		{
 			name:               "warp routing quic",
-			protocol:           "quic",
+			protocol:           "auto",
 			expectedProtocol:   QUICWarp,
 			hasFallback:        true,
 			expectedFallback:   HTTP2Warp,
@@ -254,6 +266,7 @@ func TestAutoProtocolSelectorRefresh(t *testing.T) {
 
 func TestHTTP2ProtocolSelectorRefresh(t *testing.T) {
 	fetcher := dynamicMockFetcher{}
+	// Since the user chooses http2 on purpose, we always stick to it.
 	selector, err := NewProtocolSelector("http2", noWarpRoutingEnabled, testNamedTunnelConfig, fetcher.fetch(), testNoTTL, &log)
 	assert.NoError(t, err)
 	assert.Equal(t, HTTP2, selector.Current())
@@ -269,7 +282,7 @@ func TestHTTP2ProtocolSelectorRefresh(t *testing.T) {
 
 	fetcher.protocolPercents = edgediscovery.ProtocolPercents{edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: -1}}
 	fetcher.err = nil
-	assert.Equal(t, H2mux, selector.Current())
+	assert.Equal(t, HTTP2, selector.Current())
 
 	fetcher.protocolPercents = edgediscovery.ProtocolPercents{edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 0}}
 	assert.Equal(t, HTTP2, selector.Current())
@@ -278,7 +291,7 @@ func TestHTTP2ProtocolSelectorRefresh(t *testing.T) {
 	assert.Equal(t, HTTP2, selector.Current())
 
 	fetcher.protocolPercents = edgediscovery.ProtocolPercents{edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: -1}}
-	assert.Equal(t, H2mux, selector.Current())
+	assert.Equal(t, HTTP2, selector.Current())
 }
 
 func TestProtocolSelectorRefreshTTL(t *testing.T) {

connection/quic.go

@@ -29,84 +29,111 @@ const (
 	// HTTPMethodKey is used to get or set http method in QUIC ALPN if the underlying proxy connection type is HTTP.
 	HTTPMethodKey = "HttpMethod"
 	// HTTPHostKey is used to get or set http Method in QUIC ALPN if the underlying proxy connection type is HTTP.
-	HTTPHostKey          = "HttpHost"
-	MaxDatagramFrameSize = 1220
+	HTTPHostKey = "HttpHost"
 )
 
 // QUICConnection represents the type that facilitates Proxying via QUIC streams.
 type QUICConnection struct {
-	session        quic.Session
-	logger         *zerolog.Logger
-	httpProxy      OriginProxy
-	sessionManager datagramsession.Manager
+	session              quic.Session
+	logger               *zerolog.Logger
+	httpProxy            OriginProxy
+	sessionManager       datagramsession.Manager
+	controlStreamHandler ControlStreamHandler
+	connOptions          *tunnelpogs.ConnectionOptions
 }
 
 // NewQUICConnection returns a new instance of QUICConnection.
 func NewQUICConnection(
-	ctx context.Context,
 	quicConfig *quic.Config,
 	edgeAddr net.Addr,
 	tlsConfig *tls.Config,
 	httpProxy OriginProxy,
 	connOptions *tunnelpogs.ConnectionOptions,
 	controlStreamHandler ControlStreamHandler,
-	observer *Observer,
+	logger *zerolog.Logger,
 ) (*QUICConnection, error) {
 	session, err := quic.DialAddr(edgeAddr.String(), tlsConfig, quicConfig)
 	if err != nil {
 		return nil, fmt.Errorf("failed to dial to edge: %w", err)
 	}
 
-	registrationStream, err := session.OpenStream()
-	if err != nil {
-		return nil, fmt.Errorf("failed to open a registration stream: %w", err)
-	}
-
-	err = controlStreamHandler.ServeControlStream(ctx, registrationStream, connOptions, false)
-	if err != nil {
-		// Not wrapping error here to be consistent with the http2 message.
-		return nil, err
-	}
-
 	datagramMuxer, err := quicpogs.NewDatagramMuxer(session)
 	if err != nil {
 		return nil, err
 	}
 
-	sessionManager := datagramsession.NewManager(datagramMuxer, observer.log)
+	sessionManager := datagramsession.NewManager(datagramMuxer, logger)
 
 	return &QUICConnection{
-		session:        session,
-		httpProxy:      httpProxy,
-		logger:         observer.log,
-		sessionManager: sessionManager,
+		session:              session,
+		httpProxy:            httpProxy,
+		logger:               logger,
+		sessionManager:       sessionManager,
+		controlStreamHandler: controlStreamHandler,
+		connOptions:          connOptions,
 	}, nil
 }
 
 // Serve starts a QUIC session that begins accepting streams.
 func (q *QUICConnection) Serve(ctx context.Context) error {
+	// origintunneld assumes the first stream is used for the control plane
+	controlStream, err := q.session.OpenStream()
+	if err != nil {
+		return fmt.Errorf("failed to open a registration control stream: %w", err)
+	}
+
+	// If either goroutine returns nil error, we rely on this cancellation to make sure the other goroutine exits
+	// as fast as possible as well. Nil error means we want to exit for good (caller code won't retry serving this
+	// connection).
+	// If either goroutine returns a non nil error, then the error group cancels the context, thus also canceling the
+	// other goroutine as fast as possible.
+	ctx, cancel := context.WithCancel(ctx)
 	errGroup, ctx := errgroup.WithContext(ctx)
+
+	// In the future, if cloudflared can autonomously push traffic to the edge, we have to make sure the control
+	// stream is already fully registered before the other goroutines can proceed.
 	errGroup.Go(func() error {
+		defer cancel()
+		return q.serveControlStream(ctx, controlStream)
+	})
+	errGroup.Go(func() error {
+		defer cancel()
 		return q.acceptStream(ctx)
 	})
 	errGroup.Go(func() error {
+		defer cancel()
 		return q.sessionManager.Serve(ctx)
 	})
+
 	return errGroup.Wait()
 }
 
+func (q *QUICConnection) serveControlStream(ctx context.Context, controlStream quic.Stream) error {
+	// This blocks until the control plane is done.
+	err := q.controlStreamHandler.ServeControlStream(ctx, controlStream, q.connOptions)
+	if err != nil {
+		// Not wrapping error here to be consistent with the http2 message.
+		return err
+	}
+
+	return nil
+}
+
 func (q *QUICConnection) acceptStream(ctx context.Context) error {
+	defer q.Close()
 	for {
-		stream, err := q.session.AcceptStream(ctx)
+		quicStream, err := q.session.AcceptStream(ctx)
 		if err != nil {
 			// context.Canceled is usually a user ctrl+c. We don't want to log an error here as it's intentional.
-			if errors.Is(err, context.Canceled) {
+			if errors.Is(err, context.Canceled) || q.controlStreamHandler.IsStopped() {
 				return nil
 			}
 			return fmt.Errorf("failed to accept QUIC stream: %w", err)
 		}
 		go func() {
+			stream := quicpogs.NewSafeStreamCloser(quicStream)
 			defer stream.Close()
+
 			if err = q.handleStream(stream); err != nil {
 				q.logger.Err(err).Msg("Failed to handle QUIC stream")
 			}
@@ -119,7 +146,7 @@ func (q *QUICConnection) Close() {
 	q.session.CloseWithError(0, "")
 }
 
-func (q *QUICConnection) handleStream(stream quic.Stream) error {
+func (q *QUICConnection) handleStream(stream io.ReadWriteCloser) error {
 	signature, err := quicpogs.DetermineProtocol(stream)
 	if err != nil {
 		return err
@@ -165,9 +192,10 @@ func (q *QUICConnection) handleDataStream(stream *quicpogs.RequestServerStream)
 }
 
 func (q *QUICConnection) handleRPCStream(rpcStream *quicpogs.RPCServerStream) error {
-	return rpcStream.Serve(q, q.logger)
+	return rpcStream.Serve(q, q, q.logger)
 }
 
+// RegisterUdpSession is the RPC method invoked by edge to register and run a session
 func (q *QUICConnection) RegisterUdpSession(ctx context.Context, sessionID uuid.UUID, dstIP net.IP, dstPort uint16, closeAfterIdleHint time.Duration) error {
 	// Each session is a series of datagram from an eyeball to a dstIP:dstPort.
 	// (src port, dst IP, dst port) uniquely identifies a session, so it needs a dedicated connected socket.
@@ -178,22 +206,63 @@ func (q *QUICConnection) RegisterUdpSession(ctx context.Context, sessionID uuid.
 	}
 	session, err := q.sessionManager.RegisterSession(ctx, sessionID, originProxy)
 	if err != nil {
-		q.logger.Err(err).Msgf("Failed to register udp session %s", sessionID)
+		q.logger.Err(err).Str("sessionID", sessionID.String()).Msgf("Failed to register udp session")
 		return err
 	}
-	go func() {
-		defer q.sessionManager.UnregisterSession(q.session.Context(), sessionID)
-		if err := session.Serve(q.session.Context(), closeAfterIdleHint); err != nil {
-			q.logger.Debug().Err(err).Str("sessionID", sessionID.String()).Msg("session terminated")
-		}
-	}()
+
+	go q.serveUDPSession(session, closeAfterIdleHint)
+
 	q.logger.Debug().Msgf("Registered session %v, %v, %v", sessionID, dstIP, dstPort)
 	return nil
 }
 
-func (q *QUICConnection) UnregisterUdpSession(ctx context.Context, sessionID uuid.UUID) error {
-	q.sessionManager.UnregisterSession(ctx, sessionID)
-	return nil
+func (q *QUICConnection) serveUDPSession(session *datagramsession.Session, closeAfterIdleHint time.Duration) {
+	ctx := q.session.Context()
+	closedByRemote, err := session.Serve(ctx, closeAfterIdleHint)
+	// If session is terminated by remote, then we know it has been unregistered from session manager and edge
+	if !closedByRemote {
+		if err != nil {
+			q.closeUDPSession(ctx, session.ID, err.Error())
+		} else {
+			q.closeUDPSession(ctx, session.ID, "terminated without error")
+		}
+	}
+	q.logger.Debug().Err(err).Str("sessionID", session.ID.String()).Msg("Session terminated")
+}
+
+// closeUDPSession first unregisters the session from session manager, then it tries to unregister from edge
+func (q *QUICConnection) closeUDPSession(ctx context.Context, sessionID uuid.UUID, message string) {
+	q.sessionManager.UnregisterSession(ctx, sessionID, message, false)
+	stream, err := q.session.OpenStream()
+	if err != nil {
+		// Log this at debug because this is not an error if session was closed due to lost connection
+		// with edge
+		q.logger.Debug().Err(err).Str("sessionID", sessionID.String()).
+			Msgf("Failed to open quic stream to unregister udp session with edge")
+		return
+	}
+	rpcClientStream, err := quicpogs.NewRPCClientStream(ctx, stream, q.logger)
+	if err != nil {
+		// Log this at debug because this is not an error if session was closed due to lost connection
+		// with edge
+		q.logger.Err(err).Str("sessionID", sessionID.String()).
+			Msgf("Failed to open rpc stream to unregister udp session with edge")
+		return
+	}
+	if err := rpcClientStream.UnregisterUdpSession(ctx, sessionID, message); err != nil {
+		q.logger.Err(err).Str("sessionID", sessionID.String()).
+			Msgf("Failed to unregister udp session with edge")
+	}
+}
+
+// UnregisterUdpSession is the RPC method invoked by edge to unregister and terminate a sesssion
+func (q *QUICConnection) UnregisterUdpSession(ctx context.Context, sessionID uuid.UUID, message string) error {
+	return q.sessionManager.UnregisterSession(ctx, sessionID, message, true)
+}
+
+// UpdateConfiguration is the RPC method invoked by edge when there is a new configuration
+func (q *QUICConnection) UpdateConfiguration(ctx context.Context, version int32, config []byte) (*tunnelpogs.UpdateConfigurationResponse, error) {
+	return nil, fmt.Errorf("TODO: TUN-5698")
 }
 
 // streamReadWriteAcker is a light wrapper over QUIC streams with a callback to send response back to

connection/quic_test.go

@@ -3,43 +3,41 @@ package connection
 import (
 	"bytes"
 	"context"
-	"crypto/rand"
-	"crypto/rsa"
 	"crypto/tls"
-	"crypto/x509"
-	"encoding/pem"
 	"fmt"
 	"io"
-	"math/big"
 	"net"
 	"net/http"
 	"net/url"
 	"os"
 	"sync"
 	"testing"
+	"time"
 
 	"github.com/gobwas/ws/wsutil"
+	"github.com/google/uuid"
 	"github.com/lucas-clemente/quic-go"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
+	"github.com/cloudflare/cloudflared/datagramsession"
 	quicpogs "github.com/cloudflare/cloudflared/quic"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
+var (
+	testTLSServerConfig = quicpogs.GenerateTLSConfig()
+	testQUICConfig      = &quic.Config{
+		KeepAlive:       true,
+		EnableDatagrams: true,
+	}
+)
+
 // TestQUICServer tests if a quic server accepts and responds to a quic client with the acceptance protocol.
 // It also serves as a demonstration for communication with the QUIC connection started by a cloudflared.
 func TestQUICServer(t *testing.T) {
-	quicConfig := &quic.Config{
-		KeepAlive:       true,
-		EnableDatagrams: true,
-	}
-
-	// Setup test.
-	log := zerolog.New(os.Stdout)
-
 	// Start a UDP Listener for QUIC.
 	udpAddr, err := net.ResolveUDPAddr("udp", "127.0.0.1:0")
 	require.NoError(t, err)
@@ -47,18 +45,6 @@ func TestQUICServer(t *testing.T) {
 	require.NoError(t, err)
 	defer udpListener.Close()
 
-	// Create a simple tls config.
-	tlsConfig := generateTLSConfig()
-
-	// Create a client config
-	tlsClientConfig := &tls.Config{
-		InsecureSkipVerify: true,
-		NextProtos:         []string{"argotunnel"},
-	}
-
-	// Start a mock httpProxy
-	originProxy := &mockOriginProxyWithRequest{}
-
 	// This is simply a sample websocket frame message.
 	wsBuf := &bytes.Buffer{}
 	wsutil.WriteClientText(wsBuf, []byte("Hello"))
@@ -93,7 +79,7 @@ func TestQUICServer(t *testing.T) {
 		},
 		{
 			desc:           "test http body request streaming",
-			dest:           "/echo_body",
+			dest:           "/slow_echo_body",
 			connectionType: quicpogs.ConnectionTypeHTTP,
 			metadata: []quicpogs.Metadata{
 				{
@@ -158,25 +144,13 @@ func TestQUICServer(t *testing.T) {
 			go func() {
 				defer wg.Done()
 				quicServer(
-					t, udpListener, tlsConfig, quicConfig,
+					t, udpListener, testTLSServerConfig, testQUICConfig,
 					test.dest, test.connectionType, test.metadata, test.message, test.expectedResponse,
 				)
 			}()
 
-			controlStream := fakeControlStream{}
-
-			qC, err := NewQUICConnection(
-				ctx,
-				quicConfig,
-				udpListener.LocalAddr(),
-				tlsClientConfig,
-				originProxy,
-				&tunnelpogs.ConnectionOptions{},
-				controlStream,
-				NewObserver(&log, &log, false),
-			)
-			require.NoError(t, err)
-			go qC.Serve(ctx)
+			qc := testQUICConnection(udpListener.LocalAddr(), t)
+			go qc.Serve(ctx)
 
 			wg.Wait()
 			cancel()
@@ -188,7 +162,8 @@ type fakeControlStream struct {
 	ControlStreamHandler
 }
 
-func (fakeControlStream) ServeControlStream(ctx context.Context, rw io.ReadWriteCloser, connOptions *tunnelpogs.ConnectionOptions, shouldWaitForUnregister bool) error {
+func (fakeControlStream) ServeControlStream(ctx context.Context, rw io.ReadWriteCloser, connOptions *tunnelpogs.ConnectionOptions) error {
+	<-ctx.Done()
 	return nil
 }
 func (fakeControlStream) IsStopped() bool {
@@ -215,8 +190,9 @@ func quicServer(
 	session, err := earlyListener.Accept(ctx)
 	require.NoError(t, err)
 
-	stream, err := session.OpenStreamSync(context.Background())
+	quicStream, err := session.OpenStreamSync(context.Background())
 	require.NoError(t, err)
+	stream := quicpogs.NewSafeStreamCloser(quicStream)
 
 	reqClientStream := quicpogs.RequestClientStream{ReadWriteCloser: stream}
 	err = reqClientStream.WriteConnectRequestData(dest, connectionType, metadata...)
@@ -227,42 +203,20 @@ func quicServer(
 
 	if message != nil {
 		// ALPN successful. Write data.
-		_, err := stream.Write([]byte(message))
+		_, err := stream.Write(message)
 		require.NoError(t, err)
 	}
 
 	response := make([]byte, len(expectedResponse))
-	stream.Read(response)
-	require.NoError(t, err)
+	_, err = stream.Read(response)
+	if err != io.EOF {
+		require.NoError(t, err)
+	}
 
 	// For now it is an echo server. Verify if the same data is returned.
 	assert.Equal(t, expectedResponse, response)
 }
 
-// Setup a bare-bones TLS config for the server
-func generateTLSConfig() *tls.Config {
-	key, err := rsa.GenerateKey(rand.Reader, 1024)
-	if err != nil {
-		panic(err)
-	}
-	template := x509.Certificate{SerialNumber: big.NewInt(1)}
-	certDER, err := x509.CreateCertificate(rand.Reader, &template, &template, &key.PublicKey, key)
-	if err != nil {
-		panic(err)
-	}
-	keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)})
-	certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER})
-
-	tlsCert, err := tls.X509KeyPair(certPEM, keyPEM)
-	if err != nil {
-		panic(err)
-	}
-	return &tls.Config{
-		Certificates: []tls.Certificate{tlsCert},
-		NextProtos:   []string{"argotunnel"},
-	}
-}
-
 type mockOriginProxyWithRequest struct{}
 
 func (moc *mockOriginProxyWithRequest) ProxyHTTP(w ResponseWriter, r *http.Request, isWebsocket bool) error {
@@ -284,6 +238,9 @@ func (moc *mockOriginProxyWithRequest) ProxyHTTP(w ResponseWriter, r *http.Reque
 	switch r.URL.Path {
 	case "/ok":
 		originRespEndpoint(w, http.StatusOK, []byte(http.StatusText(http.StatusOK)))
+	case "/slow_echo_body":
+		time.Sleep(5)
+		fallthrough
 	case "/echo_body":
 		resp := &http.Response{
 			StatusCode: http.StatusOK,
@@ -531,3 +488,156 @@ func (moc *mockOriginProxyWithRequest) ProxyTCP(ctx context.Context, rwa ReadWri
 	io.Copy(rwa, rwa)
 	return nil
 }
+
+func TestServeUDPSession(t *testing.T) {
+	// Start a UDP Listener for QUIC.
+	udpAddr, err := net.ResolveUDPAddr("udp", "127.0.0.1:0")
+	require.NoError(t, err)
+	udpListener, err := net.ListenUDP(udpAddr.Network(), udpAddr)
+	require.NoError(t, err)
+	defer udpListener.Close()
+
+	ctx, cancel := context.WithCancel(context.Background())
+
+	// Establish QUIC connection with edge
+	edgeQUICSessionChan := make(chan quic.Session)
+	go func() {
+		earlyListener, err := quic.Listen(udpListener, testTLSServerConfig, testQUICConfig)
+		require.NoError(t, err)
+
+		edgeQUICSession, err := earlyListener.Accept(ctx)
+		require.NoError(t, err)
+		edgeQUICSessionChan <- edgeQUICSession
+	}()
+
+	qc := testQUICConnection(udpListener.LocalAddr(), t)
+	go qc.Serve(ctx)
+
+	edgeQUICSession := <-edgeQUICSessionChan
+	serveSession(ctx, qc, edgeQUICSession, closedByOrigin, io.EOF.Error(), t)
+	serveSession(ctx, qc, edgeQUICSession, closedByTimeout, datagramsession.SessionIdleErr(time.Millisecond*50).Error(), t)
+	serveSession(ctx, qc, edgeQUICSession, closedByRemote, "eyeball closed connection", t)
+	cancel()
+}
+
+func serveSession(ctx context.Context, qc *QUICConnection, edgeQUICSession quic.Session, closeType closeReason, expectedReason string, t *testing.T) {
+	var (
+		payload = []byte(t.Name())
+	)
+	sessionID := uuid.New()
+	cfdConn, originConn := net.Pipe()
+	// Registers and run a new session
+	session, err := qc.sessionManager.RegisterSession(ctx, sessionID, cfdConn)
+	require.NoError(t, err)
+
+	sessionDone := make(chan struct{})
+	go func() {
+		qc.serveUDPSession(session, time.Millisecond*50)
+		close(sessionDone)
+	}()
+
+	// Send a message to the quic session on edge side, it should be deumx to this datagram session
+	muxedPayload := append(payload, sessionID[:]...)
+	err = edgeQUICSession.SendMessage(muxedPayload)
+	require.NoError(t, err)
+
+	readBuffer := make([]byte, len(payload)+1)
+	n, err := originConn.Read(readBuffer)
+	require.NoError(t, err)
+	require.Equal(t, len(payload), n)
+	require.True(t, bytes.Equal(payload, readBuffer[:n]))
+
+	// Close connection to terminate session
+	switch closeType {
+	case closedByOrigin:
+		originConn.Close()
+	case closedByRemote:
+		err = qc.UnregisterUdpSession(ctx, sessionID, expectedReason)
+		require.NoError(t, err)
+	case closedByTimeout:
+	}
+
+	if closeType != closedByRemote {
+		// Session was not closed by remote, so closeUDPSession should be invoked to unregister from remote
+		unregisterFromEdgeChan := make(chan struct{})
+		sessionRPCServer := &mockSessionRPCServer{
+			sessionID:            sessionID,
+			unregisterReason:     expectedReason,
+			calledUnregisterChan: unregisterFromEdgeChan,
+		}
+		go runRPCServer(ctx, edgeQUICSession, sessionRPCServer, nil, t)
+
+		<-unregisterFromEdgeChan
+	}
+
+	<-sessionDone
+}
+
+type closeReason uint8
+
+const (
+	closedByOrigin closeReason = iota
+	closedByRemote
+	closedByTimeout
+)
+
+func runRPCServer(ctx context.Context, session quic.Session, sessionRPCServer tunnelpogs.SessionManager, configRPCServer tunnelpogs.ConfigurationManager, t *testing.T) {
+	stream, err := session.AcceptStream(ctx)
+	require.NoError(t, err)
+
+	if stream.StreamID() == 0 {
+		// Skip the first stream, it's the control stream of the QUIC connection
+		stream, err = session.AcceptStream(ctx)
+		require.NoError(t, err)
+	}
+	protocol, err := quicpogs.DetermineProtocol(stream)
+	assert.NoError(t, err)
+	rpcServerStream, err := quicpogs.NewRPCServerStream(stream, protocol)
+	assert.NoError(t, err)
+
+	log := zerolog.New(os.Stdout)
+	err = rpcServerStream.Serve(sessionRPCServer, configRPCServer, &log)
+	assert.NoError(t, err)
+}
+
+type mockSessionRPCServer struct {
+	sessionID            uuid.UUID
+	unregisterReason     string
+	calledUnregisterChan chan struct{}
+}
+
+func (s mockSessionRPCServer) RegisterUdpSession(ctx context.Context, sessionID uuid.UUID, dstIP net.IP, dstPort uint16, closeIdleAfter time.Duration) error {
+	return fmt.Errorf("mockSessionRPCServer doesn't implement RegisterUdpSession")
+}
+
+func (s mockSessionRPCServer) UnregisterUdpSession(ctx context.Context, sessionID uuid.UUID, reason string) error {
+	if s.sessionID != sessionID {
+		return fmt.Errorf("expect session ID %s, got %s", s.sessionID, sessionID)
+	}
+	if s.unregisterReason != reason {
+		return fmt.Errorf("expect unregister reason %s, got %s", s.unregisterReason, reason)
+	}
+	close(s.calledUnregisterChan)
+	return nil
+}
+
+func testQUICConnection(udpListenerAddr net.Addr, t *testing.T) *QUICConnection {
+	tlsClientConfig := &tls.Config{
+		InsecureSkipVerify: true,
+		NextProtos:         []string{"argotunnel"},
+	}
+	// Start a mock httpProxy
+	originProxy := &mockOriginProxyWithRequest{}
+	log := zerolog.New(os.Stdout)
+	qc, err := NewQUICConnection(
+		testQUICConfig,
+		udpListenerAddr,
+		tlsClientConfig,
+		originProxy,
+		&tunnelpogs.ConnectionOptions{},
+		fakeControlStream{},
+		&log,
+	)
+	require.NoError(t, err)
+	return qc
+}

datagramsession/event.go

@@ -1,6 +1,7 @@
 package datagramsession
 
 import (
+	"fmt"
 	"io"
 
 	"github.com/google/uuid"
@@ -24,6 +25,22 @@ func newRegisterSessionEvent(sessionID uuid.UUID, originProxy io.ReadWriteCloser
 // unregisterSessionEvent is an event to stop tracking and terminate the session.
 type unregisterSessionEvent struct {
 	sessionID uuid.UUID
+	err       *errClosedSession
+}
+
+// ClosedSessionError represent a condition that closes the session other than I/O
+// I/O error is not included, because the side that closes the session is ambiguous.
+type errClosedSession struct {
+	message  string
+	byRemote bool
+}
+
+func (sc *errClosedSession) Error() string {
+	if sc.byRemote {
+		return fmt.Sprintf("session closed by remote due to %s", sc.message)
+	} else {
+		return fmt.Sprintf("session closed by local due to %s", sc.message)
+	}
 }
 
 // newDatagram is an event when transport receives new datagram

datagramsession/manager.go

@@ -5,6 +5,7 @@ import (
 	"io"
 
 	"github.com/google/uuid"
+	"github.com/lucas-clemente/quic-go"
 	"github.com/rs/zerolog"
 	"golang.org/x/sync/errgroup"
 )
@@ -20,7 +21,7 @@ type Manager interface {
 	// RegisterSession starts tracking a session. Caller is responsible for starting the session
 	RegisterSession(ctx context.Context, sessionID uuid.UUID, dstConn io.ReadWriteCloser) (*Session, error)
 	// UnregisterSession stops tracking the session and terminates it
-	UnregisterSession(ctx context.Context, sessionID uuid.UUID) error
+	UnregisterSession(ctx context.Context, sessionID uuid.UUID, message string, byRemote bool) error
 }
 
 type manager struct {
@@ -50,8 +51,11 @@ func (m *manager) Serve(ctx context.Context) error {
 		for {
 			sessionID, payload, err := m.transport.ReceiveFrom()
 			if err != nil {
-				m.log.Err(err).Msg("Failed to receive datagram from transport, closing session manager")
-				return err
+				if aerr, ok := err.(*quic.ApplicationError); ok && uint64(aerr.ErrorCode) == uint64(quic.NoError) {
+					return nil
+				} else {
+					return err
+				}
 			}
 			datagram := &newDatagram{
 				sessionID: sessionID,
@@ -70,7 +74,7 @@ func (m *manager) Serve(ctx context.Context) error {
 		for {
 			select {
 			case <-ctx.Done():
-				return ctx.Err()
+				return nil
 			case datagram := <-m.datagramChan:
 				m.sendToSession(datagram)
 			case registration := <-m.registrationChan:
@@ -96,13 +100,19 @@ func (m *manager) RegisterSession(ctx context.Context, sessionID uuid.UUID, orig
 }
 
 func (m *manager) registerSession(ctx context.Context, registration *registerSessionEvent) {
-	session := newSession(registration.sessionID, m.transport, registration.originProxy)
+	session := newSession(registration.sessionID, m.transport, registration.originProxy, m.log)
 	m.sessions[registration.sessionID] = session
 	registration.resultChan <- session
 }
 
-func (m *manager) UnregisterSession(ctx context.Context, sessionID uuid.UUID) error {
-	event := &unregisterSessionEvent{sessionID: sessionID}
+func (m *manager) UnregisterSession(ctx context.Context, sessionID uuid.UUID, message string, byRemote bool) error {
+	event := &unregisterSessionEvent{
+		sessionID: sessionID,
+		err: &errClosedSession{
+			message:  message,
+			byRemote: byRemote,
+		},
+	}
 	select {
 	case <-ctx.Done():
 		return ctx.Err()
@@ -115,7 +125,7 @@ func (m *manager) unregisterSession(unregistration *unregisterSessionEvent) {
 	session, ok := m.sessions[unregistration.sessionID]
 	if ok {
 		delete(m.sessions, unregistration.sessionID)
-		session.close()
+		session.close(unregistration.err)
 	}
 }
 

datagramsession/manager_test.go

@@ -17,8 +17,9 @@ import (
 
 func TestManagerServe(t *testing.T) {
 	const (
-		sessions = 20
-		msgs     = 50
+		sessions            = 20
+		msgs                = 50
+		remoteUnregisterMsg = "eyeball closed connection"
 	)
 	log := zerolog.Nop()
 	transport := &mockQUICTransport{
@@ -89,7 +90,13 @@ func TestManagerServe(t *testing.T) {
 
 			sessionDone := make(chan struct{})
 			go func() {
-				session.Serve(ctx, time.Minute*2)
+				closedByRemote, err := session.Serve(ctx, time.Minute*2)
+				closeSession := &errClosedSession{
+					message:  remoteUnregisterMsg,
+					byRemote: true,
+				}
+				require.Equal(t, closeSession, err)
+				require.True(t, closedByRemote)
 				close(sessionDone)
 			}()
 
@@ -100,7 +107,7 @@ func TestManagerServe(t *testing.T) {
 			// Make sure eyeball and origin have received all messages before unregistering the session
 			require.NoError(t, reqErrGroup.Wait())
 
-			require.NoError(t, mg.UnregisterSession(ctx, sessionID))
+			require.NoError(t, mg.UnregisterSession(ctx, sessionID, remoteUnregisterMsg, true))
 			<-sessionDone
 
 			return nil

datagramsession/session.go

@@ -2,17 +2,23 @@ package datagramsession
 
 import (
 	"context"
+	"fmt"
 	"io"
 	"time"
 
 	"github.com/google/uuid"
+	"github.com/rs/zerolog"
 )
 
 const (
 	defaultCloseIdleAfter = time.Second * 210
 )
 
-// Each Session is a bidirectional pipe of datagrams between transport and dstConn
+func SessionIdleErr(timeout time.Duration) error {
+	return fmt.Errorf("session idle for %v", timeout)
+}
+
+// Session is a bidirectional pipe of datagrams between transport and dstConn
 // Currently the only implementation of transport is quic DatagramMuxer
 // Destination can be a connection with origin or with eyeball
 // When the destination is origin:
@@ -24,47 +30,56 @@ const (
 // - Datagrams from cloudflared are read by Manager from the transport. Manager finds the corresponding Session and calls the
 //   write method of the Session to send to eyeball
 type Session struct {
-	id        uuid.UUID
+	ID        uuid.UUID
 	transport transport
 	dstConn   io.ReadWriteCloser
 	// activeAtChan is used to communicate the last read/write time
 	activeAtChan chan time.Time
-	doneChan     chan struct{}
+	closeChan    chan error
+	log          *zerolog.Logger
 }
 
-func newSession(id uuid.UUID, transport transport, dstConn io.ReadWriteCloser) *Session {
+func newSession(id uuid.UUID, transport transport, dstConn io.ReadWriteCloser, log *zerolog.Logger) *Session {
 	return &Session{
-		id:        id,
+		ID:        id,
 		transport: transport,
 		dstConn:   dstConn,
 		// activeAtChan has low capacity. It can be full when there are many concurrent read/write. markActive() will
 		// drop instead of blocking because last active time only needs to be an approximation
 		activeAtChan: make(chan time.Time, 2),
-		doneChan:     make(chan struct{}),
+		// capacity is 2 because close() and dstToTransport routine in Serve() can write to this channel
+		closeChan: make(chan error, 2),
+		log:       log,
 	}
 }
 
-func (s *Session) Serve(ctx context.Context, closeAfterIdle time.Duration) error {
-	serveCtx, cancel := context.WithCancel(ctx)
-	defer cancel()
-	go s.waitForCloseCondition(serveCtx, closeAfterIdle)
-	// QUIC implementation copies data to another buffer before returning https://github.com/lucas-clemente/quic-go/blob/v0.24.0/session.go#L1967-L1975
-	// This makes it safe to share readBuffer between iterations
-	readBuffer := make([]byte, s.transport.MTU())
-	for {
-		if err := s.dstToTransport(readBuffer); err != nil {
-			return err
+func (s *Session) Serve(ctx context.Context, closeAfterIdle time.Duration) (closedByRemote bool, err error) {
+	go func() {
+		// QUIC implementation copies data to another buffer before returning https://github.com/lucas-clemente/quic-go/blob/v0.24.0/session.go#L1967-L1975
+		// This makes it safe to share readBuffer between iterations
+		const maxPacketSize = 1500
+		readBuffer := make([]byte, maxPacketSize)
+		for {
+			if err := s.dstToTransport(readBuffer); err != nil {
+				s.closeChan <- err
+				return
+			}
 		}
+	}()
+	err = s.waitForCloseCondition(ctx, closeAfterIdle)
+	if closeSession, ok := err.(*errClosedSession); ok {
+		closedByRemote = closeSession.byRemote
 	}
+	return closedByRemote, err
 }
 
-func (s *Session) waitForCloseCondition(ctx context.Context, closeAfterIdle time.Duration) {
+func (s *Session) waitForCloseCondition(ctx context.Context, closeAfterIdle time.Duration) error {
+	// Closing dstConn cancels read so dstToTransport routine in Serve() can return
+	defer s.dstConn.Close()
 	if closeAfterIdle == 0 {
 		// provide deafult is caller doesn't specify one
 		closeAfterIdle = defaultCloseIdleAfter
 	}
-	// Closing dstConn cancels read so Serve function can return
-	defer s.dstConn.Close()
 
 	checkIdleFreq := closeAfterIdle / 8
 	checkIdleTicker := time.NewTicker(checkIdleFreq)
@@ -74,14 +89,14 @@ func (s *Session) waitForCloseCondition(ctx context.Context, closeAfterIdle time
 	for {
 		select {
 		case <-ctx.Done():
-			return
-		case <-s.doneChan:
-			return
+			return ctx.Err()
+		case reason := <-s.closeChan:
+			return reason
 		// TODO: TUN-5423 evaluate if using atomic is more efficient
 		case now := <-checkIdleTicker.C:
 			// The session is considered inactive if current time is after (last active time + allowed idle time)
 			if now.After(activeAt.Add(closeAfterIdle)) {
-				return
+				return SessionIdleErr(closeAfterIdle)
 			}
 		case activeAt = <-s.activeAtChan: // Update last active time
 		}
@@ -91,11 +106,23 @@ func (s *Session) waitForCloseCondition(ctx context.Context, closeAfterIdle time
 func (s *Session) dstToTransport(buffer []byte) error {
 	n, err := s.dstConn.Read(buffer)
 	s.markActive()
+	// https://pkg.go.dev/io#Reader suggests caller should always process n > 0 bytes
 	if n > 0 {
-		if err := s.transport.SendTo(s.id, buffer[:n]); err != nil {
-			return err
+		if n <= int(s.transport.MTU()) {
+			err = s.transport.SendTo(s.ID, buffer[:n])
+		} else {
+			// drop packet for now, eventually reply with ICMP for PMTUD
+			s.log.Debug().
+				Str("session", s.ID.String()).
+				Int("len", n).
+				Int("mtu", s.transport.MTU()).
+				Msg("dropped packet exceeding MTU")
 		}
 	}
+	// Some UDP application might send 0-size payload.
+	if err == nil && n == 0 {
+		err = s.transport.SendTo(s.ID, []byte{})
+	}
 	return err
 }
 
@@ -113,6 +140,6 @@ func (s *Session) markActive() {
 	}
 }
 
-func (s *Session) close() {
-	close(s.doneChan)
+func (s *Session) close(err *errClosedSession) {
+	s.closeChan <- err
 }

datagramsession/session_test.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"github.com/google/uuid"
+	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/require"
 	"golang.org/x/sync/errgroup"
 )
@@ -31,6 +32,12 @@ func TestCloseIdle(t *testing.T) {
 }
 
 func testSessionReturns(t *testing.T, closeBy closeMethod, closeAfterIdle time.Duration) {
+	var (
+		localCloseReason = &errClosedSession{
+			message:  "connection closed by origin",
+			byRemote: false,
+		}
+	)
 	sessionID := uuid.New()
 	cfdConn, originConn := net.Pipe()
 	payload := testPayload(sessionID)
@@ -38,12 +45,24 @@ func testSessionReturns(t *testing.T, closeBy closeMethod, closeAfterIdle time.D
 		reqChan:  newDatagramChannel(1),
 		respChan: newDatagramChannel(1),
 	}
-	session := newSession(sessionID, transport, cfdConn)
+	log := zerolog.Nop()
+	session := newSession(sessionID, transport, cfdConn, &log)
 
 	ctx, cancel := context.WithCancel(context.Background())
 	sessionDone := make(chan struct{})
 	go func() {
-		session.Serve(ctx, closeAfterIdle)
+		closedByRemote, err := session.Serve(ctx, closeAfterIdle)
+		switch closeBy {
+		case closeByContext:
+			require.Equal(t, context.Canceled, err)
+			require.False(t, closedByRemote)
+		case closeByCallingClose:
+			require.Equal(t, localCloseReason, err)
+			require.Equal(t, localCloseReason.byRemote, closedByRemote)
+		case closeByTimeout:
+			require.Equal(t, SessionIdleErr(closeAfterIdle), err)
+			require.False(t, closedByRemote)
+		}
 		close(sessionDone)
 	}()
 
@@ -64,7 +83,7 @@ func testSessionReturns(t *testing.T, closeBy closeMethod, closeAfterIdle time.D
 	case closeByContext:
 		cancel()
 	case closeByCallingClose:
-		session.close()
+		session.close(localCloseReason)
 	}
 
 	<-sessionDone
@@ -102,7 +121,8 @@ func testActiveSessionNotClosed(t *testing.T, readFromDst bool, writeToDst bool)
 		reqChan:  newDatagramChannel(100),
 		respChan: newDatagramChannel(100),
 	}
-	session := newSession(sessionID, transport, cfdConn)
+	log := zerolog.Nop()
+	session := newSession(sessionID, transport, cfdConn, &log)
 
 	startTime := time.Now()
 	activeUntil := startTime.Add(activeTime)
@@ -164,7 +184,7 @@ func testActiveSessionNotClosed(t *testing.T, readFromDst bool, writeToDst bool)
 
 func TestMarkActiveNotBlocking(t *testing.T) {
 	const concurrentCalls = 50
-	session := newSession(uuid.New(), nil, nil)
+	session := newSession(uuid.New(), nil, nil, nil)
 	var wg sync.WaitGroup
 	wg.Add(concurrentCalls)
 	for i := 0; i < concurrentCalls; i++ {
@@ -175,3 +195,40 @@ func TestMarkActiveNotBlocking(t *testing.T) {
 	}
 	wg.Wait()
 }
+
+func TestZeroBytePayload(t *testing.T) {
+	sessionID := uuid.New()
+	cfdConn, originConn := net.Pipe()
+	transport := &mockQUICTransport{
+		reqChan:  newDatagramChannel(1),
+		respChan: newDatagramChannel(1),
+	}
+	log := zerolog.Nop()
+	session := newSession(sessionID, transport, cfdConn, &log)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	errGroup, ctx := errgroup.WithContext(ctx)
+	errGroup.Go(func() error {
+		// Read from underlying conn and send to transport
+		closedByRemote, err := session.Serve(ctx, time.Minute*2)
+		require.Equal(t, context.Canceled, err)
+		require.False(t, closedByRemote)
+		return nil
+	})
+
+	errGroup.Go(func() error {
+		// Write to underlying connection
+		n, err := originConn.Write([]byte{})
+		require.NoError(t, err)
+		require.Equal(t, 0, n)
+		return nil
+	})
+
+	receivedSessionID, payload, err := transport.respChan.Receive(ctx)
+	require.NoError(t, err)
+	require.Len(t, payload, 0)
+	require.Equal(t, sessionID, receivedSessionID)
+
+	cancel()
+	require.NoError(t, errGroup.Wait())
+}

datagramsession/transport.go

@@ -8,6 +8,6 @@ type transport interface {
 	SendTo(sessionID uuid.UUID, payload []byte) error
 	// ReceiveFrom reads the next datagram from the transport
 	ReceiveFrom() (uuid.UUID, []byte, error)
-	// Max transmission unit of the transport
-	MTU() uint
+	// Max transmission unit to receive from the transport
+	MTU() int
 }

datagramsession/transport_test.go

@@ -22,8 +22,8 @@ func (mt *mockQUICTransport) ReceiveFrom() (uuid.UUID, []byte, error) {
 	return mt.reqChan.Receive(context.Background())
 }
 
-func (mt *mockQUICTransport) MTU() uint {
-	return 1220
+func (mt *mockQUICTransport) MTU() int {
+	return 1280
 }
 
 func (mt *mockQUICTransport) newRequest(ctx context.Context, sessionID uuid.UUID, payload []byte) error {

dev.Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.16.4 as builder
+FROM golang:1.17.5 as builder
 ENV GO111MODULE=on \
     CGO_ENABLED=0
 WORKDIR /go/src/github.com/cloudflare/cloudflared/

go.mod

@@ -6,7 +6,7 @@ require (
 	github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894 // indirect
 	github.com/cloudflare/brotli-go v0.0.0-20191101163834-d34379f7ff93
 	github.com/cloudflare/golibs v0.0.0-20170913112048-333127dbecfc
-	github.com/coredns/coredns v1.7.0
+	github.com/coredns/coredns v1.8.7
 	github.com/coreos/go-oidc v0.0.0-20171002155002-a93f71fdfe73
 	github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf
 	github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
@@ -22,52 +22,50 @@ require (
 	github.com/gobwas/pool v0.2.1 // indirect
 	github.com/gobwas/ws v1.0.4
 	github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3
-	github.com/google/uuid v1.1.2
-	github.com/gorilla/mux v1.7.3
+	github.com/google/uuid v1.3.0
+	github.com/gorilla/mux v1.8.0
 	github.com/gorilla/websocket v1.4.2
-	github.com/json-iterator/go v1.1.10
-	github.com/kr/text v0.2.0 // indirect
+	github.com/json-iterator/go v1.1.12
 	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/lucas-clemente/quic-go v0.24.0
 	github.com/mattn/go-colorable v0.1.8
-	github.com/miekg/dns v1.1.31
+	github.com/miekg/dns v1.1.45
 	github.com/mitchellh/go-homedir v1.1.0
-	github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
 	github.com/pkg/errors v0.9.1
 	github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 // indirect
-	github.com/prometheus/client_golang v1.7.1
+	github.com/prometheus/client_golang v1.11.0
 	github.com/prometheus/client_model v0.2.0
-	github.com/prometheus/common v0.13.0 // indirect
+	github.com/prometheus/common v0.32.1 // indirect
 	github.com/rivo/tview v0.0.0-20200712113419-c65badfc3d92
 	github.com/rs/zerolog v1.20.0
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
-	github.com/stretchr/testify v1.6.0
-	github.com/urfave/cli/v2 v2.2.0
+	github.com/stretchr/testify v1.7.0
+	github.com/urfave/cli/v2 v2.3.0
 	go.uber.org/automaxprocs v1.4.0
 	golang.org/x/crypto v0.0.0-20210921155107-089bfa567519
-	golang.org/x/net v0.0.0-20211109214657-ef0fda0de508
-	golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43 // indirect
+	golang.org/x/net v0.0.0-20220114011407-0dd24b26b47d
+	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
-	golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1
-	golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf
-	google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d // indirect
-	google.golang.org/grpc v1.32.0 // indirect
-	gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
+	golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e
+	golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b
+	google.golang.org/genproto v0.0.0-20211223182754-3ac035c7e7cb // indirect
+	google.golang.org/grpc v1.43.0 // indirect
 	gopkg.in/coreos/go-oidc.v2 v2.1.0
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
 	gopkg.in/square/go-jose.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0
-	gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
 	zombiezen.com/go/capnproto2 v2.18.0+incompatible
 )
 
 require (
 	github.com/BurntSushi/toml v0.3.1 // indirect
+	github.com/apparentlymart/go-cidr v1.1.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/caddyserver/caddy v1.0.5 // indirect
 	github.com/cespare/xxhash/v2 v2.1.1 // indirect
 	github.com/cheekybits/genny v1.0.0 // indirect
+	github.com/coredns/caddy v1.1.1 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 // indirect
 	github.com/francoispqt/gojay v1.2.13 // indirect
@@ -78,23 +76,26 @@ require (
 	github.com/lucasb-eyer/go-colorful v1.0.3 // indirect
 	github.com/marten-seemann/qtls-go1-16 v0.1.4 // indirect
 	github.com/marten-seemann/qtls-go1-17 v0.1.0 // indirect
+	github.com/marten-seemann/qtls-go1-18 v0.1.0-beta.1 // indirect
 	github.com/mattn/go-isatty v0.0.12 // indirect
 	github.com/mattn/go-runewidth v0.0.8 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
-	github.com/modern-go/reflect2 v1.0.1 // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/nxadm/tail v1.4.8 // indirect
-	github.com/onsi/ginkgo v1.16.4 // indirect
+	github.com/onsi/ginkgo v1.16.5 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/procfs v0.1.3 // indirect
+	github.com/prometheus/procfs v0.6.0 // indirect
 	github.com/rivo/uniseg v0.1.0 // indirect
 	golang.org/x/mod v0.4.2 // indirect
-	golang.org/x/text v0.3.6 // indirect
-	golang.org/x/tools v0.1.1 // indirect
+	golang.org/x/text v0.3.7 // indirect
+	golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2 // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
-	google.golang.org/appengine v1.6.6 // indirect
-	google.golang.org/protobuf v1.26.0 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/protobuf v1.27.1 // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 )
 
 replace github.com/urfave/cli/v2 => github.com/ipostelnik/cli/v2 v2.3.1-0.20210324024421-b6ea8234fe3d
+
+replace github.com/lucas-clemente/quic-go => github.com/chungthuang/quic-go v0.24.1-0.20220110095058-981dc498cb62

h2mux/h2_compressor_brotli.go

@@ -1,3 +1,4 @@
+//go:build cgo
 // +build cgo
 
 package h2mux

h2mux/h2_compressor_none.go

@@ -1,3 +1,4 @@
+//go:build !cgo
 // +build !cgo
 
 package h2mux

ingress/config.go

@@ -1,6 +1,7 @@
 package ingress
 
 import (
+	"encoding/json"
 	"time"
 
 	"github.com/urfave/cli/v2"
@@ -38,6 +39,34 @@ const (
 	socksProxy = "socks"
 )
 
+// RemoteConfig models ingress settings that can be managed remotely, for example through the dashboard.
+type RemoteConfig struct {
+	Ingress     Ingress
+	WarpRouting config.WarpRoutingConfig
+}
+
+type remoteConfigJSON struct {
+	GlobalOriginRequest config.OriginRequestConfig      `json:"originRequest"`
+	IngressRules        []config.UnvalidatedIngressRule `json:"ingress"`
+	WarpRouting         config.WarpRoutingConfig        `json:"warp-routing"`
+}
+
+func (rc *RemoteConfig) UnmarshalJSON(b []byte) error {
+	var rawConfig remoteConfigJSON
+	if err := json.Unmarshal(b, &rawConfig); err != nil {
+		return err
+	}
+	ingress, err := validateIngress(rawConfig.IngressRules, originRequestFromConfig(rawConfig.GlobalOriginRequest))
+	if err != nil {
+		return err
+	}
+
+	rc.Ingress = ingress
+	rc.WarpRouting = rawConfig.WarpRouting
+
+	return nil
+}
+
 func originRequestFromSingeRule(c *cli.Context) OriginRequestConfig {
 	var connectTimeout time.Duration = defaultConnectTimeout
 	var tlsTimeout time.Duration = defaultTLSTimeout
@@ -119,7 +148,7 @@ func originRequestFromSingeRule(c *cli.Context) OriginRequestConfig {
 	}
 }
 
-func originRequestFromYAML(y config.OriginRequestConfig) OriginRequestConfig {
+func originRequestFromConfig(c config.OriginRequestConfig) OriginRequestConfig {
 	out := OriginRequestConfig{
 		ConnectTimeout:       defaultConnectTimeout,
 		TLSTimeout:           defaultTLSTimeout,
@@ -128,50 +157,58 @@ func originRequestFromYAML(y config.OriginRequestConfig) OriginRequestConfig {
 		KeepAliveTimeout:     defaultKeepAliveTimeout,
 		ProxyAddress:         defaultProxyAddress,
 	}
-	if y.ConnectTimeout != nil {
-		out.ConnectTimeout = *y.ConnectTimeout
+	if c.ConnectTimeout != nil {
+		out.ConnectTimeout = *c.ConnectTimeout
 	}
-	if y.TLSTimeout != nil {
-		out.TLSTimeout = *y.TLSTimeout
+	if c.TLSTimeout != nil {
+		out.TLSTimeout = *c.TLSTimeout
 	}
-	if y.TCPKeepAlive != nil {
-		out.TCPKeepAlive = *y.TCPKeepAlive
+	if c.TCPKeepAlive != nil {
+		out.TCPKeepAlive = *c.TCPKeepAlive
 	}
-	if y.NoHappyEyeballs != nil {
-		out.NoHappyEyeballs = *y.NoHappyEyeballs
+	if c.NoHappyEyeballs != nil {
+		out.NoHappyEyeballs = *c.NoHappyEyeballs
 	}
-	if y.KeepAliveConnections != nil {
-		out.KeepAliveConnections = *y.KeepAliveConnections
+	if c.KeepAliveConnections != nil {
+		out.KeepAliveConnections = *c.KeepAliveConnections
 	}
-	if y.KeepAliveTimeout != nil {
-		out.KeepAliveTimeout = *y.KeepAliveTimeout
+	if c.KeepAliveTimeout != nil {
+		out.KeepAliveTimeout = *c.KeepAliveTimeout
 	}
-	if y.HTTPHostHeader != nil {
-		out.HTTPHostHeader = *y.HTTPHostHeader
+	if c.HTTPHostHeader != nil {
+		out.HTTPHostHeader = *c.HTTPHostHeader
 	}
-	if y.OriginServerName != nil {
-		out.OriginServerName = *y.OriginServerName
+	if c.OriginServerName != nil {
+		out.OriginServerName = *c.OriginServerName
 	}
-	if y.CAPool != nil {
-		out.CAPool = *y.CAPool
+	if c.CAPool != nil {
+		out.CAPool = *c.CAPool
 	}
-	if y.NoTLSVerify != nil {
-		out.NoTLSVerify = *y.NoTLSVerify
+	if c.NoTLSVerify != nil {
+		out.NoTLSVerify = *c.NoTLSVerify
 	}
-	if y.DisableChunkedEncoding != nil {
-		out.DisableChunkedEncoding = *y.DisableChunkedEncoding
+	if c.DisableChunkedEncoding != nil {
+		out.DisableChunkedEncoding = *c.DisableChunkedEncoding
 	}
-	if y.BastionMode != nil {
-		out.BastionMode = *y.BastionMode
+	if c.BastionMode != nil {
+		out.BastionMode = *c.BastionMode
 	}
-	if y.ProxyAddress != nil {
-		out.ProxyAddress = *y.ProxyAddress
+	if c.ProxyAddress != nil {
+		out.ProxyAddress = *c.ProxyAddress
 	}
-	if y.ProxyPort != nil {
-		out.ProxyPort = *y.ProxyPort
+	if c.ProxyPort != nil {
+		out.ProxyPort = *c.ProxyPort
 	}
-	if y.ProxyType != nil {
-		out.ProxyType = *y.ProxyType
+	if c.ProxyType != nil {
+		out.ProxyType = *c.ProxyType
+	}
+	if len(c.IPRules) > 0 {
+		for _, r := range c.IPRules {
+			rule, err := ipaccess.NewRuleByCIDR(r.Prefix, r.Ports, r.Allow)
+			if err == nil {
+				out.IPRules = append(out.IPRules, rule)
+			}
+		}
 	}
 	return out
 }
@@ -188,10 +225,10 @@ type OriginRequestConfig struct {
 	TCPKeepAlive time.Duration `yaml:"tcpKeepAlive"`
 	// HTTP proxy should disable "happy eyeballs" for IPv4/v6 fallback
 	NoHappyEyeballs bool `yaml:"noHappyEyeballs"`
-	// HTTP proxy maximum keepalive connection pool size
-	KeepAliveConnections int `yaml:"keepAliveConnections"`
 	// HTTP proxy timeout for closing an idle connection
 	KeepAliveTimeout time.Duration `yaml:"keepAliveTimeout"`
+	// HTTP proxy maximum keepalive connection pool size
+	KeepAliveConnections int `yaml:"keepAliveConnections"`
 	// Sets the HTTP Host header for the local webserver.
 	HTTPHostHeader string `yaml:"httpHostHeader"`
 	// Hostname on the origin server certificate.
@@ -308,6 +345,19 @@ func (defaults *OriginRequestConfig) setProxyType(overrides config.OriginRequest
 	}
 }
 
+func (defaults *OriginRequestConfig) setIPRules(overrides config.OriginRequestConfig) {
+	if val := overrides.IPRules; len(val) > 0 {
+		ipAccessRule := make([]ipaccess.Rule, len(overrides.IPRules))
+		for i, r := range overrides.IPRules {
+			rule, err := ipaccess.NewRuleByCIDR(r.Prefix, r.Ports, r.Allow)
+			if err == nil {
+				ipAccessRule[i] = rule
+			}
+		}
+		defaults.IPRules = ipAccessRule
+	}
+}
+
 // SetConfig gets config for the requests that cloudflared sends to origins.
 // Each field has a setter method which sets a value for the field by trying to find:
 //   1. The user config for this rule
@@ -332,5 +382,6 @@ func setConfig(defaults OriginRequestConfig, overrides config.OriginRequestConfi
 	cfg.setProxyPort(overrides)
 	cfg.setProxyAddress(overrides)
 	cfg.setProxyType(overrides)
+	cfg.setIPRules(overrides)
 	return cfg
 }

ingress/config_test.go

@@ -0,0 +1,422 @@
+package ingress
+
+import (
+	"encoding/json"
+	"flag"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+	"github.com/urfave/cli/v2"
+	yaml "gopkg.in/yaml.v2"
+
+	"github.com/cloudflare/cloudflared/config"
+	"github.com/cloudflare/cloudflared/ipaccess"
+)
+
+// Ensure that the nullable config from `config` package and the
+// non-nullable config from `ingress` package have the same number of
+// fields.
+// This test ensures that programmers didn't add a new field to
+// one struct and forget to add it to the other ;)
+func TestCorrespondingFields(t *testing.T) {
+	require.Equal(
+		t,
+		CountFields(t, config.OriginRequestConfig{}),
+		CountFields(t, OriginRequestConfig{}),
+	)
+}
+
+func CountFields(t *testing.T, val interface{}) int {
+	b, err := yaml.Marshal(val)
+	require.NoError(t, err)
+	m := make(map[string]interface{}, 0)
+	err = yaml.Unmarshal(b, &m)
+	require.NoError(t, err)
+	return len(m)
+}
+
+func TestUnmarshalRemoteConfigOverridesGlobal(t *testing.T) {
+	rawConfig := []byte(`
+{
+    "originRequest": {
+        "connectTimeout": 90,
+		"noHappyEyeballs": true
+    },
+    "ingress": [
+        {
+            "hostname": "jira.cfops.com",
+            "service": "http://192.16.19.1:80",
+            "originRequest": {
+                "noTLSVerify": true,
+                "connectTimeout": 10
+            }
+        },
+        {
+            "service": "http_status:404"
+        }
+    ],
+    "warp-routing": {
+        "enabled": true
+    }
+}	
+`)
+	var remoteConfig RemoteConfig
+	err := json.Unmarshal(rawConfig, &remoteConfig)
+	require.NoError(t, err)
+	require.True(t, remoteConfig.Ingress.Rules[0].Config.NoTLSVerify)
+	require.True(t, remoteConfig.Ingress.defaults.NoHappyEyeballs)
+}
+
+func TestOriginRequestConfigOverrides(t *testing.T) {
+	validate := func(ing Ingress) {
+		// Rule 0 didn't override anything, so it inherits the user-specified
+		// root-level configuration.
+		actual0 := ing.Rules[0].Config
+		expected0 := OriginRequestConfig{
+			ConnectTimeout:         1 * time.Minute,
+			TLSTimeout:             1 * time.Second,
+			TCPKeepAlive:           1 * time.Second,
+			NoHappyEyeballs:        true,
+			KeepAliveTimeout:       1 * time.Second,
+			KeepAliveConnections:   1,
+			HTTPHostHeader:         "abc",
+			OriginServerName:       "a1",
+			CAPool:                 "/tmp/path0",
+			NoTLSVerify:            true,
+			DisableChunkedEncoding: true,
+			BastionMode:            true,
+			ProxyAddress:           "127.1.2.3",
+			ProxyPort:              uint(100),
+			ProxyType:              "socks5",
+			IPRules: []ipaccess.Rule{
+				newIPRule(t, "10.0.0.0/8", []int{80, 8080}, false),
+				newIPRule(t, "fc00::/7", []int{443, 4443}, true),
+			},
+		}
+		require.Equal(t, expected0, actual0)
+
+		// Rule 1 overrode all the root-level config.
+		actual1 := ing.Rules[1].Config
+		expected1 := OriginRequestConfig{
+			ConnectTimeout:         2 * time.Minute,
+			TLSTimeout:             2 * time.Second,
+			TCPKeepAlive:           2 * time.Second,
+			NoHappyEyeballs:        false,
+			KeepAliveTimeout:       2 * time.Second,
+			KeepAliveConnections:   2,
+			HTTPHostHeader:         "def",
+			OriginServerName:       "b2",
+			CAPool:                 "/tmp/path1",
+			NoTLSVerify:            false,
+			DisableChunkedEncoding: false,
+			BastionMode:            false,
+			ProxyAddress:           "interface",
+			ProxyPort:              uint(200),
+			ProxyType:              "",
+			IPRules: []ipaccess.Rule{
+				newIPRule(t, "10.0.0.0/16", []int{3000, 3030}, false),
+				newIPRule(t, "192.16.0.0/24", []int{5000, 5050}, true),
+			},
+		}
+		require.Equal(t, expected1, actual1)
+	}
+
+	rulesYAML := `
+originRequest:
+  connectTimeout: 1m
+  tlsTimeout: 1s
+  noHappyEyeballs: true
+  tcpKeepAlive: 1s
+  keepAliveConnections: 1
+  keepAliveTimeout: 1s
+  httpHostHeader: abc
+  originServerName: a1
+  caPool: /tmp/path0
+  noTLSVerify: true
+  disableChunkedEncoding: true
+  bastionMode: True
+  proxyAddress: 127.1.2.3
+  proxyPort: 100
+  proxyType: socks5
+  ipRules:
+  - prefix: "10.0.0.0/8"
+    ports:
+    - 80
+    - 8080
+    allow: false
+  - prefix: "fc00::/7"
+    ports:
+    - 443
+    - 4443
+    allow: true
+ingress:
+- hostname: tun.example.com
+  service: https://localhost:8000
+- hostname: "*"
+  service: https://localhost:8001
+  originRequest:
+    connectTimeout: 2m
+    tlsTimeout: 2s
+    noHappyEyeballs: false
+    tcpKeepAlive: 2s
+    keepAliveConnections: 2
+    keepAliveTimeout: 2s
+    httpHostHeader: def
+    originServerName: b2
+    caPool: /tmp/path1
+    noTLSVerify: false
+    disableChunkedEncoding: false
+    bastionMode: false
+    proxyAddress: interface
+    proxyPort: 200
+    proxyType: ""
+    ipRules:
+    - prefix: "10.0.0.0/16"
+      ports:
+      - 3000
+      - 3030
+      allow: false
+    - prefix: "192.16.0.0/24"
+      ports:
+      - 5000
+      - 5050
+      allow: true
+`
+
+	ing, err := ParseIngress(MustReadIngress(rulesYAML))
+	require.NoError(t, err)
+	validate(ing)
+
+	rawConfig := []byte(`
+{
+    "originRequest": {
+        "connectTimeout": 60000000000,
+		"tlsTimeout": 1000000000,
+		"noHappyEyeballs": true,
+		"tcpKeepAlive": 1000000000,
+		"keepAliveConnections": 1,
+		"keepAliveTimeout": 1000000000,
+		"httpHostHeader": "abc",
+		"originServerName": "a1",
+		"caPool": "/tmp/path0",
+		"noTLSVerify": true,
+		"disableChunkedEncoding": true,
+		"bastionMode": true,
+		"proxyAddress": "127.1.2.3",
+		"proxyPort": 100,
+		"proxyType": "socks5",
+		"ipRules": [
+			{
+				"prefix": "10.0.0.0/8",
+				"ports": [80, 8080],
+				"allow": false
+			},
+			{
+				"prefix": "fc00::/7",
+				"ports": [443, 4443],
+				"allow": true
+			}
+		]
+    },
+    "ingress": [
+        {
+            "hostname": "tun.example.com",
+            "service": "https://localhost:8000"
+        },
+        {
+			"hostname": "*",
+            "service": "https://localhost:8001",
+			"originRequest": {
+				"connectTimeout": 120000000000,
+				"tlsTimeout": 2000000000,
+				"noHappyEyeballs": false,
+				"tcpKeepAlive": 2000000000,
+				"keepAliveConnections": 2,
+				"keepAliveTimeout": 2000000000,
+				"httpHostHeader": "def",
+				"originServerName": "b2",
+				"caPool": "/tmp/path1",
+				"noTLSVerify": false,
+				"disableChunkedEncoding": false,
+				"bastionMode": false,
+				"proxyAddress": "interface",
+				"proxyPort": 200,
+				"proxyType": "",
+				"ipRules": [
+					{
+						"prefix": "10.0.0.0/16",
+						"ports": [3000, 3030],
+						"allow": false
+					},
+					{
+						"prefix": "192.16.0.0/24",
+						"ports": [5000, 5050],
+						"allow": true
+					}
+				]
+    		}
+        }
+    ],
+    "warp-routing": {
+        "enabled": true
+    }
+}	
+`)
+	var remoteConfig RemoteConfig
+	err = json.Unmarshal(rawConfig, &remoteConfig)
+	require.NoError(t, err)
+	validate(remoteConfig.Ingress)
+}
+
+func TestOriginRequestConfigDefaults(t *testing.T) {
+	validate := func(ing Ingress) {
+		// Rule 0 didn't override anything, so it inherits the cloudflared defaults
+		actual0 := ing.Rules[0].Config
+		expected0 := OriginRequestConfig{
+			ConnectTimeout:       defaultConnectTimeout,
+			TLSTimeout:           defaultTLSTimeout,
+			TCPKeepAlive:         defaultTCPKeepAlive,
+			KeepAliveConnections: defaultKeepAliveConnections,
+			KeepAliveTimeout:     defaultKeepAliveTimeout,
+			ProxyAddress:         defaultProxyAddress,
+		}
+		require.Equal(t, expected0, actual0)
+
+		// Rule 1 overrode all defaults.
+		actual1 := ing.Rules[1].Config
+		expected1 := OriginRequestConfig{
+			ConnectTimeout:         2 * time.Minute,
+			TLSTimeout:             2 * time.Second,
+			TCPKeepAlive:           2 * time.Second,
+			NoHappyEyeballs:        false,
+			KeepAliveTimeout:       2 * time.Second,
+			KeepAliveConnections:   2,
+			HTTPHostHeader:         "def",
+			OriginServerName:       "b2",
+			CAPool:                 "/tmp/path1",
+			NoTLSVerify:            false,
+			DisableChunkedEncoding: false,
+			BastionMode:            false,
+			ProxyAddress:           "interface",
+			ProxyPort:              uint(200),
+			ProxyType:              "",
+			IPRules: []ipaccess.Rule{
+				newIPRule(t, "10.0.0.0/16", []int{3000, 3030}, false),
+				newIPRule(t, "192.16.0.0/24", []int{5000, 5050}, true),
+			},
+		}
+		require.Equal(t, expected1, actual1)
+	}
+
+	rulesYAML := `
+ingress:
+- hostname: tun.example.com
+  service: https://localhost:8000
+- hostname: "*"
+  service: https://localhost:8001
+  originRequest:
+    connectTimeout: 2m
+    tlsTimeout: 2s
+    noHappyEyeballs: false
+    tcpKeepAlive: 2s
+    keepAliveConnections: 2
+    keepAliveTimeout: 2s
+    httpHostHeader: def
+    originServerName: b2
+    caPool: /tmp/path1
+    noTLSVerify: false
+    disableChunkedEncoding: false
+    bastionMode: false
+    proxyAddress: interface
+    proxyPort: 200
+    proxyType: ""
+    ipRules:
+    - prefix: "10.0.0.0/16"
+      ports:
+      - 3000
+      - 3030
+      allow: false
+    - prefix: "192.16.0.0/24"
+      ports:
+      - 5000
+      - 5050
+      allow: true
+`
+	ing, err := ParseIngress(MustReadIngress(rulesYAML))
+	if err != nil {
+		t.Error(err)
+	}
+	validate(ing)
+
+	rawConfig := []byte(`
+{
+	"ingress": [
+        {
+            "hostname": "tun.example.com",
+            "service": "https://localhost:8000"
+        },
+        {
+			"hostname": "*",
+            "service": "https://localhost:8001",
+			"originRequest": {
+				"connectTimeout": 120000000000,
+				"tlsTimeout": 2000000000,
+				"noHappyEyeballs": false,
+				"tcpKeepAlive": 2000000000,
+				"keepAliveConnections": 2,
+				"keepAliveTimeout": 2000000000,
+				"httpHostHeader": "def",
+				"originServerName": "b2",
+				"caPool": "/tmp/path1",
+				"noTLSVerify": false,
+				"disableChunkedEncoding": false,
+				"bastionMode": false,
+				"proxyAddress": "interface",
+				"proxyPort": 200,
+				"proxyType": "",
+				"ipRules": [
+					{
+						"prefix": "10.0.0.0/16",
+						"ports": [3000, 3030],
+						"allow": false
+					},
+					{
+						"prefix": "192.16.0.0/24",
+						"ports": [5000, 5050],
+						"allow": true
+					}
+				]
+			}
+		}
+	]
+}
+`)
+
+	var remoteConfig RemoteConfig
+	err = json.Unmarshal(rawConfig, &remoteConfig)
+	require.NoError(t, err)
+	validate(remoteConfig.Ingress)
+}
+
+func TestDefaultConfigFromCLI(t *testing.T) {
+	set := flag.NewFlagSet("contrive", 0)
+	c := cli.NewContext(nil, set, nil)
+
+	expected := OriginRequestConfig{
+		ConnectTimeout:       defaultConnectTimeout,
+		TLSTimeout:           defaultTLSTimeout,
+		TCPKeepAlive:         defaultTCPKeepAlive,
+		KeepAliveConnections: defaultKeepAliveConnections,
+		KeepAliveTimeout:     defaultKeepAliveTimeout,
+		ProxyAddress:         defaultProxyAddress,
+	}
+	actual := originRequestFromSingeRule(c)
+	require.Equal(t, expected, actual)
+}
+
+func newIPRule(t *testing.T, prefix string, ports []int, allow bool) ipaccess.Rule {
+	rule, err := ipaccess.NewRuleByCIDR(&prefix, ports, allow)
+	require.NoError(t, err)
+	return rule
+}

ingress/ingress.go

@@ -163,7 +163,7 @@ func (ing Ingress) CatchAll() *Rule {
 	return &ing.Rules[len(ing.Rules)-1]
 }
 
-func validate(ingress []config.UnvalidatedIngressRule, defaults OriginRequestConfig) (Ingress, error) {
+func validateIngress(ingress []config.UnvalidatedIngressRule, defaults OriginRequestConfig) (Ingress, error) {
 	rules := make([]Rule, len(ingress))
 	for i, r := range ingress {
 		cfg := setConfig(defaults, r.OriginRequest)
@@ -290,7 +290,7 @@ func ParseIngress(conf *config.Configuration) (Ingress, error) {
 	if len(conf.Ingress) == 0 {
 		return Ingress{}, ErrNoIngressRules
 	}
-	return validate(conf.Ingress, originRequestFromYAML(conf.OriginRequest))
+	return validateIngress(conf.Ingress, originRequestFromConfig(conf.OriginRequest))
 }
 
 func isHTTPService(url *url.URL) bool {

ingress/ingress_test.go

@@ -34,7 +34,7 @@ func Test_parseIngress(t *testing.T) {
 	localhost8000 := MustParseURL(t, "https://localhost:8000")
 	localhost8001 := MustParseURL(t, "https://localhost:8001")
 	fourOhFour := newStatusCode(404)
-	defaultConfig := setConfig(originRequestFromYAML(config.OriginRequestConfig{}), config.OriginRequestConfig{})
+	defaultConfig := setConfig(originRequestFromConfig(config.OriginRequestConfig{}), config.OriginRequestConfig{})
 	require.Equal(t, defaultKeepAliveConnections, defaultConfig.KeepAliveConnections)
 	tr := true
 	type args struct {
@@ -324,7 +324,17 @@ ingress:
 				{
 					Hostname: "socks.foo.com",
 					Service:  newSocksProxyOverWSService(accessPolicy()),
-					Config:   defaultConfig,
+					Config: setConfig(originRequestFromConfig(config.OriginRequestConfig{}), config.OriginRequestConfig{IPRules: []config.IngressIPRule{
+						{
+							Prefix: ipRulePrefix("1.1.1.0/24"),
+							Ports:  []int{80, 443},
+							Allow:  true,
+						},
+						{
+							Prefix: ipRulePrefix("0.0.0.0/0"),
+							Allow:  false,
+						},
+					}}),
 				},
 				{
 					Service: &fourOhFour,
@@ -345,7 +355,7 @@ ingress:
 				{
 					Hostname: "bastion.foo.com",
 					Service:  newBastionService(),
-					Config:   setConfig(originRequestFromYAML(config.OriginRequestConfig{}), config.OriginRequestConfig{BastionMode: &tr}),
+					Config:   setConfig(originRequestFromConfig(config.OriginRequestConfig{}), config.OriginRequestConfig{BastionMode: &tr}),
 				},
 				{
 					Service: &fourOhFour,
@@ -365,7 +375,7 @@ ingress:
 				{
 					Hostname: "bastion.foo.com",
 					Service:  newBastionService(),
-					Config:   setConfig(originRequestFromYAML(config.OriginRequestConfig{}), config.OriginRequestConfig{BastionMode: &tr}),
+					Config:   setConfig(originRequestFromConfig(config.OriginRequestConfig{}), config.OriginRequestConfig{BastionMode: &tr}),
 				},
 				{
 					Service: &fourOhFour,
@@ -397,6 +407,10 @@ ingress:
 	}
 }
 
+func ipRulePrefix(s string) *string {
+	return &s
+}
+
 func TestSingleOriginSetsConfig(t *testing.T) {
 	flagSet := flag.NewFlagSet(t.Name(), flag.PanicOnError)
 	flagSet.Bool("hello-world", true, "")

ingress/origin_proxy.go

@@ -23,6 +23,7 @@ type StreamBasedOriginProxy interface {
 }
 
 func (o *unixSocketPath) RoundTrip(req *http.Request) (*http.Response, error) {
+	req.URL.Scheme = "http"
 	return o.transport.RoundTrip(req)
 }
 
@@ -40,6 +41,8 @@ func (o *httpService) RoundTrip(req *http.Request) (*http.Response, error) {
 
 	if o.hostHeader != "" {
 		// For incoming requests, the Host header is promoted to the Request.Host field and removed from the Header map.
+		// Pass the original Host header as X-Forwarded-Host.
+		req.Header.Set("X-Forwarded-Host", req.Host)
 		req.Host = o.hostHeader
 	}
 	return o.transport.RoundTrip(req)

ingress/origin_proxy_test.go

@@ -3,6 +3,7 @@ package ingress
 import (
 	"context"
 	"fmt"
+	"io/ioutil"
 	"net"
 	"net/http"
 	"net/http/httptest"
@@ -118,7 +119,9 @@ func TestHTTPServiceHostHeaderOverride(t *testing.T) {
 			w.WriteHeader(http.StatusSwitchingProtocols)
 			return
 		}
-		w.Write([]byte("ok"))
+		// return the X-Forwarded-Host header for assertions
+		// as the httptest Server URL isn't available here yet
+		w.Write([]byte(r.Header.Get("X-Forwarded-Host")))
 	}
 	origin := httptest.NewServer(http.HandlerFunc(handler))
 	defer origin.Close()
@@ -141,6 +144,10 @@ func TestHTTPServiceHostHeaderOverride(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, http.StatusOK, resp.StatusCode)
 
+	respBody, err := ioutil.ReadAll(resp.Body)
+	require.NoError(t, err)
+	require.Equal(t, respBody, []byte(originURL.Host))
+
 }
 
 func tcpListenRoutine(listener net.Listener, closeChan chan struct{}) {

ingress/origin_request_config_test.go

@@ -1,203 +0,0 @@
-package ingress
-
-import (
-	"flag"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/require"
-	"github.com/urfave/cli/v2"
-	yaml "gopkg.in/yaml.v2"
-
-	"github.com/cloudflare/cloudflared/config"
-)
-
-// Ensure that the nullable config from `config` package and the
-// non-nullable config from `ingress` package have the same number of
-// fields.
-// This test ensures that programmers didn't add a new field to
-// one struct and forget to add it to the other ;)
-func TestCorrespondingFields(t *testing.T) {
-	require.Equal(
-		t,
-		CountFields(t, config.OriginRequestConfig{}),
-		CountFields(t, OriginRequestConfig{}),
-	)
-}
-
-func CountFields(t *testing.T, val interface{}) int {
-	b, err := yaml.Marshal(val)
-	require.NoError(t, err)
-	m := make(map[string]interface{}, 0)
-	err = yaml.Unmarshal(b, &m)
-	require.NoError(t, err)
-	return len(m)
-}
-
-func TestOriginRequestConfigOverrides(t *testing.T) {
-	rulesYAML := `
-originRequest:
-  connectTimeout: 1m
-  tlsTimeout: 1s
-  noHappyEyeballs: true
-  tcpKeepAlive: 1s
-  keepAliveConnections: 1
-  keepAliveTimeout: 1s
-  httpHostHeader: abc
-  originServerName: a1
-  caPool: /tmp/path0
-  noTLSVerify: true
-  disableChunkedEncoding: true
-  bastionMode: True
-  proxyAddress: 127.1.2.3
-  proxyPort: 100
-  proxyType: socks5
-ingress:
-- hostname: tun.example.com
-  service: https://localhost:8000
-- hostname: "*"
-  service: https://localhost:8001
-  originRequest:
-    connectTimeout: 2m
-    tlsTimeout: 2s
-    noHappyEyeballs: false
-    tcpKeepAlive: 2s
-    keepAliveConnections: 2
-    keepAliveTimeout: 2s
-    httpHostHeader: def
-    originServerName: b2
-    caPool: /tmp/path1
-    noTLSVerify: false
-    disableChunkedEncoding: false
-    bastionMode: false
-    proxyAddress: interface
-    proxyPort: 200
-    proxyType: ""
-`
-	ing, err := ParseIngress(MustReadIngress(rulesYAML))
-	if err != nil {
-		t.Error(err)
-	}
-
-	// Rule 0 didn't override anything, so it inherits the user-specified
-	// root-level configuration.
-	actual0 := ing.Rules[0].Config
-	expected0 := OriginRequestConfig{
-		ConnectTimeout:         1 * time.Minute,
-		TLSTimeout:             1 * time.Second,
-		NoHappyEyeballs:        true,
-		TCPKeepAlive:           1 * time.Second,
-		KeepAliveConnections:   1,
-		KeepAliveTimeout:       1 * time.Second,
-		HTTPHostHeader:         "abc",
-		OriginServerName:       "a1",
-		CAPool:                 "/tmp/path0",
-		NoTLSVerify:            true,
-		DisableChunkedEncoding: true,
-		BastionMode:            true,
-		ProxyAddress:           "127.1.2.3",
-		ProxyPort:              uint(100),
-		ProxyType:              "socks5",
-	}
-	require.Equal(t, expected0, actual0)
-
-	// Rule 1 overrode all the root-level config.
-	actual1 := ing.Rules[1].Config
-	expected1 := OriginRequestConfig{
-		ConnectTimeout:         2 * time.Minute,
-		TLSTimeout:             2 * time.Second,
-		NoHappyEyeballs:        false,
-		TCPKeepAlive:           2 * time.Second,
-		KeepAliveConnections:   2,
-		KeepAliveTimeout:       2 * time.Second,
-		HTTPHostHeader:         "def",
-		OriginServerName:       "b2",
-		CAPool:                 "/tmp/path1",
-		NoTLSVerify:            false,
-		DisableChunkedEncoding: false,
-		BastionMode:            false,
-		ProxyAddress:           "interface",
-		ProxyPort:              uint(200),
-		ProxyType:              "",
-	}
-	require.Equal(t, expected1, actual1)
-}
-
-func TestOriginRequestConfigDefaults(t *testing.T) {
-	rulesYAML := `
-ingress:
-- hostname: tun.example.com
-  service: https://localhost:8000
-- hostname: "*"
-  service: https://localhost:8001
-  originRequest:
-    connectTimeout: 2m
-    tlsTimeout: 2s
-    noHappyEyeballs: false
-    tcpKeepAlive: 2s
-    keepAliveConnections: 2
-    keepAliveTimeout: 2s
-    httpHostHeader: def
-    originServerName: b2
-    caPool: /tmp/path1
-    noTLSVerify: false
-    disableChunkedEncoding: false
-    bastionMode: false
-    proxyAddress: interface
-    proxyPort: 200
-    proxyType: ""
-`
-	ing, err := ParseIngress(MustReadIngress(rulesYAML))
-	if err != nil {
-		t.Error(err)
-	}
-
-	// Rule 0 didn't override anything, so it inherits the cloudflared defaults
-	actual0 := ing.Rules[0].Config
-	expected0 := OriginRequestConfig{
-		ConnectTimeout:       defaultConnectTimeout,
-		TLSTimeout:           defaultTLSTimeout,
-		TCPKeepAlive:         defaultTCPKeepAlive,
-		KeepAliveConnections: defaultKeepAliveConnections,
-		KeepAliveTimeout:     defaultKeepAliveTimeout,
-		ProxyAddress:         defaultProxyAddress,
-	}
-	require.Equal(t, expected0, actual0)
-
-	// Rule 1 overrode all defaults.
-	actual1 := ing.Rules[1].Config
-	expected1 := OriginRequestConfig{
-		ConnectTimeout:         2 * time.Minute,
-		TLSTimeout:             2 * time.Second,
-		NoHappyEyeballs:        false,
-		TCPKeepAlive:           2 * time.Second,
-		KeepAliveConnections:   2,
-		KeepAliveTimeout:       2 * time.Second,
-		HTTPHostHeader:         "def",
-		OriginServerName:       "b2",
-		CAPool:                 "/tmp/path1",
-		NoTLSVerify:            false,
-		DisableChunkedEncoding: false,
-		BastionMode:            false,
-		ProxyAddress:           "interface",
-		ProxyPort:              uint(200),
-		ProxyType:              "",
-	}
-	require.Equal(t, expected1, actual1)
-}
-
-func TestDefaultConfigFromCLI(t *testing.T) {
-	set := flag.NewFlagSet("contrive", 0)
-	c := cli.NewContext(nil, set, nil)
-
-	expected := OriginRequestConfig{
-		ConnectTimeout:       defaultConnectTimeout,
-		TLSTimeout:           defaultTLSTimeout,
-		TCPKeepAlive:         defaultTCPKeepAlive,
-		KeepAliveConnections: defaultKeepAliveConnections,
-		KeepAliveTimeout:     defaultKeepAliveTimeout,
-		ProxyAddress:         defaultProxyAddress,
-	}
-	actual := originRequestFromSingeRule(c)
-	require.Equal(t, expected, actual)
-}

metrics/metrics.go

@@ -83,15 +83,15 @@ func ServeMetrics(
 	return err
 }
 
-func RegisterBuildInfo(buildTime string, version string) {
+func RegisterBuildInfo(buildType, buildTime, version string) {
 	buildInfo := prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			// Don't namespace build_info, since we want it to be consistent across all Cloudflare services
 			Name: "build_info",
 			Help: "Build and version information",
 		},
-		[]string{"goversion", "revision", "version"},
+		[]string{"goversion", "type", "revision", "version"},
 	)
 	prometheus.MustRegister(buildInfo)
-	buildInfo.WithLabelValues(runtime.Version(), buildTime, version).Set(1)
+	buildInfo.WithLabelValues(runtime.Version(), buildType, buildTime, version).Set(1)
 }

origin/proxy.go

@@ -271,11 +271,20 @@ func (wr *bidirectionalStream) Write(p []byte) (n int, err error) {
 func (p *Proxy) writeEventStream(w connection.ResponseWriter, respBody io.ReadCloser) {
 	reader := bufio.NewReader(respBody)
 	for {
-		line, err := reader.ReadBytes('\n')
-		if err != nil {
-			break
+		line, readErr := reader.ReadBytes('\n')
+
+		// We first try to write whatever we read even if an error occurred
+		// The reason for doing it is to guarantee we really push everything to the eyeball side
+		// before returning
+		if len(line) > 0 {
+			if _, writeErr := w.Write(line); writeErr != nil {
+				return
+			}
+		}
+
+		if readErr != nil {
+			return
 		}
-		_, _ = w.Write(line)
 	}
 }
 

origin/proxy_posix_test.go

@@ -1,3 +1,4 @@
+//go:build !windows
 // +build !windows
 
 package origin

origin/proxy_test.go

@@ -9,6 +9,7 @@ import (
 	"net"
 	"net/http"
 	"net/http/httptest"
+	"strings"
 	"sync"
 	"testing"
 	"time"
@@ -138,6 +139,7 @@ func TestProxySingleOrigin(t *testing.T) {
 	t.Run("testProxyHTTP", testProxyHTTP(proxy))
 	t.Run("testProxyWebsocket", testProxyWebsocket(proxy))
 	t.Run("testProxySSE", testProxySSE(proxy))
+	t.Run("testProxySSEAllData", testProxySSEAllData(proxy))
 	cancel()
 	wg.Wait()
 }
@@ -256,6 +258,21 @@ func testProxySSE(proxy connection.OriginProxy) func(t *testing.T) {
 	}
 }
 
+// Regression test to guarantee that we always write the contents downstream even if EOF is reached without
+// hitting the delimiter
+func testProxySSEAllData(proxy *Proxy) func(t *testing.T) {
+	return func(t *testing.T) {
+		eyeballReader := io.NopCloser(strings.NewReader("data\r\r"))
+		responseWriter := newMockSSERespWriter()
+
+		// responseWriter uses an unbuffered channel, so we call in a different go-routine
+		go proxy.writeEventStream(responseWriter, eyeballReader)
+
+		result := string(<-responseWriter.writeNotification)
+		require.Equal(t, "data\r\r", result)
+	}
+}
+
 func TestProxyMultipleOrigins(t *testing.T) {
 	api := httptest.NewServer(mockAPI{})
 	defer api.Close()
@@ -447,7 +464,7 @@ func TestConnections(t *testing.T) {
 		// eyeball connection type.
 		connectionType connection.Type
 
-		//requestheaders to be sent in the call to proxy.Proxy
+		// requestheaders to be sent in the call to proxy.Proxy
 		requestHeaders http.Header
 	}
 
@@ -508,7 +525,7 @@ func TestConnections(t *testing.T) {
 			args: args{
 				ingressServiceScheme: "ws://",
 				originService:        runEchoWSService,
-				//eyeballResponseWriter gets set after roundtrip dial.
+				// eyeballResponseWriter gets set after roundtrip dial.
 				eyeballRequestBody: newPipedWSRequestBody([]byte("test3")),
 				warpRoutingService: ingress.NewWarpRoutingService(),
 				requestHeaders: map[string][]string{

origin/tunnel.go

@@ -31,15 +31,6 @@ const (
 	dialTimeout              = 15 * time.Second
 	FeatureSerializedHeaders = "serialized_headers"
 	FeatureQuickReconnects   = "quick_reconnects"
-	quicHandshakeIdleTimeout = 5 * time.Second
-	quicMaxIdleTimeout       = 15 * time.Second
-)
-
-type rpcName string
-
-const (
-	reconnect    rpcName = "reconnect"
-	authenticate rpcName = " authenticate"
 )
 
 type TunnelConfig struct {
@@ -192,12 +183,11 @@ func ServeTunnelLoop(
 		case <-gracefulShutdownC:
 			return nil
 		case <-protocolFallback.BackoffTimer():
-			var idleTimeoutError *quic.IdleTimeoutError
 			if !selectNextProtocol(
 				connLog.Logger(),
 				protocolFallback,
 				config.ProtocolSelector,
-				errors.As(err, &idleTimeoutError),
+				err,
 			) {
 				return err
 			}
@@ -230,9 +220,13 @@ func selectNextProtocol(
 	connLog *zerolog.Logger,
 	protocolBackoff *protocolFallback,
 	selector connection.ProtocolSelector,
-	isNetworkActivityTimeout bool,
+	cause error,
 ) bool {
-	if protocolBackoff.ReachedMaxRetries() || isNetworkActivityTimeout {
+	var idleTimeoutError *quic.IdleTimeoutError
+	isNetworkActivityTimeout := errors.As(cause, &idleTimeoutError)
+	_, hasFallback := selector.Fallback()
+
+	if protocolBackoff.ReachedMaxRetries() || (hasFallback && isNetworkActivityTimeout) {
 		fallback, hasFallback := selector.Fallback()
 		if !hasFallback {
 			return false
@@ -527,52 +521,48 @@ func ServeQUIC(
 ) (err error, recoverable bool) {
 	tlsConfig := config.EdgeTLSConfigs[connection.QUIC]
 	quicConfig := &quic.Config{
-		HandshakeIdleTimeout:  quicHandshakeIdleTimeout,
-		MaxIdleTimeout:        quicMaxIdleTimeout,
+		HandshakeIdleTimeout:  quicpogs.HandshakeIdleTimeout,
+		MaxIdleTimeout:        quicpogs.MaxIdleTimeout,
 		MaxIncomingStreams:    connection.MaxConcurrentStreams,
 		MaxIncomingUniStreams: connection.MaxConcurrentStreams,
 		KeepAlive:             true,
 		EnableDatagrams:       true,
+		MaxDatagramFrameSize:  quicpogs.MaxDatagramFrameSize,
 		Tracer:                quicpogs.NewClientTracer(connLogger.Logger(), connIndex),
 	}
-	for {
-		select {
-		case <-ctx.Done():
-			return
-		default:
-			quicConn, err := connection.NewQUICConnection(
-				ctx,
-				quicConfig,
-				edgeAddr,
-				tlsConfig,
-				config.ConnectionConfig.OriginProxy,
-				connOptions,
-				controlStreamHandler,
-				config.Observer)
-			if err != nil {
-				connLogger.ConnAwareLogger().Err(err).Msgf("Failed to create new quic connection")
-				return err, true
-			}
 
-			errGroup, serveCtx := errgroup.WithContext(ctx)
-			errGroup.Go(func() error {
-				err := quicConn.Serve(ctx)
-				if err != nil {
-					connLogger.ConnAwareLogger().Err(err).Msg("Failed to serve quic connection")
-				}
-				return fmt.Errorf("Connection with edge closed")
-			})
-
-			errGroup.Go(func() error {
-				return listenReconnect(serveCtx, reconnectCh, gracefulShutdownC)
-			})
-
-			err = errGroup.Wait()
-			if err == nil {
-				return nil, false
-			}
-		}
+	quicConn, err := connection.NewQUICConnection(
+		quicConfig,
+		edgeAddr,
+		tlsConfig,
+		config.ConnectionConfig.OriginProxy,
+		connOptions,
+		controlStreamHandler,
+		connLogger.Logger())
+	if err != nil {
+		connLogger.ConnAwareLogger().Err(err).Msgf("Failed to create new quic connection")
+		return err, true
 	}
+
+	errGroup, serveCtx := errgroup.WithContext(ctx)
+	errGroup.Go(func() error {
+		err := quicConn.Serve(serveCtx)
+		if err != nil {
+			connLogger.ConnAwareLogger().Err(err).Msg("Failed to serve quic connection")
+		}
+		return err
+	})
+
+	errGroup.Go(func() error {
+		err := listenReconnect(serveCtx, reconnectCh, gracefulShutdownC)
+		if err != nil {
+			// forcefully break the connection (this is only used for testing)
+			quicConn.Close()
+		}
+		return err
+	})
+
+	return errGroup.Wait(), false
 }
 
 func listenReconnect(ctx context.Context, reconnectCh <-chan ReconnectSignal, gracefulShutdownCh <-chan struct{}) error {

origin/tunnel_test.go

@@ -4,6 +4,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/lucas-clemente/quic-go"
 	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/assert"
 
@@ -41,7 +42,7 @@ func TestWaitForBackoffFallback(t *testing.T) {
 	}
 	warpRoutingEnabled := false
 	protocolSelector, err := connection.NewProtocolSelector(
-		connection.HTTP2.String(),
+		"auto",
 		warpRoutingEnabled,
 		namedTunnel,
 		mockFetcher.fetch(),
@@ -53,7 +54,7 @@ func TestWaitForBackoffFallback(t *testing.T) {
 	initProtocol := protocolSelector.Current()
 	assert.Equal(t, connection.HTTP2, initProtocol)
 
-	protocolFallback := &protocolFallback{
+	protoFallback := &protocolFallback{
 		backoff,
 		initProtocol,
 		false,
@@ -61,40 +62,63 @@ func TestWaitForBackoffFallback(t *testing.T) {
 
 	// Retry #0 and #1. At retry #2, we switch protocol, so the fallback loop has one more retry than this
 	for i := 0; i < int(maxRetries-1); i++ {
-		protocolFallback.BackoffTimer() // simulate retry
-		ok := selectNextProtocol(&log, protocolFallback, protocolSelector, false)
+		protoFallback.BackoffTimer() // simulate retry
+		ok := selectNextProtocol(&log, protoFallback, protocolSelector, nil)
 		assert.True(t, ok)
-		assert.Equal(t, initProtocol, protocolFallback.protocol)
+		assert.Equal(t, initProtocol, protoFallback.protocol)
 	}
 
 	// Retry fallback protocol
 	for i := 0; i < int(maxRetries); i++ {
-		protocolFallback.BackoffTimer() // simulate retry
-		ok := selectNextProtocol(&log, protocolFallback, protocolSelector, false)
+		protoFallback.BackoffTimer() // simulate retry
+		ok := selectNextProtocol(&log, protoFallback, protocolSelector, nil)
 		assert.True(t, ok)
 		fallback, ok := protocolSelector.Fallback()
 		assert.True(t, ok)
-		assert.Equal(t, fallback, protocolFallback.protocol)
+		assert.Equal(t, fallback, protoFallback.protocol)
 	}
 
 	currentGlobalProtocol := protocolSelector.Current()
 	assert.Equal(t, initProtocol, currentGlobalProtocol)
 
 	// No protocol to fallback, return error
-	protocolFallback.BackoffTimer() // simulate retry
-	ok := selectNextProtocol(&log, protocolFallback, protocolSelector, false)
+	protoFallback.BackoffTimer() // simulate retry
+	ok := selectNextProtocol(&log, protoFallback, protocolSelector, nil)
 	assert.False(t, ok)
 
-	protocolFallback.reset()
-	protocolFallback.BackoffTimer() // simulate retry
-	ok = selectNextProtocol(&log, protocolFallback, protocolSelector, false)
+	protoFallback.reset()
+	protoFallback.BackoffTimer() // simulate retry
+	ok = selectNextProtocol(&log, protoFallback, protocolSelector, nil)
 	assert.True(t, ok)
-	assert.Equal(t, initProtocol, protocolFallback.protocol)
+	assert.Equal(t, initProtocol, protoFallback.protocol)
 
-	protocolFallback.reset()
-	protocolFallback.BackoffTimer() // simulate retry
-	ok = selectNextProtocol(&log, protocolFallback, protocolSelector, true)
+	protoFallback.reset()
+	protoFallback.BackoffTimer() // simulate retry
+	ok = selectNextProtocol(&log, protoFallback, protocolSelector, &quic.IdleTimeoutError{})
 	// Check that we get a true after the first try itself when this flag is true. This allows us to immediately
-	// switch protocols.
+	// switch protocols when there is a fallback.
 	assert.True(t, ok)
+
+	// But if there is no fallback available, then we exhaust the retries despite the type of error.
+	// The reason why there's no fallback available is because we pick a specific protocol instead of letting it be auto.
+	protocolSelector, err = connection.NewProtocolSelector(
+		"quic",
+		warpRoutingEnabled,
+		namedTunnel,
+		mockFetcher.fetch(),
+		resolveTTL,
+		&log,
+	)
+	assert.NoError(t, err)
+	protoFallback = &protocolFallback{backoff, protocolSelector.Current(), false}
+	for i := 0; i < int(maxRetries-1); i++ {
+		protoFallback.BackoffTimer() // simulate retry
+		ok := selectNextProtocol(&log, protoFallback, protocolSelector, &quic.IdleTimeoutError{})
+		assert.True(t, ok)
+		assert.Equal(t, connection.QUIC, protoFallback.protocol)
+	}
+	// And finally it fails as it should, with no fallback.
+	protoFallback.BackoffTimer()
+	ok = selectNextProtocol(&log, protoFallback, protocolSelector, &quic.IdleTimeoutError{})
+	assert.False(t, ok)
 }

quic/datagram.go

@@ -9,8 +9,7 @@ import (
 )
 
 const (
-	MaxDatagramFrameSize = 1220
-	sessionIDLen         = len(uuid.UUID{})
+	sessionIDLen = len(uuid.UUID{})
 )
 
 type DatagramMuxer struct {
@@ -32,11 +31,11 @@ func NewDatagramMuxer(quicSession quic.Session) (*DatagramMuxer, error) {
 // SendTo suffix the session ID to the payload so the other end of the QUIC session can demultiplex
 // the payload from multiple datagram sessions
 func (dm *DatagramMuxer) SendTo(sessionID uuid.UUID, payload []byte) error {
-	if len(payload) > MaxDatagramFrameSize-sessionIDLen {
+	if len(payload) > maxDatagramPayloadSize {
 		// TODO: TUN-5302 return ICMP packet too big message
-		return fmt.Errorf("origin UDP payload has %d bytes, which exceeds transport MTU %d", len(payload), MaxDatagramFrameSize-sessionIDLen)
+		return fmt.Errorf("origin UDP payload has %d bytes, which exceeds transport MTU %d", len(payload), dm.MTU())
 	}
-	msgWithID, err := SuffixSessionID(sessionID, payload)
+	msgWithID, err := suffixSessionID(sessionID, payload)
 	if err != nil {
 		return errors.Wrap(err, "Failed to suffix session ID to datagram, it will be dropped")
 	}
@@ -54,16 +53,17 @@ func (dm *DatagramMuxer) ReceiveFrom() (uuid.UUID, []byte, error) {
 	if err != nil {
 		return uuid.Nil, nil, err
 	}
-	return ExtractSessionID(msg)
+	return extractSessionID(msg)
 }
 
-func (dm *DatagramMuxer) MTU() uint {
-	return MaxDatagramFrameSize
+// Maximum application payload to send to / receive from QUIC datagram frame
+func (dm *DatagramMuxer) MTU() int {
+	return maxDatagramPayloadSize
 }
 
 // Each QUIC datagram should be suffixed with session ID.
-// ExtractSessionID extracts the session ID and a slice with only the payload
-func ExtractSessionID(b []byte) (uuid.UUID, []byte, error) {
+// extractSessionID extracts the session ID and a slice with only the payload
+func extractSessionID(b []byte) (uuid.UUID, []byte, error) {
 	msgLen := len(b)
 	if msgLen < sessionIDLen {
 		return uuid.Nil, nil, fmt.Errorf("session ID has %d bytes, but data only has %d", sessionIDLen, len(b))
@@ -79,7 +79,7 @@ func ExtractSessionID(b []byte) (uuid.UUID, []byte, error) {
 
 // SuffixSessionID appends the session ID at the end of the payload. Suffix is more performant than prefix because
 // the payload slice might already have enough capacity to append the session ID at the end
-func SuffixSessionID(sessionID uuid.UUID, b []byte) ([]byte, error) {
+func suffixSessionID(sessionID uuid.UUID, b []byte) ([]byte, error) {
 	if len(b)+len(sessionID) > MaxDatagramFrameSize {
 		return nil, fmt.Errorf("datagram size exceed %d", MaxDatagramFrameSize)
 	}

quic/datagram_test.go

@@ -1,10 +1,21 @@
 package quic
 
 import (
+	"bytes"
+	"context"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/pem"
+	"math/big"
 	"testing"
+	"time"
 
 	"github.com/google/uuid"
+	"github.com/lucas-clemente/quic-go"
 	"github.com/stretchr/testify/require"
+	"golang.org/x/sync/errgroup"
 )
 
 var (
@@ -13,11 +24,11 @@ var (
 
 func TestSuffixThenRemoveSessionID(t *testing.T) {
 	msg := []byte(t.Name())
-	msgWithID, err := SuffixSessionID(testSessionID, msg)
+	msgWithID, err := suffixSessionID(testSessionID, msg)
 	require.NoError(t, err)
 	require.Len(t, msgWithID, len(msg)+sessionIDLen)
 
-	sessionID, msgWithoutID, err := ExtractSessionID(msgWithID)
+	sessionID, msgWithoutID, err := extractSessionID(msgWithID)
 	require.NoError(t, err)
 	require.Equal(t, msg, msgWithoutID)
 	require.Equal(t, testSessionID, sessionID)
@@ -26,16 +37,117 @@ func TestSuffixThenRemoveSessionID(t *testing.T) {
 func TestRemoveSessionIDError(t *testing.T) {
 	// message is too short to contain session ID
 	msg := []byte("test")
-	_, _, err := ExtractSessionID(msg)
+	_, _, err := extractSessionID(msg)
 	require.Error(t, err)
 }
 
 func TestSuffixSessionIDError(t *testing.T) {
 	msg := make([]byte, MaxDatagramFrameSize-sessionIDLen)
-	_, err := SuffixSessionID(testSessionID, msg)
+	_, err := suffixSessionID(testSessionID, msg)
 	require.NoError(t, err)
 
 	msg = make([]byte, MaxDatagramFrameSize-sessionIDLen+1)
-	_, err = SuffixSessionID(testSessionID, msg)
+	_, err = suffixSessionID(testSessionID, msg)
 	require.Error(t, err)
 }
+
+func TestMaxDatagramPayload(t *testing.T) {
+	payload := make([]byte, maxDatagramPayloadSize)
+
+	quicConfig := &quic.Config{
+		KeepAlive:            true,
+		EnableDatagrams:      true,
+		MaxDatagramFrameSize: MaxDatagramFrameSize,
+	}
+	quicListener := newQUICListener(t, quicConfig)
+	defer quicListener.Close()
+
+	errGroup, ctx := errgroup.WithContext(context.Background())
+	// Run edge side of datagram muxer
+	errGroup.Go(func() error {
+		// Accept quic connection
+		quicSession, err := quicListener.Accept(ctx)
+		if err != nil {
+			return err
+		}
+
+		muxer, err := NewDatagramMuxer(quicSession)
+		if err != nil {
+			return err
+		}
+
+		sessionID, receivedPayload, err := muxer.ReceiveFrom()
+		if err != nil {
+			return err
+		}
+		require.Equal(t, testSessionID, sessionID)
+		require.True(t, bytes.Equal(payload, receivedPayload))
+
+		return nil
+	})
+
+	// Run cloudflared side of datagram muxer
+	errGroup.Go(func() error {
+		tlsClientConfig := &tls.Config{
+			InsecureSkipVerify: true,
+			NextProtos:         []string{"argotunnel"},
+		}
+		// Establish quic connection
+		quicSession, err := quic.DialAddrEarly(quicListener.Addr().String(), tlsClientConfig, quicConfig)
+		require.NoError(t, err)
+
+		muxer, err := NewDatagramMuxer(quicSession)
+		if err != nil {
+			return err
+		}
+
+		// Wait a few milliseconds for MTU discovery to take place
+		time.Sleep(time.Millisecond * 100)
+		err = muxer.SendTo(testSessionID, payload)
+		if err != nil {
+			return err
+		}
+
+		// Payload larger than transport MTU, should return an error
+		largePayload := make([]byte, MaxDatagramFrameSize)
+		err = muxer.SendTo(testSessionID, largePayload)
+		require.Error(t, err)
+
+		return nil
+	})
+
+	require.NoError(t, errGroup.Wait())
+}
+
+func newQUICListener(t *testing.T, config *quic.Config) quic.Listener {
+	// Create a simple tls config.
+	tlsConfig := generateTLSConfig()
+
+	listener, err := quic.ListenAddr("127.0.0.1:0", tlsConfig, config)
+	require.NoError(t, err)
+
+	return listener
+}
+
+func generateTLSConfig() *tls.Config {
+	key, err := rsa.GenerateKey(rand.Reader, 1024)
+	if err != nil {
+		panic(err)
+	}
+	template := x509.Certificate{SerialNumber: big.NewInt(1)}
+	certDER, err := x509.CreateCertificate(rand.Reader, &template, &template, &key.PublicKey, key)
+	if err != nil {
+		panic(err)
+	}
+	keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)})
+	certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER})
+
+	tlsCert, err := tls.X509KeyPair(certPEM, keyPEM)
+	if err != nil {
+		panic(err)
+	}
+	return &tls.Config{
+		Certificates: []tls.Certificate{tlsCert},
+		NextProtos:   []string{"argotunnel"},
+	}
+}

quic/param_unix.go

@@ -0,0 +1,9 @@
+//go:build !windows
+
+package quic
+
+const (
+	MaxDatagramFrameSize = 1350
+	// maxDatagramPayloadSize is the maximum packet size allowed by warp client
+	maxDatagramPayloadSize = 1280
+)

quic/param_windows.go

@@ -0,0 +1,11 @@
+//go:build windows
+
+package quic
+
+const (
+	// Due to https://github.com/lucas-clemente/quic-go/issues/3273, MTU discovery is disabled on Windows
+	// 1220 is the default value https://github.com/lucas-clemente/quic-go/blob/84e03e59760ceee37359688871bb0688fcc4e98f/internal/protocol/params.go#L138
+	MaxDatagramFrameSize = 1220
+	//  3 more bytes are reserved at https://github.com/lucas-clemente/quic-go/blob/v0.24.0/internal/wire/datagram_frame.go#L61
+	maxDatagramPayloadSize = MaxDatagramFrameSize - 3 - sessionIDLen
+)

quic/quic_protocol.go

@@ -17,8 +17,8 @@ import (
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
-// The first 6 bytes of the stream is used to distinguish the type of stream. It ensures whoever performs a handshake does
-// not write data before writing the metadata.
+// ProtocolSignature defines the first 6 bytes of the stream, which is used to distinguish the type of stream. It
+// ensures whoever performs a handshake does not write data before writing the metadata.
 type ProtocolSignature [6]byte
 
 var (
@@ -29,12 +29,15 @@ var (
 	RPCStreamProtocolSignature = ProtocolSignature{0x52, 0xBB, 0x82, 0x5C, 0xDB, 0x65}
 )
 
-const protocolVersionLength = 2
-
 type protocolVersion string
 
 const (
 	protocolV1 protocolVersion = "01"
+
+	protocolVersionLength = 2
+
+	HandshakeIdleTimeout = 5 * time.Second
+	MaxIdleTimeout       = 15 * time.Second
 )
 
 // RequestServerStream is a stream to serve requests
@@ -122,7 +125,7 @@ func (rcs *RequestClientStream) ReadConnectResponseData() (*ConnectResponse, err
 		return nil, err
 	}
 	if signature != DataStreamProtocolSignature {
-		return nil, fmt.Errorf("Wrong protocol signature %v", signature)
+		return nil, fmt.Errorf("wrong protocol signature %v", signature)
 	}
 
 	// This is a NO-OP for now. We could cause a branching if we wanted to use multiple versions.
@@ -154,14 +157,17 @@ func NewRPCServerStream(stream io.ReadWriteCloser, protocol ProtocolSignature) (
 	return &RPCServerStream{stream}, nil
 }
 
-func (s *RPCServerStream) Serve(sessionManager tunnelpogs.SessionManager, logger *zerolog.Logger) error {
-	rpcTransport := tunnelrpc.NewTransportLogger(logger, rpc.StreamTransport(s))
+func (s *RPCServerStream) Serve(sessionManager tunnelpogs.SessionManager, configManager tunnelpogs.ConfigurationManager, logger *zerolog.Logger) error {
+	// RPC logs are very robust, create a new logger that only logs error to reduce noise
+	rpcLogger := logger.Level(zerolog.ErrorLevel)
+	rpcTransport := tunnelrpc.NewTransportLogger(&rpcLogger, rpc.StreamTransport(s))
 	defer rpcTransport.Close()
 
-	main := tunnelpogs.SessionManager_ServerToClient(sessionManager)
+	main := tunnelpogs.CloudflaredServer_ServerToClient(sessionManager, configManager)
 	rpcConn := rpc.NewConn(
 		rpcTransport,
 		rpc.MainInterface(main.Client),
+		tunnelrpc.ConnLog(&rpcLogger),
 	)
 	defer rpcConn.Close()
 
@@ -179,7 +185,7 @@ func DetermineProtocol(stream io.Reader) (ProtocolSignature, error) {
 	case RPCStreamProtocolSignature:
 		return RPCStreamProtocolSignature, nil
 	default:
-		return ProtocolSignature{}, fmt.Errorf("Unknown signature %v", signature)
+		return ProtocolSignature{}, fmt.Errorf("unknown signature %v", signature)
 	}
 }
 
@@ -217,7 +223,7 @@ func writeSignature(stream io.Writer, signature ProtocolSignature) error {
 
 // RPCClientStream is a stream to call methods of SessionManager
 type RPCClientStream struct {
-	client    tunnelpogs.SessionManager_PogsClient
+	client    tunnelpogs.CloudflaredServer_PogsClient
 	transport rpc.Transport
 }
 
@@ -235,7 +241,7 @@ func NewRPCClientStream(ctx context.Context, stream io.ReadWriteCloser, logger *
 		tunnelrpc.ConnLog(logger),
 	)
 	return &RPCClientStream{
-		client:    tunnelpogs.SessionManager_PogsClient{Client: conn.Bootstrap(ctx), Conn: conn},
+		client:    tunnelpogs.NewCloudflaredServer_PogsClient(conn.Bootstrap(ctx), conn),
 		transport: transport,
 	}, nil
 }
@@ -248,8 +254,12 @@ func (rcs *RPCClientStream) RegisterUdpSession(ctx context.Context, sessionID uu
 	return resp.Err
 }
 
-func (rcs *RPCClientStream) UnregisterUdpSession(ctx context.Context, sessionID uuid.UUID) error {
-	return rcs.client.UnregisterUdpSession(ctx, sessionID)
+func (rcs *RPCClientStream) UnregisterUdpSession(ctx context.Context, sessionID uuid.UUID, message string) error {
+	return rcs.client.UnregisterUdpSession(ctx, sessionID, message)
+}
+
+func (rcs *RPCClientStream) UpdateConfiguration(ctx context.Context, version int32, config []byte) (*tunnelpogs.UpdateConfigurationResponse, error) {
+	return rcs.client.UpdateConfiguration(ctx, version, config)
 }
 
 func (rcs *RPCClientStream) Close() {

quic/quic_protocol_test.go

@@ -14,6 +14,8 @@ import (
 	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
 const (
@@ -108,17 +110,15 @@ func TestConnectResponseMeta(t *testing.T) {
 }
 
 func TestRegisterUdpSession(t *testing.T) {
-	clientReader, serverWriter := io.Pipe()
-	serverReader, clientWriter := io.Pipe()
+	clientStream, serverStream := newMockRPCStreams()
 
-	clientStream := mockRPCStream{clientReader, clientWriter}
-	serverStream := mockRPCStream{serverReader, serverWriter}
-
-	rpcServer := mockRPCServer{
-		sessionID:      uuid.New(),
-		dstIP:          net.IP{172, 16, 0, 1},
-		dstPort:        8000,
-		closeIdleAfter: testCloseIdleAfterHint,
+	unregisterMessage := "closed by eyeball"
+	sessionRPCServer := mockSessionRPCServer{
+		sessionID:         uuid.New(),
+		dstIP:             net.IP{172, 16, 0, 1},
+		dstPort:           8000,
+		closeIdleAfter:    testCloseIdleAfterHint,
+		unregisterMessage: unregisterMessage,
 	}
 	logger := zerolog.Nop()
 	sessionRegisteredChan := make(chan struct{})
@@ -127,7 +127,7 @@ func TestRegisterUdpSession(t *testing.T) {
 		assert.NoError(t, err)
 		rpcServerStream, err := NewRPCServerStream(serverStream, protocol)
 		assert.NoError(t, err)
-		err = rpcServerStream.Serve(rpcServer, &logger)
+		err = rpcServerStream.Serve(sessionRPCServer, nil, &logger)
 		assert.NoError(t, err)
 
 		serverStream.Close()
@@ -137,28 +137,70 @@ func TestRegisterUdpSession(t *testing.T) {
 	rpcClientStream, err := NewRPCClientStream(context.Background(), clientStream, &logger)
 	assert.NoError(t, err)
 
-	assert.NoError(t, rpcClientStream.RegisterUdpSession(context.Background(), rpcServer.sessionID, rpcServer.dstIP, rpcServer.dstPort, testCloseIdleAfterHint))
+	assert.NoError(t, rpcClientStream.RegisterUdpSession(context.Background(), sessionRPCServer.sessionID, sessionRPCServer.dstIP, sessionRPCServer.dstPort, testCloseIdleAfterHint))
 
 	// Different sessionID, the RPC server should reject the registraion
-	assert.Error(t, rpcClientStream.RegisterUdpSession(context.Background(), uuid.New(), rpcServer.dstIP, rpcServer.dstPort, testCloseIdleAfterHint))
+	assert.Error(t, rpcClientStream.RegisterUdpSession(context.Background(), uuid.New(), sessionRPCServer.dstIP, sessionRPCServer.dstPort, testCloseIdleAfterHint))
 
-	assert.NoError(t, rpcClientStream.UnregisterUdpSession(context.Background(), rpcServer.sessionID))
+	assert.NoError(t, rpcClientStream.UnregisterUdpSession(context.Background(), sessionRPCServer.sessionID, unregisterMessage))
 
 	// Different sessionID, the RPC server should reject the unregistraion
-	assert.Error(t, rpcClientStream.UnregisterUdpSession(context.Background(), uuid.New()))
+	assert.Error(t, rpcClientStream.UnregisterUdpSession(context.Background(), uuid.New(), unregisterMessage))
 
 	rpcClientStream.Close()
 	<-sessionRegisteredChan
 }
 
-type mockRPCServer struct {
-	sessionID      uuid.UUID
-	dstIP          net.IP
-	dstPort        uint16
-	closeIdleAfter time.Duration
+func TestManageConfiguration(t *testing.T) {
+	var (
+		version int32 = 168
+		config        = []byte(t.Name())
+	)
+	clientStream, serverStream := newMockRPCStreams()
+
+	configRPCServer := mockConfigRPCServer{
+		version: version,
+		config:  config,
+	}
+
+	logger := zerolog.Nop()
+	updatedChan := make(chan struct{})
+	go func() {
+		protocol, err := DetermineProtocol(serverStream)
+		assert.NoError(t, err)
+		rpcServerStream, err := NewRPCServerStream(serverStream, protocol)
+		assert.NoError(t, err)
+		err = rpcServerStream.Serve(nil, configRPCServer, &logger)
+		assert.NoError(t, err)
+
+		serverStream.Close()
+		close(updatedChan)
+	}()
+
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+	defer cancel()
+	rpcClientStream, err := NewRPCClientStream(ctx, clientStream, &logger)
+	assert.NoError(t, err)
+
+	result, err := rpcClientStream.UpdateConfiguration(ctx, version, config)
+	assert.NoError(t, err)
+
+	require.Equal(t, version, result.LastAppliedVersion)
+	require.NoError(t, result.Err)
+
+	rpcClientStream.Close()
+	<-updatedChan
 }
 
-func (s mockRPCServer) RegisterUdpSession(ctx context.Context, sessionID uuid.UUID, dstIP net.IP, dstPort uint16, closeIdleAfter time.Duration) error {
+type mockSessionRPCServer struct {
+	sessionID         uuid.UUID
+	dstIP             net.IP
+	dstPort           uint16
+	closeIdleAfter    time.Duration
+	unregisterMessage string
+}
+
+func (s mockSessionRPCServer) RegisterUdpSession(_ context.Context, sessionID uuid.UUID, dstIP net.IP, dstPort uint16, closeIdleAfter time.Duration) error {
 	if s.sessionID != sessionID {
 		return fmt.Errorf("expect session ID %s, got %s", s.sessionID, sessionID)
 	}
@@ -174,18 +216,45 @@ func (s mockRPCServer) RegisterUdpSession(ctx context.Context, sessionID uuid.UU
 	return nil
 }
 
-func (s mockRPCServer) UnregisterUdpSession(ctx context.Context, sessionID uuid.UUID) error {
+func (s mockSessionRPCServer) UnregisterUdpSession(_ context.Context, sessionID uuid.UUID, message string) error {
 	if s.sessionID != sessionID {
 		return fmt.Errorf("expect session ID %s, got %s", s.sessionID, sessionID)
 	}
+	if s.unregisterMessage != message {
+		return fmt.Errorf("expect unregister message %s, got %s", s.unregisterMessage, message)
+	}
 	return nil
 }
 
+type mockConfigRPCServer struct {
+	version int32
+	config  []byte
+}
+
+func (s mockConfigRPCServer) UpdateConfiguration(_ context.Context, version int32, config []byte) (*tunnelpogs.UpdateConfigurationResponse, error) {
+	if s.version != version {
+		return nil, fmt.Errorf("expect version %d, got %d", s.version, version)
+	}
+	if !bytes.Equal(s.config, config) {
+		return nil, fmt.Errorf("expect config %v, got %v", s.config, config)
+	}
+	return &tunnelpogs.UpdateConfigurationResponse{LastAppliedVersion: version}, nil
+}
+
 type mockRPCStream struct {
 	io.ReadCloser
 	io.WriteCloser
 }
 
+func newMockRPCStreams() (client io.ReadWriteCloser, server io.ReadWriteCloser) {
+	clientReader, serverWriter := io.Pipe()
+	serverReader, clientWriter := io.Pipe()
+
+	client = mockRPCStream{clientReader, clientWriter}
+	server = mockRPCStream{serverReader, serverWriter}
+	return
+}
+
 func (s mockRPCStream) Close() error {
 	_ = s.ReadCloser.Close()
 	_ = s.WriteCloser.Close()

quic/safe_stream.go

@@ -0,0 +1,43 @@
+package quic
+
+import (
+	"sync"
+	"time"
+
+	"github.com/lucas-clemente/quic-go"
+)
+
+type SafeStreamCloser struct {
+	lock   sync.Mutex
+	stream quic.Stream
+}
+
+func NewSafeStreamCloser(stream quic.Stream) *SafeStreamCloser {
+	return &SafeStreamCloser{
+		stream: stream,
+	}
+}
+
+func (s *SafeStreamCloser) Read(p []byte) (n int, err error) {
+	return s.stream.Read(p)
+}
+
+func (s *SafeStreamCloser) Write(p []byte) (n int, err error) {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+	return s.stream.Write(p)
+}
+
+func (s *SafeStreamCloser) Close() error {
+	// Make sure a possible writer does not block the lock forever. We need it, so we can close the writer
+	// side of the stream safely.
+	_ = s.stream.SetWriteDeadline(time.Now())
+
+	// This lock is eventually acquired despite Write also acquiring it, because we set a deadline to writes.
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	// We have to clean up the receiving stream ourselves since the Close in the bottom does not handle that.
+	s.stream.CancelRead(0)
+	return s.stream.Close()
+}

quic/safe_stream_test.go

@@ -0,0 +1,142 @@
+package quic
+
+import (
+	"context"
+	"crypto/tls"
+	"io"
+	"net"
+	"sync"
+	"testing"
+
+	"github.com/lucas-clemente/quic-go"
+	"github.com/stretchr/testify/require"
+)
+
+var (
+	testTLSServerConfig = GenerateTLSConfig()
+	testQUICConfig      = &quic.Config{
+		KeepAlive:       true,
+		EnableDatagrams: true,
+	}
+	exchanges       = 1000
+	msgsPerExchange = 10
+	testMsg         = "Ok message"
+)
+
+func TestSafeStreamClose(t *testing.T) {
+	udpAddr, err := net.ResolveUDPAddr("udp", "127.0.0.1:0")
+	require.NoError(t, err)
+	udpListener, err := net.ListenUDP(udpAddr.Network(), udpAddr)
+	require.NoError(t, err)
+	defer udpListener.Close()
+
+	var serverReady sync.WaitGroup
+	serverReady.Add(1)
+
+	var done sync.WaitGroup
+	done.Add(1)
+	go func() {
+		defer done.Done()
+		quicServer(t, &serverReady, udpListener)
+	}()
+
+	done.Add(1)
+	go func() {
+		serverReady.Wait()
+		defer done.Done()
+		quicClient(t, udpListener.LocalAddr())
+	}()
+
+	done.Wait()
+}
+
+func quicClient(t *testing.T, addr net.Addr) {
+	tlsConf := &tls.Config{
+		InsecureSkipVerify: true,
+		NextProtos:         []string{"argotunnel"},
+	}
+	session, err := quic.DialAddr(addr.String(), tlsConf, testQUICConfig)
+	require.NoError(t, err)
+
+	var wg sync.WaitGroup
+	for exchange := 0; exchange < exchanges; exchange++ {
+		quicStream, err := session.AcceptStream(context.Background())
+		require.NoError(t, err)
+		wg.Add(1)
+
+		go func(iter int) {
+			defer wg.Done()
+
+			stream := NewSafeStreamCloser(quicStream)
+			defer stream.Close()
+
+			// Do a bunch of round trips over this stream that should work.
+			for msg := 0; msg < msgsPerExchange; msg++ {
+				clientRoundTrip(t, stream, true)
+			}
+			// And one that won't work necessarily, but shouldn't break other streams in the session.
+			if iter%2 == 0 {
+				clientRoundTrip(t, stream, false)
+			}
+		}(exchange)
+	}
+
+	wg.Wait()
+}
+
+func quicServer(t *testing.T, serverReady *sync.WaitGroup, conn net.PacketConn) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	earlyListener, err := quic.Listen(conn, testTLSServerConfig, testQUICConfig)
+	require.NoError(t, err)
+
+	serverReady.Done()
+	session, err := earlyListener.Accept(ctx)
+	require.NoError(t, err)
+
+	var wg sync.WaitGroup
+	for exchange := 0; exchange < exchanges; exchange++ {
+		quicStream, err := session.OpenStreamSync(context.Background())
+		require.NoError(t, err)
+		wg.Add(1)
+
+		go func(iter int) {
+			defer wg.Done()
+
+			stream := NewSafeStreamCloser(quicStream)
+			defer stream.Close()
+
+			// Do a bunch of round trips over this stream that should work.
+			for msg := 0; msg < msgsPerExchange; msg++ {
+				serverRoundTrip(t, stream, true)
+			}
+			// And one that won't work necessarily, but shouldn't break other streams in the session.
+			if iter%2 == 1 {
+				serverRoundTrip(t, stream, false)
+			}
+		}(exchange)
+	}
+
+	wg.Wait()
+}
+
+func clientRoundTrip(t *testing.T, stream io.ReadWriteCloser, mustWork bool) {
+	response := make([]byte, len(testMsg))
+	_, err := stream.Read(response)
+	if !mustWork {
+		return
+	}
+	if err != io.EOF {
+		require.NoError(t, err)
+	}
+	require.Equal(t, testMsg, string(response))
+}
+
+func serverRoundTrip(t *testing.T, stream io.ReadWriteCloser, mustWork bool) {
+	_, err := stream.Write([]byte(testMsg))
+	if !mustWork {
+		return
+	}
+	require.NoError(t, err)
+}

quic/test_utils.go

@@ -0,0 +1,34 @@
+package quic
+
+import (
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/pem"
+	"math/big"
+)
+
+// GenerateTLSConfig sets up a bare-bones TLS config for a QUIC server
+func GenerateTLSConfig() *tls.Config {
+	key, err := rsa.GenerateKey(rand.Reader, 1024)
+	if err != nil {
+		panic(err)
+	}
+	template := x509.Certificate{SerialNumber: big.NewInt(1)}
+	certDER, err := x509.CreateCertificate(rand.Reader, &template, &template, &key.PublicKey, key)
+	if err != nil {
+		panic(err)
+	}
+	keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)})
+	certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER})
+
+	tlsCert, err := tls.X509KeyPair(certPEM, keyPEM)
+	if err != nil {
+		panic(err)
+	}
+	return &tls.Config{
+		Certificates: []tls.Certificate{tlsCert},
+		NextProtos:   []string{"argotunnel"},
+	}
+}

sshgen/sshgen_test.go

@@ -1,3 +1,4 @@
+//go:build !windows
 // +build !windows
 
 package sshgen

tlsconfig/tlsconfig.go

@@ -19,6 +19,8 @@ type TLSParameters struct {
 	RootCAs          []string
 	ServerName       string
 	CurvePreferences []tls.CurveID
+	MinVersion       uint16 // min tls version. If zero, TLS1.0 is defined as minimum.
+	MaxVersion       uint16 // max tls version. If zero, last TLS version is used defined as limit (currently TLS1.3)
 }
 
 // GetConfig returns a TLS configuration according to the Config set by the user.
@@ -72,6 +74,9 @@ func GetConfig(p *TLSParameters) (*tls.Config, error) {
 		tlsconfig.CurvePreferences = []tls.CurveID{tls.CurveP256}
 	}
 
+	tlsconfig.MinVersion = p.MinVersion
+	tlsconfig.MaxVersion = p.MaxVersion
+
 	return tlsconfig, nil
 }
 

token/launch_browser_darwin.go

@@ -1,4 +1,5 @@
-//+build darwin
+//go:build darwin
+// +build darwin
 
 package token
 

token/launch_browser_other.go

@@ -1,4 +1,5 @@
-//+build !windows,!darwin,!linux,!netbsd,!freebsd,!openbsd
+//go:build !windows && !darwin && !linux && !netbsd && !freebsd && !openbsd
+// +build !windows,!darwin,!linux,!netbsd,!freebsd,!openbsd
 
 package token
 

token/launch_browser_unix.go

@@ -1,4 +1,5 @@
-//+build linux freebsd openbsd netbsd
+//go:build linux || freebsd || openbsd || netbsd
+// +build linux freebsd openbsd netbsd
 
 package token
 

token/launch_browser_windows.go

@@ -1,4 +1,5 @@
-//+build windows
+//go:build windows
+// +build windows
 
 package token
 

token/token_test.go

@@ -1,3 +1,4 @@
+//go:build linux
 // +build linux
 
 package token

tunneldns/metrics.go

@@ -2,7 +2,6 @@ package tunneldns
 
 import (
 	"context"
-	"sync"
 
 	"github.com/coredns/coredns/plugin"
 	"github.com/coredns/coredns/plugin/metrics"
@@ -11,10 +10,11 @@ import (
 	"github.com/coredns/coredns/plugin/pkg/rcode"
 	"github.com/coredns/coredns/request"
 	"github.com/miekg/dns"
-	"github.com/prometheus/client_golang/prometheus"
 )
 
-var once sync.Once
+const (
+	pluginName = "cloudflared"
+)
 
 // MetricsPlugin is an adapter for CoreDNS and built-in metrics
 type MetricsPlugin struct {
@@ -23,14 +23,6 @@ type MetricsPlugin struct {
 
 // NewMetricsPlugin creates a plugin with configured metrics
 func NewMetricsPlugin(next plugin.Handler) *MetricsPlugin {
-	once.Do(func() {
-		prometheus.MustRegister(vars.RequestCount)
-		prometheus.MustRegister(vars.RequestDuration)
-		prometheus.MustRegister(vars.RequestSize)
-		prometheus.MustRegister(vars.RequestDo)
-		prometheus.MustRegister(vars.ResponseSize)
-		prometheus.MustRegister(vars.ResponseRcode)
-	})
 	return &MetricsPlugin{Next: next}
 }
 
@@ -43,7 +35,7 @@ func (p MetricsPlugin) ServeDNS(ctx context.Context, w dns.ResponseWriter, r *dn
 
 	// Update built-in metrics
 	server := metrics.WithServer(ctx)
-	vars.Report(server, state, ".", rcode.ToString(rw.Rcode), rw.Len, rw.Start)
+	vars.Report(server, state, ".", rcode.ToString(rw.Rcode), pluginName, rw.Len, rw.Start)
 
 	return status, err
 }

tunnelrpc/pogs/cloudflaredrpc.go

@@ -0,0 +1,53 @@
+package pogs
+
+import (
+	"github.com/cloudflare/cloudflared/tunnelrpc"
+	capnp "zombiezen.com/go/capnproto2"
+	"zombiezen.com/go/capnproto2/rpc"
+)
+
+type CloudflaredServer interface {
+	SessionManager
+	ConfigurationManager
+}
+
+type CloudflaredServer_PogsImpl struct {
+	SessionManager_PogsImpl
+	ConfigurationManager_PogsImpl
+}
+
+func CloudflaredServer_ServerToClient(s SessionManager, c ConfigurationManager) tunnelrpc.CloudflaredServer {
+	return tunnelrpc.CloudflaredServer_ServerToClient(CloudflaredServer_PogsImpl{
+		SessionManager_PogsImpl:       SessionManager_PogsImpl{s},
+		ConfigurationManager_PogsImpl: ConfigurationManager_PogsImpl{c},
+	})
+}
+
+type CloudflaredServer_PogsClient struct {
+	SessionManager_PogsClient
+	ConfigurationManager_PogsClient
+	Client capnp.Client
+	Conn   *rpc.Conn
+}
+
+func NewCloudflaredServer_PogsClient(client capnp.Client, conn *rpc.Conn) CloudflaredServer_PogsClient {
+	sessionManagerClient := SessionManager_PogsClient{
+		Client: client,
+		Conn:   conn,
+	}
+	configManagerClient := ConfigurationManager_PogsClient{
+		Client: client,
+		Conn:   conn,
+	}
+	return CloudflaredServer_PogsClient{
+		SessionManager_PogsClient:       sessionManagerClient,
+		ConfigurationManager_PogsClient: configManagerClient,
+		Client:                          client,
+		Conn:                            conn,
+	}
+}
+
+func (c CloudflaredServer_PogsClient) Close() error {
+	c.Client.Close()
+	return c.Conn.Close()
+}

tunnelrpc/pogs/configurationrpc.go

@@ -0,0 +1,99 @@
+package pogs
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/cloudflare/cloudflared/tunnelrpc"
+	capnp "zombiezen.com/go/capnproto2"
+	"zombiezen.com/go/capnproto2/rpc"
+	"zombiezen.com/go/capnproto2/server"
+)
+
+type ConfigurationManager interface {
+	UpdateConfiguration(ctx context.Context, version int32, config []byte) (*UpdateConfigurationResponse, error)
+}
+
+type ConfigurationManager_PogsImpl struct {
+	impl ConfigurationManager
+}
+
+func ConfigurationManager_ServerToClient(c ConfigurationManager) tunnelrpc.ConfigurationManager {
+	return tunnelrpc.ConfigurationManager_ServerToClient(ConfigurationManager_PogsImpl{c})
+}
+
+func (i ConfigurationManager_PogsImpl) UpdateConfiguration(p tunnelrpc.ConfigurationManager_updateConfiguration) error {
+	server.Ack(p.Options)
+
+	version := p.Params.Version()
+	config, err := p.Params.Config()
+	if err != nil {
+		return err
+	}
+
+	updateResp, err := i.impl.UpdateConfiguration(p.Ctx, version, config)
+	if err != nil {
+		return err
+	}
+
+	result, err := p.Results.NewResult()
+	if err != nil {
+		return err
+	}
+
+	return updateResp.Marshal(result)
+}
+
+type ConfigurationManager_PogsClient struct {
+	Client capnp.Client
+	Conn   *rpc.Conn
+}
+
+func (c ConfigurationManager_PogsClient) Close() error {
+	c.Client.Close()
+	return c.Conn.Close()
+}
+
+func (c ConfigurationManager_PogsClient) UpdateConfiguration(ctx context.Context, version int32, config []byte) (*UpdateConfigurationResponse, error) {
+	client := tunnelrpc.ConfigurationManager{Client: c.Client}
+	promise := client.UpdateConfiguration(ctx, func(p tunnelrpc.ConfigurationManager_updateConfiguration_Params) error {
+		p.SetVersion(version)
+		return p.SetConfig(config)
+	})
+	result, err := promise.Result().Struct()
+	if err != nil {
+		return nil, wrapRPCError(err)
+	}
+	response := new(UpdateConfigurationResponse)
+
+	err = response.Unmarshal(result)
+	if err != nil {
+		return nil, err
+	}
+	return response, nil
+}
+
+type UpdateConfigurationResponse struct {
+	LastAppliedVersion int32 `json:"lastAppliedVersion"`
+	Err                error `json:"err"`
+}
+
+func (p *UpdateConfigurationResponse) Marshal(s tunnelrpc.UpdateConfigurationResponse) error {
+	s.SetLatestAppliedVersion(p.LastAppliedVersion)
+	if p.Err != nil {
+		return s.SetErr(p.Err.Error())
+	}
+	return nil
+}
+
+func (p *UpdateConfigurationResponse) Unmarshal(s tunnelrpc.UpdateConfigurationResponse) error {
+	p.LastAppliedVersion = s.LatestAppliedVersion()
+	respErr, err := s.Err()
+	if err != nil {
+		return err
+	}
+	if respErr != "" {
+		p.Err = fmt.Errorf(respErr)
+	}
+	return nil
+}

tunnelrpc/pogs/sessionrpc.go

@@ -15,7 +15,7 @@ import (
 
 type SessionManager interface {
 	RegisterUdpSession(ctx context.Context, sessionID uuid.UUID, dstIP net.IP, dstPort uint16, closeAfterIdleHint time.Duration) error
-	UnregisterUdpSession(ctx context.Context, sessionID uuid.UUID) error
+	UnregisterUdpSession(ctx context.Context, sessionID uuid.UUID, message string) error
 }
 
 type SessionManager_PogsImpl struct {
@@ -76,7 +76,12 @@ func (i SessionManager_PogsImpl) UnregisterUdpSession(p tunnelrpc.SessionManager
 		return err
 	}
 
-	return i.impl.UnregisterUdpSession(p.Ctx, sessionID)
+	message, err := p.Params.Message()
+	if err != nil {
+		return err
+	}
+
+	return i.impl.UnregisterUdpSession(p.Ctx, sessionID, message)
 }
 
 type RegisterUdpSessionResponse struct {
@@ -137,12 +142,15 @@ func (c SessionManager_PogsClient) RegisterUdpSession(ctx context.Context, sessi
 	return response, nil
 }
 
-func (c SessionManager_PogsClient) UnregisterUdpSession(ctx context.Context, sessionID uuid.UUID) error {
+func (c SessionManager_PogsClient) UnregisterUdpSession(ctx context.Context, sessionID uuid.UUID, message string) error {
 	client := tunnelrpc.SessionManager{Client: c.Client}
 	promise := client.UnregisterUdpSession(ctx, func(p tunnelrpc.SessionManager_unregisterUdpSession_Params) error {
 		if err := p.SetSessionId(sessionID[:]); err != nil {
 			return err
 		}
+		if err := p.SetMessage(message); err != nil {
+			return err
+		}
 		return nil
 	})
 	_, err := promise.Struct()

tunnelrpc/tunnelrpc.capnp

@@ -150,5 +150,20 @@ struct RegisterUdpSessionResponse {
 interface SessionManager {
     # Let the edge decide closeAfterIdle to make sure cloudflared doesn't close session before the edge closes its side
     registerUdpSession @0 (sessionId :Data, dstIp :Data, dstPort: UInt16, closeAfterIdleHint: Int64) -> (result :RegisterUdpSessionResponse);
-    unregisterUdpSession @1 (sessionId :Data) -> ();
-}
+    unregisterUdpSession @1 (sessionId :Data, message: Text) -> ();
+}
+
+struct UpdateConfigurationResponse {
+    # Latest configuration that was applied successfully. The err field might be populated at the same time to indicate
+    # that cloudflared is using an older configuration because the latest cannot be applied
+    latestAppliedVersion @0 :Int32;
+    # Any error encountered when trying to apply the last configuration
+    err @1 :Text;
+}
+
+# ConfigurationManager defines RPC to manage cloudflared configuration remotely
+interface ConfigurationManager {
+    updateConfiguration @0 (version :Int32, config :Data) -> (result: UpdateConfigurationResponse);
+}
+
+interface CloudflaredServer extends(SessionManager, ConfigurationManager) {}

tunnelrpc/tunnelrpc.capnp.go

@@ -3485,7 +3485,7 @@ func (c SessionManager) UnregisterUdpSession(ctx context.Context, params func(Se
 		Options: capnp.NewCallOptions(opts),
 	}
 	if params != nil {
-		call.ParamsSize = capnp.ObjectSize{DataSize: 0, PointerCount: 1}
+		call.ParamsSize = capnp.ObjectSize{DataSize: 0, PointerCount: 2}
 		call.ParamsFunc = func(s capnp.Struct) error { return params(SessionManager_unregisterUdpSession_Params{Struct: s}) }
 	}
 	return SessionManager_unregisterUdpSession_Results_Promise{Pipeline: capnp.NewPipeline(c.Client.Call(call))}
@@ -3743,12 +3743,12 @@ type SessionManager_unregisterUdpSession_Params struct{ capnp.Struct }
 const SessionManager_unregisterUdpSession_Params_TypeID = 0x96b74375ce9b0ef6
 
 func NewSessionManager_unregisterUdpSession_Params(s *capnp.Segment) (SessionManager_unregisterUdpSession_Params, error) {
-	st, err := capnp.NewStruct(s, capnp.ObjectSize{DataSize: 0, PointerCount: 1})
+	st, err := capnp.NewStruct(s, capnp.ObjectSize{DataSize: 0, PointerCount: 2})
 	return SessionManager_unregisterUdpSession_Params{st}, err
 }
 
 func NewRootSessionManager_unregisterUdpSession_Params(s *capnp.Segment) (SessionManager_unregisterUdpSession_Params, error) {
-	st, err := capnp.NewRootStruct(s, capnp.ObjectSize{DataSize: 0, PointerCount: 1})
+	st, err := capnp.NewRootStruct(s, capnp.ObjectSize{DataSize: 0, PointerCount: 2})
 	return SessionManager_unregisterUdpSession_Params{st}, err
 }
 
@@ -3776,12 +3776,31 @@ func (s SessionManager_unregisterUdpSession_Params) SetSessionId(v []byte) error
 	return s.Struct.SetData(0, v)
 }
 
+func (s SessionManager_unregisterUdpSession_Params) Message() (string, error) {
+	p, err := s.Struct.Ptr(1)
+	return p.Text(), err
+}
+
+func (s SessionManager_unregisterUdpSession_Params) HasMessage() bool {
+	p, err := s.Struct.Ptr(1)
+	return p.IsValid() || err != nil
+}
+
+func (s SessionManager_unregisterUdpSession_Params) MessageBytes() ([]byte, error) {
+	p, err := s.Struct.Ptr(1)
+	return p.TextBytes(), err
+}
+
+func (s SessionManager_unregisterUdpSession_Params) SetMessage(v string) error {
+	return s.Struct.SetText(1, v)
+}
+
 // SessionManager_unregisterUdpSession_Params_List is a list of SessionManager_unregisterUdpSession_Params.
 type SessionManager_unregisterUdpSession_Params_List struct{ capnp.List }
 
 // NewSessionManager_unregisterUdpSession_Params creates a new list of SessionManager_unregisterUdpSession_Params.
 func NewSessionManager_unregisterUdpSession_Params_List(s *capnp.Segment, sz int32) (SessionManager_unregisterUdpSession_Params_List, error) {
-	l, err := capnp.NewCompositeList(s, capnp.ObjectSize{DataSize: 0, PointerCount: 1}, sz)
+	l, err := capnp.NewCompositeList(s, capnp.ObjectSize{DataSize: 0, PointerCount: 2}, sz)
 	return SessionManager_unregisterUdpSession_Params_List{l}, err
 }
 
@@ -3861,203 +3880,661 @@ func (p SessionManager_unregisterUdpSession_Results_Promise) Struct() (SessionMa
 	return SessionManager_unregisterUdpSession_Results{s}, err
 }
 
-const schema_db8274f9144abc7e = "x\xda\xccY{p\x14e\xb6?\xa7{&\x9d@\x86" +
-	"IW\x0f\x04\xa6\xe4\xe6^\x0a\xcaK\x14\x14\xb8\xdeB" +
-	"\xae\xde\x04L\xb8&\xf2H\xcf\x90\xbb\x96\xa0eg\xe6" +
-	"#tv\xa6{\xe8\xee\x89\x04A\x1e\x82\x88\xe5\x0b\x04" +
-	"E\x94\x95\xc5r\xb7D\xdd\x85U\xd7eKke\xd7" +
-	"\x17\xa5\xa8X\xb8\x05\x8a\xb5\xab\xc8>(\\W\xc4\xb5" +
-	"\xdcu\xed\xad\xd3=\xfd\xc8$$A\xf6\x8f\xfdor" +
-	"\xfa\xfb\xcew\xce\xef\xfc\xbes\xcewr\xe9w*\x1b" +
-	"\xb9)\xd1\xda\x18\x80\xbc-Za\xb3\xfaw\x96\xef\x9c" +
-	"\xf0\xab\xb5 '\x11\xed[\x9eoM|e\xad}\x1f" +
-	"\xa2\xbc\x000mi\xc5r\x94\xd6W\x08\x00\xd2\x9a\x8a" +
-	"\xdf\x03\xda\xb7\x8d\xda\xf3\xc8c\xcd[n\x051\xc9\x07" +
-	"\x8b\x01\xa71\xa1\x15\xa5\x1e\x81V\x16\x85\x0d\xd2\xbb\xf4" +
-	"\xcb\xbeF\xbcda\xe2\xed7iuXu\x84T\xbf" +
-	" \xd4\xa3t\xd0\xd9p@ \xd5W\xe4\xdf\xda\xf5\xdf" +
-	"[__\x07b\x92\xeb\xa5\xfa\xd9\xca\xe5(\x1d\xa8\xa4" +
-	"\x95/W\xce\x07\xb4?\xdf2\xfa\x89\xef\xbf\xf9\xdaz" +
-	"\x10/D(Y\xfaA\xe5{\x08(}V\xf9c@" +
-	"\xfb\xe0\x17\x0b\xcf<\xf3\xcae\xb7\x818\x91\x16 -" +
-	"\xd8T5\x8e\x03\x94\x1e\xafj\x00\xb4O\x9e\xfa\xdb\x86" +
-	"\x9b'\xce\xbb\x17\xe4\x89\xc8\x01D9Zq\xa0*I" +
-	"+>\xac\"k\x1af\x1f\xdc\x97\x9cv\xff\x962\xd3" +
-	"\x9d\x85\xfb\x87\xd5\xa3th\x18\x19tp\xd8M\x80\xf6" +
-	"_F<\xf4f\xf1\xaa\xe7\xee\x0f\x9f7ex=i" +
-	"k\x19N\xe7\x8d\xeb\x9ep\xe3/_~\xfa\x01\x90'" +
-	"!\xda\xc7:.z\x97\xdf\xb1\xfb}hG\x81\x8e\x9f" +
-	"\x96\x1f\xbe\x8b\x8c_9\x9c\x94\xbdu\xf1\xf3?\xbf\xf7" +
-	"\xe9\x0d\x0f\x81|!\"\x80\x03\xd6\x87\xc3\xffJ\x0b\xbe" +
-	"p\x94m9\xf2\xc2\xbc\xfc\xa6\xed\xbb\\\xf7\x9d\xefc" +
-	"\xab9\x0e\"\xf6\xba\x96/\xf3\xed\x8f\xa6\x1f-\x01\x13" +
-	"\xa5Ob\xf5i\x04\x9c6\xa1\xba\x0e\x01\xed\xcb\xde;" +
-	"1\x7f\xeeO\x16\xff0\xb4wfl9\xed\xdd\xb0\xf8" +
-	"\xf4\xfe\x9aT\xfe\x892\x87\x1d_\xae\x8c\xedF\xa9=" +
-	"F\x0e\xcb12\xe1\xa9\x7f\xbb\xa6j\xd9\x89\xd9{@" +
-	"\x9c\xe4\xa9Y\x1aK\x91\x9a\xc8\xf5\xfc7\xca\xb6_<" +
-	"SN'\x07\xb8|\xac\x03\xa55\xa4g\xda\xca\x98c" +
-	"\xcf\x1d\xfb\xb7_T\xf9\xc8\xe7\xcf\xf6\x07\xf3c#:" +
-	"P\xda7\x82N}v\x04!3\xb2\x05\x8f\xbd8%" +
-	"\xf2\\8\xee#\xe3'\x09\x99\x89q\x8a\xfb\xd8Of" +
-	"\xc5\xb4O\xd7\xbeX\xa6\xcdYx \xde\x8a\xd2\x07q" +
-	"\xd2v\xd4Y\xdc\xba\xf0\xbe\xcd\xd1\x13\xf7\xbdJ\x96\x86" +
-	"\x08\x17%\xa2M+\xd6\x18(m\xac\xa1\x9f\xebkj" +
-	"y@;\xb9\xe7\x7f~4+{\xf4\xf5~,\x95\xa2" +
-	"\x89\xd3\x92\x98\xa0_\xb1\x04\x19z|\xd2\xde\x9b\xffx" +
-	"\xd7\xa1\xc3%C\x1d\x0c\xd5\x84\x13\xc2\x95\x09\xc2\xcfg" +
-	"@\x19J\xce\xca\x1d\x89.\x94\xf6:\xea\x9erVs" +
-	"'\x941\xab\x7f\xfd\xbf\xc7BA\xdb\x9b\xf8\x08!b" +
-	"\xcf\xfb\xff\x85]U+\x8f\x1f\x0f\x1f\xf4X\xc2Ad" +
-	"\x9f\xb3\xf5O?8y\xcf\xa9|\xf6w\x0e\xf1<\xcc" +
-	"\x8e&f\x105?K\x10\xd1k\xebb\xcd\xe3\x8e\xb4" +
-	"\x9dtC\xe9\xaa88r\x16-81\x92T\\v" +
-	"\xe3L\xb6h\xfa\xb5'\xfb\\\xf9\xe8\xa8\x19(\x8d\x1c" +
-	"\xe5\x90l\xd4\x06\x94&\xd6\xd6\x02\xd8\xdd?\xddt\xed" +
-	"\x13/\xcd;\xed\xde\x05\xc7\xd81\xb5S\x89\x1aw\xdf" +
-	"\xd24\xff\xf2q\xfbO\x87\x8d\x15k\x89\x9d\xd2\x84Z" +
-	":i\xf1\xf4S\xff7\xe1\xeeWN\xf7G\xc1\xe6\xda" +
-	"z\x94\xdak\x1d\x0a\xd2\xe2Og\x7f\xefp2\x9e<" +
-	"S\x06`\x85\x13\xbc\xda.\x946\xd6:\xc1\xab}\x95" +
-	"hv\xdb\xfb7,{\xe7\xd6\xcf\xbf(\x8f\xb5\xa3\xba" +
-	"gL\x0a\xa5\xbb\xc6\x90\xea\x8dc\x88\x19\x0f,\xf8\xc3" +
-	"\xaaS[G}\xd9\xc7\xe3\x89\xc9.\x94\xaeL\xd2\xca" +
-	"\xcb\x93\x1b\xa4\x07\xe9\x97\xfd\xb6\xf0\xe8\x94\xa6U\xaf\x7f" +
-	"\x15\xba\x0bk\x92\xad\xe4\xf0\xfd\xc2\xc3\xc7W\xff\xe6\x86" +
-	"\xaf\xc3\x0e\xafL~D\x0eoJ\x92\xc3+>}\xf0" +
-	"\xea{\x16=\xf9M8\xb0\xc9\xb5\xb4\xd5*j\x1a\xcb" +
-	"\x19\x85H\xe6\x12\xefgfrF)h\x85\x193\x8b" +
-	"\xd6\x12\xa6YjF\xb1X\x8a5\x98\x05]3Y\x1b" +
-	"\xa2\\\xc3G\x00\"\x08 *]\x00\xf2\x8d<\xca9" +
-	"\x0eE\xc4\x04\x85^TI\xb8\x84G\xd9\xe2P\xe4\xb8" +
-	"\x04e\x1eq\xe98\x009\xc7\xa3\xbc\x8cC\xe4\x13\xc8" +
-	"\x03\x88\xc5\xcd\x00\xf22\x1e\xe5u\x1c\xda\x05f\xe4\x15" +
-	"\x8di\x10\xb7\x9a\x0d\x03\xab\x81\xc3j@\xdb`\x96\xd1" +
-	"\xa3t\xe4 \xceBb\xa1\xeb&\x0bc\xc0a\x0c\xd0" +
-	"^\xa2\x17\x0d\xb3]\xb3P\xcd\xa5\xd8b\x83\x99\xb8\x04" +
-	"+\x80\xc3\x0a\xc0\x81\xdcK3\xd3Tum\xae\xa2)" +
-	"\x9d\xcc\x00 \xcf*\xf9(\x80\x9f\xb4\xd1K\xef\xe2\x94" +
-	"\xed\xc0\x89\x93\x04\x0c20z\xf4\x13\xffc7p\xe2" +
-	"X\xc16X\xa7jZ\xcc\xc0\xf6l\xc1\xd1\xcd\xebZ" +
-	"#\xdaE\xcd\xfd\x80\xccp?\xc4\xe9\xd4Fl\xc3\xc0" +
-	":\xbe\xafuW\xe5T\xa6Y\xf1\x16m\xb1^\x06y" +
-	"k\x7f\x90\xb7\x96 _\x17\x82|\xcd,\x00y\x05\x8f" +
-	"\xf2\xed\x1c\x8a|\x09\xf3\xf5\xf5\x00\xf2j\x1e\xe5;9" +
-	"\xb43\xce!-Y\x00\xf0\xd1\\\xcc\x14\xabh0\x93" +
-	"d#\x00\xdbxt@\x1f\x01\xb8\xaa\x9b\x19d\xbb\x17" +
-	"\x84\xb8bd\x96\xf8\x81\x1a\x00\xe9\xe6e\xaai\xa9Z" +
-	"\xe7\x02G\xde\xd0\xa6\xe7\xd4L\x0fyU\xed\xd89v" +
-	"\x06\x00\xa28\xf2:\x00\xe4Dq\x16@\x83\xda\xa9\xe9" +
-	"\x06\xb3\xb3\xaa\x99\xd15\x8d\x01\x9f\xb1Vu(9E" +
-	"\xcb0\xff\xa0\x8a\xbe\x07\xb9\x07\xa4\x99\xd1\xcd\x8c\xc9J" +
-	"\x88\xbe\xe3\xdb\x14C\xe1\xf3\xa6\\\xed\xe3\xd8|\x1d\x80" +
-	"\xdc\xc4\xa3\xdc\x16\xc2q.\xe18\x87G\xf9\xda\x10\x8e" +
-	"\xed\x84c\x1b\x8f\xf2\"\x0em\xddP;U\xed*\x06" +
-	"\xbc\x11f\xa0iiJ\x9e\x11f%<V\xe9\x05K" +
-	"\xd55\x13k\x82\xfc\x0f\x885!\xa4\x84\xc189\xd9" +
-	"\xa3\x94\xc7(]\x1b\x9fbfQ\xc8Y\xa6\x1c\xf1=" +
-	"\x89\xcd\x00\x90+y\x94\x13\x1c6\x18\xcc,\xe6,\xac" +
-	"\x09\xca\xec?\xe3T\x0f\xbe\x10\x0dS\xfd\xd1p*\x80" +
-	"\x9c\xe5Q.p\x88%\xf4\xf2\xb3B\xd9\x80G\x97\x85" +
-	"K\xb7\x03\xc8\x16\x8f\xf2j\x0em\xd3=\xa4\x050\xeb" +
-	"!Z\x975\xad\x96\x82\xf7\xd7\xaa\xaci\xb5\xe9\x86\x85" +
-	"\x02p(\x00\xf1V7\xd9\xcc\xc5t\xa7Z\xb29v" +
-	"\xb5\xcak\x16F\x81\xc3(\x0cx\xa9\\~\xc4)\xb1" +
-	"\xb9\xb7\xdd\xf3f\"\x91\xe1?y\x94\xff+\xe4\xcd\x14" +
-	"\xcac\x97\xf2(_\xc1\xa1\xadd2zQ\xb3\x16\x00" +
-	"\xaft\x96q>\xcd \x9e1X@\x87\xa1C\xed%" +
-	"\x872\xb0\xe3\x86\x92\xef\x15a\x02\xbb\x9aGyt\xff" +
-	"p\xf9'F\xfbI$t\x812\xc4\xc5\x14sS\xf8" +
-	"d\x83\x99B1g\x91\xff\xd5\xb6\xed\x02@\x91\x1b\xcf" +
-	"\xa3|)\x871\xfc\xc6v\x11\x98\xb49@\xa0\x8e\x19" +
-	"\x86n`MP\xe2J\xc4\xca\x94\x0e@]kb\x96" +
-	"\xa2\xe6\x90H\xef\xf7[e\xf4\x1b\xec\xd6\x06\x88\xb8\xe2" +
-	"\xf1\x0d\xc4\xbd\xdeh\x10yjx\x94/\xe0\xd0\xee4" +
-	"\x94\x0ckc\x06\xaazv\x9e\xa2\xe9i\x9ee\xfaP" +
-	"a\xc4\xb9\x1e\x9arn\x91\x09\xfe\xae\x81\xf7\x1b\xac\x04" +
-	"Bi{[\x9dks\xc2\xb7y\xe5\xb8\xa0\xd4\xf9\x04" +
-	"[\xd3\x11\xe4b?\xdbl$*\xde\xce\xa3\xbc%\x94" +
-	"\xb57Q^\xba\x97G\xf9a\x0e\xc5H$\x81\x11\x00" +
-	"\xf1A\xbaY[x\x94wr\xbd\x0b\"\xebf\x9a\xd5" +
-	"\xa4v\x82\xc0\xcc@J&6\xa9\x9d\x0cx\xf3|3" +
-	"W\xe5 x\xe8\x1d\xa6\x9ec\x16kb\x99\x9cb(" +
-	"\x96\xda\xcd\xdc\xef%2zA\x1d\x88\xb7\xa9>\x17\x83" +
-	"\xf8\x1b\xf7z\x90\x10\x1d\xc6\x05\xe9O`\xa1\xd6a\x00" +
-	"k]\xe5d\x99\xae\xf5\xe1@pcJ<@s\xa0" +
-	"\xea\x16,\x9f_\xb0TA\xd7L\xb2/\x14\xfa\x19\xfd" +
-	"\x85\xde\x08B\xefe\xca\x8dk\xc3\x91/e\xcaM\xdb" +
-	"\x83 \x8b\x11\xce\x8d\xfc\x8e]\x00\xf2N\x1e\xe5'9" +
-	"lp\x8b8\xd6\x04\x8f\xe0R\xb4\xdcR5G\x87\xba" +
-	"\x8c\x92\x0b\xb2\xa9m\xb0BN\xc9\xb0f,\x95e@" +
-	"\x04\x0e\xd1\xa1H\xbe`0\xd3DU\xd7\xe4\xa2\x92S" +
-	"y\xab\xc7o\xa5\xb4b\xbe\xcd`\xdd*\xeaEs\xa6" +
-	"e\xb1\xbcP\xb0\xcc\xa14Z\x01@\x94\x1f\x045g" +
-	"\x96%\xdf\xfa \xf7\xf8\x00M\xa2\xe4{1\x8f\xf2t" +
-	"\x0e\xe3\xc5\xa2\x1a\xe4\xba\x9c\x9eq\xe2\x06\xf1yJ\x9e" +
-	"\xf5\x89v\xc5\xa0w\xb5\xd7M\xf7\x92\xed\xbfRc0" +
-	"p/N\xae;\xcdj\xc8d\xba\x02\x8d<\xcasB" +
-	"&\xb7L\x0d\xf9\xe1\x99<\xb7#\xf0C\xf8.\xeb\xf1" +
-	"\xac\xaacy\xca\xdc\x1e\x98%gf\x82pM\xb0f" +
-	" \xfb\xc2\x17j~\xa1\xce\xf1\x90l\x9c\xee\xd9(\xf5" +
-	"`+@z\x19\xf2\x98^\x87\x81\x99\xd2\x1a\x9c\x05\x90" +
-	"^A\xf2\xdb1\xb0TZ\x8fI\x80\xf4j\x92\xdf\x89" +
-	"\xfe\x9bA\xda\x88\xbb\x01\xd2w\x92x\x1b-\x8f\xf0\xce" +
-	"\x95\x90\xb6:\xea\xb7\x90|'\xc9\xa3\x91\x04F\x01\xa4" +
-	"\x1dX\x0f\x90\xdeF\xf2gH^\xc1%\xb0\x02@\xda" +
-	"\x8b]\x00\xe9=$\x7f\x9e\xe4B4A\xcf&i\x1f" +
-	"\x1a\x00\xe9\x9f\x91\xfc%\x92W\x8eN`%\x80\xb4\xdf" +
-	"\x91\xbfH\xf27H^5&\x81U\x00\xd2\x01\\\x0b" +
-	"\x90~\x8d\xe4\x87I>\x0c\x138\x0c@:\x84\xdb\x01" +
-	"\xd2\x87I\xfe[\x92\x0f\xafH\xe0p\x00\xe9\x03\xc7\x9e" +
-	"#$\xff\x98\xe4\xd5\x91\x04V\x03H\x1f\xe2.\x80\xf4" +
-	"\xc7$\xff3\xc9cB\x02c\x00\xd2'\x8e_\xa7H" +
-	"^\xc9\x95\xb5\xec\x1e\xa3\xca\xfar^7\xfd\x90\xb1\xd2" +
-	"\x1dG\x97\xeemz\x9czo\x8c\x07C0@\x8c\x03" +
-	"\xda\x05]\xcf\xcd\xeb\xcd\xd4\xb8\xa5t\x9a\xde\x1b\xa0&" +
-	"\x98K\x00\x92\xd0\xaf\xfb\x10\xd7\xb5\x96\xac\x9f\x08\xca\xb3" +
-	"\x8eg\x89j\xce,Zz\xb1\x00uY\xc5bY?" +
-	"\xe7\x18Em\xb6\xa1\xe7\x17 3\xf2\xaa\xa6\xe4\x06\xc9" +
-	"FU\xc0a\x15\x94R\x82\xa7{\xe0\xd4t\xf6\x17\x8d" +
-	"\xcfh\xae\x9c\xd1u\x85\x19\x0b\x94\xce\xa1\xe4\xa9\xa9A" +
-	"\xe7\x18\xd7B\x09\xa9\xae[\xc9\x15\xbfMz\xea\xddJ" +
-	"\xa4\x1a\xdcVd\xb0~\xdf\x1bS\x0c\x9eJz7\x84" +
-	"\xbd\x0b*\x86&\x88t\x0eW\xd2?d\xf3;\x99\xe5" +
-	"\xfe\xa2\x87+=\x1b\x84p\x99?\xb7\xdd)f\xc6\x87" +
-	"\xe2z0\xce\x19\xfc\xa9\xd3O\xe1\xef\xa7\xec{=g" +
-	"\xe8\xb9C\xb1_\xc4\xa3\xbc$\x14{\xd6\xda\xcfs'" +
-	"\x15\xcc9D\x9e+\x0d:\xa8P\x14x\x94Wp\x18" +
-	"\xa7w)\xd6\x04s\xdf^F\xf7~\x8b\x13\x15Z\xb4" +
-	",\x03\\\xe6\xb19T>\xfc\x09\xe8\xe0\xdd\xd9\xd0\xdc" +
-	"\xf6\xba\xdeA\x01\xf7\xa7\x8ae'\x9f\xf5\xc9\xd5\xe0\x1e" +
-	"J<\x1b\xed\x8cX\xbc\x09+z\xb3:q\xefr\xe0" +
-	"\xc4\xc7\x05\x0c\xa6\x90\xe8\x0d\x1d\xc5\x1d\x06p\xe2V\x01" +
-	"9\x7ff\x8d\xdelZ\xdcx\x07p\xe2z\x01y\x7f" +
-	"\xe4\x8c\xde\xb4kJ\xcf0\x04N\\)`\xc4\x1f\xe5" +
-	"\xa37+\x13\x97v\x01'\xaa\x02F\xfdi6z\xe3" +
-	"T\xf1\xfa\xb5\xc0\x89\xed\xc1L\x07\x1a\\?\x1a\xd1\xf6" +
-	"8\x0au\x0eK{Ox\xdcU\x00\x8dh{=0" +
-	"\x7f\xb6&\xd8Y\xe5\x0d) \x9eQ,\xd6H\xcd\x99" +
-	"{\xff\xb1\x94\x00\xa0\x11\xe5\x08\x86F\x85\x00\xe7\xfb\xbe" +
-	"L\xb1:'\xce\xdf\xb6e\xf2\xf6\x7f\xcb\x94\xc4\xf7g" +
-	"5\x9d\xe3\x0f\xbbBz\xbbB\x0f\xdfA\x1a\xbf\xc8\xd9" +
-	"\xbc\xf0\xc8\x1f\xa7\xcd\xa4\xff\xdf}\xfd\x87\xa8qz\x83" +
-	"G\xf9H\xe8Z\xbfK\xc2\xb7y\x94\x8f\x85\x1a\xa7\xa3" +
-	"t\xd7\x8f\xf0(\x9f\x09\xe6\x97\x9f\xdd\x01 \x9f\xe11" +
-	"\x15jD\xc4\xbf\xd3\xc2\xaf\xa9\\;m\x08\xbamH" +
-	"\x147\x03\xa4+\xa9\x8c'\x9c6$\xe2\xb6!\"v" +
-	"\x00\xa4kH~A\xb8\x0d\x19\x83\xd7\x01\xa4G\x93|" +
-	"<\xf6~\xd7\x08E#h\xd4rz\xe7\x1cU\xeb\xb7" +
-	"\xb6y\x03U\xb4f+j\xaeh0\x08Jk)\xd9" +
-	"4\x85\xaa\xbd;iu\x87*i\"a\x16M\x7f\xe0" +
-	"r\x0e/\xca!U\x9ef\xc3\xd0\xd1(kb\xa7\x06" +
-	"M\xac\xdf\xc3R/~5\x8f\xf2\x02\x0aE\xa3\x1b\x0a" +
-	"\xb9#h\xbb\xeb2J\xd1d}|\x00\x9e\x19\xfe\x14" +
-	"\xc0\\\xa2\x17s\xd9\x14\x03\xc12z\xca \x18\xb4\x99" +
-	"M\xb3\xb8\x97\xb9\xdc\xe1\xb0\xf7\x8f\x0e\xf4\xfe\x9f\x11\x1a" +
-	"\x0e{\x13z\xf4\xfem\xd5w8\xeca\xd0g8\xec" +
-	"~p8\xda{8|\x1e\xcfW\xb7\x8c\x852\xc69" +
-	"\xcdL\x87<j\xf4\xff\xb3[v\xd3\xab\xcewL\xe0" +
-	"\x15\xa4\x7f\x04\x00\x00\xff\xff\xa5\x0ed\xc9"
+type UpdateConfigurationResponse struct{ capnp.Struct }
+
+// UpdateConfigurationResponse_TypeID is the unique identifier for the type UpdateConfigurationResponse.
+const UpdateConfigurationResponse_TypeID = 0xdb58ff694ba05cf9
+
+func NewUpdateConfigurationResponse(s *capnp.Segment) (UpdateConfigurationResponse, error) {
+	st, err := capnp.NewStruct(s, capnp.ObjectSize{DataSize: 8, PointerCount: 1})
+	return UpdateConfigurationResponse{st}, err
+}
+
+func NewRootUpdateConfigurationResponse(s *capnp.Segment) (UpdateConfigurationResponse, error) {
+	st, err := capnp.NewRootStruct(s, capnp.ObjectSize{DataSize: 8, PointerCount: 1})
+	return UpdateConfigurationResponse{st}, err
+}
+
+func ReadRootUpdateConfigurationResponse(msg *capnp.Message) (UpdateConfigurationResponse, error) {
+	root, err := msg.RootPtr()
+	return UpdateConfigurationResponse{root.Struct()}, err
+}
+
+func (s UpdateConfigurationResponse) String() string {
+	str, _ := text.Marshal(0xdb58ff694ba05cf9, s.Struct)
+	return str
+}
+
+func (s UpdateConfigurationResponse) LatestAppliedVersion() int32 {
+	return int32(s.Struct.Uint32(0))
+}
+
+func (s UpdateConfigurationResponse) SetLatestAppliedVersion(v int32) {
+	s.Struct.SetUint32(0, uint32(v))
+}
+
+func (s UpdateConfigurationResponse) Err() (string, error) {
+	p, err := s.Struct.Ptr(0)
+	return p.Text(), err
+}
+
+func (s UpdateConfigurationResponse) HasErr() bool {
+	p, err := s.Struct.Ptr(0)
+	return p.IsValid() || err != nil
+}
+
+func (s UpdateConfigurationResponse) ErrBytes() ([]byte, error) {
+	p, err := s.Struct.Ptr(0)
+	return p.TextBytes(), err
+}
+
+func (s UpdateConfigurationResponse) SetErr(v string) error {
+	return s.Struct.SetText(0, v)
+}
+
+// UpdateConfigurationResponse_List is a list of UpdateConfigurationResponse.
+type UpdateConfigurationResponse_List struct{ capnp.List }
+
+// NewUpdateConfigurationResponse creates a new list of UpdateConfigurationResponse.
+func NewUpdateConfigurationResponse_List(s *capnp.Segment, sz int32) (UpdateConfigurationResponse_List, error) {
+	l, err := capnp.NewCompositeList(s, capnp.ObjectSize{DataSize: 8, PointerCount: 1}, sz)
+	return UpdateConfigurationResponse_List{l}, err
+}
+
+func (s UpdateConfigurationResponse_List) At(i int) UpdateConfigurationResponse {
+	return UpdateConfigurationResponse{s.List.Struct(i)}
+}
+
+func (s UpdateConfigurationResponse_List) Set(i int, v UpdateConfigurationResponse) error {
+	return s.List.SetStruct(i, v.Struct)
+}
+
+func (s UpdateConfigurationResponse_List) String() string {
+	str, _ := text.MarshalList(0xdb58ff694ba05cf9, s.List)
+	return str
+}
+
+// UpdateConfigurationResponse_Promise is a wrapper for a UpdateConfigurationResponse promised by a client call.
+type UpdateConfigurationResponse_Promise struct{ *capnp.Pipeline }
+
+func (p UpdateConfigurationResponse_Promise) Struct() (UpdateConfigurationResponse, error) {
+	s, err := p.Pipeline.Struct()
+	return UpdateConfigurationResponse{s}, err
+}
+
+type ConfigurationManager struct{ Client capnp.Client }
+
+// ConfigurationManager_TypeID is the unique identifier for the type ConfigurationManager.
+const ConfigurationManager_TypeID = 0xb48edfbdaa25db04
+
+func (c ConfigurationManager) UpdateConfiguration(ctx context.Context, params func(ConfigurationManager_updateConfiguration_Params) error, opts ...capnp.CallOption) ConfigurationManager_updateConfiguration_Results_Promise {
+	if c.Client == nil {
+		return ConfigurationManager_updateConfiguration_Results_Promise{Pipeline: capnp.NewPipeline(capnp.ErrorAnswer(capnp.ErrNullClient))}
+	}
+	call := &capnp.Call{
+		Ctx: ctx,
+		Method: capnp.Method{
+			InterfaceID:   0xb48edfbdaa25db04,
+			MethodID:      0,
+			InterfaceName: "tunnelrpc/tunnelrpc.capnp:ConfigurationManager",
+			MethodName:    "updateConfiguration",
+		},
+		Options: capnp.NewCallOptions(opts),
+	}
+	if params != nil {
+		call.ParamsSize = capnp.ObjectSize{DataSize: 8, PointerCount: 1}
+		call.ParamsFunc = func(s capnp.Struct) error { return params(ConfigurationManager_updateConfiguration_Params{Struct: s}) }
+	}
+	return ConfigurationManager_updateConfiguration_Results_Promise{Pipeline: capnp.NewPipeline(c.Client.Call(call))}
+}
+
+type ConfigurationManager_Server interface {
+	UpdateConfiguration(ConfigurationManager_updateConfiguration) error
+}
+
+func ConfigurationManager_ServerToClient(s ConfigurationManager_Server) ConfigurationManager {
+	c, _ := s.(server.Closer)
+	return ConfigurationManager{Client: server.New(ConfigurationManager_Methods(nil, s), c)}
+}
+
+func ConfigurationManager_Methods(methods []server.Method, s ConfigurationManager_Server) []server.Method {
+	if cap(methods) == 0 {
+		methods = make([]server.Method, 0, 1)
+	}
+
+	methods = append(methods, server.Method{
+		Method: capnp.Method{
+			InterfaceID:   0xb48edfbdaa25db04,
+			MethodID:      0,
+			InterfaceName: "tunnelrpc/tunnelrpc.capnp:ConfigurationManager",
+			MethodName:    "updateConfiguration",
+		},
+		Impl: func(c context.Context, opts capnp.CallOptions, p, r capnp.Struct) error {
+			call := ConfigurationManager_updateConfiguration{c, opts, ConfigurationManager_updateConfiguration_Params{Struct: p}, ConfigurationManager_updateConfiguration_Results{Struct: r}}
+			return s.UpdateConfiguration(call)
+		},
+		ResultsSize: capnp.ObjectSize{DataSize: 0, PointerCount: 1},
+	})
+
+	return methods
+}
+
+// ConfigurationManager_updateConfiguration holds the arguments for a server call to ConfigurationManager.updateConfiguration.
+type ConfigurationManager_updateConfiguration struct {
+	Ctx     context.Context
+	Options capnp.CallOptions
+	Params  ConfigurationManager_updateConfiguration_Params
+	Results ConfigurationManager_updateConfiguration_Results
+}
+
+type ConfigurationManager_updateConfiguration_Params struct{ capnp.Struct }
+
+// ConfigurationManager_updateConfiguration_Params_TypeID is the unique identifier for the type ConfigurationManager_updateConfiguration_Params.
+const ConfigurationManager_updateConfiguration_Params_TypeID = 0xb177ca2526a3ca76
+
+func NewConfigurationManager_updateConfiguration_Params(s *capnp.Segment) (ConfigurationManager_updateConfiguration_Params, error) {
+	st, err := capnp.NewStruct(s, capnp.ObjectSize{DataSize: 8, PointerCount: 1})
+	return ConfigurationManager_updateConfiguration_Params{st}, err
+}
+
+func NewRootConfigurationManager_updateConfiguration_Params(s *capnp.Segment) (ConfigurationManager_updateConfiguration_Params, error) {
+	st, err := capnp.NewRootStruct(s, capnp.ObjectSize{DataSize: 8, PointerCount: 1})
+	return ConfigurationManager_updateConfiguration_Params{st}, err
+}
+
+func ReadRootConfigurationManager_updateConfiguration_Params(msg *capnp.Message) (ConfigurationManager_updateConfiguration_Params, error) {
+	root, err := msg.RootPtr()
+	return ConfigurationManager_updateConfiguration_Params{root.Struct()}, err
+}
+
+func (s ConfigurationManager_updateConfiguration_Params) String() string {
+	str, _ := text.Marshal(0xb177ca2526a3ca76, s.Struct)
+	return str
+}
+
+func (s ConfigurationManager_updateConfiguration_Params) Version() int32 {
+	return int32(s.Struct.Uint32(0))
+}
+
+func (s ConfigurationManager_updateConfiguration_Params) SetVersion(v int32) {
+	s.Struct.SetUint32(0, uint32(v))
+}
+
+func (s ConfigurationManager_updateConfiguration_Params) Config() ([]byte, error) {
+	p, err := s.Struct.Ptr(0)
+	return []byte(p.Data()), err
+}
+
+func (s ConfigurationManager_updateConfiguration_Params) HasConfig() bool {
+	p, err := s.Struct.Ptr(0)
+	return p.IsValid() || err != nil
+}
+
+func (s ConfigurationManager_updateConfiguration_Params) SetConfig(v []byte) error {
+	return s.Struct.SetData(0, v)
+}
+
+// ConfigurationManager_updateConfiguration_Params_List is a list of ConfigurationManager_updateConfiguration_Params.
+type ConfigurationManager_updateConfiguration_Params_List struct{ capnp.List }
+
+// NewConfigurationManager_updateConfiguration_Params creates a new list of ConfigurationManager_updateConfiguration_Params.
+func NewConfigurationManager_updateConfiguration_Params_List(s *capnp.Segment, sz int32) (ConfigurationManager_updateConfiguration_Params_List, error) {
+	l, err := capnp.NewCompositeList(s, capnp.ObjectSize{DataSize: 8, PointerCount: 1}, sz)
+	return ConfigurationManager_updateConfiguration_Params_List{l}, err
+}
+
+func (s ConfigurationManager_updateConfiguration_Params_List) At(i int) ConfigurationManager_updateConfiguration_Params {
+	return ConfigurationManager_updateConfiguration_Params{s.List.Struct(i)}
+}
+
+func (s ConfigurationManager_updateConfiguration_Params_List) Set(i int, v ConfigurationManager_updateConfiguration_Params) error {
+	return s.List.SetStruct(i, v.Struct)
+}
+
+func (s ConfigurationManager_updateConfiguration_Params_List) String() string {
+	str, _ := text.MarshalList(0xb177ca2526a3ca76, s.List)
+	return str
+}
+
+// ConfigurationManager_updateConfiguration_Params_Promise is a wrapper for a ConfigurationManager_updateConfiguration_Params promised by a client call.
+type ConfigurationManager_updateConfiguration_Params_Promise struct{ *capnp.Pipeline }
+
+func (p ConfigurationManager_updateConfiguration_Params_Promise) Struct() (ConfigurationManager_updateConfiguration_Params, error) {
+	s, err := p.Pipeline.Struct()
+	return ConfigurationManager_updateConfiguration_Params{s}, err
+}
+
+type ConfigurationManager_updateConfiguration_Results struct{ capnp.Struct }
+
+// ConfigurationManager_updateConfiguration_Results_TypeID is the unique identifier for the type ConfigurationManager_updateConfiguration_Results.
+const ConfigurationManager_updateConfiguration_Results_TypeID = 0x958096448eb3373e
+
+func NewConfigurationManager_updateConfiguration_Results(s *capnp.Segment) (ConfigurationManager_updateConfiguration_Results, error) {
+	st, err := capnp.NewStruct(s, capnp.ObjectSize{DataSize: 0, PointerCount: 1})
+	return ConfigurationManager_updateConfiguration_Results{st}, err
+}
+
+func NewRootConfigurationManager_updateConfiguration_Results(s *capnp.Segment) (ConfigurationManager_updateConfiguration_Results, error) {
+	st, err := capnp.NewRootStruct(s, capnp.ObjectSize{DataSize: 0, PointerCount: 1})
+	return ConfigurationManager_updateConfiguration_Results{st}, err
+}
+
+func ReadRootConfigurationManager_updateConfiguration_Results(msg *capnp.Message) (ConfigurationManager_updateConfiguration_Results, error) {
+	root, err := msg.RootPtr()
+	return ConfigurationManager_updateConfiguration_Results{root.Struct()}, err
+}
+
+func (s ConfigurationManager_updateConfiguration_Results) String() string {
+	str, _ := text.Marshal(0x958096448eb3373e, s.Struct)
+	return str
+}
+
+func (s ConfigurationManager_updateConfiguration_Results) Result() (UpdateConfigurationResponse, error) {
+	p, err := s.Struct.Ptr(0)
+	return UpdateConfigurationResponse{Struct: p.Struct()}, err
+}
+
+func (s ConfigurationManager_updateConfiguration_Results) HasResult() bool {
+	p, err := s.Struct.Ptr(0)
+	return p.IsValid() || err != nil
+}
+
+func (s ConfigurationManager_updateConfiguration_Results) SetResult(v UpdateConfigurationResponse) error {
+	return s.Struct.SetPtr(0, v.Struct.ToPtr())
+}
+
+// NewResult sets the result field to a newly
+// allocated UpdateConfigurationResponse struct, preferring placement in s's segment.
+func (s ConfigurationManager_updateConfiguration_Results) NewResult() (UpdateConfigurationResponse, error) {
+	ss, err := NewUpdateConfigurationResponse(s.Struct.Segment())
+	if err != nil {
+		return UpdateConfigurationResponse{}, err
+	}
+	err = s.Struct.SetPtr(0, ss.Struct.ToPtr())
+	return ss, err
+}
+
+// ConfigurationManager_updateConfiguration_Results_List is a list of ConfigurationManager_updateConfiguration_Results.
+type ConfigurationManager_updateConfiguration_Results_List struct{ capnp.List }
+
+// NewConfigurationManager_updateConfiguration_Results creates a new list of ConfigurationManager_updateConfiguration_Results.
+func NewConfigurationManager_updateConfiguration_Results_List(s *capnp.Segment, sz int32) (ConfigurationManager_updateConfiguration_Results_List, error) {
+	l, err := capnp.NewCompositeList(s, capnp.ObjectSize{DataSize: 0, PointerCount: 1}, sz)
+	return ConfigurationManager_updateConfiguration_Results_List{l}, err
+}
+
+func (s ConfigurationManager_updateConfiguration_Results_List) At(i int) ConfigurationManager_updateConfiguration_Results {
+	return ConfigurationManager_updateConfiguration_Results{s.List.Struct(i)}
+}
+
+func (s ConfigurationManager_updateConfiguration_Results_List) Set(i int, v ConfigurationManager_updateConfiguration_Results) error {
+	return s.List.SetStruct(i, v.Struct)
+}
+
+func (s ConfigurationManager_updateConfiguration_Results_List) String() string {
+	str, _ := text.MarshalList(0x958096448eb3373e, s.List)
+	return str
+}
+
+// ConfigurationManager_updateConfiguration_Results_Promise is a wrapper for a ConfigurationManager_updateConfiguration_Results promised by a client call.
+type ConfigurationManager_updateConfiguration_Results_Promise struct{ *capnp.Pipeline }
+
+func (p ConfigurationManager_updateConfiguration_Results_Promise) Struct() (ConfigurationManager_updateConfiguration_Results, error) {
+	s, err := p.Pipeline.Struct()
+	return ConfigurationManager_updateConfiguration_Results{s}, err
+}
+
+func (p ConfigurationManager_updateConfiguration_Results_Promise) Result() UpdateConfigurationResponse_Promise {
+	return UpdateConfigurationResponse_Promise{Pipeline: p.Pipeline.GetPipeline(0)}
+}
+
+type CloudflaredServer struct{ Client capnp.Client }
+
+// CloudflaredServer_TypeID is the unique identifier for the type CloudflaredServer.
+const CloudflaredServer_TypeID = 0xf548cef9dea2a4a1
+
+func (c CloudflaredServer) RegisterUdpSession(ctx context.Context, params func(SessionManager_registerUdpSession_Params) error, opts ...capnp.CallOption) SessionManager_registerUdpSession_Results_Promise {
+	if c.Client == nil {
+		return SessionManager_registerUdpSession_Results_Promise{Pipeline: capnp.NewPipeline(capnp.ErrorAnswer(capnp.ErrNullClient))}
+	}
+	call := &capnp.Call{
+		Ctx: ctx,
+		Method: capnp.Method{
+			InterfaceID:   0x839445a59fb01686,
+			MethodID:      0,
+			InterfaceName: "tunnelrpc/tunnelrpc.capnp:SessionManager",
+			MethodName:    "registerUdpSession",
+		},
+		Options: capnp.NewCallOptions(opts),
+	}
+	if params != nil {
+		call.ParamsSize = capnp.ObjectSize{DataSize: 16, PointerCount: 2}
+		call.ParamsFunc = func(s capnp.Struct) error { return params(SessionManager_registerUdpSession_Params{Struct: s}) }
+	}
+	return SessionManager_registerUdpSession_Results_Promise{Pipeline: capnp.NewPipeline(c.Client.Call(call))}
+}
+func (c CloudflaredServer) UnregisterUdpSession(ctx context.Context, params func(SessionManager_unregisterUdpSession_Params) error, opts ...capnp.CallOption) SessionManager_unregisterUdpSession_Results_Promise {
+	if c.Client == nil {
+		return SessionManager_unregisterUdpSession_Results_Promise{Pipeline: capnp.NewPipeline(capnp.ErrorAnswer(capnp.ErrNullClient))}
+	}
+	call := &capnp.Call{
+		Ctx: ctx,
+		Method: capnp.Method{
+			InterfaceID:   0x839445a59fb01686,
+			MethodID:      1,
+			InterfaceName: "tunnelrpc/tunnelrpc.capnp:SessionManager",
+			MethodName:    "unregisterUdpSession",
+		},
+		Options: capnp.NewCallOptions(opts),
+	}
+	if params != nil {
+		call.ParamsSize = capnp.ObjectSize{DataSize: 0, PointerCount: 2}
+		call.ParamsFunc = func(s capnp.Struct) error { return params(SessionManager_unregisterUdpSession_Params{Struct: s}) }
+	}
+	return SessionManager_unregisterUdpSession_Results_Promise{Pipeline: capnp.NewPipeline(c.Client.Call(call))}
+}
+func (c CloudflaredServer) UpdateConfiguration(ctx context.Context, params func(ConfigurationManager_updateConfiguration_Params) error, opts ...capnp.CallOption) ConfigurationManager_updateConfiguration_Results_Promise {
+	if c.Client == nil {
+		return ConfigurationManager_updateConfiguration_Results_Promise{Pipeline: capnp.NewPipeline(capnp.ErrorAnswer(capnp.ErrNullClient))}
+	}
+	call := &capnp.Call{
+		Ctx: ctx,
+		Method: capnp.Method{
+			InterfaceID:   0xb48edfbdaa25db04,
+			MethodID:      0,
+			InterfaceName: "tunnelrpc/tunnelrpc.capnp:ConfigurationManager",
+			MethodName:    "updateConfiguration",
+		},
+		Options: capnp.NewCallOptions(opts),
+	}
+	if params != nil {
+		call.ParamsSize = capnp.ObjectSize{DataSize: 8, PointerCount: 1}
+		call.ParamsFunc = func(s capnp.Struct) error { return params(ConfigurationManager_updateConfiguration_Params{Struct: s}) }
+	}
+	return ConfigurationManager_updateConfiguration_Results_Promise{Pipeline: capnp.NewPipeline(c.Client.Call(call))}
+}
+
+type CloudflaredServer_Server interface {
+	RegisterUdpSession(SessionManager_registerUdpSession) error
+
+	UnregisterUdpSession(SessionManager_unregisterUdpSession) error
+
+	UpdateConfiguration(ConfigurationManager_updateConfiguration) error
+}
+
+func CloudflaredServer_ServerToClient(s CloudflaredServer_Server) CloudflaredServer {
+	c, _ := s.(server.Closer)
+	return CloudflaredServer{Client: server.New(CloudflaredServer_Methods(nil, s), c)}
+}
+
+func CloudflaredServer_Methods(methods []server.Method, s CloudflaredServer_Server) []server.Method {
+	if cap(methods) == 0 {
+		methods = make([]server.Method, 0, 3)
+	}
+
+	methods = append(methods, server.Method{
+		Method: capnp.Method{
+			InterfaceID:   0x839445a59fb01686,
+			MethodID:      0,
+			InterfaceName: "tunnelrpc/tunnelrpc.capnp:SessionManager",
+			MethodName:    "registerUdpSession",
+		},
+		Impl: func(c context.Context, opts capnp.CallOptions, p, r capnp.Struct) error {
+			call := SessionManager_registerUdpSession{c, opts, SessionManager_registerUdpSession_Params{Struct: p}, SessionManager_registerUdpSession_Results{Struct: r}}
+			return s.RegisterUdpSession(call)
+		},
+		ResultsSize: capnp.ObjectSize{DataSize: 0, PointerCount: 1},
+	})
+
+	methods = append(methods, server.Method{
+		Method: capnp.Method{
+			InterfaceID:   0x839445a59fb01686,
+			MethodID:      1,
+			InterfaceName: "tunnelrpc/tunnelrpc.capnp:SessionManager",
+			MethodName:    "unregisterUdpSession",
+		},
+		Impl: func(c context.Context, opts capnp.CallOptions, p, r capnp.Struct) error {
+			call := SessionManager_unregisterUdpSession{c, opts, SessionManager_unregisterUdpSession_Params{Struct: p}, SessionManager_unregisterUdpSession_Results{Struct: r}}
+			return s.UnregisterUdpSession(call)
+		},
+		ResultsSize: capnp.ObjectSize{DataSize: 0, PointerCount: 0},
+	})
+
+	methods = append(methods, server.Method{
+		Method: capnp.Method{
+			InterfaceID:   0xb48edfbdaa25db04,
+			MethodID:      0,
+			InterfaceName: "tunnelrpc/tunnelrpc.capnp:ConfigurationManager",
+			MethodName:    "updateConfiguration",
+		},
+		Impl: func(c context.Context, opts capnp.CallOptions, p, r capnp.Struct) error {
+			call := ConfigurationManager_updateConfiguration{c, opts, ConfigurationManager_updateConfiguration_Params{Struct: p}, ConfigurationManager_updateConfiguration_Results{Struct: r}}
+			return s.UpdateConfiguration(call)
+		},
+		ResultsSize: capnp.ObjectSize{DataSize: 0, PointerCount: 1},
+	})
+
+	return methods
+}
+
+const schema_db8274f9144abc7e = "x\xda\xccZ{t\x14\xe7u\xbfwfW#\x81V" +
+	"\xab\xf1\xac\xd1\x03T\xb5:P\x179\xd8\x06Jk\xab" +
+	"9\xd1\xc3\x12\xb1d\x03\x9a]\x94\xe3cC\x8eG\xbb" +
+	"\x9f\xa4Qwg\x96\x99Y\x19\x11\x130\x01c\xfb\xb8" +
+	"\x8eq\xc0\xb1Ih0.\xed\x01\xdb\xad\x89\xdd\xa6\xee" +
+	"1\xa7\xa6\xcd\xabq\xc0&\x87\xf4\x90@\x9a&\x84>" +
+	"8\xb8\xae14\x876\xf1\xf4\xdc\x99\x9d\x87v\x17\x09" +
+	"\x8c\xff\xc8\x7f\xab;\xdf\xe3\xde\xdf\xf7\xbb\x8f\xef~\xba" +
+	"\xed\xe6\x9a.nq\xf4\xed:\x00\xf9\xa5h\x95\xcd\xda" +
+	"\x7f\xb0a\xef\x82\x7f\xdc\x02r3\xa2\xfd\xf97\x06\x12" +
+	"\x97\xad-\xa7 \xca\x0b\x00KW\x08\x1bPR\x04\x01" +
+	"@Z+\xfc;\xa0\xfd\xc8\x9cW\xbe\xb6\xbfo\xe7\x17" +
+	"@l\xe6\x83\xc1\x80K\xbb\xab\x07P\x1a\xaa\xa6\x91r" +
+	"\xf5v\xe9\x10\xfd\xb2\xef\x16o\xbd?\xf1\xce1\x1a\x1d" +
+	"^:BK?W\xdd\x8e\xd2\x01g\xc2\xfejZ\xfa" +
+	"\x93\xb9\xb7\xf7\xfd\xc1\xae\xb7\xb6\x82\xd8\xccMYzG" +
+	"\xcd\x06\x94\xf6\xd7\xd0\xc8\xe7kV\x01\xda\x1f\xecl|" +
+	"\xf1\xf9c\xdf\xdd\x06\xe2M\x08EM_\xaf\xf91\x02" +
+	"JGk\xfe\x0a\xd0>z\xe9\xfe\x8b\xaf}{\xd9#" +
+	" .\xa4\x01H\x03r\xb3\xda8@i\xdb\xacN@" +
+	"\xfb\xdc\xf9\xff\xdb\xfe\xb9\x85+\x9f\x02y!r\x00Q" +
+	"\x8eF\xec\x9f\xd5L#\x0e\xcf\"m:\x97\x1f}\xbd" +
+	"y\xe93;KTw\x06\xee\x99\xdd\x8e\xd2\xcb\xb3I" +
+	"\xa1\x03\xb3\x1f\x04\xb4?\xf5\x87\xaf>\xd9\xfb\xcc\xe6]" +
+	" \xde\xea\xef\x17\xab\xbd\x8fV[XK\xfb\xfdO\xdd" +
+	"W\x8e\x15\xee\xfc\xc63E\x85\x9cU\xfak\xdbi\x80" +
+	"RK+\xb4M,x\xe0\x1f\xbe\xf5\xea\x97A^\x84" +
+	"h\x9f\x1e\xbe\xf9\x87\xfc\x9e\x83\xa7`\x08\x05\xd2o\xe9" +
+	"\x91\xda}d\xddqg\xec\xdb\x9fx\xe3\xef\x9ezu" +
+	"\xfbW@\xbe\x09\x11\xc0AsY\xec\x7fi@\x7f\x8c" +
+	"v\xdby\xf2\xf0\xca\xdc\x8e\xdd\xfb\\|\x9c\xef\xebb" +
+	"\x1c\x07\x11{k\xff/sC/\xa4^(\"\x17\xa5" +
+	"O,v\x01\x01\x97N\xc6Z\x11\xd0^\xf6\xe3\xb3\xab" +
+	"V|}\xe4/Bsw\xd5m\xa0\xb9\xdbG.\x1c" +
+	"\xa9O\xe6^,A\xc41vG\xddA\x94\x0e\xd49" +
+	"\x87YG*\xbc\xfc[w\xd7\xac?\xbb\xfc\x15\x10\x17" +
+	"y\xcb|\xab.I\xcbL|\xef\x85\xdf]\xf0\xbd\x07" +
+	"\x0f\x81|+\xfa`\x1d\xa1o(\xfd\xa4\x8e\xec\x8b\x9c" +
+	"Zp\xf0\xf0O\x9f|\xad\x8ccw\xc47\xa0\xb4\"" +
+	"N\xbb\xf4\xc7?-M\xd2/;\xb2\x96\xffPy\xf6" +
+	"\xef_+\xe5\xaf\x83\xb1\x12\x1fF\xa9\x10w\x10\x88;" +
+	"\xf6=~d\xf7\xcd\xd5_\xfb\xe0\xaf+\x9d\xebs\xf5" +
+	"\xc3(\xbd\\\xef\x9ck=irc?\x9e~sq" +
+	"\xe4\x1ba\xa2\xd5\x88\xe7\x08\xe9\x16\x91\x88\xd6\xf2nO" +
+	"L{o\xcb\x9b%\xab9\x03\x0f\x8b\x03(\x1d\x17i" +
+	"\xb5\xa3\xce\xe0\x81\xfb\xbf\xf4t\xf4\xec\x97\xbeC\x9a\x86" +
+	"\x18\x1e%\x1fX\xaa\xde`\xa0\xb4\xf1\x06\xfa9yC" +
+	"\x03\x0fh7\xbf\xf2G\x7f\xd9\x93\xf9\xd1[\x154\x95" +
+	".\xdfxA\x8a\xce\xa1_8\x87\x14=\xb3\xe8\xd0\xe7" +
+	"\xfe\xf3O\x8e\x9f(*\xea`\xbav\x8eC\x89us" +
+	"\xe8<.\xaf\xd9{\xb7j\xdf{\xaa\x14%\xf7\xf4\xe6" +
+	"|\x1d\xa5\x03\xcer\xfb\x9d\xe5|\xfeU\x1a]\xd30" +
+	"\x8eRK\x03\x8dnj\xa0\xb5\xb9\xb3J\xd3\xe6\x7f\xfe" +
+	"\xd4\xe9\x10eZ\x1a~\x8e\x10\xb1W~\xe6\xfe\xf1\x9a" +
+	"\x8dg\xce\x84\xd5\x12\x1b\x1c\xfc\x168S\xff\xeb\xcf\xcf" +
+	"}\xf1|.\xf3o\x0e\xed=\x84\xfb\x1a:\x88\x0ck" +
+	"\x1b\xc8\x0f\x1bZc}m'\x07\xcf\xb9Dr\x97\xb8" +
+	"\xa3\xb1\x87\x06\xc8\x8d\xb4\xc4\xb2\x07\xba\xd9\x9a\xdb\xef=" +
+	"W\xc6\x96u\x8d\x1d(=\xdcH\x1366nGi" +
+	"WS\x03\x80=\xf17;\xee}\xf1\x9b+/\xb8\x9e" +
+	"\xe8(\xbb\xadi\x09\x11\xf3\xc9\xcf\xf7\xae\xba\xa3\xed\xc8" +
+	"\x85\xb0\xb2\x1b\x9b\xc87\xa4\x1dM\xb4\xd3\xc8\xed\xe7?" +
+	"\xbd\xe0\xc9o_\xa8\xe4\x00\x87\x9a\xdaQ:\xd2D\xa0" +
+	"\x1c\xa6\xc1\xef-\xff\xd3\x13\xcd\xf1\xe6\x8b%\x00V\xd1" +
+	"\xd8\x9f5\x8d\xa3t\x89\xc6.}\xbf\xe9;D\xca\xe7" +
+	"\xffl\xdf\xbf\\>v\xd7\xa52\x1b\xce\xce\x1dF\xe9" +
+	"\xf2\\Z\xf6\xd2\\A\xba4\xf7&\x00\xfb\x91S\x9f" +
+	"]\xff\x83/|p\xa9\x94G\x8e\"\xef\xceM\xa2\x84" +
+	"\xf3h\xc6\xaf\xe7\x12\xeb\xbe\xbc\xfa?6\x9d\xdf5\xe7" +
+	"\x97ek\xef\x997\x8e\xd2!g\xe4\xcb\xf3\xb6K\xb1" +
+	"\x16\xf2\xa6w\x84\x17\x16\xf7nz\xebr\xc8o/\xcd" +
+	"\x1b x\x9e\x11\xbezf\xf3O?\xfb\xab0<\xef" +
+	"\xcf\xfb9\xc1\x13m!x\x1ez\xef\xb9\xbb\xbe\xb8\xe6" +
+	"\xa5\x0fC4X\xd0\xb2\x85\xa6Z\x05McY#\x1f" +
+	"I\xdf\xea\xfdL\xdf\x92V\xf2Z\xbe\xa3\xbb`\x8d1" +
+	"\xcdR\xd3\x8a\xc5\x92\xac\xd3\xcc\xeb\x9a\xc9\x06\x11\xe5z" +
+	">\x02\x10A\x00Q\x19\x07\x90\x1f\xe0Q\xcer(\"" +
+	"&\x88(\xa2J\xc21\x1ee\x8bC\x91\xe3\x12\x14%" +
+	"\xc5um\x00r\x96Gy=\x87\xc8'\x90\x07\x10\x0b" +
+	"O\x03\xc8\xeby\x94\xb7rh\xe7\x99\x91S4\xa6A" +
+	"\xdc\xea3\x0c\xac\x05\x0ek\x01m\x83Y\xc6\xa42\x9c" +
+	"\x858\x0b\x89\x85\xf1\x07-\x8c\x01\x871@{L/" +
+	"\x18\xe6\x90f\xa1\x9aM\xb2\x11\x83\x998\x86U\xc0a" +
+	"\x15\xe0t\xe6\xa5\x98i\xaa\xba\xb6B\xd1\x94Qf\x00" +
+	"\x90e\xd5|\x14\xc0\xcf@\xe8\xe5*q\xf1n\xe0\xc4" +
+	"E\x02\x06\xd9\x02=\xb2\x8a\xbfs\x108\xb1E\xb0\x0d" +
+	"6\xaa\x9a\x163p(\x93w\xd6\xe6u\xad\x0b\xed\x82" +
+	"\xe6~@f\xb8\x1f\xe2\xb4k\x17\x0eb\xa0\x1d_\xae" +
+	"\xdd\x9dY\x95iV\xbc_\x1b\xd1K \x1f\xa8\x04\xf9" +
+	"@\x11\xf2\xad!\xc8\x1f\xee\x01\x90\x1f\xe2Q~\x94C" +
+	"\x91/b\xbe\xad\x1d@\xde\xcc\xa3\xfc\x04\x87v\xda\xd9" +
+	"\xa4?\x03\x00>\x9a#L\xb1\x0a\x063IV\x078" +
+	"\xc8\xa3\x03z\x1d\xe0\xa6\x09f\x90\xee\xde!\xc4\x15#" +
+	"=\xe6\x1f\xd44H\xf7\xadWMK\xd5FW;\xf2" +
+	"\xceA=\xab\xa6'\xc9\xaaZG\xcf\x96\x0e\x00D\xf1" +
+	"\xc6\xfb\x00\x90\x13\xc5\x1e\x80NuT\xd3\x0dfgT" +
+	"3\xadk\x1a\x03>mm\x1aV\xb2\x8a\x96f\xfeF" +
+	"U\xe5\x1b\xb9\x1b\xa4\x981\xc1\x8c[\x94\x10}\xe7\x0f" +
+	"*\x86\xc2\xe7L\xb9\xd6\xc7\xb1\xef>\x00\xb9\x97Gy" +
+	"0\x84\xe3\x0a\xc2\xf1\x1e\x1e\xe5{C8\x0e\x11\x8e\x83" +
+	"<\xcak8\xb4uC\x1dU\xb5;\x19\xf0F\x98\x81" +
+	"\xa6\xa5)9F\x98\x15\xf1\xd8\xa4\xe7-U\xd7L\xac" +
+	"\x0fr\x0b \xd6\x87\x90\x12f\xe2\xe4-\x1e\xa5<F" +
+	"\xe9\xda\xfc$3\x0bB\xd62\xe5\x88oI\xac\x03@" +
+	"\xae\xe6QNp\xd8i0\xb3\x90\xb5\xb0>(\x09>" +
+	"\x8e]=\xf8B4LV\xa2\xe1\x12\x009\xc3\xa3\x9c" +
+	"\xe7\x10\x8b\xe8\xe5zB\xd1\x80G\x97\x85\xebv\x03\xc8" +
+	"\x16\x8f\xf2f\x0em\xd3\xdd\xa4\x1f0\xe3!\xda\x9a1" +
+	"\xad\xfe\xbc\xf7\xd7\xa6\x8ci\x0d\xea\x86\x85\x02p(\x00" +
+	"\xf1V7Y\xf7\x08\xf9T\x7f&\xcb\xeeRy\xcd\xc2" +
+	"(p\x18\x85i\x9d\xca\xe5G\x9c\x02\x9b\xeb\xed\x9e5" +
+	"\x0b\x89\x0c\xbf\xc7\xa3\xfc\xfb!k\x16S\x1c\xbb\x8dG" +
+	"\xf9\x93\x1c\xdaJ:\xad\x174k5\xf0\xcah\x09\xe7" +
+	"S\x0c\xe2i\x83\x05t\xf0\xb6\xad\xae\xe0\xd6\xba6\xa2" +
+	"\x8e\x16\x0c\xc5\x0a\x01^\xc8g\x14\x8bM\xf9\xe4\x9cs" +
+	"\x96\xbf\x8as\xf6\xab\x87k>g/2\x95\x9ct\xdc" +
+	"Prf\x18\x9bd%l\xe8T?\xc1\xa3|{\xe5" +
+	"\x03\xdc\x94c\xa6\xa9\x8c\xb2\xb2\xf0\x10\xad\x88\x89\xc6\xd2" +
+	"du\x92\xb9I\xe6\x16\x83\x99B!k\x91\x16\xb5\xb6" +
+	"\xed\xaaA\xdc\x9a\xcf\xa3|\x1b\x871\xfc\xd0v\xf5X" +
+	"\xf4tpF\xad\xcc0t\x03\xeb\x83$\\\x84$]" +
+	"\xdc\x00u\xad\x97Y\x8a\x9aErK\xbf\xda,\x01n" +
+	"\xa6\xb8\x12\xc0\xe6\x8a\xe7w\x92w\xe4\xa6\x9c\x14\xd1\xbb" +
+	"\x9eGy\x1e\x87\xf6\xa8\xa1\xa4\xd9 3P\xd53+" +
+	"\x15MO\xf1,]F\xd6\xbak\xdd\xd4\xe1\x87e\x82" +
+	"?k\xfa\xf9\x06+\x82P\x9c>\xd8\xea\xea\x9c\xf0u" +
+	"\xde\xd8\x16$c\xff\x98\x1f\x1e\x0e\xb2\x85\x1f\x0f\x1f#" +
+	"gy\x94Gyg(\xaf\xec\xa0\xc8\xf9\x14\x8f\xf2W" +
+	"9\x14#\x91\x04F\x00\xc4\xe7\x88%;y\x94\xf7r" +
+	"SS6\x9b`\x9a\xd5\xab\x8e\x82\xc0\xcc@J*\xf6" +
+	"\xaa\xa3\x0cx\xf3zck\xf5\x0cx\xe8\xc3\xa6\x9ee" +
+	"\x16\xebe\xe9\xacB.7\xc1\xdc\xefE2z\x87:" +
+	"\x1do\x93e\xdeC\xfc\x8d{UR\x88\x0em\x81\xe3" +
+	"\x0a,T\xdcL\xa3\xad\xbb\xb8\x1b\x0c\xca8\x10xL" +
+	"\x91\x07h~,A\xc7\xb1\x19\xa78\x7fO\xe0u\x1e" +
+	")\x16u\x04\x01\xc1\xaf\x09\"\xc0a\x04\xb03\xed," +
+	"X\x16\x0a#3i\xd5\xe9\xaa\xe5\x02GE\x98w\x17" +
+	"E\xef\x02/\x8a\xfb\x80\x13c\x82\xedi\x8e\xde|\xa1" +
+	"\xac\xa0\x8aL\x17eV\xe5-U\xd05\x93\xf6\x0a\xf1" +
+	"\xbf\xa3\x12\xff\x8d\x80\xff^B{lK\x98\xfe\xc5\x84" +
+	"\xb6cw\xc0t1\xc2\xb9\xf4\xdf\xb3\x0f@\xde\xcb\xa3" +
+	"\xfc\x12\x87\x9dn\xad\x85\xf5A\xe3\xa5HY\xb7\xa2\xb8" +
+	"G\x87\xd6\xb4\x92\x0d\x92\x9em\xb0|VI\xb3>," +
+	"VO\x80\x08\x1c\xa2\xe3'\xb9\xbc\xc1L\x13U]\x93" +
+	"\x0bJV\xe5\xadI\xbf\xe2\xd5\x0a\xb9A\x83M\xa8\xa8" +
+	"\x17\xccn\xcbb9!o\x99WS\x0f\x07\x00Q\x90" +
+	"\x14\xd4\xacY\x92#\xdb\x03*\xf8\x00-\x1a\x0f\xf2@" +
+	"\xbcPP\xfd\x04`g\xf5\xb4s\xb2\x10_\xa9\xe4\xca" +
+	"\xf3@\xd5\x8c\x01kJ\xb8\xf3\xd2\xd2oR\xfd6\xfd" +
+	"\x95\x89Lw\xee\x14!\x95)\x0et\xf1(\xdf\x13R" +
+	"\xb9\x7fI\xc8\x0eO\xe5\x15\xc3\x81\x1d\xc2\x1f\xb3IO" +
+	"\xabV\x96\xa3\xf4\xe5\x81Y4\xa6\x1b\x84\xbb\x831\xd3" +
+	"\xe9\x17\x8e*\xab\xf2\xad\x8e\x85\xa4\xe3\xed\x9e\x8e\xd2$" +
+	"\x0e\x00\xa4\xd6#\x8f\xa9\xad\x18\xa8)=\x8c=\x00\xa9" +
+	"\x87H\xfe(\x06\x9aJ\xdb\xb0\x19 \xb5\x99\xe4O\xa0" +
+	"\x7f\xb5\x93\x1e\xc3\x83\x00\xa9'H\xfc,\x0d\x8f\xf0\x8e" +
+	"KH\xbb\x9c\xe5w\x92|/\xc9\xa3\x91\x04F\x01\xa4" +
+	"=\xd8\x0e\x90z\x96\xe4\xaf\x91\xbc\x8aK`\x15\x80t" +
+	"\x08\xc7\x01R\xaf\x90\xfc\x0d\x92\x0b\xd1\x04\xddn\xa5\xd7" +
+	"\xd1\x00H\xfd-\xc9\xbfI\xf2\xea\xc6\x04V\x03HG" +
+	"\x1c\xf9\x9b$\xff>\xc9k\x9a\x12X\x03 \xfd\x13n" +
+	"\x01H}\x97\xe4'H>\x0b\x138\x0b@:\x8e\xbb" +
+	"\x01R'H\xfe\xaf$\x9f]\x95\xc0\xd9\x00\xd2O\x1c" +
+	"}N\x92\xfc\x17$\xaf\x8d$\xb0\x16@\xfa\x19\xee\x03" +
+	"H\xfd\x82\xe4\xffM\xf2\x98\x90\xc0\x18\x80\xf4\xaec\xd7" +
+	"y\x92Ws%7+\x8fQ%\xd7'^7\xfd#" +
+	"cE\x1fG\x97\xee\x83z\x9c\xaeH\x18\x0f\x1a\xaf\x80" +
+	"\x18\x07\xb4\xf3\xba\x9e]9\x95\xa9qK\x195\xbd\xab" +
+	"Z}\xd0\x9a\x02$\xa1_\xfc@\\\xd7\xfa3~ " +
+	"(\x8d:\x9e&\xaa\xd9]\xb0\xf4B\x1eZ)\xc8f" +
+	"\xfc\x98c\x14\xb4\xe5\x86\x9e[\x8d\xcc\xc8\xa9\x9a\x92\x9d" +
+	"!\x1a\xd5\x00\x875P\x0c\x09\xde\xda\xd3\x87\xa6+_" +
+	"<}Fs\xa5\x8cn\xcdw\xacVF\xaf&N-" +
+	"\x09rV\\\x0b\x05\xa4\xd6\x09%[\xf8(\xe1ij" +
+	"=\x95\xect\xeb\xb1\x99\xcau\xaf\xf7T\x12J*T" +
+	"\x17C\xe5\xf99\xc9\xccV\xbf\x09\x132\xf8`\x10\x83" +
+	"={\x97\xb5\x85\xee.Y\xc5b\xa6\xd5\x9d\xc7|V" +
+	"e\x99\xcf0#\x1eN\xd9\x15+\x92\xc8Le\xfa\xd4" +
+	"2\x07C]r2\x9c+\x1a|\xd5x\x8e2\xcb\xfd" +
+	"\xd5\xaf\x8d\xe8T\x87\x08\xe1\xe2\xeb\xdaf'\x99\x19\xbf" +
+	"\x9a\xb3\x08\x9a\x863_\x9d*\x94c\x15\x8a1\xef&" +
+	"\x10\xba&\x13\x19\xd7\xf0(\x8f\x85\xc8\xc8\x06*\\\x93" +
+	"\x93A\x7fL\xe4\xb9b\x83\x8c2W\x9eG\xf9!\x0e" +
+	"\xe3J\xc1\x1a\xc3\xfa\xe0\xf1c\x8a\xd2S{8\xc4\xcd" +
+	"~-\xc3\x00\xd7{\xee\x15\xcag~W~\xe6\x9a\xf9" +
+	"\xea\xcc\xf6\xee\"3\x02\xee\xf7\xaeKv\xbe\xe2U\xbd" +
+	"\xd3\xdd\x94x\xd6\xe8T\x85^\xd7\x1f\xbd\x8e\xb0xh" +
+	"\x03p\xe2\x01\x01\x83^7z\xadmq\x8f\x01\x9c\xb8" +
+	"K@\xce\x7f\x97A\xef\xfdE|\xecq\xe0\xc4m\x02" +
+	"\xf2\xfe\xb3\x0az]\xd2\xc5\x93\xb3\x108q\xa3\x80\x11" +
+	"\xff=\x0b\xbd\x1e\xab\xb8n\x1c8Q\x150\xea\xbf\xd8" +
+	"\xa0\xd7\xe2\x17\xd7n\x01N\x1c\x0az\x81\xd0\xe9\xda\xd1" +
+	"\x85\xb6\xc7QhuX:\xb53\xe8\x8e\x02\xe8B\xdb" +
+	"\xbb\x99\xf0W\xba\x9a8\xa3\xbc\xe6\x16\xc4\xd3\x8a\xc5\xba" +
+	"\xa8Zt\x03\x12\x16#\x12t\xa1\x1c\xc1P\x8b\x19\xe0" +
+	"z[\x03I\xd6\xea\x9c\xf3G\xad\xe1\xbc\xf9\x1f1F" +
+	"\xf2\x95\xb4\xa6}\xfc&ih]*Kky\x94\x1b" +
+	"\xb9\x19+\xd1\xc8\x95\xac\xf0\xc8\x1f\xa7\xc9\xb4\xfeo\xfb" +
+	"\xeb\x1f\xa7\xf0\xfa}\x1e\xe5\x93!\xb7\xfe!\x09\xdf\xe1" +
+	"Q>\x1d\xaa\xe4~D\xbe~\x92G\xf9b\xd0\xf7~" +
+	"\xffq\x00\xf9\"\x8f\xc9Pe$\xfe\x9a\x06\xfe\x8a\xea" +
+	"\x07\xa7.B\xb7.\x8a\xe2\xd3\x00\xa9j\xaa+\x12N" +
+	"]\x14q\xeb\"\x11\x87\x01R\xf5$\x9f\x17\xae\x8b\x9a" +
+	"\xf0>\x80T#\xc9\xe7\xe3\xd4\xdb\xa6P0\x82\xca1" +
+	"\xab\x8f\xde\xa3j\x15\x93\xad\xd7\x88Gk\xb9\xa2f\x0b" +
+	"\x06\x83 \xd7\x17\x83Mo\xa8\xfcp;\xf4n3." +
+	"E$\xcc\xa0\xe97\xea\xae\xe1\x9e?]\xe6\xc9\xea\x85" +
+	"\xccHV1X&\xc5\x0c\xc1\x0d\x08\x83|T\xae\xc6" +
+	"\xd0\xab7@\xf0:\x19\"\xfb\xb4\x99\xac\xcf0t4" +
+	"J\xaa\xf4%A\x95\xee\x17\xe9t\xd9\xb8\x8bGy5" +
+	"\x1dm\x97{\xb4\xf2pp\xafhM+\x05\x93\x95a" +
+	"\x02<3\xfc^\x8f9\xa6\x17\xb2\x99$\x03\xc12&" +
+	"K \x9d\xb1ZO\xb1\xb8\x17\x09\xddG\x0a\xefy\x0e" +
+	"\xbdW\xb8\xd0#\x85\xf7R\x84\xdeSo\xf9#\x85\x87" +
+	"A\xd9#\x85\xfb\xc1\xe1\xfc\xd4;\xf5u4)\xdc\xb4" +
+	"\x18:\x94k\xea\xdd_u\xcb\xdb\xffw\x89\x92\xc8Q" +
+	"s\xbd\xcd /\xc1\xfd\x7f\x00\x00\x00\xff\xff\xf1\xc3d" +
+	"\xc6"
 
 func init() {
 	schemas.Register(schema_db8274f9144abc7e,
@@ -4069,6 +4546,7 @@ func init() {
 		0x8635c6b4f45bf5cd,
 		0x904e297b87fbecea,
 		0x9496331ab9cd463f,
+		0x958096448eb3373e,
 		0x96b74375ce9b0ef6,
 		0x97b3c5c260257622,
 		0x9b87b390babc2ccf,
@@ -4077,6 +4555,8 @@ func init() {
 		0xa766b24d4fe5da35,
 		0xab6d5210c1f26687,
 		0xb046e578094b1ead,
+		0xb177ca2526a3ca76,
+		0xb48edfbdaa25db04,
 		0xb4bf9861fe035d04,
 		0xb5f39f082b9ac18a,
 		0xb70431c0dc014915,
@@ -4084,6 +4564,7 @@ func init() {
 		0xc793e50592935b4a,
 		0xcbd96442ae3bb01a,
 		0xd4d18de97bb12de3,
+		0xdb58ff694ba05cf9,
 		0xdbaa9d03d52b62dc,
 		0xdc3ed6801961e502,
 		0xe3e37d096a5b564e,
@@ -4094,6 +4575,7 @@ func init() {
 		0xf2c122394f447e8e,
 		0xf2c68e2547ec3866,
 		0xf41a0f001ad49e46,
+		0xf548cef9dea2a4a1,
 		0xf5f383d2785edb86,
 		0xf71695ec7fe85497,
 		0xf9cb7f4431a307d0,

tunnelstore/cleanup_params.go

@@ -1,25 +0,0 @@
-package tunnelstore
-
-import (
-	"net/url"
-
-	"github.com/google/uuid"
-)
-
-type CleanupParams struct {
-	queryParams url.Values
-}
-
-func NewCleanupParams() *CleanupParams {
-	return &CleanupParams{
-		queryParams: url.Values{},
-	}
-}
-
-func (cp *CleanupParams) ForClient(clientID uuid.UUID) {
-	cp.queryParams.Set("client_id", clientID.String())
-}
-
-func (cp CleanupParams) encode() string {
-	return cp.queryParams.Encode()
-}