Release 2021.3.6

This commit reverts part of 1cf6ae37eb.

CHANGES.md

@@ -1,5 +1,52 @@
 **Experimental**: This is a new format for release notes. The format and availability is subject to change.
 
+## 2021.3.6
+
+### Bug Fixes
+
+- Reverted 2021.3.5 improvement to use HTTP/2 in a best-effort manner between cloudflared and origin services because
+  it was found to break in some cases.
+
+## 2021.3.5
+
+### Improvements
+
+ - HTTP/2 transport is now always chosen if origin server supports it and the service url scheme is HTTPS.
+   This was previously done in a best attempt manner.
+
+### Bug Fixes
+
+ - The MacOS binaries were not successfully released in 2021.3.3 and 2021.3.4. This release is aimed at addressing that.
+
+## 2021.3.3
+
+### Improvements
+
+- Tunnel create command, as well as, running ad-hoc tunnels using `cloudflared tunnel -name NAME`, will not overwrite
+  existing files when writing tunnel credentials. 
+
+### Bug Fixes
+
+- Tunnel create and delete commands no longer use path to credentials from the configuration file.
+  If you need ot place tunnel credentials file at a specific location, you must use `--credentials-file` flag.
+- Access ssh-gen creates properly named keys for SSH short lived certs.
+
+
+## 2021.3.2
+
+### New Features
+
+- It is now possible to obtain more detailed information about the cloudflared connectors to Cloudflare Edge via
+  `cloudflared tunnel info <name/uuid>`. It is possible to sort the output as well as output in different formats,
+  such as: `cloudflared tunnel info --sort-by version --invert-sort --output json <name/uuid>`.
+  You can obtain more information via `cloudflared tunnel info --help`.
+
+### Bug Fixes
+
+- Don't look for configuration file in default paths when `--config FILE` flag is present after `tunnel` subcommand.
+- cloudflared access token command now functions correctly with the new token-per-app change from 2021.3.0.
+
+
 ## 2021.3.0
 
 ### New Features

Makefile

@@ -254,3 +254,7 @@ vet:
 .PHONY: msi
 msi: cloudflared
 	go-msi make --msi cloudflared.msi --version $(MSI_VERSION)
+
+.PHONY: goimports
+goimports:
+	for d in $$(go list -mod=readonly -f '{{.Dir}}' -a ./... | fgrep -v tunnelrpc) ; do goimports -format-only -local github.com/cloudflare/cloudflared -w $$d ; done

RELEASE_NOTES

@@ -1,3 +1,48 @@
+2021.3.6
+- 2021-03-30 TUN-4150: Only show the connector table in 'tunnel info' if there are connectors. Don't show rows with zero connections.
+- 2021-03-31 TUN-4153: Revert best-effort HTTP2 usage when talking to origins
+- 2021-03-26 TUN-4141: Better error messages for tunnel info subcommand.
+- 2021-03-29 TUN-4146: Unhide and document grace-period
+- 2021-03-25 TUN-3863: Consolidate header handling logic in the connection package; move headers definitions from h2mux to packages that manage them; cleanup header conversions
+
+2021.3.5
+- 2021-03-26 TUN-3896: http-service and tunnelstore client use http2 transport.
+- 2021-03-25 TUN-4125: Change component tests to run in CI with its own dedicated resources
+- 2021-03-26 Publish change log for 2021.3.5
+
+2021.3.4
+
+
+2021.3.3
+- 2021-03-23 TUN-4111: Warn the user if both properties "tunnel" and "hostname" are used
+- 2021-03-23 TUN-4082: Test logging when running as a service
+- 2021-03-23 TUN-4112: Skip testing graceful shutdown with SIGINT on Windows
+- 2021-03-23 TUN-4116: Ingore credentials-file setting in configuration file during tunnel create and delete opeations.
+- 2021-03-23 TUN-4118: Don't overwrite existing file with tunnel credentials. For ad-hoc tunnels, this means tunnel won't start if there's a file in the way.
+- 2021-03-24 TUN-4123: Don't capture output in reconnect componet test
+- 2021-03-23 TUN-4067: Reformat code for consistent import order, grouping, and fix formatting. Added goimports target to the Makefile to make this easier in the future.
+- 2021-03-24 AUTH-3455: Generate short-lived ssh cert per hostname
+- 2021-03-25 Update changelog 2021.3.3
+
+2021.3.2
+- 2021-03-23 TUN-4042: Capture cloudflared output to debug component tests
+- 2021-03-23 Publish changelog for 2021.3.2
+- 2021-03-16 TUN-4089: Address flakiness in component tests for termination
+- 2021-03-16 TUN-4060: Fix Go Vet warnings (new with go 1.16) where t.Fatalf is called from a test goroutine
+- 2021-03-16 TUN-4091: Use flaky decorator to rerun reconnect component tests when they fail
+- 2021-03-12 TUN-4081: Update log severities to use Zerolog's levels
+- 2021-03-16 TUN-4094: Don't read configuration file for access commands
+- 2021-03-15 TUN-3993: New `cloudflared tunnel info` to obtain details about the active connectors for a tunnel
+- 2021-03-17 TUN-3715: Apply input source to the correct context
+- 2021-03-17 AUTH-3394: Ensure scheme on token command
+- 2021-03-18 TUN-4096: Reduce tunnel not connected assertion backoff to address flaky termination tests
+- 2021-03-19 TUN-3998: Allow to cleanup the connections of a tunnel limited to a single client
+- 2021-02-04 TUN-3715: Only read config file once, right before invoking the command
+
+2021.3.1
+- 2021-03-11 TUN-4051: Add component-tests to test graceful shutdown
+- 2021-03-12 TUN-4052: Add component tests to assert service mode behavior
+
 2021.3.0
 - 2021-03-10 TUN-4075: Dedup test should not compare order of list
 - 2021-03-10 Revert "AUTH-3394: Creates a token per app instead of per path"

carrier/carrier.go

@@ -5,20 +5,25 @@ package carrier
 
 import (
 	"crypto/tls"
+	"fmt"
 	"io"
 	"net"
 	"net/http"
+	"net/url"
 	"os"
 	"strings"
 
-	"github.com/cloudflare/cloudflared/h2mux"
-	"github.com/cloudflare/cloudflared/token"
-
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
+
+	"github.com/cloudflare/cloudflared/token"
 )
 
-const LogFieldOriginURL = "originURL"
+const (
+	LogFieldOriginURL       = "originURL"
+	CFAccessTokenHeader     = "Cf-Access-Token"
+	cfJumpDestinationHeader = "Cf-Access-Jump-Destination"
+)
 
 type StartOptions struct {
 	AppInfo         *token.AppInfo
@@ -32,15 +37,11 @@ type StartOptions struct {
 type Connection interface {
 	// ServeStream is used to forward data from the client to the edge
 	ServeStream(*StartOptions, io.ReadWriter) error
-
-	// StartServer is used to listen for incoming connections from the edge to the origin
-	StartServer(net.Listener, string, <-chan struct{}) error
 }
 
 // StdinoutStream is empty struct for wrapping stdin/stdout
 // into a single ReadWriter
-type StdinoutStream struct {
-}
+type StdinoutStream struct{}
 
 // Read will read from Stdin
 func (c *StdinoutStream) Read(p []byte) (int, error) {
@@ -149,7 +150,7 @@ func BuildAccessRequest(options *StartOptions, log *zerolog.Logger) (*http.Reque
 	if err != nil {
 		return nil, err
 	}
-	originRequest.Header.Set(h2mux.CFAccessTokenHeader, token)
+	originRequest.Header.Set(CFAccessTokenHeader, token)
 
 	for k, v := range options.Headers {
 		if len(v) >= 1 {
@@ -159,3 +160,26 @@ func BuildAccessRequest(options *StartOptions, log *zerolog.Logger) (*http.Reque
 
 	return originRequest, nil
 }
+
+func SetBastionDest(header http.Header, destination string) {
+	if destination != "" {
+		header.Set(cfJumpDestinationHeader, destination)
+	}
+}
+
+func ResolveBastionDest(r *http.Request) (string, error) {
+	jumpDestination := r.Header.Get(cfJumpDestinationHeader)
+	if jumpDestination == "" {
+		return "", fmt.Errorf("Did not receive final destination from client. The --destination flag is likely not set on the client side")
+	}
+	// Strip scheme and path set by client. Without a scheme
+	// Parsing a hostname and path without scheme might not return an error due to parsing ambiguities
+	if jumpURL, err := url.Parse(jumpDestination); err == nil && jumpURL.Host != "" {
+		return removePath(jumpURL.Host), nil
+	}
+	return removePath(jumpDestination), nil
+}
+
+func removePath(dest string) string {
+	return strings.SplitN(dest, "/", 2)[0]
+}

carrier/carrier_test.go

@@ -81,7 +81,8 @@ func TestStartServer(t *testing.T) {
 	go func() {
 		err := Serve(wsConn, listener, shutdownC, options)
 		if err != nil {
-			t.Fatalf("Error running server: %v", err)
+			t.Errorf("Error running server: %v", err)
+			return
 		}
 	}()
 
@@ -155,3 +156,99 @@ func testRequest(t *testing.T, url string, stream io.ReadWriter) *http.Request {
 
 	return req
 }
+
+func TestBastionDestination(t *testing.T) {
+	tests := []struct {
+		name         string
+		header       http.Header
+		expectedDest string
+		wantErr      bool
+	}{
+		{
+			name: "hostname destination",
+			header: http.Header{
+				cfJumpDestinationHeader: []string{"localhost"},
+			},
+			expectedDest: "localhost",
+		},
+		{
+			name: "hostname destination with port",
+			header: http.Header{
+				cfJumpDestinationHeader: []string{"localhost:9000"},
+			},
+			expectedDest: "localhost:9000",
+		},
+		{
+			name: "hostname destination with scheme and port",
+			header: http.Header{
+				cfJumpDestinationHeader: []string{"ssh://localhost:9000"},
+			},
+			expectedDest: "localhost:9000",
+		},
+		{
+			name: "full hostname url",
+			header: http.Header{
+				cfJumpDestinationHeader: []string{"ssh://localhost:9000/metrics"},
+			},
+			expectedDest: "localhost:9000",
+		},
+		{
+			name: "hostname destination with port and path",
+			header: http.Header{
+				cfJumpDestinationHeader: []string{"localhost:9000/metrics"},
+			},
+			expectedDest: "localhost:9000",
+		},
+		{
+			name: "ip destination",
+			header: http.Header{
+				cfJumpDestinationHeader: []string{"127.0.0.1"},
+			},
+			expectedDest: "127.0.0.1",
+		},
+		{
+			name: "ip destination with port",
+			header: http.Header{
+				cfJumpDestinationHeader: []string{"127.0.0.1:9000"},
+			},
+			expectedDest: "127.0.0.1:9000",
+		},
+		{
+			name: "ip destination with port and path",
+			header: http.Header{
+				cfJumpDestinationHeader: []string{"127.0.0.1:9000/metrics"},
+			},
+			expectedDest: "127.0.0.1:9000",
+		},
+		{
+			name: "ip destination with schem and port",
+			header: http.Header{
+				cfJumpDestinationHeader: []string{"tcp://127.0.0.1:9000"},
+			},
+			expectedDest: "127.0.0.1:9000",
+		},
+		{
+			name: "full ip url",
+			header: http.Header{
+				cfJumpDestinationHeader: []string{"ssh://127.0.0.1:9000/metrics"},
+			},
+			expectedDest: "127.0.0.1:9000",
+		},
+		{
+			name:    "no destination",
+			wantErr: true,
+		},
+	}
+	for _, test := range tests {
+		r := &http.Request{
+			Header: test.header,
+		}
+		dest, err := ResolveBastionDest(r)
+		if test.wantErr {
+			assert.Error(t, err, "Test %s expects error", test.name)
+		} else {
+			assert.NoError(t, err, "Test %s expects no error, got error %v", test.name, err)
+			assert.Equal(t, test.expectedDest, dest, "Test %s expect dest %s, got %s", test.name, test.expectedDest, dest)
+		}
+	}
+}

carrier/websocket.go

@@ -1,19 +1,15 @@
 package carrier
 
 import (
-	"fmt"
 	"io"
-	"net"
 	"net/http"
 	"net/http/httputil"
 
-	"github.com/cloudflare/cloudflared/ingress"
-	"github.com/cloudflare/cloudflared/socks"
-	"github.com/cloudflare/cloudflared/token"
-	cfwebsocket "github.com/cloudflare/cloudflared/websocket"
-
 	"github.com/gorilla/websocket"
 	"github.com/rs/zerolog"
+
+	"github.com/cloudflare/cloudflared/token"
+	cfwebsocket "github.com/cloudflare/cloudflared/websocket"
 )
 
 // Websocket is used to carry data via WS binary frames over the tunnel from client to the origin
@@ -23,20 +19,6 @@ type Websocket struct {
 	isSocks bool
 }
 
-type wsdialer struct {
-	conn *cfwebsocket.GorillaConn
-}
-
-func (d *wsdialer) Dial(address string) (io.ReadWriteCloser, *socks.AddrSpec, error) {
-	local, ok := d.conn.LocalAddr().(*net.TCPAddr)
-	if !ok {
-		return nil, nil, fmt.Errorf("not a tcp connection")
-	}
-
-	addr := socks.AddrSpec{IP: local.IP, Port: local.Port}
-	return d.conn, &addr, nil
-}
-
 // NewWSConnection returns a new connection object
 func NewWSConnection(log *zerolog.Logger) Connection {
 	return &Websocket{
@@ -54,16 +36,10 @@ func (ws *Websocket) ServeStream(options *StartOptions, conn io.ReadWriter) erro
 	}
 	defer wsConn.Close()
 
-	ingress.Stream(wsConn, conn, ws.log)
+	cfwebsocket.Stream(wsConn, conn, ws.log)
 	return nil
 }
 
-// StartServer creates a Websocket server to listen for connections.
-// This is used on the origin (tunnel) side to take data from the muxer and send it to the origin
-func (ws *Websocket) StartServer(listener net.Listener, remote string, shutdownC <-chan struct{}) error {
-	return cfwebsocket.StartProxyServer(ws.log, listener, remote, shutdownC, ingress.DefaultStreamHandler)
-}
-
 // createWebsocketStream will create a WebSocket connection to stream data over
 // It also handles redirects from Access and will present that flow if
 // the token is not present on the request

cfsetup.yaml

@@ -195,13 +195,15 @@ stretch: &stretch
       - *pinned_go_fips
       - build-essential
       - gotest-to-teamcity
+    pre-cache:
+      - go get golang.org/x/tools/cmd/goimports
+      - go get github.com/BurntSushi/go-sumtype
     post-cache:
       - export GOOS=linux
       - export GOARCH=amd64
       - export FIPS=true
-      # cd to a non-module directory: https://github.com/golang/go/issues/24250
-      - (cd / && go get github.com/BurntSushi/go-sumtype)
       - export PATH="$HOME/go/bin:$PATH"
+      - ./fmt-check.sh
       - make test | gotest-to-teamcity
   component-test:
     build_dir: *build_dir
@@ -210,14 +212,19 @@ stretch: &stretch
       - python3.7
       - python3-pip
       - python3-setuptools
+      # procps installs the ps command which is needed in test_sysv_service because the init script
+      # uses ps pid to determine if the agent is running
+      - procps
     pre-cache-copy-paths:
       - component-tests/requirements.txt
     pre-cache:
       - sudo pip3 install --upgrade -r component-tests/requirements.txt
     post-cache:
-      # Constructs config file from env vars
-      - python3 component-tests/config.py
+      # Creates and routes a Named Tunnel for this build. Also constructs config file from env vars.
+      - python3 component-tests/setup.py --type create
       - pytest component-tests
+      # The Named Tunnel is deleted and its route unprovisioned here.
+      - python3 component-tests/setup.py --type cleanup
   update-homebrew:
     builddeps:
       - openssh-client

cmd/cloudflared/access/carrier.go

@@ -6,19 +6,20 @@ import (
 	"net/http"
 	"strings"
 
-	"github.com/cloudflare/cloudflared/carrier"
-	"github.com/cloudflare/cloudflared/config"
-	"github.com/cloudflare/cloudflared/h2mux"
-	"github.com/cloudflare/cloudflared/logger"
-	"github.com/cloudflare/cloudflared/validation"
-
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
 	"github.com/urfave/cli/v2"
+
+	"github.com/cloudflare/cloudflared/carrier"
+	"github.com/cloudflare/cloudflared/config"
+	"github.com/cloudflare/cloudflared/logger"
+	"github.com/cloudflare/cloudflared/validation"
 )
 
 const (
-	LogFieldHost = "host"
+	LogFieldHost               = "host"
+	cfAccessClientIDHeader     = "Cf-Access-Client-Id"
+	cfAccessClientSecretHeader = "Cf-Access-Client-Secret"
 )
 
 // StartForwarder starts a client side websocket forward
@@ -31,16 +32,14 @@ func StartForwarder(forwarder config.Forwarder, shutdown <-chan struct{}, log *z
 	// get the headers from the config file and add to the request
 	headers := make(http.Header)
 	if forwarder.TokenClientID != "" {
-		headers.Set(h2mux.CFAccessClientIDHeader, forwarder.TokenClientID)
+		headers.Set(cfAccessClientIDHeader, forwarder.TokenClientID)
 	}
 
 	if forwarder.TokenSecret != "" {
-		headers.Set(h2mux.CFAccessClientSecretHeader, forwarder.TokenSecret)
+		headers.Set(cfAccessClientSecretHeader, forwarder.TokenSecret)
 	}
 
-	if forwarder.Destination != "" {
-		headers.Add(h2mux.CFJumpDestinationHeader, forwarder.Destination)
-	}
+	carrier.SetBastionDest(headers, forwarder.Destination)
 
 	options := &carrier.StartOptions{
 		OriginURL: forwarder.URL,
@@ -72,16 +71,13 @@ func ssh(c *cli.Context) error {
 	// get the headers from the cmdline and add them
 	headers := buildRequestHeaders(c.StringSlice(sshHeaderFlag))
 	if c.IsSet(sshTokenIDFlag) {
-		headers.Set(h2mux.CFAccessClientIDHeader, c.String(sshTokenIDFlag))
+		headers.Set(cfAccessClientIDHeader, c.String(sshTokenIDFlag))
 	}
 	if c.IsSet(sshTokenSecretFlag) {
-		headers.Set(h2mux.CFAccessClientSecretHeader, c.String(sshTokenSecretFlag))
+		headers.Set(cfAccessClientSecretHeader, c.String(sshTokenSecretFlag))
 	}
 
-	destination := c.String(sshDestinationFlag)
-	if destination != "" {
-		headers.Add(h2mux.CFJumpDestinationHeader, destination)
-	}
+	carrier.SetBastionDest(headers, c.String(sshDestinationFlag))
 
 	options := &carrier.StartOptions{
 		OriginURL: originURL,

cmd/cloudflared/access/cmd.go

@@ -11,19 +11,18 @@ import (
 	"text/template"
 	"time"
 
-	"github.com/cloudflare/cloudflared/carrier"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
-	"github.com/cloudflare/cloudflared/h2mux"
-	"github.com/cloudflare/cloudflared/logger"
-	"github.com/cloudflare/cloudflared/sshgen"
-	"github.com/cloudflare/cloudflared/token"
-	"github.com/cloudflare/cloudflared/validation"
-
 	"github.com/getsentry/raven-go"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
 	"github.com/urfave/cli/v2"
 	"golang.org/x/net/idna"
+
+	"github.com/cloudflare/cloudflared/carrier"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
+	"github.com/cloudflare/cloudflared/logger"
+	"github.com/cloudflare/cloudflared/sshgen"
+	"github.com/cloudflare/cloudflared/token"
+	"github.com/cloudflare/cloudflared/validation"
 )
 
 const (
@@ -40,7 +39,7 @@ Add to your {{.Home}}/.ssh/config:
 
 Host {{.Hostname}}
 {{- if .ShortLivedCerts}}
-  ProxyCommand bash -c '{{.Cloudflared}} access ssh-gen --hostname %h; ssh -tt %r@cfpipe-{{.Hostname}} >&2 <&1' 
+  ProxyCommand bash -c '{{.Cloudflared}} access ssh-gen --hostname %h; ssh -tt %r@cfpipe-{{.Hostname}} >&2 <&1'
 
 Host cfpipe-{{.Hostname}}
   HostName {{.Hostname}}
@@ -77,30 +76,24 @@ func Commands() []*cli.Command {
 			Aliases:  []string{"forward"},
 			Category: "Access",
 			Usage:    "access <subcommand>",
-			Description: `Cloudflare Access protects internal resources by securing, authenticating and monitoring access 
-			per-user and by application. With Cloudflare Access, only authenticated users with the required permissions are 
-			able to reach sensitive resources. The commands provided here allow you to interact with Access protected 
+			Description: `Cloudflare Access protects internal resources by securing, authenticating and monitoring access
+			per-user and by application. With Cloudflare Access, only authenticated users with the required permissions are
+			able to reach sensitive resources. The commands provided here allow you to interact with Access protected
 			applications from the command line.`,
 			Subcommands: []*cli.Command{
 				{
 					Name:   "login",
-					Action: cliutil.ErrorHandler(login),
+					Action: cliutil.Action(login),
 					Usage:  "login <url of access application>",
 					Description: `The login subcommand initiates an authentication flow with your identity provider.
 					The subcommand will launch a browser. For headless systems, a url is provided.
 					Once authenticated with your identity provider, the login command will generate a JSON Web Token (JWT)
-					scoped to your identity, the application you intend to reach, and valid for a session duration set by your 
+					scoped to your identity, the application you intend to reach, and valid for a session duration set by your
 					administrator. cloudflared stores the token in local storage.`,
-					Flags: []cli.Flag{
-						&cli.StringFlag{
-							Name:   "url",
-							Hidden: true,
-						},
-					},
 				},
 				{
 					Name:   "curl",
-					Action: cliutil.ErrorHandler(curl),
+					Action: cliutil.Action(curl),
 					Usage:  "curl [--allow-request, -ar] <url> [<curl args>...]",
 					Description: `The curl subcommand wraps curl and automatically injects the JWT into a cf-access-token
 					header when using curl to reach an application behind Access.`,
@@ -109,7 +102,7 @@ func Commands() []*cli.Command {
 				},
 				{
 					Name:        "token",
-					Action:      cliutil.ErrorHandler(generateToken),
+					Action:      cliutil.Action(generateToken),
 					Usage:       "token -app=<url of access application>",
 					ArgsUsage:   "url of Access application",
 					Description: `The token subcommand produces a JWT which can be used to authenticate requests.`,
@@ -121,7 +114,7 @@ func Commands() []*cli.Command {
 				},
 				{
 					Name:        "tcp",
-					Action:      cliutil.ErrorHandler(ssh),
+					Action:      cliutil.Action(ssh),
 					Aliases:     []string{"rdp", "ssh", "smb"},
 					Usage:       "",
 					ArgsUsage:   "",
@@ -164,7 +157,7 @@ func Commands() []*cli.Command {
 						&cli.StringFlag{
 							Name:    logger.LogSSHLevelFlag,
 							Aliases: []string{"loglevel"}, //added to match the tunnel side
-							Usage:   "Application logging level {fatal, error, info, debug}. ",
+							Usage:   "Application logging level {debug, info, warn, error, fatal}. ",
 						},
 						&cli.StringFlag{
 							Name:   sshConnectTo,
@@ -175,7 +168,7 @@ func Commands() []*cli.Command {
 				},
 				{
 					Name:        "ssh-config",
-					Action:      cliutil.ErrorHandler(sshConfig),
+					Action:      cliutil.Action(sshConfig),
 					Usage:       "",
 					Description: `Prints an example configuration ~/.ssh/config`,
 					Flags: []cli.Flag{
@@ -191,7 +184,7 @@ func Commands() []*cli.Command {
 				},
 				{
 					Name:        "ssh-gen",
-					Action:      cliutil.ErrorHandler(sshGen),
+					Action:      cliutil.Action(sshGen),
 					Usage:       "",
 					Description: `Generates a short lived certificate for given hostname`,
 					Flags: []cli.Flag{
@@ -292,11 +285,10 @@ func curl(c *cli.Context) error {
 	}
 
 	cmdArgs = append(cmdArgs, "-H")
-	cmdArgs = append(cmdArgs, fmt.Sprintf("%s: %s", h2mux.CFAccessTokenHeader, tok))
+	cmdArgs = append(cmdArgs, fmt.Sprintf("%s: %s", carrier.CFAccessTokenHeader, tok))
 	return run("curl", cmdArgs...)
 }
 
-
 // run kicks off a shell task and pipe the results to the respective std pipes
 func run(cmd string, args ...string) error {
 	c := exec.Command(cmd, args...)
@@ -323,7 +315,7 @@ func generateToken(c *cli.Context) error {
 	if err := raven.SetDSN(sentryDSN); err != nil {
 		return err
 	}
-	appURL, err := url.Parse(c.String("app"))
+	appURL, err := url.Parse(ensureURLScheme(c.String("app")))
 	if err != nil || c.NumFlags() < 1 {
 		fmt.Fprintln(os.Stderr, "Please provide a url.")
 		return err
@@ -394,7 +386,7 @@ func sshGen(c *cli.Context) error {
 		return err
 	}
 
-	if err := sshgen.GenerateShortLivedCertificate(appInfo, cfdToken); err != nil {
+	if err := sshgen.GenerateShortLivedCertificate(originURL, cfdToken); err != nil {
 		return err
 	}
 
@@ -479,10 +471,10 @@ func isFileThere(candidate string) bool {
 func verifyTokenAtEdge(appUrl *url.URL, appInfo *token.AppInfo, c *cli.Context, log *zerolog.Logger) error {
 	headers := buildRequestHeaders(c.StringSlice(sshHeaderFlag))
 	if c.IsSet(sshTokenIDFlag) {
-		headers.Add(h2mux.CFAccessClientIDHeader, c.String(sshTokenIDFlag))
+		headers.Add(cfAccessClientIDHeader, c.String(sshTokenIDFlag))
 	}
 	if c.IsSet(sshTokenSecretFlag) {
-		headers.Add(h2mux.CFAccessClientSecretHeader, c.String(sshTokenSecretFlag))
+		headers.Add(cfAccessClientSecretHeader, c.String(sshTokenSecretFlag))
 	}
 	options := &carrier.StartOptions{AppInfo: appInfo, OriginURL: appUrl.String(), Headers: headers}
 

cmd/cloudflared/app_forward_service.go

@@ -1,10 +1,10 @@
 package main
 
 import (
+	"github.com/rs/zerolog"
+
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/access"
 	"github.com/cloudflare/cloudflared/config"
-
-	"github.com/rs/zerolog"
 )
 
 // ForwardServiceType is used to identify what kind of overwatch service this is

cmd/cloudflared/app_resolver_service.go

@@ -1,10 +1,10 @@
 package main
 
 import (
+	"github.com/rs/zerolog"
+
 	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/tunneldns"
-
-	"github.com/rs/zerolog"
 )
 
 const (

cmd/cloudflared/app_service.go

@@ -1,10 +1,10 @@
 package main
 
 import (
+	"github.com/rs/zerolog"
+
 	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/overwatch"
-
-	"github.com/rs/zerolog"
 )
 
 // AppService is the main service that runs when no command lines flags are passed to cloudflared

cmd/cloudflared/buildinfo/build_info.go

@@ -1,9 +1,10 @@
 package buildinfo
 
 import (
-	"github.com/rs/zerolog"
 	"fmt"
 	"runtime"
+
+	"github.com/rs/zerolog"
 )
 
 type BuildInfo struct {

cmd/cloudflared/cliutil/deprecated.go

@@ -9,13 +9,13 @@ import (
 func RemovedCommand(name string) *cli.Command {
 	return &cli.Command{
 		Name: name,
-		Action: ErrorHandler(func(context *cli.Context) error {
+		Action: func(context *cli.Context) error {
 			return cli.Exit(
 				fmt.Sprintf("%s command is no longer supported by cloudflared. Consult Argo Tunnel documentation for possible alternative solutions.", name),
 				-1,
 			)
-		}),
+		},
 		Description: fmt.Sprintf("%s is deprecated", name),
-		Hidden: true,
+		Hidden:      true,
 	}
 }

cmd/cloudflared/cliutil/errors.go

@@ -2,6 +2,7 @@ package cliutil
 
 import (
 	"fmt"
+
 	"github.com/urfave/cli/v2"
 )
 
@@ -21,7 +22,7 @@ func UsageError(format string, args ...interface{}) error {
 }
 
 // Ensures exit with error code if actionFunc returns an error
-func ErrorHandler(actionFunc cli.ActionFunc) cli.ActionFunc {
+func WithErrorHandler(actionFunc cli.ActionFunc) cli.ActionFunc {
 	return func(ctx *cli.Context) error {
 		err := actionFunc(ctx)
 		if err != nil {

cmd/cloudflared/cliutil/handler.go

@@ -0,0 +1,39 @@
+package cliutil
+
+import (
+	"github.com/urfave/cli/v2"
+	"github.com/urfave/cli/v2/altsrc"
+
+	"github.com/cloudflare/cloudflared/config"
+	"github.com/cloudflare/cloudflared/logger"
+)
+
+func Action(actionFunc cli.ActionFunc) cli.ActionFunc {
+	return WithErrorHandler(actionFunc)
+}
+
+func ConfiguredAction(actionFunc cli.ActionFunc) cli.ActionFunc {
+	return WithErrorHandler(func(c *cli.Context) error {
+		if err := setFlagsFromConfigFile(c); err != nil {
+			return err
+		}
+		return actionFunc(c)
+	})
+}
+
+func setFlagsFromConfigFile(c *cli.Context) error {
+	const errorExitCode = 1
+	log := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
+	inputSource, err := config.ReadConfigFile(c, log)
+	if err != nil {
+		if err == config.ErrNoConfigFile {
+			return nil
+		}
+		return cli.Exit(err, errorExitCode)
+	}
+
+	if err := altsrc.ApplyInputSource(c, inputSource); err != nil {
+		return cli.Exit(err, errorExitCode)
+	}
+	return nil
+}

cmd/cloudflared/linux_service.go

@@ -7,13 +7,13 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/rs/zerolog"
+	"github.com/urfave/cli/v2"
+
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/tunnel"
 	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/logger"
-
-	"github.com/rs/zerolog"
-	"github.com/urfave/cli/v2"
 )
 
 func runApp(app *cli.App, graceShutdownC chan struct{}) {
@@ -24,7 +24,7 @@ func runApp(app *cli.App, graceShutdownC chan struct{}) {
 			{
 				Name:   "install",
 				Usage:  "Install Argo Tunnel as a system service",
-				Action: cliutil.ErrorHandler(installLinuxService),
+				Action: cliutil.ConfiguredAction(installLinuxService),
 				Flags: []cli.Flag{
 					&cli.BoolFlag{
 						Name:  "legacy",
@@ -35,7 +35,7 @@ func runApp(app *cli.App, graceShutdownC chan struct{}) {
 			{
 				Name:   "uninstall",
 				Usage:  "Uninstall the Argo Tunnel service",
-				Action: cliutil.ErrorHandler(uninstallLinuxService),
+				Action: cliutil.ConfiguredAction(uninstallLinuxService),
 			},
 		},
 	})

cmd/cloudflared/macos_service.go

@@ -6,11 +6,11 @@ import (
 	"fmt"
 	"os"
 
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
-	"github.com/cloudflare/cloudflared/logger"
-
 	"github.com/pkg/errors"
 	"github.com/urfave/cli/v2"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
+	"github.com/cloudflare/cloudflared/logger"
 )
 
 const (
@@ -25,12 +25,12 @@ func runApp(app *cli.App, graceShutdownC chan struct{}) {
 			{
 				Name:   "install",
 				Usage:  "Install Argo Tunnel as an user launch agent",
-				Action: cliutil.ErrorHandler(installLaunchd),
+				Action: cliutil.ConfiguredAction(installLaunchd),
 			},
 			{
 				Name:   "uninstall",
 				Usage:  "Uninstall the Argo Tunnel launch agent",
-				Action: cliutil.ErrorHandler(uninstallLaunchd),
+				Action: cliutil.ConfiguredAction(uninstallLaunchd),
 			},
 		},
 	})

cmd/cloudflared/main.go

@@ -6,21 +6,21 @@ import (
 	"strings"
 	"time"
 
+	"github.com/getsentry/raven-go"
+	homedir "github.com/mitchellh/go-homedir"
+	"github.com/pkg/errors"
+	"github.com/urfave/cli/v2"
+
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/access"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
-	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/proxydns"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/tunnel"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/updater"
+	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/metrics"
 	"github.com/cloudflare/cloudflared/overwatch"
 	"github.com/cloudflare/cloudflared/watcher"
-
-	"github.com/getsentry/raven-go"
-	"github.com/mitchellh/go-homedir"
-	"github.com/pkg/errors"
-	"github.com/urfave/cli/v2"
 )
 
 const (
@@ -77,7 +77,6 @@ func main() {
 	See https://developers.cloudflare.com/argo-tunnel/ for more in-depth documentation.`
 	app.Flags = flags()
 	app.Action = action(graceShutdownC)
-	app.Before = tunnel.SetFlagsFromConfigFile
 	app.Commands = commands(cli.ShowVersion)
 
 	tunnel.Init(Version, graceShutdownC) // we need this to support the tunnel sub command...
@@ -90,7 +89,7 @@ func commands(version func(c *cli.Context)) []*cli.Command {
 	cmds := []*cli.Command{
 		{
 			Name:   "update",
-			Action: cliutil.ErrorHandler(updater.Update),
+			Action: cliutil.ConfiguredAction(updater.Update),
 			Usage:  "Update the agent if a new version exists",
 			Flags: []cli.Flag{
 				&cli.BoolFlag{
@@ -145,7 +144,7 @@ func isEmptyInvocation(c *cli.Context) bool {
 }
 
 func action(graceShutdownC chan struct{}) cli.ActionFunc {
-	return cliutil.ErrorHandler(func(c *cli.Context) (err error) {
+	return cliutil.ConfiguredAction(func(c *cli.Context) (err error) {
 		if isEmptyInvocation(c) {
 			return handleServiceMode(c, graceShutdownC)
 		}

cmd/cloudflared/proxydns/cmd.go

@@ -17,8 +17,9 @@ import (
 func Command(hidden bool) *cli.Command {
 	return &cli.Command{
 		Name:   "proxy-dns",
-		Action: cliutil.ErrorHandler(Run),
-		Usage:  "Run a DNS over HTTPS proxy server.",
+		Action: cliutil.ConfiguredAction(Run),
+
+		Usage: "Run a DNS over HTTPS proxy server.",
 		Flags: []cli.Flag{
 			&cli.StringFlag{
 				Name:    "metrics",
@@ -111,5 +112,3 @@ func Run(c *cli.Context) error {
 	}
 	return err
 }
-
-

cmd/cloudflared/service_template.go

@@ -10,7 +10,7 @@ import (
 	"os/exec"
 	"text/template"
 
-	"github.com/mitchellh/go-homedir"
+	homedir "github.com/mitchellh/go-homedir"
 
 	"github.com/cloudflare/cloudflared/config"
 )

cmd/cloudflared/tunnel/cmd.go

@@ -7,12 +7,20 @@ import (
 	"io/ioutil"
 	"net/url"
 	"os"
-	"reflect"
 	"runtime/trace"
 	"strings"
 	"sync"
 	"time"
 
+	"github.com/coreos/go-systemd/daemon"
+	"github.com/facebookgo/grace/gracenet"
+	"github.com/getsentry/raven-go"
+	homedir "github.com/mitchellh/go-homedir"
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog"
+	"github.com/urfave/cli/v2"
+	"github.com/urfave/cli/v2/altsrc"
+
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/proxydns"
@@ -28,15 +36,6 @@ import (
 	"github.com/cloudflare/cloudflared/tlsconfig"
 	"github.com/cloudflare/cloudflared/tunneldns"
 	"github.com/cloudflare/cloudflared/tunnelstore"
-
-	"github.com/coreos/go-systemd/daemon"
-	"github.com/facebookgo/grace/gracenet"
-	"github.com/getsentry/raven-go"
-	"github.com/mitchellh/go-homedir"
-	"github.com/pkg/errors"
-	"github.com/rs/zerolog"
-	"github.com/urfave/cli/v2"
-	"github.com/urfave/cli/v2/altsrc"
 )
 
 const (
@@ -75,8 +74,8 @@ const (
 	// uiFlag is to enable launching cloudflared in interactive UI mode
 	uiFlag = "ui"
 
-	debugLevelWarning = "At debug level, request URL, method, protocol, content legnth and header will be logged. " +
-		"Response status, content length and header will also be logged in debug level."
+	debugLevelWarning = "At debug level cloudflared will log request URL, method, protocol, content length, as well as, all request and response headers. " +
+		"This can expose sensitive information in your logs."
 
 	LogFieldCommand             = "command"
 	LogFieldExpandedPath        = "expandedPath"
@@ -101,6 +100,7 @@ func Commands() []*cli.Command {
 		buildRouteCommand(),
 		buildRunCommand(),
 		buildListCommand(),
+		buildInfoCommand(),
 		buildIngressSubcommand(),
 		buildDeleteCommand(),
 		buildCleanupCommand(),
@@ -120,8 +120,7 @@ func Commands() []*cli.Command {
 func buildTunnelCommand(subcommands []*cli.Command) *cli.Command {
 	return &cli.Command{
 		Name:      "tunnel",
-		Action:    cliutil.ErrorHandler(TunnelCommand),
-		Before:    SetFlagsFromConfigFile,
+		Action:    cliutil.ConfiguredAction(TunnelCommand),
 		Category:  "Tunnel",
 		Usage:     "Make a locally-running web service accessible over the internet using Argo Tunnel.",
 		ArgsUsage: " ",
@@ -369,26 +368,6 @@ func StartServer(
 	return waitToShutdown(&wg, cancel, errC, graceShutdownC, c.Duration("grace-period"), log)
 }
 
-func SetFlagsFromConfigFile(c *cli.Context) error {
-	const exitCode = 1
-	log := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
-	inputSource, err := config.ReadConfigFile(c, log)
-	if err != nil {
-		if err == config.ErrNoConfigFile {
-			return nil
-		}
-		return cli.Exit(err, exitCode)
-	}
-	targetFlags := c.Command.Flags
-	if c.Command.Name == "" {
-		targetFlags = c.App.Flags
-	}
-	if err := altsrc.ApplyInputSourceValues(c, inputSource, targetFlags); err != nil {
-		return cli.Exit(err, exitCode)
-	}
-	return nil
-}
-
 func waitToShutdown(wg *sync.WaitGroup,
 	cancelServerContext func(),
 	errC <-chan error,
@@ -477,33 +456,6 @@ func addPortIfMissing(uri *url.URL, port int) string {
 	return fmt.Sprintf("%s:%d", uri.Hostname(), port)
 }
 
-// appendFlags will append extra flags to a slice of flags.
-//
-// The cli package will panic if two flags exist with the same name,
-// so if extraFlags contains a flag that was already defined, modify the
-// original flags to use the extra version.
-func appendFlags(flags []cli.Flag, extraFlags ...cli.Flag) []cli.Flag {
-	for _, extra := range extraFlags {
-		var found bool
-
-		// Check if an extra flag overrides an existing flag.
-		for i, flag := range flags {
-			if reflect.DeepEqual(extra.Names(), flag.Names()) {
-				flags[i] = extra
-				found = true
-				break
-			}
-		}
-
-		// Append the extra flag if it has nothing to override.
-		if !found {
-			flags = append(flags, extra)
-		}
-	}
-
-	return flags
-}
-
 func tunnelFlags(shouldHide bool) []cli.Flag {
 	flags := configureCloudflaredFlags(shouldHide)
 	flags = append(flags, configureProxyFlags(shouldHide)...)
@@ -513,7 +465,7 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 		credentialsFileFlag,
 		altsrc.NewBoolFlag(&cli.BoolFlag{
 			Name:   "is-autoupdated",
-			Usage:  "Signal the new process that Argo Tunnel client has been autoupdated",
+			Usage:  "Signal the new process that Argo Tunnel connector has been autoupdated",
 			Value:  false,
 			Hidden: true,
 		}),
@@ -613,10 +565,10 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 		}),
 		altsrc.NewDurationFlag(&cli.DurationFlag{
 			Name:    "grace-period",
-			Usage:   "Duration to accept new requests after cloudflared receives first SIGINT/SIGTERM. A second SIGINT/SIGTERM will force cloudflared to shutdown immediately.",
+			Usage:   "When cloudflared receives SIGINT/SIGTERM it will stop accepting new requests, wait for in-progress requests to terminate, then shutdown. Waiting for in-progress requests will timeout after this grace period, or when a second SIGTERM/SIGINT is received.",
 			Value:   time.Second * 30,
 			EnvVars: []string{"TUNNEL_GRACE_PERIOD"},
-			Hidden:  true,
+			Hidden:  shouldHide,
 		}),
 		// Note TUN-3758 , we use Int because UInt is not supported with altsrc
 		altsrc.NewIntFlag(&cli.IntFlag{
@@ -652,6 +604,7 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			Aliases: []string{"n"},
 			EnvVars: []string{"TUNNEL_NAME"},
 			Usage:   "Stable name to identify the tunnel. Using this flag will create, route and run a tunnel. For production usage, execute each command separately",
+			Hidden:  shouldHide,
 		}),
 		altsrc.NewBoolFlag(&cli.BoolFlag{
 			Name:   uiFlag,
@@ -920,7 +873,7 @@ func configureLoggingFlags(shouldHide bool) []cli.Flag {
 		altsrc.NewStringFlag(&cli.StringFlag{
 			Name:    logger.LogLevelFlag,
 			Value:   "info",
-			Usage:   "Application logging level {fatal, error, info, debug}. " + debugLevelWarning,
+			Usage:   "Application logging level {debug, info, warn, error, fatal}. " + debugLevelWarning,
 			EnvVars: []string{"TUNNEL_LOGLEVEL"},
 			Hidden:  shouldHide,
 		}),
@@ -928,7 +881,7 @@ func configureLoggingFlags(shouldHide bool) []cli.Flag {
 			Name:    logger.LogTransportLevelFlag,
 			Aliases: []string{"proto-loglevel"}, // This flag used to be called proto-loglevel
 			Value:   "info",
-			Usage:   "Transport logging level(previously called protocol logging level) {fatal, error, info, debug}",
+			Usage:   "Transport logging level(previously called protocol logging level) {debug, info, warn, error, fatal}",
 			EnvVars: []string{"TUNNEL_PROTO_LOGLEVEL", "TUNNEL_TRANSPORT_LOGLEVEL"},
 			Hidden:  shouldHide,
 		}),

cmd/cloudflared/tunnel/configuration.go

@@ -8,6 +8,13 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/google/uuid"
+	homedir "github.com/mitchellh/go-homedir"
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog"
+	"github.com/urfave/cli/v2"
+	"golang.org/x/crypto/ssh/terminal"
+
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
 	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/connection"
@@ -18,13 +25,6 @@ import (
 	"github.com/cloudflare/cloudflared/tlsconfig"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 	"github.com/cloudflare/cloudflared/validation"
-
-	"github.com/google/uuid"
-	"github.com/mitchellh/go-homedir"
-	"github.com/pkg/errors"
-	"github.com/rs/zerolog"
-	"github.com/urfave/cli/v2"
-	"golang.org/x/crypto/ssh/terminal"
 )
 
 const LogFieldOriginCertPath = "originCertPath"
@@ -160,6 +160,10 @@ func prepareTunnelConfig(
 	if err != nil {
 		log.Err(err).Str(LogFieldHostname, configHostname).Msg("Invalid hostname")
 		return nil, ingress.Ingress{}, errors.Wrap(err, "Invalid hostname")
+	} else if hostname != "" && isNamedTunnel {
+		log.Warn().Msg("The property `hostname` in your configuration is ignored because you configured a Named Tunnel " +
+			"in the property `tunnel`. Make sure to provision the routing (e.g. via `cloudflared tunnel route`) or else " +
+			"your origin will not be reachable. You should remove the `hostname` property to avoid this warning.")
 	}
 	isFreeTunnel := hostname == ""
 	clientID := c.String("id")
@@ -199,9 +203,9 @@ func prepareTunnelConfig(
 	if isNamedTunnel {
 		clientUUID, err := uuid.NewRandom()
 		if err != nil {
-			return nil, ingress.Ingress{}, errors.Wrap(err, "can't generate clientUUID")
+			return nil, ingress.Ingress{}, errors.Wrap(err, "can't generate connector UUID")
 		}
-		log.Info().Msgf("Generated Client ID: %s", clientUUID)
+		log.Info().Msgf("Generated Connector ID: %s", clientUUID)
 		features := append(c.StringSlice("features"), origin.FeatureSerializedHeaders)
 		namedTunnel.Client = tunnelpogs.ClientInfo{
 			ClientID: clientUUID[:],

cmd/cloudflared/tunnel/info.go

@@ -0,0 +1,16 @@
+package tunnel
+
+import (
+	"time"
+
+	"github.com/google/uuid"
+
+	"github.com/cloudflare/cloudflared/tunnelstore"
+)
+
+type Info struct {
+	ID         uuid.UUID                   `json:"id"`
+	Name       string                      `json:"name"`
+	CreatedAt  time.Time                   `json:"createdAt"`
+	Connectors []*tunnelstore.ActiveClient `json:"conns"`
+}

cmd/cloudflared/tunnel/ingress_subcommands.go

@@ -45,7 +45,7 @@ func buildIngressSubcommand() *cli.Command {
 func buildValidateIngressCommand() *cli.Command {
 	return &cli.Command{
 		Name:        "validate",
-		Action:      cliutil.ErrorHandler(validateIngressCommand),
+		Action:      cliutil.ConfiguredAction(validateIngressCommand),
 		Usage:       "Validate the ingress configuration ",
 		UsageText:   "cloudflared tunnel [--config FILEPATH] ingress validate",
 		Description: "Validates the configuration file, ensuring your ingress rules are OK.",
@@ -55,7 +55,7 @@ func buildValidateIngressCommand() *cli.Command {
 func buildTestURLCommand() *cli.Command {
 	return &cli.Command{
 		Name:      "rule",
-		Action:    cliutil.ErrorHandler(testURLCommand),
+		Action:    cliutil.ConfiguredAction(testURLCommand),
 		Usage:     "Check which ingress rule matches a given request URL",
 		UsageText: "cloudflared tunnel [--config FILEPATH] ingress rule URL",
 		ArgsUsage: "URL",

cmd/cloudflared/tunnel/login.go

@@ -8,7 +8,7 @@ import (
 	"path/filepath"
 	"syscall"
 
-	"github.com/mitchellh/go-homedir"
+	homedir "github.com/mitchellh/go-homedir"
 	"github.com/pkg/errors"
 	"github.com/urfave/cli/v2"
 
@@ -26,16 +26,10 @@ const (
 func buildLoginSubcommand(hidden bool) *cli.Command {
 	return &cli.Command{
 		Name:      "login",
-		Action:    cliutil.ErrorHandler(login),
+		Action:    cliutil.ConfiguredAction(login),
 		Usage:     "Generate a configuration file with your login details",
 		ArgsUsage: " ",
-		Flags: []cli.Flag{
-			&cli.StringFlag{
-				Name:   "url",
-				Hidden: true,
-			},
-		},
-		Hidden: hidden,
+		Hidden:    hidden,
 	}
 }
 

cmd/cloudflared/tunnel/signal.go

@@ -20,4 +20,4 @@ func waitForSignal(graceShutdownC chan struct{}, logger *zerolog.Logger) {
 		close(graceShutdownC)
 	case <-graceShutdownC:
 	}
-}
+}

cmd/cloudflared/tunnel/signal_test.go

@@ -101,4 +101,3 @@ func TestWaitForShutdown(t *testing.T) {
 	assert.True(t, contextCancelled)
 	assert.True(t, time.Now().Sub(startTime) < time.Second) // check that wait ended early
 }
-

cmd/cloudflared/tunnel/subcommand_context.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
+	"path/filepath"
 	"strings"
 
 	"github.com/google/uuid"
@@ -147,7 +148,7 @@ func (sc *subcommandContext) readTunnelCredentials(credFinder CredFinder) (conne
 	return credentials, nil
 }
 
-func (sc *subcommandContext) create(name string, credentialsOutputPath string) (*tunnelstore.Tunnel, error) {
+func (sc *subcommandContext) create(name string, credentialsFilePath string) (*tunnelstore.Tunnel, error) {
 	client, err := sc.client()
 	if err != nil {
 		return nil, errors.Wrap(err, "couldn't create client to talk to Argo Tunnel backend")
@@ -173,27 +174,40 @@ func (sc *subcommandContext) create(name string, credentialsOutputPath string) (
 		TunnelID:     tunnel.ID,
 		TunnelName:   name,
 	}
-	filePath, writeFileErr := writeTunnelCredentials(credential.certPath, credentialsOutputPath, &tunnelCredentials)
+	usedCertPath := false
+	if credentialsFilePath == "" {
+		originCertDir := filepath.Dir(credential.certPath)
+		credentialsFilePath, err = tunnelFilePath(tunnelCredentials.TunnelID, originCertDir)
+		if err != nil {
+			return nil, err
+		}
+		usedCertPath = true
+	}
+	writeFileErr := writeTunnelCredentials(credentialsFilePath, &tunnelCredentials)
 	if writeFileErr != nil {
 		var errorLines []string
-		errorLines = append(errorLines, fmt.Sprintf("Your tunnel '%v' was created with ID %v. However, cloudflared couldn't write to the tunnel credentials file at %v.json.", tunnel.Name, tunnel.ID, tunnel.ID))
+		errorLines = append(errorLines, fmt.Sprintf("Your tunnel '%v' was created with ID %v. However, cloudflared couldn't write tunnel credentials to %s.", tunnel.Name, tunnel.ID, credentialsFilePath))
 		errorLines = append(errorLines, fmt.Sprintf("The file-writing error is: %v", writeFileErr))
 		if deleteErr := client.DeleteTunnel(tunnel.ID); deleteErr != nil {
 			errorLines = append(errorLines, fmt.Sprintf("Cloudflared tried to delete the tunnel for you, but encountered an error. You should use `cloudflared tunnel delete %v` to delete the tunnel yourself, because the tunnel can't be run without the tunnelfile.", tunnel.ID))
 			errorLines = append(errorLines, fmt.Sprintf("The delete tunnel error is: %v", deleteErr))
 		} else {
-			errorLines = append(errorLines, fmt.Sprintf("The tunnel was deleted, because the tunnel can't be run without the tunnelfile"))
+			errorLines = append(errorLines, fmt.Sprintf("The tunnel was deleted, because the tunnel can't be run without the credentials file"))
 		}
 		errorMsg := strings.Join(errorLines, "\n")
 		return nil, errors.New(errorMsg)
 	}
-	sc.log.Info().Msgf("Tunnel credentials written to %v. cloudflared chose this file based on where your origin certificate was found. Keep this file secret. To revoke these credentials, delete the tunnel.", filePath)
 
 	if outputFormat := sc.c.String(outputFormatFlag.Name); outputFormat != "" {
 		return nil, renderOutput(outputFormat, &tunnel)
 	}
 
-	sc.log.Info().Msgf("Created tunnel %s with id %s", tunnel.Name, tunnel.ID)
+	fmt.Printf("Tunnel credentials written to %v.", credentialsFilePath)
+	if usedCertPath {
+		fmt.Print(" cloudflared chose this file based on where your origin certificate was found.")
+	}
+	fmt.Println(" Keep this file secret. To revoke these credentials, delete the tunnel.")
+	fmt.Printf("\nCreated tunnel %s with id %s\n", tunnel.Name, tunnel.ID)
 	return tunnel, nil
 }
 
@@ -224,7 +238,7 @@ func (sc *subcommandContext) delete(tunnelIDs []uuid.UUID) error {
 			return fmt.Errorf("Tunnel %s has already been deleted", tunnel.ID)
 		}
 		if forceFlagSet {
-			if err := client.CleanupConnections(tunnel.ID); err != nil {
+			if err := client.CleanupConnections(tunnel.ID, tunnelstore.NewCleanupParams()); err != nil {
 				return errors.Wrapf(err, "Error cleaning up connections for tunnel %s", tunnel.ID)
 			}
 		}
@@ -276,13 +290,24 @@ func (sc *subcommandContext) run(tunnelID uuid.UUID) error {
 }
 
 func (sc *subcommandContext) cleanupConnections(tunnelIDs []uuid.UUID) error {
+	params := tunnelstore.NewCleanupParams()
+	extraLog := ""
+	if connector := sc.c.String("connector-id"); connector != "" {
+		connectorID, err := uuid.Parse(connector)
+		if err != nil {
+			return errors.Wrapf(err, "%s is not a valid client ID (must be a UUID)", connector)
+		}
+		params.ForClient(connectorID)
+		extraLog = fmt.Sprintf(" for connector-id %s", connectorID.String())
+	}
+
 	client, err := sc.client()
 	if err != nil {
 		return err
 	}
 	for _, tunnelID := range tunnelIDs {
-		sc.log.Info().Msgf("Cleanup connection for tunnel %s", tunnelID)
-		if err := client.CleanupConnections(tunnelID); err != nil {
+		sc.log.Info().Msgf("Cleanup connection for tunnel %s%s", tunnelID, extraLog)
+		if err := client.CleanupConnections(tunnelID, params); err != nil {
 			sc.log.Error().Msgf("Error cleaning up connections for tunnel %v, error :%v", tunnelID, err)
 		}
 	}
@@ -343,6 +368,12 @@ func (sc *subcommandContext) findID(input string) (uuid.UUID, error) {
 // one Tunnelstore API call.
 func (sc *subcommandContext) findIDs(inputs []string) ([]uuid.UUID, error) {
 
+	// Shortcut without Tunnelstore call if we find that all inputs are already UUIDs.
+	uuids, err := convertNamesToUuids(inputs, make(map[string]uuid.UUID))
+	if err == nil {
+		return uuids, nil
+	}
+
 	// First, look up all tunnels the user has
 	filter := tunnelstore.NewFilter()
 	filter.NoDeleted()
@@ -362,7 +393,10 @@ func findIDs(tunnels []*tunnelstore.Tunnel, inputs []string) ([]uuid.UUID, error
 		nameToID[tunnel.Name] = tunnel.ID
 	}
 
-	// For each input, try to find the tunnel ID.
+	return convertNamesToUuids(inputs, nameToID)
+}
+
+func convertNamesToUuids(inputs []string, nameToID map[string]uuid.UUID) ([]uuid.UUID, error) {
 	tunnelIDs := make([]uuid.UUID, len(inputs))
 	var badInputs []string
 	for i, input := range inputs {

cmd/cloudflared/tunnel/subcommand_context_teamnet.go

@@ -3,8 +3,9 @@ package tunnel
 import (
 	"net"
 
-	"github.com/cloudflare/cloudflared/teamnet"
 	"github.com/pkg/errors"
+
+	"github.com/cloudflare/cloudflared/teamnet"
 )
 
 const noClientMsg = "error while creating backend client"

cmd/cloudflared/tunnel/subcommand_context_test.go

@@ -4,16 +4,17 @@ import (
 	"encoding/base64"
 	"flag"
 	"fmt"
-	"github.com/rs/zerolog"
 	"reflect"
 	"testing"
 	"time"
 
-	"github.com/cloudflare/cloudflared/connection"
-	"github.com/cloudflare/cloudflared/tunnelstore"
 	"github.com/google/uuid"
 	"github.com/pkg/errors"
+	"github.com/rs/zerolog"
 	"github.com/urfave/cli/v2"
+
+	"github.com/cloudflare/cloudflared/connection"
+	"github.com/cloudflare/cloudflared/tunnelstore"
 )
 
 func Test_findIDs(t *testing.T) {
@@ -260,7 +261,7 @@ func (d *deleteMockTunnelStore) DeleteTunnel(tunnelID uuid.UUID) error {
 	return nil
 }
 
-func (d *deleteMockTunnelStore) CleanupConnections(tunnelID uuid.UUID) error {
+func (d *deleteMockTunnelStore) CleanupConnections(tunnelID uuid.UUID, _ *tunnelstore.CleanupParams) error {
 	tunnel, ok := d.mockTunnels[tunnelID]
 	if !ok {
 		return fmt.Errorf("Couldn't find tunnel: %v", tunnelID)

cmd/cloudflared/tunnel/subcommands.go

@@ -14,12 +14,12 @@ import (
 	"time"
 
 	"github.com/google/uuid"
-	"github.com/mitchellh/go-homedir"
+	homedir "github.com/mitchellh/go-homedir"
 	"github.com/pkg/errors"
 	"github.com/urfave/cli/v2"
 	"github.com/urfave/cli/v2/altsrc"
 	"golang.org/x/net/idna"
-	"gopkg.in/yaml.v2"
+	yaml "gopkg.in/yaml.v2"
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/updater"
@@ -29,9 +29,10 @@ import (
 )
 
 const (
-	allSortByOptions  = "name, id, createdAt, deletedAt, numConnections"
-	CredFileFlagAlias = "cred-file"
-	CredFileFlag      = "credentials-file"
+	allSortByOptions   = "name, id, createdAt, deletedAt, numConnections"
+	connsSortByOptions = "id, startedAt, numConnections, version"
+	CredFileFlagAlias  = "cred-file"
+	CredFileFlag       = "credentials-file"
 
 	LogFieldTunnelID = "tunnelID"
 )
@@ -64,11 +65,11 @@ var (
 		Aliases: []string{"rd"},
 		Usage:   "Include connections that have recently disconnected in the list",
 	}
-	outputFormatFlag = altsrc.NewStringFlag(&cli.StringFlag{
+	outputFormatFlag = &cli.StringFlag{
 		Name:    "output",
 		Aliases: []string{"o"},
 		Usage:   "Render output using given `FORMAT`. Valid options are 'json' or 'yaml'",
-	})
+	}
 	sortByFlag = &cli.StringFlag{
 		Name:    "sort-by",
 		Value:   "name",
@@ -93,13 +94,14 @@ var (
 		Aliases: []string{"F"},
 		Usage:   "Opt into various features that are still being developed or tested.",
 	})
-	credentialsFileFlag = altsrc.NewStringFlag(&cli.StringFlag{
+	credentialsFileFlagCLIOnly = &cli.StringFlag{
 		Name:    CredFileFlag,
 		Aliases: []string{CredFileFlagAlias},
 		Usage:   "Filepath at which to read/write the tunnel credentials",
 		EnvVars: []string{"TUNNEL_CRED_FILE"},
-	})
-	forceDeleteFlag = &cli.BoolFlag{
+	}
+	credentialsFileFlag = altsrc.NewStringFlag(credentialsFileFlagCLIOnly)
+	forceDeleteFlag     = &cli.BoolFlag{
 		Name:    "force",
 		Aliases: []string{"f"},
 		Usage: "Cleans up any stale connections before the tunnel is deleted. cloudflared will not " +
@@ -114,12 +116,29 @@ var (
 		EnvVars: []string{"TUNNEL_TRANSPORT_PROTOCOL"},
 		Hidden:  true,
 	})
+	sortInfoByFlag = &cli.StringFlag{
+		Name:    "sort-by",
+		Value:   "createdAt",
+		Usage:   fmt.Sprintf("Sorts the list of connections of a tunnel by the given field. Valid options are {%s}", connsSortByOptions),
+		EnvVars: []string{"TUNNEL_INFO_SORT_BY"},
+	}
+	invertInfoSortFlag = &cli.BoolFlag{
+		Name:    "invert-sort",
+		Usage:   "Inverts the sort order of the tunnel info.",
+		EnvVars: []string{"TUNNEL_INFO_INVERT_SORT"},
+	}
+	cleanupClientFlag = &cli.StringFlag{
+		Name:    "connector-id",
+		Aliases: []string{"c"},
+		Usage:   `Constraints the cleanup to stop the connections of a single Connector (by its ID). You can find the various Connectors (and their IDs) currently connected to your tunnel via 'cloudflared tunnel info <name>'.`,
+		EnvVars: []string{"TUNNEL_CLEANUP_CONNECTOR"},
+	}
 )
 
 func buildCreateCommand() *cli.Command {
 	return &cli.Command{
 		Name:      "create",
-		Action:    cliutil.ErrorHandler(createCommand),
+		Action:    cliutil.ConfiguredAction(createCommand),
 		Usage:     "Create a new tunnel with given name",
 		UsageText: "cloudflared tunnel [tunnel command options] create [subcommand options] NAME",
 		Description: `Creates a tunnel, registers it with Cloudflare edge and generates credential file used to run this tunnel.
@@ -128,7 +147,7 @@ func buildCreateCommand() *cli.Command {
   For example, to create a tunnel named 'my-tunnel' run:
 
   $ cloudflared tunnel create my-tunnel`,
-		Flags:              []cli.Flag{outputFormatFlag, credentialsFileFlag},
+		Flags:              []cli.Flag{outputFormatFlag, credentialsFileFlagCLIOnly},
 		CustomHelpTemplate: commandHelpTemplate(),
 	}
 }
@@ -164,33 +183,26 @@ func tunnelFilePath(tunnelID uuid.UUID, directory string) (string, error) {
 	return homedir.Expand(filePath)
 }
 
-// If an `outputFile` is given, write the credentials there.
-// Otherwise, write it to the same directory as the originCert,
-// with the filename `<tunnel id>.json`.
-func writeTunnelCredentials(
-	originCertPath, outputFile string,
-	credentials *connection.Credentials,
-) (filePath string, err error) {
-	filePath = outputFile
-	if outputFile == "" {
-		originCertDir := filepath.Dir(originCertPath)
-		filePath, err = tunnelFilePath(credentials.TunnelID, originCertDir)
+// writeTunnelCredentials saves `credentials` as a JSON into `filePath`, only if
+// the file does not exist already
+func writeTunnelCredentials(filePath string, credentials *connection.Credentials) error {
+	if _, err := os.Stat(filePath); !os.IsNotExist(err) {
+		if err == nil {
+			return fmt.Errorf("%s already exists", filePath)
+		}
+		return err
 	}
-	if err != nil {
-		return "", err
-	}
-	// Write the name and ID to the file too
 	body, err := json.Marshal(credentials)
 	if err != nil {
-		return "", errors.Wrap(err, "Unable to marshal tunnel credentials to JSON")
+		return errors.Wrap(err, "Unable to marshal tunnel credentials to JSON")
 	}
-	return filePath, ioutil.WriteFile(filePath, body, 400)
+	return ioutil.WriteFile(filePath, body, 400)
 }
 
 func buildListCommand() *cli.Command {
 	return &cli.Command{
 		Name:        "list",
-		Action:      cliutil.ErrorHandler(listCommand),
+		Action:      cliutil.ConfiguredAction(listCommand),
 		Usage:       "List existing tunnels",
 		UsageText:   "cloudflared tunnel [tunnel command options] list [subcommand options]",
 		Description: "cloudflared tunnel list will display all active tunnels, their created time and associated connections. Use -d flag to include deleted tunnels. See the list of options to filter the list",
@@ -214,6 +226,9 @@ func listCommand(c *cli.Context) error {
 		return err
 	}
 
+	warningChecker := updater.StartWarningCheck(c)
+	defer warningChecker.LogWarningIfAny(sc.log)
+
 	filter := tunnelstore.NewFilter()
 	if !c.Bool("show-deleted") {
 		filter.NoDeleted()
@@ -232,9 +247,6 @@ func listCommand(c *cli.Context) error {
 		filter.ByTunnelID(tunnelID)
 	}
 
-	warningChecker := updater.StartWarningCheck(c)
-	defer warningChecker.LogWarningIfAny(sc.log)
-
 	tunnels, err := sc.list(filter)
 	if err != nil {
 		return err
@@ -284,17 +296,11 @@ func listCommand(c *cli.Context) error {
 }
 
 func formatAndPrintTunnelList(tunnels []*tunnelstore.Tunnel, showRecentlyDisconnected bool) {
-	const (
-		minWidth = 0
-		tabWidth = 8
-		padding  = 1
-		padChar  = ' '
-		flags    = 0
-	)
-
-	writer := tabwriter.NewWriter(os.Stdout, minWidth, tabWidth, padding, padChar, flags)
+	writer := tabWriter()
 	defer writer.Flush()
 
+	_, _ = fmt.Fprintln(writer, "You can obtain more detailed information for each tunnel with `cloudflared tunnel info <name/uuid>`")
+
 	// Print column headers with tabbed columns
 	_, _ = fmt.Fprintln(writer, "ID\tNAME\tCREATED\tCONNECTIONS\t")
 
@@ -336,14 +342,176 @@ func fmtConnections(connections []tunnelstore.Connection, showRecentlyDisconnect
 	return strings.Join(output, ", ")
 }
 
+func buildInfoCommand() *cli.Command {
+	return &cli.Command{
+		Name:        "info",
+		Action:      cliutil.ConfiguredAction(tunnelInfo),
+		Usage:       "List details about the active connectors for a tunnel",
+		UsageText:   "cloudflared tunnel [tunnel command options] info [subcommand options] [TUNNEL]",
+		Description: "cloudflared tunnel info displays details about the active connectors for a given tunnel (identified by name or uuid).",
+		Flags: []cli.Flag{
+			outputFormatFlag,
+			showRecentlyDisconnected,
+			sortInfoByFlag,
+			invertInfoSortFlag,
+		},
+		CustomHelpTemplate: commandHelpTemplate(),
+	}
+}
+
+func tunnelInfo(c *cli.Context) error {
+	sc, err := newSubcommandContext(c)
+	if err != nil {
+		return err
+	}
+
+	warningChecker := updater.StartWarningCheck(c)
+	defer warningChecker.LogWarningIfAny(sc.log)
+
+	if c.NArg() != 1 {
+		return cliutil.UsageError(`"cloudflared tunnel info" accepts exactly one argument, the ID or name of the tunnel to get info about.`)
+	}
+	tunnelID, err := sc.findID(c.Args().First())
+	if err != nil {
+		return errors.Wrap(err, "error parsing tunnel ID")
+	}
+
+	client, err := sc.client()
+	if err != nil {
+		return err
+	}
+
+	clients, err := client.ListActiveClients(tunnelID)
+	if err != nil {
+		return err
+	}
+
+	sortBy := c.String("sort-by")
+	invalidSortField := false
+	sort.Slice(clients, func(i, j int) bool {
+		cmp := func() bool {
+			switch sortBy {
+			case "id":
+				return clients[i].ID.String() < clients[j].ID.String()
+			case "createdAt":
+				return clients[i].RunAt.Unix() < clients[j].RunAt.Unix()
+			case "numConnections":
+				return len(clients[i].Connections) < len(clients[j].Connections)
+			case "version":
+				return clients[i].Version < clients[j].Version
+			default:
+				invalidSortField = true
+				return clients[i].RunAt.Unix() < clients[j].RunAt.Unix()
+			}
+		}()
+		if c.Bool("invert-sort") {
+			return !cmp
+		}
+		return cmp
+	})
+	if invalidSortField {
+		sc.log.Error().Msgf("%s is not a valid sort field. Valid sort fields are %s. Defaulting to 'name'.", sortBy, connsSortByOptions)
+	}
+
+	tunnel, err := getTunnel(sc, tunnelID)
+	if err != nil {
+		return err
+	}
+	info := Info{
+		tunnel.ID,
+		tunnel.Name,
+		tunnel.CreatedAt,
+		clients,
+	}
+
+	if outputFormat := c.String(outputFormatFlag.Name); outputFormat != "" {
+		return renderOutput(outputFormat, info)
+	}
+
+	if len(clients) > 0 {
+		formatAndPrintConnectionsList(info, c.Bool("show-recently-disconnected"))
+	} else {
+		fmt.Printf("Your tunnel %s does not have any active connection.\n", tunnelID)
+	}
+
+	return nil
+}
+
+func getTunnel(sc *subcommandContext, tunnelID uuid.UUID) (*tunnelstore.Tunnel, error) {
+	filter := tunnelstore.NewFilter()
+	filter.ByTunnelID(tunnelID)
+	tunnels, err := sc.list(filter)
+	if err != nil {
+		return nil, err
+	}
+	if len(tunnels) != 1 {
+		return nil, errors.Errorf("Expected to find a single tunnel with uuid %v but found %d tunnels.", tunnelID, len(tunnels))
+	}
+	return tunnels[0], nil
+}
+
+func formatAndPrintConnectionsList(tunnelInfo Info, showRecentlyDisconnected bool) {
+	writer := tabWriter()
+	defer writer.Flush()
+
+	// Print the general tunnel info table
+	_, _ = fmt.Fprintf(writer, "NAME:     %s\nID:       %s\nCREATED:  %s\n\n", tunnelInfo.Name, tunnelInfo.ID, tunnelInfo.CreatedAt)
+
+	// Determine whether to print the connector table
+	shouldDisplayTable := false
+	for _, c := range tunnelInfo.Connectors {
+		conns := fmtConnections(c.Connections, showRecentlyDisconnected)
+		if len(conns) > 0 {
+			shouldDisplayTable = true
+		}
+	}
+	if !shouldDisplayTable {
+		fmt.Println("This tunnel has no active connectors.")
+		return
+	}
+
+	// Print the connector table
+	_, _ = fmt.Fprintln(writer, "CONNECTOR ID\tCREATED\tARCHITECTURE\tVERSION\tORIGIN IP\tEDGE\t")
+	for _, c := range tunnelInfo.Connectors {
+		conns := fmtConnections(c.Connections, showRecentlyDisconnected)
+		if len(conns) == 0 {
+			continue
+		}
+		originIp := c.Connections[0].OriginIP.String()
+		formattedStr := fmt.Sprintf(
+			"%s\t%s\t%s\t%s\t%s\t%s\t",
+			c.ID,
+			c.RunAt.Format(time.RFC3339),
+			c.Arch,
+			c.Version,
+			originIp,
+			conns,
+		)
+		_, _ = fmt.Fprintln(writer, formattedStr)
+	}
+}
+
+func tabWriter() *tabwriter.Writer {
+	const (
+		minWidth = 0
+		tabWidth = 8
+		padding  = 1
+		padChar  = ' '
+		flags    = 0
+	)
+
+	writer := tabwriter.NewWriter(os.Stdout, minWidth, tabWidth, padding, padChar, flags)
+	return writer
+}
+
 func buildDeleteCommand() *cli.Command {
 	return &cli.Command{
 		Name:               "delete",
-		Action:             cliutil.ErrorHandler(deleteCommand),
+		Action:             cliutil.ConfiguredAction(deleteCommand),
 		Usage:              "Delete existing tunnel by UUID or name",
 		UsageText:          "cloudflared tunnel [tunnel command options] delete [subcommand options] TUNNEL",
 		Description:        "cloudflared tunnel delete will delete tunnels with the given tunnel UUIDs or names. A tunnel cannot be deleted if it has active connections. To delete the tunnel unconditionally, use -f flag.",
-		Flags:              []cli.Flag{credentialsFileFlag, forceDeleteFlag},
+		Flags:              []cli.Flag{credentialsFileFlagCLIOnly, forceDeleteFlag},
 		CustomHelpTemplate: commandHelpTemplate(),
 	}
 }
@@ -392,8 +560,7 @@ func buildRunCommand() *cli.Command {
 	flags = append(flags, configureProxyFlags(false)...)
 	return &cli.Command{
 		Name:      "run",
-		Action:    cliutil.ErrorHandler(runCommand),
-		Before:    SetFlagsFromConfigFile,
+		Action:    cliutil.ConfiguredAction(runCommand),
 		Usage:     "Proxy a local web server by running the given tunnel",
 		UsageText: "cloudflared tunnel [tunnel command options] run [subcommand options] [TUNNEL]",
 		Description: `Runs the tunnel identified by name or UUUD, creating highly available connections
@@ -445,10 +612,11 @@ func runNamedTunnel(sc *subcommandContext, tunnelRef string) error {
 func buildCleanupCommand() *cli.Command {
 	return &cli.Command{
 		Name:               "cleanup",
-		Action:             cliutil.ErrorHandler(cleanupCommand),
+		Action:             cliutil.ConfiguredAction(cleanupCommand),
 		Usage:              "Cleanup tunnel connections",
 		UsageText:          "cloudflared tunnel [tunnel command options] cleanup [subcommand options] TUNNEL",
 		Description:        "Delete connections for tunnels with the given UUIDs or names.",
+		Flags:              []cli.Flag{cleanupClientFlag},
 		CustomHelpTemplate: commandHelpTemplate(),
 	}
 }
@@ -474,7 +642,7 @@ func cleanupCommand(c *cli.Context) error {
 func buildRouteCommand() *cli.Command {
 	return &cli.Command{
 		Name:      "route",
-		Action:    cliutil.ErrorHandler(routeCommand),
+		Action:    cliutil.ConfiguredAction(routeCommand),
 		Usage:     "Define which traffic routed from Cloudflare edge to this tunnel: requests to a DNS hostname, to a Cloudflare Load Balancer, or traffic originating from Cloudflare WARP clients",
 		UsageText: "cloudflared tunnel [tunnel command options] route [subcommand options] [dns TUNNEL HOSTNAME]|[lb TUNNEL HOSTNAME LB-POOL]|[ip NETWORK TUNNEL]",
 		Description: `The route command defines how Cloudflare will proxy requests to this tunnel.

cmd/cloudflared/tunnel/subcommands_test.go

@@ -5,10 +5,11 @@ import (
 	"path/filepath"
 	"testing"
 
-	"github.com/cloudflare/cloudflared/tunnelstore"
 	"github.com/google/uuid"
-	"github.com/mitchellh/go-homedir"
+	homedir "github.com/mitchellh/go-homedir"
 	"github.com/stretchr/testify/assert"
+
+	"github.com/cloudflare/cloudflared/tunnelstore"
 )
 
 func Test_fmtConnections(t *testing.T) {

cmd/cloudflared/tunnel/teamnet_subcommands.go

@@ -6,10 +6,11 @@ import (
 	"os"
 	"text/tabwriter"
 
+	"github.com/pkg/errors"
+
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/updater"
 	"github.com/cloudflare/cloudflared/teamnet"
-	"github.com/pkg/errors"
 
 	"github.com/urfave/cli/v2"
 )
@@ -25,7 +26,7 @@ Cloudflare WARP client. You can also build rules to determine who can reach cert
 		Subcommands: []*cli.Command{
 			{
 				Name:      "add",
-				Action:    cliutil.ErrorHandler(addRouteCommand),
+				Action:    cliutil.ConfiguredAction(addRouteCommand),
 				Usage:     "Add any new network to the routing table reachable via the tunnel",
 				UsageText: "cloudflared tunnel [--config FILEPATH] route ip add [CIDR] [TUNNEL] [COMMENT?]",
 				Description: `Adds any network route space (represented as a CIDR) to your routing table.
@@ -38,7 +39,7 @@ reachable from the tunnel.`,
 			{
 				Name:        "show",
 				Aliases:     []string{"list"},
-				Action:      cliutil.ErrorHandler(showRoutesCommand),
+				Action:      cliutil.ConfiguredAction(showRoutesCommand),
 				Usage:       "Show the routing table",
 				UsageText:   "cloudflared tunnel [--config FILEPATH] route ip show [flags]",
 				Description: `Shows your organization private routing table. You can use flags to filter the results.`,
@@ -46,7 +47,7 @@ reachable from the tunnel.`,
 			},
 			{
 				Name:      "delete",
-				Action:    cliutil.ErrorHandler(deleteRouteCommand),
+				Action:    cliutil.ConfiguredAction(deleteRouteCommand),
 				Usage:     "Delete a row from your organization's private routing table",
 				UsageText: "cloudflared tunnel [--config FILEPATH] route ip delete [CIDR]",
 				Description: `Deletes the row for a given CIDR from your routing table. That portion
@@ -54,7 +55,7 @@ of your network will no longer be reachable by the WARP clients.`,
 			},
 			{
 				Name:      "get",
-				Action:    cliutil.ErrorHandler(getRouteByIPCommand),
+				Action:    cliutil.ConfiguredAction(getRouteByIPCommand),
 				Usage:     "Check which row of the routing table matches a given IP.",
 				UsageText: "cloudflared tunnel [--config FILEPATH] route ip get [IP]",
 				Description: `Checks which row of the routing table will be used to proxy a given IP.

cmd/cloudflared/ui/launch_ui.go

@@ -5,12 +5,12 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/cloudflare/cloudflared/connection"
-	"github.com/cloudflare/cloudflared/ingress"
-
 	"github.com/gdamore/tcell"
 	"github.com/rivo/tview"
 	"github.com/rs/zerolog"
+
+	"github.com/cloudflare/cloudflared/connection"
+	"github.com/cloudflare/cloudflared/ingress"
 )
 
 type connState struct {

cmd/cloudflared/windows_service.go

@@ -12,13 +12,15 @@ import (
 	"time"
 	"unsafe"
 
-	"github.com/cloudflare/cloudflared/logger"
-
+	"github.com/pkg/errors"
 	"github.com/urfave/cli/v2"
 	"golang.org/x/sys/windows"
 	"golang.org/x/sys/windows/svc"
 	"golang.org/x/sys/windows/svc/eventlog"
 	"golang.org/x/sys/windows/svc/mgr"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
+	"github.com/cloudflare/cloudflared/logger"
 )
 
 const (
@@ -48,13 +50,13 @@ func runApp(app *cli.App, graceShutdownC chan struct{}) {
 			{
 				Name:   "install",
 				Usage:  "Install Argo Tunnel as a Windows service",
-				Action: installWindowsService,
-						},
+				Action: cliutil.ConfiguredAction(installWindowsService),
+			},
 			{
 				Name:   "uninstall",
 				Usage:  "Uninstall the Argo Tunnel service",
-				Action: uninstallWindowsService,
-						},
+				Action: cliutil.ConfiguredAction(uninstallWindowsService),
+			},
 		},
 	})
 
@@ -177,35 +179,30 @@ func installWindowsService(c *cli.Context) error {
 	zeroLogger.Info().Msg("Installing Argo Tunnel Windows service")
 	exepath, err := os.Executable()
 	if err != nil {
-		zeroLogger.Err(err).Msg("Cannot find path name that start the process")
-		return err
+		return errors.Wrap(err, "Cannot find path name that start the process")
 	}
 	m, err := mgr.Connect()
 	if err != nil {
-		zeroLogger.Err(err).Msg("Cannot establish a connection to the service control manager")
-		return err
+		return errors.Wrap(err, "Cannot establish a connection to the service control manager")
 	}
 	defer m.Disconnect()
 	s, err := m.OpenService(windowsServiceName)
 	log := zeroLogger.With().Str(LogFieldWindowsServiceName, windowsServiceName).Logger()
 	if err == nil {
 		s.Close()
-		log.Err(err).Msg("service already exists")
-		return fmt.Errorf("service %s already exists", windowsServiceName)
+		return fmt.Errorf("Service %s already exists", windowsServiceName)
 	}
 	config := mgr.Config{StartType: mgr.StartAutomatic, DisplayName: windowsServiceDescription}
 	s, err = m.CreateService(windowsServiceName, exepath, config)
 	if err != nil {
-		log.Error().Msg("Cannot install service")
-		return err
+		return errors.Wrap(err, "Cannot install service")
 	}
 	defer s.Close()
 	log.Info().Msg("Argo Tunnel agent service is installed")
 	err = eventlog.InstallAsEventCreate(windowsServiceName, eventlog.Error|eventlog.Warning|eventlog.Info)
 	if err != nil {
 		s.Delete()
-		log.Err(err).Msg("Cannot install event logger")
-		return fmt.Errorf("SetupEventLogSource() failed: %s", err)
+		return errors.Wrap(err, "Cannot install event logger")
 	}
 	err = configRecoveryOption(s.Handle)
 	if err != nil {
@@ -223,26 +220,22 @@ func uninstallWindowsService(c *cli.Context) error {
 	log.Info().Msg("Uninstalling Argo Tunnel Windows Service")
 	m, err := mgr.Connect()
 	if err != nil {
-		log.Error().Msg("Cannot establish a connection to the service control manager")
-		return err
+		return errors.Wrap(err, "Cannot establish a connection to the service control manager")
 	}
 	defer m.Disconnect()
 	s, err := m.OpenService(windowsServiceName)
 	if err != nil {
-		log.Error().Msg("service is not installed")
-		return fmt.Errorf("service %s is not installed", windowsServiceName)
+		return fmt.Errorf("Service %s is not installed", windowsServiceName)
 	}
 	defer s.Close()
 	err = s.Delete()
 	if err != nil {
-		log.Error().Msg("Cannot delete service")
-		return err
+		return errors.Wrap(err, "Cannot delete service")
 	}
 	log.Info().Msg("Argo Tunnel agent service is uninstalled")
 	err = eventlog.Remove(windowsServiceName)
 	if err != nil {
-		log.Error().Msg("Cannot remove event logger")
-		return fmt.Errorf("RemoveEventLogSource() failed: %s", err)
+		return errors.Wrap(err, "Cannot remove event logger")
 	}
 	return nil
 }

component-tests/README.md

@@ -31,6 +31,9 @@ If you are using Visual Studio, follow https://code.visualstudio.com/docs/python
 to turn on formatter and https://marketplace.visualstudio.com/items?itemName=cbrevik.toggle-format-on-save
 to turn on format on save.
 
+6. If you have cloudflared running as a service on your machine, you can either stop the service or ignore the service tests
+via `--ignore test_service.py`
+
 # How to run
 Specify path to config file via env var `COMPONENT_TESTS_CONFIG`. This is required.
 ## All tests

component-tests/config.py

@@ -1,15 +1,10 @@
 #!/usr/bin/env python
-import base64
 import copy
-import os
-import yaml
 
 from dataclasses import dataclass, InitVar
 
 from constants import METRICS_PORT
 
-from util import LOGGER
-
 # frozen=True raises exception when assigning to fields. This emulates immutability
 
 
@@ -98,35 +93,3 @@ class ClassicTunnelConfig(ClassicTunnelBaseConfig):
 
     def get_url(self):
         return "https://" + self.hostname
-
-
-def build_config_from_env():
-    config_path = get_env("COMPONENT_TESTS_CONFIG")
-    config_content = base64.b64decode(
-        get_env("COMPONENT_TESTS_CONFIG_CONTENT")).decode('utf-8')
-    config_yaml = yaml.safe_load(config_content)
-
-    credentials_file = get_env("COMPONENT_TESTS_CREDENTIALS_FILE")
-    write_file(credentials_file, config_yaml["credentials_file"])
-
-    origincert = get_env("COMPONENT_TESTS_ORIGINCERT")
-    write_file(origincert, config_yaml["origincert"])
-
-    write_file(config_content, config_path)
-
-
-def write_file(content, path):
-    with open(path, 'w') as outfile:
-        outfile.write(content)
-        outfile.close
-
-
-def get_env(env_name):
-    val = os.getenv(env_name)
-    if val is None:
-        raise Exception(f"{env_name} is not set")
-    return val
-
-
-if __name__ == '__main__':
-    build_config_from_env()

component-tests/constants.py

@@ -1,3 +1,3 @@
 METRICS_PORT = 51000
-MAX_RETRIES = 3
-BACKOFF_SECS = 5
+MAX_RETRIES = 5
+BACKOFF_SECS = 7

component-tests/requirements.txt

@@ -1,3 +1,5 @@
+cloudflare==2.8.15
+flaky==3.7.0
 pytest==6.2.2
 pyyaml==5.4.1
 requests==2.25.1

component-tests/setup.py

@@ -0,0 +1,165 @@
+#!/usr/bin/env python
+import argparse
+import base64
+import json
+import os
+import subprocess
+import uuid
+
+import CloudFlare
+import yaml
+from retrying import retry
+
+from constants import MAX_RETRIES, BACKOFF_SECS
+from util import LOGGER
+
+
+def get_config_from_env():
+    config_content = base64.b64decode(get_env("COMPONENT_TESTS_CONFIG_CONTENT")).decode('utf-8')
+    return yaml.safe_load(config_content)
+
+
+def get_config_from_file():
+    config_path = get_env("COMPONENT_TESTS_CONFIG")
+    with open(config_path, 'r') as infile:
+        return yaml.safe_load(infile)
+
+
+def persist_config(config):
+    config_path = get_env("COMPONENT_TESTS_CONFIG")
+    with open(config_path, 'w') as outfile:
+        yaml.safe_dump(config, outfile)
+
+
+def persist_origin_cert(config):
+    origincert = get_env("COMPONENT_TESTS_ORIGINCERT")
+    path = config["origincert"]
+    with open(path, 'w') as outfile:
+        outfile.write(origincert)
+    return path
+
+
+@retry(stop_max_attempt_number=MAX_RETRIES, wait_fixed=BACKOFF_SECS * 1000)
+def create_tunnel(config, origincert_path, random_uuid):
+    # Delete any previous existing credentials file. If the agent keeps files around (that's the case in Windows) then
+    # cloudflared tunnel create will refuse to create the tunnel because it does not want to overwrite credentials
+    # files.
+    credentials_path = config["credentials_file"]
+    try:
+        os.remove(credentials_path)
+    except OSError:
+        pass
+
+    tunnel_name = "cfd_component_test-" + random_uuid
+    create_cmd = [config["cloudflared_binary"], "tunnel", "--origincert", origincert_path, "create",
+                  "--credentials-file", credentials_path, tunnel_name]
+    LOGGER.info(f"Creating tunnel with {create_cmd}")
+    subprocess.run(create_cmd, check=True)
+
+    list_cmd = [config["cloudflared_binary"], "tunnel", "--origincert", origincert_path, "list", "--name",
+                tunnel_name, "--output", "json"]
+    LOGGER.info(f"Listing tunnel with {list_cmd}")
+    cloudflared = subprocess.run(list_cmd, check=True, capture_output=True)
+    return json.loads(cloudflared.stdout)[0]["id"]
+
+
+@retry(stop_max_attempt_number=MAX_RETRIES, wait_fixed=BACKOFF_SECS * 1000)
+def delete_tunnel(config):
+    credentials_path = config["credentials_file"]
+    delete_cmd = [config["cloudflared_binary"], "tunnel", "--origincert", config["origincert"], "delete",
+                  "--credentials-file", credentials_path, "-f", config["tunnel"]]
+    LOGGER.info(f"Deleting tunnel with {delete_cmd}")
+    subprocess.run(delete_cmd, check=True)
+
+
+@retry(stop_max_attempt_number=MAX_RETRIES, wait_fixed=BACKOFF_SECS * 1000)
+def create_dns(config, hostname, type, content):
+    cf = CloudFlare.CloudFlare(debug=True, token=get_env("DNS_API_TOKEN"))
+    cf.zones.dns_records.post(
+        config["zone_tag"],
+        data={'name': hostname, 'type': type, 'content': content, 'proxied': True}
+    )
+
+
+def create_classic_dns(config, random_uuid):
+    classic_hostname = "classic-" + random_uuid + "." + config["zone_domain"]
+    create_dns(config, classic_hostname, "AAAA", "fd10:aec2:5dae::")
+    return classic_hostname
+
+
+def create_named_dns(config, random_uuid):
+    hostname = "named-" + random_uuid + "." + config["zone_domain"]
+    create_dns(config, hostname, "CNAME", config["tunnel"] + ".cfargotunnel.com")
+    return hostname
+
+
+@retry(stop_max_attempt_number=MAX_RETRIES, wait_fixed=BACKOFF_SECS * 1000)
+def delete_dns(config, hostname):
+    cf = CloudFlare.CloudFlare(debug=True, token=get_env("DNS_API_TOKEN"))
+    zone_tag = config["zone_tag"]
+    dns_records = cf.zones.dns_records.get(zone_tag, params={'name': hostname})
+    if len(dns_records) > 0:
+        cf.zones.dns_records.delete(zone_tag, dns_records[0]['id'])
+
+
+def write_file(content, path):
+    with open(path, 'w') as outfile:
+        outfile.write(content)
+
+
+def get_env(env_name):
+    val = os.getenv(env_name)
+    if val is None:
+        raise Exception(f"{env_name} is not set")
+    return val
+
+
+def create():
+    """
+    Creates the necessary resources for the components test to run.
+     - Creates a named tunnel with a random name.
+     - Creates a random CNAME DNS entry for that tunnel.
+     - Creates a random AAAA DNS entry for a classic tunnel.
+
+    Those created resources are added to the config (obtained from an environment variable).
+    The resulting configuration is persisted for the tests to use.
+    """
+    config = get_config_from_env()
+    origincert_path = persist_origin_cert(config)
+
+    random_uuid = str(uuid.uuid4())
+    config["tunnel"] = create_tunnel(config, origincert_path, random_uuid)
+    config["classic_hostname"] = create_classic_dns(config, random_uuid)
+    config["ingress"] = [
+        {
+            "hostname": create_named_dns(config, random_uuid),
+            "service": "hello_world"
+        },
+        {
+            "service": "http_status:404"
+        }
+    ]
+
+    persist_config(config)
+
+
+def cleanup():
+    """
+    Reads the persisted configuration that was created previously.
+    Deletes the resources that were created there.
+    """
+    config = get_config_from_file()
+    delete_tunnel(config)
+    delete_dns(config, config["classic_hostname"])
+    delete_dns(config, config["ingress"][0]["hostname"])
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='setup component tests')
+    parser.add_argument('--type', choices=['create', 'cleanup'], default='create')
+    args = parser.parse_args()
+
+    if args.type == 'create':
+        create()
+    else:
+        cleanup()

component-tests/test_logging.py

@@ -1,32 +1,87 @@
 #!/usr/bin/env python
 import json
 import os
-from util import start_cloudflared, wait_tunnel_ready, send_requests, LOGGER
+
+from util import start_cloudflared, wait_tunnel_ready, send_requests
+
+# Rolling logger rotate log files after 1 MB
+rotate_after_size = 1000 * 1000
+default_log_file = "cloudflared.log"
+expect_message = "Starting tunnel"
+
+
+def assert_log_to_terminal(cloudflared):
+    stderr = cloudflared.stderr.read(200)
+    # cloudflared logs the following when it first starts
+    # 2021-03-10T12:30:39Z INF Starting tunnel tunnelID=<tunnel ID>
+    assert expect_message.encode() in stderr, f"{stderr} doesn't contain {expect_message}"
+
+
+def assert_log_in_file(file):
+    with open(file, "r") as f:
+        log = f.read(200)
+        # cloudflared logs the following when it first starts
+        # {"level":"info","tunnelID":"<tunnel ID>","time":"2021-03-10T12:21:13Z","message":"Starting tunnel"}
+        assert expect_message in log, f"{log} doesn't contain {expect_message}"
+
+
+def assert_json_log(file):
+    def assert_in_json(j, key):
+        assert key in j, f"{key} is not in j"
+
+    with open(file, "r") as f:
+        line = f.readline()
+        json_log = json.loads(line)
+        assert_in_json(json_log, "level")
+        assert_in_json(json_log, "time")
+        assert_in_json(json_log, "message")
+
+
+def assert_log_to_dir(config, log_dir):
+    max_batches = 3
+    batch_requests = 1000
+    for _ in range(max_batches):
+        send_requests(config.get_url(),
+                      batch_requests, require_ok=False)
+        files = os.listdir(log_dir)
+        if len(files) == 2:
+            current_log_file_index = files.index(default_log_file)
+            current_file = log_dir / files[current_log_file_index]
+            stats = os.stat(current_file)
+            assert stats.st_size > 0
+            assert_json_log(current_file)
+
+            # One file is the current log file, the other is the rotated log file
+            rotated_log_file_index = 0 if current_log_file_index == 1 else 1
+            rotated_file = log_dir / files[rotated_log_file_index]
+            stats = os.stat(rotated_file)
+            assert stats.st_size > rotate_after_size
+            assert_log_in_file(rotated_file)
+            assert_json_log(current_file)
+            return
+
+    raise Exception(
+        f"Log file isn't rotated after sending {max_batches * batch_requests} requests")
 
 
 class TestLogging:
-    # Rolling logger rotate log files after 1 MB
-    rotate_after_size = 1000 * 1000
-    default_log_file = "cloudflared.log"
-    expect_message = "Starting tunnel"
-
     def test_logging_to_terminal(self, tmp_path, component_tests_config):
         config = component_tests_config()
         with start_cloudflared(tmp_path, config, new_process=True) as cloudflared:
-            wait_tunnel_ready()
-            self.assert_log_to_terminal(cloudflared)
+            wait_tunnel_ready(tunnel_url=config.get_url())
+            assert_log_to_terminal(cloudflared)
 
     def test_logging_to_file(self, tmp_path, component_tests_config):
-        log_file = tmp_path / self.default_log_file
+        log_file = tmp_path / default_log_file
         extra_config = {
             # Convert from pathlib.Path to str
             "logfile": str(log_file),
         }
         config = component_tests_config(extra_config)
         with start_cloudflared(tmp_path, config, new_process=True, capture_output=False):
-            wait_tunnel_ready()
-            self.assert_log_in_file(log_file)
-            self.assert_json_log(log_file)
+            wait_tunnel_ready(tunnel_url=config.get_url())
+            assert_log_in_file(log_file)
+            assert_json_log(log_file)
 
     def test_logging_to_dir(self, tmp_path, component_tests_config):
         log_dir = tmp_path / "logs"
@@ -37,56 +92,5 @@ class TestLogging:
         }
         config = component_tests_config(extra_config)
         with start_cloudflared(tmp_path, config, new_process=True, capture_output=False):
-            wait_tunnel_ready()
-            self.assert_log_to_dir(config, log_dir)
-
-    def assert_log_to_terminal(self, cloudflared):
-        stderr = cloudflared.stderr.read(200)
-        # cloudflared logs the following when it first starts
-        # 2021-03-10T12:30:39Z INF Starting tunnel tunnelID=<tunnel ID>
-        assert self.expect_message.encode(
-        ) in stderr, f"{stderr} doesn't contain {self.expect_message}"
-
-    def assert_log_in_file(self, file, expect_message=""):
-        with open(file, "r") as f:
-            log = f.read(200)
-            # cloudflared logs the following when it first starts
-            # {"level":"info","tunnelID":"<tunnel ID>","time":"2021-03-10T12:21:13Z","message":"Starting tunnel"}
-            assert self.expect_message in log, f"{log} doesn't contain {self.expect_message}"
-
-    def assert_json_log(self, file):
-        with open(file, "r") as f:
-            line = f.readline()
-            json_log = json.loads(line)
-            self.assert_in_json(json_log, "level")
-            self.assert_in_json(json_log, "time")
-            self.assert_in_json(json_log, "message")
-
-    def assert_in_json(self, j, key):
-        assert key in j, f"{key} is not in j"
-
-    def assert_log_to_dir(self, config, log_dir):
-        max_batches = 3
-        batch_requests = 1000
-        for _ in range(max_batches):
-            send_requests(config.get_url(),
-                          batch_requests, require_ok=False)
-            files = os.listdir(log_dir)
-            if len(files) == 2:
-                current_log_file_index = files.index(self.default_log_file)
-                current_file = log_dir / files[current_log_file_index]
-                stats = os.stat(current_file)
-                assert stats.st_size > 0
-                self.assert_json_log(current_file)
-
-                # One file is the current log file, the other is the rotated log file
-                rotated_log_file_index = 0 if current_log_file_index == 1 else 1
-                rotated_file = log_dir / files[rotated_log_file_index]
-                stats = os.stat(rotated_file)
-                assert stats.st_size > self.rotate_after_size
-                self.assert_log_in_file(rotated_file)
-                self.assert_json_log(current_file)
-                return
-
-        raise Exception(
-            f"Log file isn't rotated after sending {max_batches * batch_requests} requests")
+            wait_tunnel_ready(tunnel_url=config.get_url())
+            assert_log_to_dir(config, log_dir)

component-tests/test_reconnect.py

@@ -1,12 +1,13 @@
 #!/usr/bin/env python
 import copy
-
-from retrying import retry
 from time import sleep
 
-from util import start_cloudflared, wait_tunnel_ready, check_tunnel_not_ready, send_requests
+from flaky import flaky
+
+from util import start_cloudflared, wait_tunnel_ready, check_tunnel_not_connected
 
 
+@flaky(max_runs=3, min_passes=1)
 class TestReconnect():
     default_ha_conns = 4
     default_reconnect_secs = 5
@@ -16,7 +17,7 @@ class TestReconnect():
 
     def test_named_reconnect(self, tmp_path, component_tests_config):
         config = component_tests_config(self.extra_config)
-        with start_cloudflared(tmp_path, config, new_process=True, allow_input=True) as cloudflared:
+        with start_cloudflared(tmp_path, config, new_process=True, allow_input=True, capture_output=False) as cloudflared:
             # Repeat the test multiple times because some issues only occur after multiple reconnects
             self.assert_reconnect(config, cloudflared, 5)
 
@@ -25,7 +26,7 @@ class TestReconnect():
         extra_config["hello-world"] = True
         config = component_tests_config(
             additional_config=extra_config, named_tunnel=False)
-        with start_cloudflared(tmp_path, config, cfd_args=[], new_process=True, allow_input=True) as cloudflared:
+        with start_cloudflared(tmp_path, config, cfd_args=[], new_process=True, allow_input=True, capture_output=False) as cloudflared:
             self.assert_reconnect(config, cloudflared, 1)
 
     def send_reconnect(self, cloudflared, secs):
@@ -35,16 +36,19 @@ class TestReconnect():
         cloudflared.stdin.flush()
 
     def assert_reconnect(self, config, cloudflared, repeat):
-        wait_tunnel_ready()
+        wait_tunnel_ready(tunnel_url=config.get_url())
         for _ in range(repeat):
             for i in range(self.default_ha_conns):
                 self.send_reconnect(cloudflared, self.default_reconnect_secs)
                 expect_connections = self.default_ha_conns-i-1
                 if expect_connections > 0:
-                    wait_tunnel_ready(expect_connections=expect_connections)
+                    # Don't check if tunnel returns 200 here because there is a race condition between wait_tunnel_ready
+                    # retrying to get 200 response and reconnecting
+                    wait_tunnel_ready(
+                        require_min_connections=expect_connections)
                 else:
-                    check_tunnel_not_ready()
+                    check_tunnel_not_connected()
 
             sleep(self.default_reconnect_secs + 10)
-            wait_tunnel_ready()
-            send_requests(config.get_url(), 1)
+            wait_tunnel_ready(tunnel_url=config.get_url(),
+                              require_min_connections=self.default_ha_conns)

component-tests/test_service.py

@@ -0,0 +1,139 @@
+#!/usr/bin/env python
+import os
+import platform
+import subprocess
+from contextlib import contextmanager
+from pathlib import Path
+
+import pytest
+
+import test_logging
+from util import start_cloudflared, wait_tunnel_ready
+
+
+def select_platform(plat):
+    return pytest.mark.skipif(
+        platform.system() != plat, reason=f"Only runs on {plat}")
+
+
+def default_config_dir():
+    return os.path.join(Path.home(), ".cloudflared")
+
+
+def default_config_file():
+    return os.path.join(default_config_dir(), "config.yml")
+
+
+class TestServiceMode:
+    @select_platform("Darwin")
+    @pytest.mark.skipif(os.path.exists(default_config_file()), reason=f"There is already a config file in default path")
+    def test_launchd_service_log_to_file(self, tmp_path, component_tests_config):
+        log_file = tmp_path / test_logging.default_log_file
+        additional_config = {
+            # On Darwin cloudflared service defaults to run classic tunnel command
+            "hello-world": True,
+            "logfile": str(log_file),
+        }
+        config = component_tests_config(additional_config=additional_config, named_tunnel=False)
+
+        def assert_log_file():
+            test_logging.assert_log_in_file(log_file)
+            test_logging.assert_json_log(log_file)
+
+        self.launchd_service_scenario(config, assert_log_file)
+
+    @select_platform("Darwin")
+    @pytest.mark.skipif(os.path.exists(default_config_file()), reason=f"There is already a config file in default path")
+    def test_launchd_service_rotating_log(self, tmp_path, component_tests_config):
+        log_dir = tmp_path / "logs"
+        additional_config = {
+            # On Darwin cloudflared service defaults to run classic tunnel command
+            "hello-world": True,
+            "loglevel": "debug",
+            "log-directory": str(log_dir),
+        }
+        config = component_tests_config(additional_config=additional_config, named_tunnel=False)
+
+        def assert_rotating_log():
+            test_logging.assert_log_to_dir(config, log_dir)
+
+        self.launchd_service_scenario(config, assert_rotating_log)
+
+    def launchd_service_scenario(self, config, extra_assertions):
+        with self.run_service(Path(default_config_dir()), config):
+            self.launchctl_cmd("list")
+            self.launchctl_cmd("start")
+            wait_tunnel_ready(tunnel_url=config.get_url())
+            extra_assertions()
+            self.launchctl_cmd("stop")
+
+        os.remove(default_config_file())
+        self.launchctl_cmd("list", success=False)
+
+    @select_platform("Linux")
+    @pytest.mark.skipif(os.path.exists("/etc/cloudflared/config.yml"),
+                        reason=f"There is already a config file in default path")
+    def test_sysv_service_log_to_file(self, tmp_path, component_tests_config):
+        log_file = tmp_path / test_logging.default_log_file
+        additional_config = {
+            "logfile": str(log_file),
+        }
+        config = component_tests_config(additional_config=additional_config)
+
+        def assert_log_file():
+            test_logging.assert_log_in_file(log_file)
+            test_logging.assert_json_log(log_file)
+
+        self.sysv_service_scenario(config, tmp_path, assert_log_file)
+
+    @select_platform("Linux")
+    @pytest.mark.skipif(os.path.exists("/etc/cloudflared/config.yml"),
+                        reason=f"There is already a config file in default path")
+    def test_sysv_service_rotating_log(self, tmp_path, component_tests_config):
+        log_dir = tmp_path / "logs"
+        additional_config = {
+            "loglevel": "debug",
+            "log-directory": str(log_dir),
+        }
+        config = component_tests_config(additional_config=additional_config)
+
+        def assert_rotating_log():
+            # We need the folder to have executable permissions for the "stat" command in the assertions to work.
+            subprocess.check_call(['sudo', 'chmod', 'o+x', log_dir])
+            test_logging.assert_log_to_dir(config, log_dir)
+
+        self.sysv_service_scenario(config, tmp_path, assert_rotating_log)
+
+    def sysv_service_scenario(self, config, tmp_path, extra_assertions):
+        with self.run_service(tmp_path, config, root=True):
+            self.sysv_cmd("start")
+            self.sysv_cmd("status")
+            wait_tunnel_ready(tunnel_url=config.get_url())
+            extra_assertions()
+            self.sysv_cmd("stop")
+
+        # Service install copies config file to /etc/cloudflared/config.yml
+        subprocess.run(["sudo", "rm", "/etc/cloudflared/config.yml"])
+        self.sysv_cmd("status", success=False)
+
+    @contextmanager
+    def run_service(self, tmp_path, config, root=False):
+        try:
+            service = start_cloudflared(
+                tmp_path, config, cfd_args=["service", "install"], cfd_pre_args=[], capture_output=False, root=root)
+            yield service
+        finally:
+            start_cloudflared(
+                tmp_path, config, cfd_args=["service", "uninstall"], cfd_pre_args=[], capture_output=False, root=root)
+
+    def launchctl_cmd(self, action, success=True):
+        cmd = subprocess.run(
+            ["launchctl", action, "com.cloudflare.cloudflared"], check=success)
+        if not success:
+            assert cmd.returncode != 0, f"Expect {cmd.args} to fail, but it succeed"
+
+    def sysv_cmd(self, action, success=True):
+        cmd = subprocess.run(
+            ["sudo", "service", "cloudflared", action], check=success)
+        if not success:
+            assert cmd.returncode != 0, f"Expect {cmd.args} to fail, but it succeed"

component-tests/test_termination.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python
+from contextlib import contextmanager
+import platform
+import signal
+import threading
+import time
+
+import pytest
+import requests
+
+from util import start_cloudflared, wait_tunnel_ready, check_tunnel_not_connected
+
+
+def supported_signals():
+    if platform.system() == "Windows":
+        return [signal.SIGTERM]
+    return [signal.SIGTERM, signal.SIGINT]
+
+
+class TestTermination():
+    grace_period = 5
+    timeout = 10
+    extra_config = {
+        "grace-period": f"{grace_period}s",
+    }
+    sse_endpoint = "/sse?freq=1s"
+
+    @pytest.mark.parametrize("signal", supported_signals())
+    def test_graceful_shutdown(self, tmp_path, component_tests_config, signal):
+        config = component_tests_config(self.extra_config)
+        with start_cloudflared(
+                tmp_path, config, new_process=True, capture_output=False) as cloudflared:
+            wait_tunnel_ready(tunnel_url=config.get_url())
+
+            connected = threading.Condition()
+            in_flight_req = threading.Thread(
+                target=self.stream_request, args=(config, connected, False, ))
+            in_flight_req.start()
+
+            with connected:
+                connected.wait(self.timeout)
+            # Send signal after the SSE connection is established
+            self.terminate_by_signal(cloudflared, signal)
+            self.wait_eyeball_thread(
+                in_flight_req, self.grace_period + self.timeout)
+
+    # test cloudflared terminates before grace period expires when all eyeball
+    # connections are drained
+    @pytest.mark.parametrize("signal", supported_signals())
+    def test_shutdown_once_no_connection(self, tmp_path, component_tests_config, signal):
+        config = component_tests_config(self.extra_config)
+        with start_cloudflared(
+                tmp_path, config, new_process=True, capture_output=False) as cloudflared:
+            wait_tunnel_ready(tunnel_url=config.get_url())
+
+            connected = threading.Condition()
+            in_flight_req = threading.Thread(
+                target=self.stream_request, args=(config, connected, True, ))
+            in_flight_req.start()
+
+            with connected:
+                connected.wait(self.timeout)
+            with self.within_grace_period():
+                # Send signal after the SSE connection is established
+                self.terminate_by_signal(cloudflared, signal)
+                self.wait_eyeball_thread(in_flight_req, self.grace_period)
+
+    @pytest.mark.parametrize("signal", supported_signals())
+    def test_no_connection_shutdown(self, tmp_path, component_tests_config, signal):
+        config = component_tests_config(self.extra_config)
+        with start_cloudflared(
+                tmp_path, config, new_process=True, capture_output=False) as cloudflared:
+            wait_tunnel_ready(tunnel_url=config.get_url())
+            with self.within_grace_period():
+                self.terminate_by_signal(cloudflared, signal)
+
+    def terminate_by_signal(self, cloudflared, sig):
+        cloudflared.send_signal(sig)
+        check_tunnel_not_connected()
+        cloudflared.wait()
+
+    def wait_eyeball_thread(self, thread, timeout):
+        thread.join(timeout)
+        assert thread.is_alive() == False, "eyeball thread is still alive"
+
+    # Using this context asserts logic within the context is executed within grace period
+    @contextmanager
+    def within_grace_period(self):
+        try:
+            start = time.time()
+            yield
+        finally:
+            duration = time.time() - start
+            assert duration < self.grace_period
+
+    def stream_request(self, config, connected, early_terminate):
+        expected_terminate_message = "502 Bad Gateway"
+        url = config.get_url() + self.sse_endpoint
+
+        with requests.get(url, timeout=5, stream=True) as resp:
+            with connected:
+                connected.notifyAll()
+            lines = 0
+            for line in resp.iter_lines():
+                if expected_terminate_message.encode() == line:
+                    break
+                lines += 1
+                if early_terminate and lines == 2:
+                    return
+            # /sse returns count followed by 2 new lines
+            assert lines >= (self.grace_period * 2)

component-tests/util.py

@@ -12,26 +12,34 @@ from constants import METRICS_PORT, MAX_RETRIES, BACKOFF_SECS
 LOGGER = logging.getLogger(__name__)
 
 
-def write_config(path, config):
-    config_path = path / "config.yaml"
+def write_config(directory, config):
+    config_path = directory / "config.yml"
     with open(config_path, 'w') as outfile:
         yaml.dump(config, outfile)
     return config_path
 
 
-def start_cloudflared(path, config, cfd_args=["run"], cfd_pre_args=["tunnel"], new_process=False, allow_input=False, capture_output=True):
-    config_path = write_config(path, config.full_config)
-    cmd = [config.cloudflared_binary]
-    cmd += cfd_pre_args
-    cmd += ["--config", config_path]
-    cmd += cfd_args
-    LOGGER.info(f"Run cmd {cmd} with config {config}")
+def start_cloudflared(directory, config, cfd_args=["run"], cfd_pre_args=["tunnel"], new_process=False, allow_input=False, capture_output=True, root=False):
+    config_path = write_config(directory, config.full_config)
+    cmd = cloudflared_cmd(config, config_path, cfd_args, cfd_pre_args, root)
     if new_process:
         return run_cloudflared_background(cmd, allow_input, capture_output)
     # By setting check=True, it will raise an exception if the process exits with non-zero exit code
     return subprocess.run(cmd, check=True, capture_output=capture_output)
 
 
+def cloudflared_cmd(config, config_path, cfd_args, cfd_pre_args, root):
+    cmd = []
+    if root:
+        cmd += ["sudo"]
+    cmd += [config.cloudflared_binary]
+    cmd += cfd_pre_args
+    cmd += ["--config", config_path]
+    cmd += cfd_args
+    LOGGER.info(f"Run cmd {cmd} with config {config}")
+    return cmd
+
+
 @contextmanager
 def run_cloudflared_background(cmd, allow_input, capture_output):
     output = subprocess.PIPE if capture_output else subprocess.DEVNULL
@@ -41,28 +49,36 @@ def run_cloudflared_background(cmd, allow_input, capture_output):
         yield cfd
     finally:
         cfd.terminate()
+        if capture_output:
+            LOGGER.info(
+                f"cloudflared log: {cfd.stderr.read()}")
 
 
 @retry(stop_max_attempt_number=MAX_RETRIES, wait_fixed=BACKOFF_SECS * 1000)
-def wait_tunnel_ready(expect_connections=4):
-    url = f'http://localhost:{METRICS_PORT}/ready'
+def wait_tunnel_ready(tunnel_url=None, require_min_connections=1):
+    metrics_url = f'http://localhost:{METRICS_PORT}/ready'
 
     with requests.Session() as s:
-        resp = send_request(s, url, True)
+        resp = send_request(s, metrics_url, True)
         assert resp.json()[
-            "readyConnections"] == expect_connections, f"Ready endpoint returned {resp.json()} but we expect {expect_connections} ready connections"
+            "readyConnections"] >= require_min_connections, f"Ready endpoint returned {resp.json()} but we expect at least {require_min_connections} connections"
+        if tunnel_url is not None:
+            send_request(s, tunnel_url, True)
 
 
-@retry(stop_max_attempt_number=MAX_RETRIES, wait_fixed=BACKOFF_SECS * 1000)
-def check_tunnel_not_ready():
+@retry(stop_max_attempt_number=MAX_RETRIES * BACKOFF_SECS, wait_fixed=1000)
+def check_tunnel_not_connected():
     url = f'http://localhost:{METRICS_PORT}/ready'
 
-    resp = requests.get(url, timeout=1)
-    assert resp.status_code == 503, f"Expect {url} returns 503, got {resp.status_code}"
+    try:
+        resp = requests.get(url, timeout=1)
+        assert resp.status_code == 503, f"Expect {url} returns 503, got {resp.status_code}"
+    # cloudflared might already terminate
+    except requests.exceptions.ConnectionError as e:
+        LOGGER.warning(f"Failed to connect to {url}, error: {e}")
+
 
 # In some cases we don't need to check response status, such as when sending batch requests to generate logs
-
-
 def send_requests(url, count, require_ok=True):
     errors = 0
     with requests.Session() as s:

config/configuration.go

@@ -9,11 +9,11 @@ import (
 	"runtime"
 	"time"
 
-	"github.com/mitchellh/go-homedir"
+	homedir "github.com/mitchellh/go-homedir"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
 	"github.com/urfave/cli/v2"
-	"gopkg.in/yaml.v2"
+	yaml "gopkg.in/yaml.v2"
 
 	"github.com/cloudflare/cloudflared/validation"
 )

config/configuration_test.go

@@ -5,7 +5,7 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/assert"
-	"gopkg.in/yaml.v2"
+	yaml "gopkg.in/yaml.v2"
 )
 
 func TestConfigFileSettings(t *testing.T) {

config/manager.go

@@ -4,11 +4,11 @@ import (
 	"io"
 	"os"
 
-	"github.com/cloudflare/cloudflared/watcher"
-
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
-	"gopkg.in/yaml.v2"
+	yaml "gopkg.in/yaml.v2"
+
+	"github.com/cloudflare/cloudflared/watcher"
 )
 
 // Notifier sends out config updates

config/manager_test.go

@@ -4,10 +4,10 @@ import (
 	"os"
 	"testing"
 
-	"github.com/cloudflare/cloudflared/watcher"
-
 	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/assert"
+
+	"github.com/cloudflare/cloudflared/watcher"
 )
 
 type mockNotifier struct {

connection/connection.go

@@ -8,8 +8,9 @@ import (
 	"strings"
 	"time"
 
-	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 	"github.com/google/uuid"
+
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
 const LogFieldConnIndex = "connIndex"

connection/h2mux.go

@@ -7,13 +7,13 @@ import (
 	"net/http"
 	"time"
 
-	"github.com/cloudflare/cloudflared/h2mux"
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
-	"github.com/cloudflare/cloudflared/websocket"
-
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
 	"golang.org/x/sync/errgroup"
+
+	"github.com/cloudflare/cloudflared/h2mux"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/cloudflare/cloudflared/websocket"
 )
 
 const (
@@ -234,7 +234,7 @@ func (h *h2muxConnection) newRequest(stream *h2mux.MuxedStream) (*http.Request,
 	if err != nil {
 		return nil, errors.Wrap(err, "Unexpected error from http.NewRequest")
 	}
-	err = h2mux.H2RequestHeadersToH1Request(stream.Headers, req)
+	err = H2RequestHeadersToH1Request(stream.Headers, req)
 	if err != nil {
 		return nil, errors.Wrap(err, "invalid request received")
 	}
@@ -246,15 +246,15 @@ type h2muxRespWriter struct {
 }
 
 func (rp *h2muxRespWriter) WriteRespHeaders(status int, header http.Header) error {
-	headers := h2mux.H1ResponseToH2ResponseHeaders(status, header)
-	headers = append(headers, h2mux.Header{Name: ResponseMetaHeaderField, Value: responseMetaHeaderOrigin})
+	headers := H1ResponseToH2ResponseHeaders(status, header)
+	headers = append(headers, h2mux.Header{Name: ResponseMetaHeader, Value: responseMetaHeaderOrigin})
 	return rp.WriteHeaders(headers)
 }
 
 func (rp *h2muxRespWriter) WriteErrorResponse() {
 	_ = rp.WriteHeaders([]h2mux.Header{
 		{Name: ":status", Value: "502"},
-		{Name: ResponseMetaHeaderField, Value: responseMetaHeaderCfd},
+		{Name: ResponseMetaHeader, Value: responseMetaHeaderCfd},
 	})
 	_, _ = rp.Write([]byte("502 Bad Gateway"))
 }

connection/h2mux_test.go

@@ -38,7 +38,7 @@ func newH2MuxConnection(t require.TestingT) (*h2muxConnection, *h2mux.Muxer) {
 				// we only expect RPC traffic in client->edge direction, provide minimal support for mocking
 				require.True(t, stream.IsRPCStream())
 				return stream.WriteHeaders([]h2mux.Header{
-					{Name:  ":status", Value: "200"},
+					{Name: ":status", Value: "200"},
 				})
 			}),
 		}
@@ -115,9 +115,9 @@ func TestServeStreamHTTP(t *testing.T) {
 		require.True(t, hasHeader(stream, ":status", strconv.Itoa(test.expectedStatus)))
 
 		if test.isProxyError {
-			assert.True(t, hasHeader(stream, ResponseMetaHeaderField, responseMetaHeaderCfd))
+			assert.True(t, hasHeader(stream, ResponseMetaHeader, responseMetaHeaderCfd))
 		} else {
-			assert.True(t, hasHeader(stream, ResponseMetaHeaderField, responseMetaHeaderOrigin))
+			assert.True(t, hasHeader(stream, ResponseMetaHeader, responseMetaHeaderOrigin))
 			body := make([]byte, len(test.expectedBody))
 			_, err = stream.Read(body)
 			require.NoError(t, err)
@@ -164,7 +164,7 @@ func TestServeStreamWS(t *testing.T) {
 	require.NoError(t, err)
 
 	require.True(t, hasHeader(stream, ":status", strconv.Itoa(http.StatusSwitchingProtocols)))
-	assert.True(t, hasHeader(stream, ResponseMetaHeaderField, responseMetaHeaderOrigin))
+	assert.True(t, hasHeader(stream, ResponseMetaHeader, responseMetaHeaderOrigin))
 
 	data := []byte("test websocket")
 	err = wsutil.WriteClientText(writePipe, data)
@@ -268,7 +268,7 @@ func benchmarkServeStreamHTTPSimple(b *testing.B, test testRequest) {
 		b.StopTimer()
 
 		require.NoError(b, openstreamErr)
-		assert.True(b, hasHeader(stream, ResponseMetaHeaderField, responseMetaHeaderOrigin))
+		assert.True(b, hasHeader(stream, ResponseMetaHeader, responseMetaHeaderOrigin))
 		require.True(b, hasHeader(stream, ":status", strconv.Itoa(http.StatusOK)))
 		require.NoError(b, readBodyErr)
 		require.Equal(b, test.expectedBody, body)

connection/header.go

@@ -1,21 +1,33 @@
 package connection
 
 import (
+	"encoding/base64"
 	"fmt"
 	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+
+	"github.com/pkg/errors"
 
 	"github.com/cloudflare/cloudflared/h2mux"
 )
 
-const (
-	ResponseMetaHeaderField = "cf-cloudflared-response-meta"
+var (
+	// h2mux-style special headers
+	RequestUserHeaders  = "cf-cloudflared-request-headers"
+	ResponseUserHeaders = "cf-cloudflared-response-headers"
+	ResponseMetaHeader  = "cf-cloudflared-response-meta"
+
+	// h2mux-style special headers
+	CanonicalResponseUserHeaders = http.CanonicalHeaderKey(ResponseUserHeaders)
+	CanonicalResponseMetaHeader  = http.CanonicalHeaderKey(ResponseMetaHeader)
 )
 
 var (
-	canonicalResponseUserHeadersField = http.CanonicalHeaderKey(h2mux.ResponseUserHeadersField)
-	canonicalResponseMetaHeaderField  = http.CanonicalHeaderKey(ResponseMetaHeaderField)
-	responseMetaHeaderCfd             = mustInitRespMetaHeader("cloudflared")
-	responseMetaHeaderOrigin          = mustInitRespMetaHeader("origin")
+	// pre-generate possible values for res
+	responseMetaHeaderCfd    = mustInitRespMetaHeader("cloudflared")
+	responseMetaHeaderOrigin = mustInitRespMetaHeader("origin")
 )
 
 type responseMetaHeader struct {
@@ -29,3 +41,204 @@ func mustInitRespMetaHeader(src string) string {
 	}
 	return string(header)
 }
+
+var headerEncoding = base64.RawStdEncoding
+
+// note: all h2mux headers should be lower-case (http/2 style)
+const ()
+
+// H2RequestHeadersToH1Request converts the HTTP/2 headers coming from origintunneld
+// to an HTTP/1 Request object destined for the local origin web service.
+// This operation includes conversion of the pseudo-headers into their closest
+// HTTP/1 equivalents. See https://tools.ietf.org/html/rfc7540#section-8.1.2.3
+func H2RequestHeadersToH1Request(h2 []h2mux.Header, h1 *http.Request) error {
+	for _, header := range h2 {
+		name := strings.ToLower(header.Name)
+		if !IsControlHeader(name) {
+			continue
+		}
+
+		switch name {
+		case ":method":
+			h1.Method = header.Value
+		case ":scheme":
+			// noop - use the preexisting scheme from h1.URL
+		case ":authority":
+			// Otherwise the host header will be based on the origin URL
+			h1.Host = header.Value
+		case ":path":
+			// We don't want to be an "opinionated" proxy, so ideally we would use :path as-is.
+			// However, this HTTP/1 Request object belongs to the Go standard library,
+			// whose URL package makes some opinionated decisions about the encoding of
+			// URL characters: see the docs of https://godoc.org/net/url#URL,
+			// in particular the EscapedPath method https://godoc.org/net/url#URL.EscapedPath,
+			// which is always used when computing url.URL.String(), whether we'd like it or not.
+			//
+			// Well, not *always*. We could circumvent this by using url.URL.Opaque. But
+			// that would present unusual difficulties when using an HTTP proxy: url.URL.Opaque
+			// is treated differently when HTTP_PROXY is set!
+			// See https://github.com/golang/go/issues/5684#issuecomment-66080888
+			//
+			// This means we are subject to the behavior of net/url's function `shouldEscape`
+			// (as invoked with mode=encodePath): https://github.com/golang/go/blob/go1.12.7/src/net/url/url.go#L101
+
+			if header.Value == "*" {
+				h1.URL.Path = "*"
+				continue
+			}
+			// Due to the behavior of validation.ValidateUrl, h1.URL may
+			// already have a partial value, with or without a trailing slash.
+			base := h1.URL.String()
+			base = strings.TrimRight(base, "/")
+			// But we know :path begins with '/', because we handled '*' above - see RFC7540
+			requestURL, err := url.Parse(base + header.Value)
+			if err != nil {
+				return errors.Wrap(err, fmt.Sprintf("invalid path '%v'", header.Value))
+			}
+			h1.URL = requestURL
+		case "content-length":
+			contentLength, err := strconv.ParseInt(header.Value, 10, 64)
+			if err != nil {
+				return fmt.Errorf("unparseable content length")
+			}
+			h1.ContentLength = contentLength
+		case RequestUserHeaders:
+			// Do not forward the serialized headers to the origin -- deserialize them, and ditch the serialized version
+			// Find and parse user headers serialized into a single one
+			userHeaders, err := DeserializeHeaders(header.Value)
+			if err != nil {
+				return errors.Wrap(err, "Unable to parse user headers")
+			}
+			for _, userHeader := range userHeaders {
+				h1.Header.Add(userHeader.Name, userHeader.Value)
+			}
+		default:
+			// All other control headers shall just be proxied transparently
+			h1.Header.Add(header.Name, header.Value)
+		}
+	}
+
+	return nil
+}
+
+func IsControlHeader(headerName string) bool {
+	return headerName == "content-length" ||
+		headerName == "connection" || headerName == "upgrade" || // Websocket headers
+		strings.HasPrefix(headerName, ":") ||
+		strings.HasPrefix(headerName, "cf-")
+}
+
+// isWebsocketClientHeader returns true if the header name is required by the client to upgrade properly
+func IsWebsocketClientHeader(headerName string) bool {
+	return headerName == "sec-websocket-accept" ||
+		headerName == "connection" ||
+		headerName == "upgrade"
+}
+
+func H1ResponseToH2ResponseHeaders(status int, h1 http.Header) (h2 []h2mux.Header) {
+	h2 = []h2mux.Header{
+		{Name: ":status", Value: strconv.Itoa(status)},
+	}
+	userHeaders := make(http.Header, len(h1))
+	for header, values := range h1 {
+		h2name := strings.ToLower(header)
+		if h2name == "content-length" {
+			// This header has meaning in HTTP/2 and will be used by the edge,
+			// so it should be sent as an HTTP/2 response header.
+
+			// Since these are http2 headers, they're required to be lowercase
+			h2 = append(h2, h2mux.Header{Name: "content-length", Value: values[0]})
+		} else if !IsControlHeader(h2name) || IsWebsocketClientHeader(h2name) {
+			// User headers, on the other hand, must all be serialized so that
+			// HTTP/2 header validation won't be applied to HTTP/1 header values
+			userHeaders[header] = values
+		}
+	}
+
+	// Perform user header serialization and set them in the single header
+	h2 = append(h2, h2mux.Header{Name: ResponseUserHeaders, Value: SerializeHeaders(userHeaders)})
+	return h2
+}
+
+// Serialize HTTP1.x headers by base64-encoding each header name and value,
+// and then joining them in the format of [key:value;]
+func SerializeHeaders(h1Headers http.Header) string {
+	// compute size of the fully serialized value and largest temp buffer we will need
+	serializedLen := 0
+	maxTempLen := 0
+	for headerName, headerValues := range h1Headers {
+		for _, headerValue := range headerValues {
+			nameLen := headerEncoding.EncodedLen(len(headerName))
+			valueLen := headerEncoding.EncodedLen(len(headerValue))
+			const delims = 2
+			serializedLen += delims + nameLen + valueLen
+			if nameLen > maxTempLen {
+				maxTempLen = nameLen
+			}
+			if valueLen > maxTempLen {
+				maxTempLen = valueLen
+			}
+		}
+	}
+	var buf strings.Builder
+	buf.Grow(serializedLen)
+
+	temp := make([]byte, maxTempLen)
+	writeB64 := func(s string) {
+		n := headerEncoding.EncodedLen(len(s))
+		if n > len(temp) {
+			temp = make([]byte, n)
+		}
+		headerEncoding.Encode(temp[:n], []byte(s))
+		buf.Write(temp[:n])
+	}
+
+	for headerName, headerValues := range h1Headers {
+		for _, headerValue := range headerValues {
+			if buf.Len() > 0 {
+				buf.WriteByte(';')
+			}
+			writeB64(headerName)
+			buf.WriteByte(':')
+			writeB64(headerValue)
+		}
+	}
+
+	return buf.String()
+}
+
+// Deserialize headers serialized by `SerializeHeader`
+func DeserializeHeaders(serializedHeaders string) ([]h2mux.Header, error) {
+	const unableToDeserializeErr = "Unable to deserialize headers"
+
+	var deserialized []h2mux.Header
+	for _, serializedPair := range strings.Split(serializedHeaders, ";") {
+		if len(serializedPair) == 0 {
+			continue
+		}
+
+		serializedHeaderParts := strings.Split(serializedPair, ":")
+		if len(serializedHeaderParts) != 2 {
+			return nil, errors.New(unableToDeserializeErr)
+		}
+
+		serializedName := serializedHeaderParts[0]
+		serializedValue := serializedHeaderParts[1]
+		deserializedName := make([]byte, headerEncoding.DecodedLen(len(serializedName)))
+		deserializedValue := make([]byte, headerEncoding.DecodedLen(len(serializedValue)))
+
+		if _, err := headerEncoding.Decode(deserializedName, []byte(serializedName)); err != nil {
+			return nil, errors.Wrap(err, unableToDeserializeErr)
+		}
+		if _, err := headerEncoding.Decode(deserializedValue, []byte(serializedValue)); err != nil {
+			return nil, errors.Wrap(err, unableToDeserializeErr)
+		}
+
+		deserialized = append(deserialized, h2mux.Header{
+			Name:  string(deserializedName),
+			Value: string(deserializedValue),
+		})
+	}
+
+	return deserialized, nil
+}

connection/header_test.go

@@ -1,4 +1,4 @@
-package h2mux
+package connection
 
 import (
 	"fmt"
@@ -14,9 +14,11 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+
+	"github.com/cloudflare/cloudflared/h2mux"
 )
 
-type ByName []Header
+type ByName []h2mux.Header
 
 func (a ByName) Len() int      { return len(a) }
 func (a ByName) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
@@ -37,16 +39,16 @@ func TestH2RequestHeadersToH1Request_RegularHeaders(t *testing.T) {
 		"Mock header 2": {"Mock value 2"},
 	}
 
-	headersConversionErr := H2RequestHeadersToH1Request(createSerializedHeaders(RequestUserHeadersField, mockHeaders), request)
+	headersConversionErr := H2RequestHeadersToH1Request(createSerializedHeaders(RequestUserHeaders, mockHeaders), request)
 
 	assert.True(t, reflect.DeepEqual(mockHeaders, request.Header))
 	assert.NoError(t, headersConversionErr)
 }
 
-func createSerializedHeaders(headersField string, headers http.Header) []Header {
-	return []Header{{
-		headersField,
-		SerializeHeaders(headers),
+func createSerializedHeaders(headersField string, headers http.Header) []h2mux.Header {
+	return []h2mux.Header{{
+		Name:  headersField,
+		Value: SerializeHeaders(headers),
 	}}
 }
 
@@ -54,15 +56,16 @@ func TestH2RequestHeadersToH1Request_NoHeaders(t *testing.T) {
 	request, err := http.NewRequest(http.MethodGet, "http://example.com", nil)
 	assert.NoError(t, err)
 
+	emptyHeaders := make(http.Header)
 	headersConversionErr := H2RequestHeadersToH1Request(
-		[]Header{{
-			RequestUserHeadersField,
-			SerializeHeaders(http.Header{}),
+		[]h2mux.Header{{
+			Name:  RequestUserHeaders,
+			Value: SerializeHeaders(emptyHeaders),
 		}},
 		request,
 	)
 
-	assert.True(t, reflect.DeepEqual(http.Header{}, request.Header))
+	assert.True(t, reflect.DeepEqual(emptyHeaders, request.Header))
 	assert.NoError(t, headersConversionErr)
 }
 
@@ -70,9 +73,9 @@ func TestH2RequestHeadersToH1Request_InvalidHostPath(t *testing.T) {
 	request, err := http.NewRequest(http.MethodGet, "http://example.com", nil)
 	assert.NoError(t, err)
 
-	mockRequestHeaders := []Header{
+	mockRequestHeaders := []h2mux.Header{
 		{Name: ":path", Value: "//bad_path/"},
-		{Name: RequestUserHeadersField, Value: SerializeHeaders(http.Header{"Mock header": {"Mock value"}})},
+		{Name: RequestUserHeaders, Value: SerializeHeaders(http.Header{"Mock header": {"Mock value"}})},
 	}
 
 	headersConversionErr := H2RequestHeadersToH1Request(mockRequestHeaders, request)
@@ -90,9 +93,9 @@ func TestH2RequestHeadersToH1Request_HostPathWithQuery(t *testing.T) {
 	request, err := http.NewRequest(http.MethodGet, "http://example.com/", nil)
 	assert.NoError(t, err)
 
-	mockRequestHeaders := []Header{
+	mockRequestHeaders := []h2mux.Header{
 		{Name: ":path", Value: "/?query=mock%20value"},
-		{Name: RequestUserHeadersField, Value: SerializeHeaders(http.Header{"Mock header": {"Mock value"}})},
+		{Name: RequestUserHeaders, Value: SerializeHeaders(http.Header{"Mock header": {"Mock value"}})},
 	}
 
 	headersConversionErr := H2RequestHeadersToH1Request(mockRequestHeaders, request)
@@ -110,9 +113,9 @@ func TestH2RequestHeadersToH1Request_HostPathWithURLEncoding(t *testing.T) {
 	request, err := http.NewRequest(http.MethodGet, "http://example.com/", nil)
 	assert.NoError(t, err)
 
-	mockRequestHeaders := []Header{
+	mockRequestHeaders := []h2mux.Header{
 		{Name: ":path", Value: "/mock%20path"},
-		{Name: RequestUserHeadersField, Value: SerializeHeaders(http.Header{"Mock header": {"Mock value"}})},
+		{Name: RequestUserHeaders, Value: SerializeHeaders(http.Header{"Mock header": {"Mock value"}})},
 	}
 
 	headersConversionErr := H2RequestHeadersToH1Request(mockRequestHeaders, request)
@@ -267,9 +270,9 @@ func TestH2RequestHeadersToH1Request_WeirdURLs(t *testing.T) {
 		request, err := http.NewRequest(http.MethodGet, requestURL, nil)
 		assert.NoError(t, err)
 
-		mockRequestHeaders := []Header{
+		mockRequestHeaders := []h2mux.Header{
 			{Name: ":path", Value: testCase.path},
-			{Name: RequestUserHeadersField, Value: SerializeHeaders(http.Header{"Mock header": {"Mock value"}})},
+			{Name: RequestUserHeaders, Value: SerializeHeaders(http.Header{"Mock header": {"Mock value"}})},
 		}
 
 		headersConversionErr := H2RequestHeadersToH1Request(mockRequestHeaders, request)
@@ -337,12 +340,12 @@ func TestH2RequestHeadersToH1Request_QuickCheck(t *testing.T) {
 				const expectedMethod = "POST"
 				const expectedHostname = "request.hostname.example.com"
 
-				h2 := []Header{
+				h2 := []h2mux.Header{
 					{Name: ":method", Value: expectedMethod},
 					{Name: ":scheme", Value: testScheme},
 					{Name: ":authority", Value: expectedHostname},
 					{Name: ":path", Value: testPath},
-					{Name: RequestUserHeadersField, Value: ""},
+					{Name: RequestUserHeaders, Value: ""},
 				}
 				h1, err := http.NewRequest("GET", testOrigin.url, nil)
 				require.NoError(t, err)
@@ -424,10 +427,10 @@ func randomHTTP2Path(t *testing.T, rand *rand.Rand) string {
 	return result
 }
 
-func stdlibHeaderToH2muxHeader(headers http.Header) (h2muxHeaders []Header) {
+func stdlibHeaderToH2muxHeader(headers http.Header) (h2muxHeaders []h2mux.Header) {
 	for name, values := range headers {
 		for _, value := range values {
-			h2muxHeaders = append(h2muxHeaders, Header{name, value})
+			h2muxHeaders = append(h2muxHeaders, h2mux.Header{Name: name, Value: value})
 		}
 	}
 
@@ -515,14 +518,14 @@ func TestParseHeaders(t *testing.T) {
 		"Mock-Header-Three": {"3"},
 	}
 
-	mockHeaders := []Header{
+	mockHeaders := []h2mux.Header{
 		{Name: "One", Value: "1"}, // will be dropped
 		{Name: "Cf-Two", Value: "cf-value-1"},
 		{Name: "Cf-Two", Value: "cf-value-2"},
-		{Name: RequestUserHeadersField, Value: SerializeHeaders(mockUserHeadersToSerialize)},
+		{Name: RequestUserHeaders, Value: SerializeHeaders(mockUserHeadersToSerialize)},
 	}
 
-	expectedHeaders := []Header{
+	expectedHeaders := []h2mux.Header{
 		{Name: "Cf-Two", Value: "cf-value-1"},
 		{Name: "Cf-Two", Value: "cf-value-2"},
 		{Name: "Mock-Header-One", Value: "1"},
@@ -583,7 +586,7 @@ func TestH1ResponseToH2ResponseHeaders(t *testing.T) {
 
 	serializedHeadersIndex := -1
 	for i, header := range headers {
-		if header.Name == ResponseUserHeadersField {
+		if header.Name == ResponseUserHeaders {
 			serializedHeadersIndex = i
 			break
 		}
@@ -593,7 +596,7 @@ func TestH1ResponseToH2ResponseHeaders(t *testing.T) {
 		headers[:serializedHeadersIndex],
 		headers[serializedHeadersIndex+1:]...,
 	)
-	expectedControlHeaders := []Header{
+	expectedControlHeaders := []h2mux.Header{
 		{Name: ":status", Value: "200"},
 		{Name: "content-length", Value: "123"},
 	}
@@ -601,7 +604,7 @@ func TestH1ResponseToH2ResponseHeaders(t *testing.T) {
 	assert.ElementsMatch(t, expectedControlHeaders, actualControlHeaders)
 
 	actualUserHeaders, err := DeserializeHeaders(headers[serializedHeadersIndex].Value)
-	expectedUserHeaders := []Header{
+	expectedUserHeaders := []h2mux.Header{
 		{Name: "User-header-one", Value: ""},
 		{Name: "User-header-two", Value: "1"},
 		{Name: "User-header-two", Value: "2"},
@@ -630,7 +633,7 @@ func TestHeaderSize(t *testing.T) {
 	}
 
 	for _, header := range serializedHeaders {
-		if header.Name != ResponseUserHeadersField {
+		if header.Name != ResponseUserHeaders {
 			continue
 		}
 

connection/http2.go

@@ -10,18 +10,18 @@ import (
 	"strings"
 	"sync"
 
-	"github.com/cloudflare/cloudflared/h2mux"
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
-
 	"github.com/rs/zerolog"
 	"golang.org/x/net/http2"
+
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
+// note: these constants are exported so we can reuse them in the edge-side code
 const (
-	internalUpgradeHeader = "Cf-Cloudflared-Proxy-Connection-Upgrade"
-	tcpStreamHeader       = "Cf-Cloudflared-Proxy-Src"
-	websocketUpgrade      = "websocket"
-	controlStreamUpgrade  = "control-stream"
+	InternalUpgradeHeader     = "Cf-Cloudflared-Proxy-Connection-Upgrade"
+	InternalTCPProxySrcHeader = "Cf-Cloudflared-Proxy-Src"
+	WebsocketUpgrade          = "websocket"
+	ControlStreamUpgrade      = "control-stream"
 )
 
 var errEdgeConnectionClosed = fmt.Errorf("connection with edge closed")
@@ -178,25 +178,23 @@ func newHTTP2RespWriter(r *http.Request, w http.ResponseWriter, connType Type) (
 func (rp *http2RespWriter) WriteRespHeaders(status int, header http.Header) error {
 	dest := rp.w.Header()
 	userHeaders := make(http.Header, len(header))
-	for header, values := range header {
+	for name, values := range header {
 		// Since these are http2 headers, they're required to be lowercase
-		h2name := strings.ToLower(header)
-		for _, v := range values {
-			if h2name == "content-length" {
-				// This header has meaning in HTTP/2 and will be used by the edge,
-				// so it should be sent as an HTTP/2 response header.
-				dest.Add(h2name, v)
-				// Since these are http2 headers, they're required to be lowercase
-			} else if !h2mux.IsControlHeader(h2name) || h2mux.IsWebsocketClientHeader(h2name) {
-				// User headers, on the other hand, must all be serialized so that
-				// HTTP/2 header validation won't be applied to HTTP/1 header values
-				userHeaders.Add(h2name, v)
-			}
+		h2name := strings.ToLower(name)
+		if h2name == "content-length" {
+			// This header has meaning in HTTP/2 and will be used by the edge,
+			// so it should be sent as an HTTP/2 response header.
+			dest[name] = values
+			// Since these are http2 headers, they're required to be lowercase
+		} else if !IsControlHeader(h2name) || IsWebsocketClientHeader(h2name) {
+			// User headers, on the other hand, must all be serialized so that
+			// HTTP/2 header validation won't be applied to HTTP/1 header values
+			userHeaders[name] = values
 		}
 	}
 
 	// Perform user header serialization and set them in the single header
-	dest.Set(canonicalResponseUserHeadersField, h2mux.SerializeHeaders(userHeaders))
+	dest.Set(CanonicalResponseUserHeaders, SerializeHeaders(userHeaders))
 	rp.setResponseMetaHeader(responseMetaHeaderOrigin)
 	// HTTP2 removes support for 101 Switching Protocols https://tools.ietf.org/html/rfc7540#section-8.1.1
 	if status == http.StatusSwitchingProtocols {
@@ -218,7 +216,7 @@ func (rp *http2RespWriter) WriteErrorResponse() {
 }
 
 func (rp *http2RespWriter) setResponseMetaHeader(value string) {
-	rp.w.Header().Set(canonicalResponseMetaHeaderField, value)
+	rp.w.Header().Set(CanonicalResponseMetaHeader, value)
 }
 
 func (rp *http2RespWriter) Read(p []byte) (n int, err error) {
@@ -258,18 +256,18 @@ func determineHTTP2Type(r *http.Request) Type {
 }
 
 func isControlStreamUpgrade(r *http.Request) bool {
-	return r.Header.Get(internalUpgradeHeader) == controlStreamUpgrade
+	return r.Header.Get(InternalUpgradeHeader) == ControlStreamUpgrade
 }
 
 func isWebsocketUpgrade(r *http.Request) bool {
-	return r.Header.Get(internalUpgradeHeader) == websocketUpgrade
+	return r.Header.Get(InternalUpgradeHeader) == WebsocketUpgrade
 }
 
 // IsTCPStream discerns if the connection request needs a tcp stream proxy.
 func IsTCPStream(r *http.Request) bool {
-	return r.Header.Get(tcpStreamHeader) != ""
+	return r.Header.Get(InternalTCPProxySrcHeader) != ""
 }
 
 func stripWebsocketUpgradeHeader(r *http.Request) {
-	r.Header.Del(internalUpgradeHeader)
+	r.Header.Del(InternalUpgradeHeader)
 }

connection/http2_test.go

@@ -12,15 +12,14 @@ import (
 	"testing"
 	"time"
 
+	"github.com/gobwas/ws/wsutil"
+	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/net/http2"
 
 	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
-
-	"github.com/gobwas/ws/wsutil"
-	"github.com/rs/zerolog"
-	"github.com/stretchr/testify/require"
-	"golang.org/x/net/http2"
 )
 
 var (
@@ -104,9 +103,9 @@ func TestServeHTTP(t *testing.T) {
 			require.Equal(t, test.expectedBody, respBody)
 		}
 		if test.isProxyError {
-			require.Equal(t, responseMetaHeaderCfd, resp.Header.Get(ResponseMetaHeaderField))
+			require.Equal(t, responseMetaHeaderCfd, resp.Header.Get(ResponseMetaHeader))
 		} else {
-			require.Equal(t, responseMetaHeaderOrigin, resp.Header.Get(ResponseMetaHeaderField))
+			require.Equal(t, responseMetaHeaderOrigin, resp.Header.Get(ResponseMetaHeader))
 		}
 	}
 	cancel()
@@ -192,7 +191,7 @@ func TestServeWS(t *testing.T) {
 
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://localhost:8080/ws", readPipe)
 	require.NoError(t, err)
-	req.Header.Set(internalUpgradeHeader, websocketUpgrade)
+	req.Header.Set(InternalUpgradeHeader, WebsocketUpgrade)
 
 	wg.Add(1)
 	go func() {
@@ -212,7 +211,7 @@ func TestServeWS(t *testing.T) {
 	resp := respWriter.Result()
 	// http2RespWriter should rewrite status 101 to 200
 	require.Equal(t, http.StatusOK, resp.StatusCode)
-	require.Equal(t, responseMetaHeaderOrigin, resp.Header.Get(ResponseMetaHeaderField))
+	require.Equal(t, responseMetaHeaderOrigin, resp.Header.Get(ResponseMetaHeader))
 
 	wg.Wait()
 }
@@ -236,7 +235,7 @@ func TestServeControlStream(t *testing.T) {
 
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://localhost:8080/", nil)
 	require.NoError(t, err)
-	req.Header.Set(internalUpgradeHeader, controlStreamUpgrade)
+	req.Header.Set(InternalUpgradeHeader, ControlStreamUpgrade)
 
 	edgeHTTP2Conn, err := testTransport.NewClientConn(edgeConn)
 	require.NoError(t, err)
@@ -275,7 +274,7 @@ func TestFailRegistration(t *testing.T) {
 
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://localhost:8080/", nil)
 	require.NoError(t, err)
-	req.Header.Set(internalUpgradeHeader, controlStreamUpgrade)
+	req.Header.Set(InternalUpgradeHeader, ControlStreamUpgrade)
 
 	edgeHTTP2Conn, err := testTransport.NewClientConn(edgeConn)
 	require.NoError(t, err)
@@ -311,7 +310,7 @@ func TestGracefulShutdownHTTP2(t *testing.T) {
 
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://localhost:8080/", nil)
 	require.NoError(t, err)
-	req.Header.Set(internalUpgradeHeader, controlStreamUpgrade)
+	req.Header.Set(InternalUpgradeHeader, ControlStreamUpgrade)
 
 	edgeHTTP2Conn, err := testTransport.NewClientConn(edgeConn)
 	require.NoError(t, err)

connection/metrics.go

@@ -4,8 +4,9 @@ import (
 	"sync"
 	"time"
 
-	"github.com/cloudflare/cloudflared/h2mux"
 	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/cloudflare/cloudflared/h2mux"
 )
 
 const (
@@ -414,4 +415,4 @@ func newTunnelMetrics() *tunnelMetrics {
 		tunnelMetricsInternal.metrics = initTunnelMetrics()
 	})
 	return tunnelMetricsInternal.metrics
-}
+}

connection/observer.go

@@ -5,9 +5,9 @@ import (
 	"net/url"
 	"strings"
 
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
-
 	"github.com/rs/zerolog"
+
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
 const (

connection/observer_test.go

@@ -67,10 +67,9 @@ func TestObserverEventsDontBlock(t *testing.T) {
 	}
 }
 
-
 type eventCollectorSink struct {
 	observedEvents []Event
-	mu sync.Mutex
+	mu             sync.Mutex
 }
 
 func (s *eventCollectorSink) OnTunnelEvent(event Event) {
@@ -83,4 +82,4 @@ func (s *eventCollectorSink) assertSawEvent(t *testing.T, event Event) {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 	assert.Contains(t, s.observedEvents, event)
-}
+}

connection/rpc.go

@@ -6,11 +6,11 @@ import (
 	"io"
 	"time"
 
-	"github.com/cloudflare/cloudflared/tunnelrpc"
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
-
 	"github.com/rs/zerolog"
 	"zombiezen.com/go/capnproto2/rpc"
+
+	"github.com/cloudflare/cloudflared/tunnelrpc"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
 type tunnelServerClient struct {

edgediscovery/edgediscovery.go

@@ -5,8 +5,9 @@ import (
 	"net"
 	"sync"
 
-	"github.com/cloudflare/cloudflared/edgediscovery/allregions"
 	"github.com/rs/zerolog"
+
+	"github.com/cloudflare/cloudflared/edgediscovery/allregions"
 )
 
 const (

fmt-check.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -e -o pipefail
+
+OUTPUT=$(for d in $(go list -mod=vendor -f '{{.Dir}}' -a ./... | fgrep -v tunnelrpc) ; do goimports -format-only -local github.com/cloudflare/cloudflared -d $d ; done)
+
+if [ -n "$OUTPUT" ] ; then
+  PAGER=$(which colordiff || echo cat)
+  echo
+  echo "Code formatting issues found, use 'goimports -format-only -local github.com/cloudflare/cloudflared' to correct them"
+  echo
+  echo "$OUTPUT" | $PAGER
+  exit 1
+fi

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/coredns/coredns v1.7.0
 	github.com/coreos/go-oidc v0.0.0-20171002155002-a93f71fdfe73
 	github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf
+	github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
 	github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 // indirect
 	github.com/facebookgo/freeport v0.0.0-20150612182905-d4adf43b75b9 // indirect
 	github.com/facebookgo/grace v0.0.0-20180706040059-75cf19382434
@@ -39,6 +40,7 @@ require (
 	github.com/prometheus/common v0.13.0 // indirect
 	github.com/rivo/tview v0.0.0-20200712113419-c65badfc3d92
 	github.com/rs/zerolog v1.20.0
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/stretchr/testify v1.6.0
 	github.com/urfave/cli/v2 v2.2.0
 	golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a
@@ -58,4 +60,4 @@ require (
 	zombiezen.com/go/capnproto2 v2.18.0+incompatible
 )
 
-replace github.com/urfave/cli/v2 => github.com/ipostelnik/cli/v2 v2.3.1-0.20210223162147-c8b4f97735e4
+replace github.com/urfave/cli/v2 => github.com/ipostelnik/cli/v2 v2.3.1-0.20210324024421-b6ea8234fe3d

go.sum

@@ -140,8 +140,9 @@ github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+
 github.com/coreos/license-bill-of-materials v0.0.0-20190913234955-13baff47494e/go.mod h1:4xMOusJ7xxc84WclVxKT8+lNfGYDwojOUC2OQNCwcj4=
 github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/cpu/goacmedns v0.0.1/go.mod h1:sesf/pNnCYwUevQEQfEwY0Y3DydlQWSGZbaMElOWxok=
-github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
+github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM=
+github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -347,8 +348,8 @@ github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJ
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo=
 github.com/infobloxopen/go-trees v0.0.0-20190313150506-2af4e13f9062/go.mod h1:PcNJqIlcX/dj3DTG/+QQnRvSgTMG6CLpRMjWcv4+J6w=
-github.com/ipostelnik/cli/v2 v2.3.1-0.20210223162147-c8b4f97735e4 h1:U2OxGKZHC5puhKZnGo6wWWjuWusJCM6J/S1eF7Vt+Uk=
-github.com/ipostelnik/cli/v2 v2.3.1-0.20210223162147-c8b4f97735e4/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
+github.com/ipostelnik/cli/v2 v2.3.1-0.20210324024421-b6ea8234fe3d h1:PRDnysJ9dF1vUMmEzBu6aHQeUluSQy4eWH3RsSSy/vI=
+github.com/ipostelnik/cli/v2 v2.3.1-0.20210324024421-b6ea8234fe3d/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
 github.com/jimstudt/http-authentication v0.0.0-20140401203705-3eca13d6893a/go.mod h1:wK6yTYYcgjHE1Z1QtXACPDjcFJyBskHEdagmnq3vsP8=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik=
@@ -544,14 +545,14 @@ github.com/rs/zerolog v1.20.0 h1:38k9hgtUBdxFwE34yS8rTHmHBa4eN16E4DJlv177LNs=
 github.com/rs/zerolog v1.20.0/go.mod h1:IzD0RJ65iWH0w97OQQebJEvTZYvsCUm9WVLWBQrJRjo=
 github.com/russross/blackfriday v0.0.0-20170610170232-067529f716f4 h1:S9YlS71UNJIyS61OqGAmLXv3w5zclSidN+qwr80XxKs=
 github.com/russross/blackfriday v0.0.0-20170610170232-067529f716f4/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
-github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/sacloud/libsacloud v1.26.1/go.mod h1:79ZwATmHLIFZIMd7sxA3LwzVy/B77uj3LDoToVTxDoQ=
 github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
-github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=

h2mux/h2mux_test.go

@@ -663,7 +663,8 @@ func AssertIfPipeReadable(t *testing.T, pipe io.ReadCloser) {
 		b := []byte{0}
 		n, err := pipe.Read(b)
 		if n > 0 {
-			t.Fatalf("read pipe was not empty")
+			t.Errorf("read pipe was not empty")
+			return
 		}
 		errC <- err
 	}()

h2mux/header.go

@@ -1,234 +0,0 @@
-package h2mux
-
-import (
-	"encoding/base64"
-	"fmt"
-	"net/http"
-	"net/url"
-	"strconv"
-	"strings"
-
-	"github.com/pkg/errors"
-)
-
-type Header struct {
-	Name, Value string
-}
-
-var headerEncoding = base64.RawStdEncoding
-
-const (
-	RequestUserHeadersField  = "cf-cloudflared-request-headers"
-	ResponseUserHeadersField = "cf-cloudflared-response-headers"
-
-	CFAccessTokenHeader        = "cf-access-token"
-	CFJumpDestinationHeader    = "CF-Access-Jump-Destination"
-	CFAccessClientIDHeader     = "CF-Access-Client-Id"
-	CFAccessClientSecretHeader = "CF-Access-Client-Secret"
-)
-
-// H2RequestHeadersToH1Request converts the HTTP/2 headers coming from origintunneld
-// to an HTTP/1 Request object destined for the local origin web service.
-// This operation includes conversion of the pseudo-headers into their closest
-// HTTP/1 equivalents. See https://tools.ietf.org/html/rfc7540#section-8.1.2.3
-func H2RequestHeadersToH1Request(h2 []Header, h1 *http.Request) error {
-	for _, header := range h2 {
-		name := strings.ToLower(header.Name)
-		if !IsControlHeader(name) {
-			continue
-		}
-
-		switch name {
-		case ":method":
-			h1.Method = header.Value
-		case ":scheme":
-			// noop - use the preexisting scheme from h1.URL
-		case ":authority":
-			// Otherwise the host header will be based on the origin URL
-			h1.Host = header.Value
-		case ":path":
-			// We don't want to be an "opinionated" proxy, so ideally we would use :path as-is.
-			// However, this HTTP/1 Request object belongs to the Go standard library,
-			// whose URL package makes some opinionated decisions about the encoding of
-			// URL characters: see the docs of https://godoc.org/net/url#URL,
-			// in particular the EscapedPath method https://godoc.org/net/url#URL.EscapedPath,
-			// which is always used when computing url.URL.String(), whether we'd like it or not.
-			//
-			// Well, not *always*. We could circumvent this by using url.URL.Opaque. But
-			// that would present unusual difficulties when using an HTTP proxy: url.URL.Opaque
-			// is treated differently when HTTP_PROXY is set!
-			// See https://github.com/golang/go/issues/5684#issuecomment-66080888
-			//
-			// This means we are subject to the behavior of net/url's function `shouldEscape`
-			// (as invoked with mode=encodePath): https://github.com/golang/go/blob/go1.12.7/src/net/url/url.go#L101
-
-			if header.Value == "*" {
-				h1.URL.Path = "*"
-				continue
-			}
-			// Due to the behavior of validation.ValidateUrl, h1.URL may
-			// already have a partial value, with or without a trailing slash.
-			base := h1.URL.String()
-			base = strings.TrimRight(base, "/")
-			// But we know :path begins with '/', because we handled '*' above - see RFC7540
-			requestURL, err := url.Parse(base + header.Value)
-			if err != nil {
-				return errors.Wrap(err, fmt.Sprintf("invalid path '%v'", header.Value))
-			}
-			h1.URL = requestURL
-		case "content-length":
-			contentLength, err := strconv.ParseInt(header.Value, 10, 64)
-			if err != nil {
-				return fmt.Errorf("unparseable content length")
-			}
-			h1.ContentLength = contentLength
-		case RequestUserHeadersField:
-			// Do not forward the serialized headers to the origin -- deserialize them, and ditch the serialized version
-			// Find and parse user headers serialized into a single one
-			userHeaders, err := ParseUserHeaders(RequestUserHeadersField, h2)
-			if err != nil {
-				return errors.Wrap(err, "Unable to parse user headers")
-			}
-			for _, userHeader := range userHeaders {
-				h1.Header.Add(http.CanonicalHeaderKey(userHeader.Name), userHeader.Value)
-			}
-		default:
-			// All other control headers shall just be proxied transparently
-			h1.Header.Add(http.CanonicalHeaderKey(header.Name), header.Value)
-		}
-	}
-
-	return nil
-}
-
-func ParseUserHeaders(headerNameToParseFrom string, headers []Header) ([]Header, error) {
-	for _, header := range headers {
-		if header.Name == headerNameToParseFrom {
-			return DeserializeHeaders(header.Value)
-		}
-	}
-
-	return nil, fmt.Errorf("%v header not found", RequestUserHeadersField)
-}
-
-func IsControlHeader(headerName string) bool {
-	return headerName == "content-length" ||
-		headerName == "connection" || headerName == "upgrade" || // Websocket headers
-		strings.HasPrefix(headerName, ":") ||
-		strings.HasPrefix(headerName, "cf-")
-}
-
-// isWebsocketClientHeader returns true if the header name is required by the client to upgrade properly
-func IsWebsocketClientHeader(headerName string) bool {
-	return headerName == "sec-websocket-accept" ||
-		headerName == "connection" ||
-		headerName == "upgrade"
-}
-
-func H1ResponseToH2ResponseHeaders(status int, h1 http.Header) (h2 []Header) {
-	h2 = []Header{
-		{Name: ":status", Value: strconv.Itoa(status)},
-	}
-	userHeaders := make(http.Header, len(h1))
-	for header, values := range h1 {
-		h2name := strings.ToLower(header)
-		if h2name == "content-length" {
-			// This header has meaning in HTTP/2 and will be used by the edge,
-			// so it should be sent as an HTTP/2 response header.
-
-			// Since these are http2 headers, they're required to be lowercase
-			h2 = append(h2, Header{Name: "content-length", Value: values[0]})
-		} else if !IsControlHeader(h2name) || IsWebsocketClientHeader(h2name) {
-			// User headers, on the other hand, must all be serialized so that
-			// HTTP/2 header validation won't be applied to HTTP/1 header values
-			userHeaders[header] = values
-		}
-	}
-
-	// Perform user header serialization and set them in the single header
-	h2 = append(h2, Header{ResponseUserHeadersField, SerializeHeaders(userHeaders)})
-	return h2
-}
-
-// Serialize HTTP1.x headers by base64-encoding each header name and value,
-// and then joining them in the format of [key:value;]
-func SerializeHeaders(h1Headers http.Header) string {
-	// compute size of the fully serialized value and largest temp buffer we will need
-	serializedLen := 0
-	maxTempLen := 0
-	for headerName, headerValues := range h1Headers {
-		for _, headerValue := range headerValues {
-			nameLen := headerEncoding.EncodedLen(len(headerName))
-			valueLen := headerEncoding.EncodedLen(len(headerValue))
-			const delims = 2
-			serializedLen += delims + nameLen + valueLen
-			if nameLen > maxTempLen {
-				maxTempLen = nameLen
-			}
-			if valueLen > maxTempLen {
-				maxTempLen = valueLen
-			}
-		}
-	}
-	var buf strings.Builder
-	buf.Grow(serializedLen)
-
-	temp := make([]byte, maxTempLen)
-	writeB64 := func(s string) {
-		n := headerEncoding.EncodedLen(len(s))
-		if n > len(temp) {
-			temp = make([]byte, n)
-		}
-		headerEncoding.Encode(temp[:n], []byte(s))
-		buf.Write(temp[:n])
-	}
-
-	for headerName, headerValues := range h1Headers {
-		for _, headerValue := range headerValues {
-			if buf.Len() > 0 {
-				buf.WriteByte(';')
-			}
-			writeB64(headerName)
-			buf.WriteByte(':')
-			writeB64(headerValue)
-		}
-	}
-
-	return buf.String()
-}
-
-// Deserialize headers serialized by `SerializeHeader`
-func DeserializeHeaders(serializedHeaders string) ([]Header, error) {
-	const unableToDeserializeErr = "Unable to deserialize headers"
-
-	var deserialized []Header
-	for _, serializedPair := range strings.Split(serializedHeaders, ";") {
-		if len(serializedPair) == 0 {
-			continue
-		}
-
-		serializedHeaderParts := strings.Split(serializedPair, ":")
-		if len(serializedHeaderParts) != 2 {
-			return nil, errors.New(unableToDeserializeErr)
-		}
-
-		serializedName := serializedHeaderParts[0]
-		serializedValue := serializedHeaderParts[1]
-		deserializedName := make([]byte, headerEncoding.DecodedLen(len(serializedName)))
-		deserializedValue := make([]byte, headerEncoding.DecodedLen(len(serializedValue)))
-
-		if _, err := headerEncoding.Decode(deserializedName, []byte(serializedName)); err != nil {
-			return nil, errors.Wrap(err, unableToDeserializeErr)
-		}
-		if _, err := headerEncoding.Decode(deserializedValue, []byte(serializedValue)); err != nil {
-			return nil, errors.Wrap(err, unableToDeserializeErr)
-		}
-
-		deserialized = append(deserialized, Header{
-			Name:  string(deserializedName),
-			Value: string(deserializedValue),
-		})
-	}
-
-	return deserialized, nil
-}

h2mux/muxedstream.go

@@ -23,6 +23,10 @@ type MuxedStreamDataSignaller interface {
 	Signal(ID uint32)
 }
 
+type Header struct {
+	Name, Value string
+}
+
 // MuxedStream is logically an HTTP/2 stream, with an additional buffer for outgoing data.
 type MuxedStream struct {
 	streamID uint32
@@ -74,8 +78,6 @@ type MuxedStream struct {
 	sentEOF bool
 	// true if the peer sent us an EOF
 	receivedEOF bool
-	// If valid, tunnelHostname is used to identify which origin service is the intended recipient of the request
-	tunnelHostname TunnelHostname
 	// Compression-related fields
 	receivedUseDict bool
 	method          string
@@ -252,10 +254,6 @@ func (s *MuxedStream) IsRPCStream() bool {
 	return true
 }
 
-func (s *MuxedStream) TunnelHostname() TunnelHostname {
-	return s.tunnelHostname
-}
-
 // Block until a value is sent on writeBufferHasSpace.
 // Must be called while holding writeLock
 func (s *MuxedStream) awaitWriteBufferHasSpace() {

h2mux/muxreader.go

@@ -12,10 +12,6 @@ import (
 	"golang.org/x/net/http2"
 )
 
-const (
-	CloudflaredProxyTunnelHostnameHeader = "cf-cloudflared-proxy-tunnel-hostname"
-)
-
 type MuxReader struct {
 	// f is used to read HTTP2 frames.
 	f *http2.Framer
@@ -252,8 +248,6 @@ func (r *MuxReader) receiveHeaderData(frame *http2.MetaHeadersFrame) error {
 			if r.dictionaries.write != nil {
 				continue
 			}
-		case CloudflaredProxyTunnelHostnameHeader:
-			stream.tunnelHostname = TunnelHostname(header.Value)
 		}
 		headers = append(headers, Header{Name: header.Name, Value: header.Value})
 	}

h2mux/muxreader_test.go

@@ -21,10 +21,6 @@ var (
 		Name:  ":path",
 		Value: "/api/tunnels",
 	}
-	tunnelHostnameHeader = Header{
-		Name:  CloudflaredProxyTunnelHostnameHeader,
-		Value: "tunnel.example.com",
-	}
 	respStatusHeader = Header{
 		Name:  ":status",
 		Value: "200",
@@ -42,15 +38,6 @@ func (mosh *mockOriginStreamHandler) ServeStream(stream *MuxedStream) error {
 	return nil
 }
 
-func getCloudflaredProxyTunnelHostnameHeader(stream *MuxedStream) string {
-	for _, header := range stream.Headers {
-		if header.Name == CloudflaredProxyTunnelHostnameHeader {
-			return header.Value
-		}
-	}
-	return ""
-}
-
 func assertOpenStreamSucceed(t *testing.T, stream *MuxedStream, err error) {
 	assert.NoError(t, err)
 	assert.Len(t, stream.Headers, 1)
@@ -72,13 +59,11 @@ func TestMissingHeaders(t *testing.T) {
 		},
 	}
 
-	// Request doesn't contain CloudflaredProxyTunnelHostnameHeader
 	stream, err := muxPair.EdgeMux.OpenStream(ctx, reqHeaders, nil)
 	assertOpenStreamSucceed(t, stream, err)
 
 	assert.Empty(t, originHandler.stream.method)
 	assert.Empty(t, originHandler.stream.path)
-	assert.False(t, originHandler.stream.TunnelHostname().IsSet())
 }
 
 func TestReceiveHeaderData(t *testing.T) {
@@ -90,18 +75,14 @@ func TestReceiveHeaderData(t *testing.T) {
 		methodHeader,
 		schemeHeader,
 		pathHeader,
-		tunnelHostnameHeader,
 	}
 
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
 	defer cancel()
 
-	reqHeaders = append(reqHeaders, tunnelHostnameHeader)
 	stream, err := muxPair.EdgeMux.OpenStream(ctx, reqHeaders, nil)
 	assertOpenStreamSucceed(t, stream, err)
 
 	assert.Equal(t, methodHeader.Value, originHandler.stream.method)
 	assert.Equal(t, pathHeader.Value, originHandler.stream.path)
-	assert.True(t, originHandler.stream.TunnelHostname().IsSet())
-	assert.Equal(t, tunnelHostnameHeader.Value, originHandler.stream.TunnelHostname().String())
 }

h2mux/muxwriter.go

@@ -3,10 +3,11 @@ package h2mux
 import (
 	"bytes"
 	"encoding/binary"
-	"github.com/rs/zerolog"
 	"io"
 	"time"
 
+	"github.com/rs/zerolog"
+
 	"golang.org/x/net/http2"
 	"golang.org/x/net/http2/hpack"
 )

h2mux/shared_buffer_test.go

@@ -81,7 +81,8 @@ func TestSharedBufferConcurrentReadWrite(t *testing.T) {
 				expectedResult.Write(block[:blockSize])
 				n, err := b.Write(block[:blockSize])
 				if n != blockSize || err != nil {
-					t.Fatalf("write error: %d %s", n, err)
+					t.Errorf("write error: %d %s", n, err)
+					return
 				}
 			}
 		}
@@ -94,7 +95,8 @@ func TestSharedBufferConcurrentReadWrite(t *testing.T) {
 			for i := 0; i < 256; i++ {
 				n, err := io.ReadFull(b, block[:blockSize])
 				if n != blockSize || err != nil {
-					t.Fatalf("read error: %d %s", n, err)
+					t.Errorf("read error: %d %s", n, err)
+					return
 				}
 				actualResult.Write(block[:blockSize])
 			}

ingress/ingress.go

@@ -9,12 +9,12 @@ import (
 	"strings"
 	"sync"
 
-	"github.com/cloudflare/cloudflared/config"
-	"github.com/cloudflare/cloudflared/ipaccess"
-
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
 	"github.com/urfave/cli/v2"
+
+	"github.com/cloudflare/cloudflared/config"
+	"github.com/cloudflare/cloudflared/ipaccess"
 )
 
 var (

ingress/ingress_test.go

@@ -12,7 +12,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"github.com/urfave/cli/v2"
-	"gopkg.in/yaml.v2"
+	yaml "gopkg.in/yaml.v2"
 
 	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/ipaccess"

ingress/origin_connection.go

@@ -7,11 +7,12 @@ import (
 	"net"
 	"net/http"
 
+	gws "github.com/gorilla/websocket"
+	"github.com/rs/zerolog"
+
 	"github.com/cloudflare/cloudflared/ipaccess"
 	"github.com/cloudflare/cloudflared/socks"
 	"github.com/cloudflare/cloudflared/websocket"
-	gws "github.com/gorilla/websocket"
-	"github.com/rs/zerolog"
 )
 
 // OriginConnection is a way to stream to a service running on the user's origin.
@@ -24,34 +25,10 @@ type OriginConnection interface {
 
 type streamHandlerFunc func(originConn io.ReadWriter, remoteConn net.Conn, log *zerolog.Logger)
 
-// Stream copies copy data to & from provided io.ReadWriters.
-func Stream(conn, backendConn io.ReadWriter, log *zerolog.Logger) {
-	proxyDone := make(chan struct{}, 2)
-
-	go func() {
-		_, err := io.Copy(conn, backendConn)
-		if err != nil {
-			log.Debug().Msgf("conn to backendConn copy: %v", err)
-		}
-		proxyDone <- struct{}{}
-	}()
-
-	go func() {
-		_, err := io.Copy(backendConn, conn)
-		if err != nil {
-			log.Debug().Msgf("backendConn to conn copy: %v", err)
-		}
-		proxyDone <- struct{}{}
-	}()
-
-	// If one side is done, we are done.
-	<-proxyDone
-}
-
 // DefaultStreamHandler is an implementation of streamHandlerFunc that
 // performs a two way io.Copy between originConn and remoteConn.
 func DefaultStreamHandler(originConn io.ReadWriter, remoteConn net.Conn, log *zerolog.Logger) {
-	Stream(originConn, remoteConn, log)
+	websocket.Stream(originConn, remoteConn, log)
 }
 
 // tcpConnection is an OriginConnection that directly streams to raw TCP.
@@ -60,7 +37,7 @@ type tcpConnection struct {
 }
 
 func (tc *tcpConnection) Stream(ctx context.Context, tunnelConn io.ReadWriter, log *zerolog.Logger) {
-	Stream(tunnelConn, tc.conn, log)
+	websocket.Stream(tunnelConn, tc.conn, log)
 }
 
 func (tc *tcpConnection) Close() {
@@ -88,7 +65,7 @@ type wsConnection struct {
 }
 
 func (wsc *wsConnection) Stream(ctx context.Context, tunnelConn io.ReadWriter, log *zerolog.Logger) {
-	Stream(tunnelConn, wsc.wsConn.UnderlyingConn(), log)
+	websocket.Stream(tunnelConn, wsc.wsConn.UnderlyingConn(), log)
 }
 
 func (wsc *wsConnection) Close() {

ingress/origin_connection_test.go

@@ -13,14 +13,16 @@ import (
 	"testing"
 	"time"
 
-	"github.com/cloudflare/cloudflared/logger"
-	"github.com/cloudflare/cloudflared/socks"
 	"github.com/gobwas/ws/wsutil"
 	gorillaWS "github.com/gorilla/websocket"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"golang.org/x/net/proxy"
 	"golang.org/x/sync/errgroup"
+
+	"github.com/cloudflare/cloudflared/logger"
+	"github.com/cloudflare/cloudflared/socks"
+	"github.com/cloudflare/cloudflared/websocket"
 )
 
 const (
@@ -156,7 +158,7 @@ func TestSocksStreamWSOverTCPConnection(t *testing.T) {
 			require.NoError(t, err)
 			defer wsForwarderInConn.Close()
 
-			Stream(wsForwarderInConn, &wsEyeball{wsForwarderOutConn}, testLogger)
+			websocket.Stream(wsForwarderInConn, &wsEyeball{wsForwarderOutConn}, testLogger)
 			return nil
 		})
 

ingress/origin_proxy.go

@@ -4,12 +4,11 @@ import (
 	"fmt"
 	"net"
 	"net/http"
-	"net/url"
-	"strings"
 
-	"github.com/cloudflare/cloudflared/h2mux"
-	"github.com/cloudflare/cloudflared/websocket"
 	"github.com/pkg/errors"
+
+	"github.com/cloudflare/cloudflared/carrier"
+	"github.com/cloudflare/cloudflared/websocket"
 )
 
 var (
@@ -105,7 +104,7 @@ func (o *tcpOverWSService) EstablishConnection(r *http.Request) (OriginConnectio
 	var err error
 	dest := o.dest
 	if o.isBastion {
-		dest, err = o.bastionDest(r)
+		dest, err = carrier.ResolveBastionDest(r)
 		if err != nil {
 			return nil, nil, err
 		}
@@ -129,23 +128,6 @@ func (o *tcpOverWSService) EstablishConnection(r *http.Request) (OriginConnectio
 
 }
 
-func (o *tcpOverWSService) bastionDest(r *http.Request) (string, error) {
-	jumpDestination := r.Header.Get(h2mux.CFJumpDestinationHeader)
-	if jumpDestination == "" {
-		return "", fmt.Errorf("Did not receive final destination from client. The --destination flag is likely not set on the client side")
-	}
-	// Strip scheme and path set by client. Without a scheme
-	// Parsing a hostname and path without scheme might not return an error due to parsing ambiguities
-	if jumpURL, err := url.Parse(jumpDestination); err == nil && jumpURL.Host != "" {
-		return removePath(jumpURL.Host), nil
-	}
-	return removePath(jumpDestination), nil
-}
-
-func removePath(dest string) string {
-	return strings.SplitN(dest, "/", 2)[0]
-}
-
 func (o *socksProxyOverWSService) EstablishConnection(r *http.Request) (OriginConnection, *http.Response, error) {
 	originConn := o.conn
 	resp := &http.Response{

ingress/origin_proxy_test.go

@@ -11,10 +11,11 @@ import (
 	"sync"
 	"testing"
 
-	"github.com/cloudflare/cloudflared/h2mux"
-	"github.com/cloudflare/cloudflared/websocket"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+
+	"github.com/cloudflare/cloudflared/carrier"
+	"github.com/cloudflare/cloudflared/websocket"
 )
 
 // TestEstablishConnectionResponse ensures each implementation of StreamBasedOriginProxy returns
@@ -125,7 +126,7 @@ func TestTCPOverWSServiceEstablishConnection(t *testing.T) {
 	baseReq.Header.Set("Sec-Websocket-Key", "dGhlIHNhbXBsZSBub25jZQ==")
 
 	bastionReq := baseReq.Clone(context.Background())
-	bastionReq.Header.Set(h2mux.CFJumpDestinationHeader, originListener.Addr().String())
+	carrier.SetBastionDest(bastionReq.Header, originListener.Addr().String())
 
 	expectHeader := http.Header{
 		"Connection":           {"Upgrade"},
@@ -134,19 +135,23 @@ func TestTCPOverWSServiceEstablishConnection(t *testing.T) {
 	}
 
 	tests := []struct {
+		testCase  string
 		service   *tcpOverWSService
 		req       *http.Request
 		expectErr bool
 	}{
 		{
-			service: newTCPOverWSService(originURL),
-			req:     baseReq,
+			testCase: "specific TCP service",
+			service:  newTCPOverWSService(originURL),
+			req:      baseReq,
 		},
 		{
-			service: newBastionService(),
-			req:     bastionReq,
+			testCase: "bastion service",
+			service:  newBastionService(),
+			req:      bastionReq,
 		},
 		{
+			testCase:  "invalid bastion request",
 			service:   newBastionService(),
 			req:       baseReq,
 			expectErr: true,
@@ -154,13 +159,15 @@ func TestTCPOverWSServiceEstablishConnection(t *testing.T) {
 	}
 
 	for _, test := range tests {
-		if test.expectErr {
-			_, resp, err := test.service.EstablishConnection(test.req)
-			assert.Error(t, err)
-			assert.Nil(t, resp)
-		} else {
-			assertEstablishConnectionResponse(t, test.service, test.req, expectHeader)
-		}
+		t.Run(test.testCase, func(t *testing.T) {
+			if test.expectErr {
+				_, resp, err := test.service.EstablishConnection(test.req)
+				assert.Error(t, err)
+				assert.Nil(t, resp)
+			} else {
+				assertEstablishConnectionResponse(t, test.service, test.req, expectHeader)
+			}
+		})
 	}
 
 	originListener.Close()
@@ -174,104 +181,6 @@ func TestTCPOverWSServiceEstablishConnection(t *testing.T) {
 	}
 }
 
-func TestBastionDestination(t *testing.T) {
-	canonicalJumpDestHeader := http.CanonicalHeaderKey(h2mux.CFJumpDestinationHeader)
-	tests := []struct {
-		name         string
-		header       http.Header
-		expectedDest string
-		wantErr      bool
-	}{
-		{
-			name: "hostname destination",
-			header: http.Header{
-				canonicalJumpDestHeader: []string{"localhost"},
-			},
-			expectedDest: "localhost",
-		},
-		{
-			name: "hostname destination with port",
-			header: http.Header{
-				canonicalJumpDestHeader: []string{"localhost:9000"},
-			},
-			expectedDest: "localhost:9000",
-		},
-		{
-			name: "hostname destination with scheme and port",
-			header: http.Header{
-				canonicalJumpDestHeader: []string{"ssh://localhost:9000"},
-			},
-			expectedDest: "localhost:9000",
-		},
-		{
-			name: "full hostname url",
-			header: http.Header{
-				canonicalJumpDestHeader: []string{"ssh://localhost:9000/metrics"},
-			},
-			expectedDest: "localhost:9000",
-		},
-		{
-			name: "hostname destination with port and path",
-			header: http.Header{
-				canonicalJumpDestHeader: []string{"localhost:9000/metrics"},
-			},
-			expectedDest: "localhost:9000",
-		},
-		{
-			name: "ip destination",
-			header: http.Header{
-				canonicalJumpDestHeader: []string{"127.0.0.1"},
-			},
-			expectedDest: "127.0.0.1",
-		},
-		{
-			name: "ip destination with port",
-			header: http.Header{
-				canonicalJumpDestHeader: []string{"127.0.0.1:9000"},
-			},
-			expectedDest: "127.0.0.1:9000",
-		},
-		{
-			name: "ip destination with port and path",
-			header: http.Header{
-				canonicalJumpDestHeader: []string{"127.0.0.1:9000/metrics"},
-			},
-			expectedDest: "127.0.0.1:9000",
-		},
-		{
-			name: "ip destination with schem and port",
-			header: http.Header{
-				canonicalJumpDestHeader: []string{"tcp://127.0.0.1:9000"},
-			},
-			expectedDest: "127.0.0.1:9000",
-		},
-		{
-			name: "full ip url",
-			header: http.Header{
-				canonicalJumpDestHeader: []string{"ssh://127.0.0.1:9000/metrics"},
-			},
-			expectedDest: "127.0.0.1:9000",
-		},
-		{
-			name:    "no destination",
-			wantErr: true,
-		},
-	}
-	s := newBastionService()
-	for _, test := range tests {
-		r := &http.Request{
-			Header: test.header,
-		}
-		dest, err := s.bastionDest(r)
-		if test.wantErr {
-			assert.Error(t, err, "Test %s expects error", test.name)
-		} else {
-			assert.NoError(t, err, "Test %s expects no error, got error %v", test.name, err)
-			assert.Equal(t, test.expectedDest, dest, "Test %s expect dest %s, got %s", test.name, test.expectedDest, dest)
-		}
-	}
-}
-
 func TestHTTPServiceHostHeaderOverride(t *testing.T) {
 	cfg := OriginRequestConfig{
 		HTTPHostHeader: t.Name(),

ingress/origin_request_config.go

@@ -3,10 +3,10 @@ package ingress
 import (
 	"time"
 
-	"github.com/cloudflare/cloudflared/ipaccess"
 	"github.com/urfave/cli/v2"
 
 	"github.com/cloudflare/cloudflared/config"
+	"github.com/cloudflare/cloudflared/ipaccess"
 	"github.com/cloudflare/cloudflared/tlsconfig"
 )
 

ingress/origin_request_config_test.go

@@ -5,10 +5,11 @@ import (
 	"testing"
 	"time"
 
-	"github.com/cloudflare/cloudflared/config"
 	"github.com/stretchr/testify/require"
 	"github.com/urfave/cli/v2"
-	"gopkg.in/yaml.v2"
+	yaml "gopkg.in/yaml.v2"
+
+	"github.com/cloudflare/cloudflared/config"
 )
 
 // Ensure that the nullable config from `config` package and the

ingress/origin_service.go

@@ -11,14 +11,15 @@ import (
 	"sync"
 	"time"
 
+	gws "github.com/gorilla/websocket"
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog"
+
 	"github.com/cloudflare/cloudflared/hello"
 	"github.com/cloudflare/cloudflared/ipaccess"
 	"github.com/cloudflare/cloudflared/socks"
 	"github.com/cloudflare/cloudflared/tlsconfig"
 	"github.com/cloudflare/cloudflared/websocket"
-	gws "github.com/gorilla/websocket"
-	"github.com/pkg/errors"
-	"github.com/rs/zerolog"
 )
 
 // originService is something a tunnel can proxy traffic to.

logger/create.go

@@ -64,6 +64,8 @@ func (t resilientMultiWriter) Write(p []byte) (n int, err error) {
 	return len(p), nil
 }
 
+var levelErrorLogged = false
+
 func newZerolog(loggerConfig *Config) *zerolog.Logger {
 	var writers []io.Writer
 
@@ -91,11 +93,15 @@ func newZerolog(loggerConfig *Config) *zerolog.Logger {
 
 	multi := resilientMultiWriter{writers}
 
-	level, err := zerolog.ParseLevel(loggerConfig.MinLevel)
-	if err != nil {
-		return fallbackLogger(err)
+	level, levelErr := zerolog.ParseLevel(loggerConfig.MinLevel)
+	if levelErr != nil {
+		level = zerolog.InfoLevel
 	}
 	log := zerolog.New(multi).With().Timestamp().Logger().Level(level)
+	if !levelErrorLogged && levelErr != nil {
+		log.Error().Msgf("Failed to parse log level %q, using %q instead", loggerConfig.MinLevel, level)
+		levelErrorLogged = true
+	}
 
 	return &log
 }
@@ -151,8 +157,8 @@ func Create(loggerConfig *Config) *zerolog.Logger {
 func createConsoleLogger(config ConsoleConfig) io.Writer {
 	consoleOut := os.Stderr
 	return zerolog.ConsoleWriter{
-		Out:     colorable.NewColorable(consoleOut),
-		NoColor: config.noColor || !term.IsTerminal(int(consoleOut.Fd())),
+		Out:        colorable.NewColorable(consoleOut),
+		NoColor:    config.noColor || !term.IsTerminal(int(consoleOut.Fd())),
 		TimeFormat: consoleTimeFormat,
 	}
 }

logger/create_test.go

@@ -1,11 +1,12 @@
 package logger
 
 import (
+	"io"
+	"testing"
+
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/assert"
-	"io"
-	"testing"
 )
 
 var writeCalls int
@@ -31,45 +32,45 @@ func TestResilientMultiWriter(t *testing.T) {
 		writers []io.Writer
 	}{
 		{
-			name:    "All valid writers",
+			name: "All valid writers",
 			writers: []io.Writer{
-				mockedWriter {
+				mockedWriter{
 					wantErr: false,
 				},
-				mockedWriter {
+				mockedWriter{
 					wantErr: false,
 				},
 			},
 		},
 		{
-			name:    "All invalid writers",
+			name: "All invalid writers",
 			writers: []io.Writer{
-				mockedWriter {
+				mockedWriter{
 					wantErr: true,
 				},
-				mockedWriter {
+				mockedWriter{
 					wantErr: true,
 				},
 			},
 		},
 		{
-			name:    "First invalid writer",
+			name: "First invalid writer",
 			writers: []io.Writer{
-				mockedWriter {
+				mockedWriter{
 					wantErr: true,
 				},
-				mockedWriter {
+				mockedWriter{
 					wantErr: false,
 				},
 			},
 		},
 		{
-			name:    "First valid writer",
+			name: "First valid writer",
 			writers: []io.Writer{
-				mockedWriter {
+				mockedWriter{
 					wantErr: false,
 				},
-				mockedWriter {
+				mockedWriter{
 					wantErr: true,
 				},
 			},
@@ -86,4 +87,4 @@ func TestResilientMultiWriter(t *testing.T) {
 		assert.Equal(t, len(writers), writeCalls)
 		writeCalls = 0
 	}
-}
+}

metrics/timer.go

@@ -1,47 +1,48 @@
 package metrics
 
 import (
-  "time"
-  "github.com/prometheus/client_golang/prometheus"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
 )
 
 // Timer assumes the metrics is partitioned by one label
 type Timer struct {
-  startTime   map[string]time.Time
-  metrics     *prometheus.HistogramVec
-  measureUnit time.Duration
-  labelKey    string
+	startTime   map[string]time.Time
+	metrics     *prometheus.HistogramVec
+	measureUnit time.Duration
+	labelKey    string
 }
 
 func NewTimer(metrics *prometheus.HistogramVec, unit time.Duration, labelKey string) *Timer {
-  return &Timer{
-    startTime:   make(map[string]time.Time),
-    measureUnit: unit,
-    metrics:     metrics,
-    labelKey:    labelKey,
-  }
+	return &Timer{
+		startTime:   make(map[string]time.Time),
+		measureUnit: unit,
+		metrics:     metrics,
+		labelKey:    labelKey,
+	}
 }
 
 func (i *Timer) Start(labelVal string) {
-  i.startTime[labelVal] = time.Now()
+	i.startTime[labelVal] = time.Now()
 }
 
 func (i *Timer) End(labelVal string) time.Duration {
-  if start, ok := i.startTime[labelVal]; ok {
-    return Latency(start, time.Now())
-  }
-  return 0
+	if start, ok := i.startTime[labelVal]; ok {
+		return Latency(start, time.Now())
+	}
+	return 0
 }
 
 func (i *Timer) Observe(measurement time.Duration, labelVal string) {
-  metricsLabels := prometheus.Labels{i.labelKey: labelVal}
-  i.metrics.With(metricsLabels).Observe(float64(measurement / i.measureUnit))
+	metricsLabels := prometheus.Labels{i.labelKey: labelVal}
+	i.metrics.With(metricsLabels).Observe(float64(measurement / i.measureUnit))
 }
 
 func (i *Timer) EndAndObserve(labelVal string) {
-  i.Observe(i.End(labelVal), labelVal)
+	i.Observe(i.End(labelVal), labelVal)
 }
 
 func Latency(startTime, endTime time.Time) time.Duration {
-  return endTime.Sub(startTime)
+	return endTime.Sub(startTime)
 }

metrics/timer_test.go

@@ -1,24 +1,24 @@
 package metrics
 
 import (
-  "testing"
-  "time"
+	"testing"
+	"time"
 
-  "github.com/prometheus/client_golang/prometheus"
-  "github.com/stretchr/testify/assert"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/stretchr/testify/assert"
 )
 
 func TestEnd(t *testing.T) {
-  m := prometheus.NewHistogramVec(
-    prometheus.HistogramOpts{
-      Namespace: "TestCallLatencyWithoutMeasurement",
-      Name:      "Latency",
-      Buckets: prometheus.LinearBuckets(0, 50, 100),
-    },
-    []string{"key"},
-  )
-  timer := NewTimer(m, time.Millisecond, "key")
-  assert.Equal(t, time.Duration(0), timer.End("dne"))
-  timer.Start("test")
-  assert.NotEqual(t, time.Duration(0), timer.End("test"))
+	m := prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Namespace: "TestCallLatencyWithoutMeasurement",
+			Name:      "Latency",
+			Buckets:   prometheus.LinearBuckets(0, 50, 100),
+		},
+		[]string{"key"},
+	)
+	timer := NewTimer(m, time.Millisecond, "key")
+	assert.Equal(t, time.Duration(0), timer.End("dne"))
+	timer.Start("test")
+	assert.NotEqual(t, time.Duration(0), timer.End("test"))
 }

origin/metrics.go

@@ -1,8 +1,9 @@
 package origin
 
 import (
-	"github.com/cloudflare/cloudflared/connection"
 	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/cloudflare/cloudflared/connection"
 )
 
 // Metrics uses connection.MetricsNamespace(aka cloudflared) as namespace and connection.TunnelSubsystem

origin/proxy_test.go

@@ -13,20 +13,19 @@ import (
 	"testing"
 	"time"
 
-	"github.com/cloudflare/cloudflared/logger"
+	"github.com/gobwas/ws/wsutil"
+	gorillaWS "github.com/gorilla/websocket"
+	"github.com/rs/zerolog"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/urfave/cli/v2"
 
 	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/hello"
 	"github.com/cloudflare/cloudflared/ingress"
+	"github.com/cloudflare/cloudflared/logger"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
-	gorillaWS "github.com/gorilla/websocket"
-	"github.com/urfave/cli/v2"
-
-	"github.com/gobwas/ws/wsutil"
-	"github.com/rs/zerolog"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
 )
 
 var (

origin/reconnect.go

@@ -7,8 +7,10 @@ import (
 	"sync"
 	"time"
 
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/cloudflare/cloudflared/retry"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
 var (
@@ -102,7 +104,7 @@ func (cm *reconnectCredentialManager) SetConnDigest(connID uint8, digest []byte)
 
 func (cm *reconnectCredentialManager) RefreshAuth(
 	ctx context.Context,
-	backoff *BackoffHandler,
+	backoff *retry.BackoffHandler,
 	authenticate func(ctx context.Context, numPreviousAttempts int) (tunnelpogs.AuthOutcome, error),
 ) (retryTimer <-chan time.Time, err error) {
 	authOutcome, err := authenticate(ctx, backoff.Retries())
@@ -120,11 +122,11 @@ func (cm *reconnectCredentialManager) RefreshAuth(
 	case tunnelpogs.AuthSuccess:
 		cm.SetReconnectToken(outcome.JWT())
 		cm.authSuccess.Inc()
-		return timeAfter(outcome.RefreshAfter()), nil
+		return retry.Clock.After(outcome.RefreshAfter()), nil
 	case tunnelpogs.AuthUnknown:
 		duration := outcome.RefreshAfter()
 		cm.authFail.WithLabelValues(outcome.Error()).Inc()
-		return timeAfter(duration), nil
+		return retry.Clock.After(duration), nil
 	case tunnelpogs.AuthFail:
 		cm.authFail.WithLabelValues(outcome.Error()).Inc()
 		return nil, outcome

origin/reconnect_test.go

@@ -10,6 +10,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
+	"github.com/cloudflare/cloudflared/retry"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
@@ -17,11 +18,11 @@ func TestRefreshAuthBackoff(t *testing.T) {
 	rcm := newReconnectCredentialManager(t.Name(), t.Name(), 4)
 
 	var wait time.Duration
-	timeAfter = func(d time.Duration) <-chan time.Time {
+	retry.Clock.After = func(d time.Duration) <-chan time.Time {
 		wait = d
 		return time.After(d)
 	}
-	backoff := &BackoffHandler{MaxRetries: 3}
+	backoff := &retry.BackoffHandler{MaxRetries: 3}
 	auth := func(ctx context.Context, n int) (tunnelpogs.AuthOutcome, error) {
 		return nil, fmt.Errorf("authentication failure")
 	}
@@ -45,7 +46,7 @@ func TestRefreshAuthBackoff(t *testing.T) {
 
 	// The backoff timer should have been reset. To confirm this, make timeNow
 	// return a value after the backoff timer's grace period
-	timeNow = func() time.Time {
+	retry.Clock.Now = func() time.Time {
 		expectedGracePeriod := time.Duration(time.Second * 2 << backoff.MaxRetries)
 		return time.Now().Add(expectedGracePeriod * 2)
 	}
@@ -57,12 +58,12 @@ func TestRefreshAuthSuccess(t *testing.T) {
 	rcm := newReconnectCredentialManager(t.Name(), t.Name(), 4)
 
 	var wait time.Duration
-	timeAfter = func(d time.Duration) <-chan time.Time {
+	retry.Clock.After = func(d time.Duration) <-chan time.Time {
 		wait = d
 		return time.After(d)
 	}
 
-	backoff := &BackoffHandler{MaxRetries: 3}
+	backoff := &retry.BackoffHandler{MaxRetries: 3}
 	auth := func(ctx context.Context, n int) (tunnelpogs.AuthOutcome, error) {
 		return tunnelpogs.NewAuthSuccess([]byte("jwt"), 19), nil
 	}
@@ -81,12 +82,12 @@ func TestRefreshAuthUnknown(t *testing.T) {
 	rcm := newReconnectCredentialManager(t.Name(), t.Name(), 4)
 
 	var wait time.Duration
-	timeAfter = func(d time.Duration) <-chan time.Time {
+	retry.Clock.After = func(d time.Duration) <-chan time.Time {
 		wait = d
 		return time.After(d)
 	}
 
-	backoff := &BackoffHandler{MaxRetries: 3}
+	backoff := &retry.BackoffHandler{MaxRetries: 3}
 	auth := func(ctx context.Context, n int) (tunnelpogs.AuthOutcome, error) {
 		return tunnelpogs.NewAuthUnknown(errors.New("auth unknown"), 19), nil
 	}
@@ -104,7 +105,7 @@ func TestRefreshAuthUnknown(t *testing.T) {
 func TestRefreshAuthFail(t *testing.T) {
 	rcm := newReconnectCredentialManager(t.Name(), t.Name(), 4)
 
-	backoff := &BackoffHandler{MaxRetries: 3}
+	backoff := &retry.BackoffHandler{MaxRetries: 3}
 	auth := func(ctx context.Context, n int) (tunnelpogs.AuthOutcome, error) {
 		return tunnelpogs.NewAuthFail(errors.New("auth fail")), nil
 	}

origin/supervisor.go

@@ -7,14 +7,15 @@ import (
 	"net"
 	"time"
 
+	"github.com/google/uuid"
+	"github.com/rs/zerolog"
+
 	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/edgediscovery"
 	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/retry"
 	"github.com/cloudflare/cloudflared/signal"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
-
-	"github.com/google/uuid"
-	"github.com/rs/zerolog"
 )
 
 const (
@@ -30,12 +31,6 @@ const (
 	refreshAuthMaxBackoff = 10
 	// Waiting time before retrying a failed 'Authenticate' connection
 	refreshAuthRetryDuration = time.Second * 10
-	// Maximum time to make an Authenticate RPC
-	authTokenTimeout = time.Second * 30
-)
-
-var (
-	errEventDigestUnset = errors.New("event digest unset")
 )
 
 // Supervisor manages non-declarative tunnels. Establishes TCP connections with the edge, and
@@ -118,10 +113,10 @@ func (s *Supervisor) Run(
 	var tunnelsWaiting []int
 	tunnelsActive := s.config.HAConnections
 
-	backoff := BackoffHandler{MaxRetries: s.config.Retries, BaseTime: tunnelRetryDuration, RetryForever: true}
+	backoff := retry.BackoffHandler{MaxRetries: s.config.Retries, BaseTime: tunnelRetryDuration, RetryForever: true}
 	var backoffTimer <-chan time.Time
 
-	refreshAuthBackoff := &BackoffHandler{MaxRetries: refreshAuthMaxBackoff, BaseTime: refreshAuthRetryDuration, RetryForever: true}
+	refreshAuthBackoff := &retry.BackoffHandler{MaxRetries: refreshAuthMaxBackoff, BaseTime: refreshAuthRetryDuration, RetryForever: true}
 	var refreshAuthBackoffTimer <-chan time.Time
 
 	if s.useReconnectToken {

origin/tunnel.go

@@ -18,6 +18,7 @@ import (
 	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/edgediscovery"
 	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/retry"
 	"github.com/cloudflare/cloudflared/signal"
 	"github.com/cloudflare/cloudflared/tunnelrpc"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
@@ -138,7 +139,7 @@ func ServeTunnelLoop(
 	connLog := config.Log.With().Uint8(connection.LogFieldConnIndex, connIndex).Logger()
 
 	protocolFallback := &protocolFallback{
-		BackoffHandler{MaxRetries: config.Retries},
+		retry.BackoffHandler{MaxRetries: config.Retries},
 		config.ProtocolSelector.Current(),
 		false,
 	}
@@ -195,18 +196,18 @@ func ServeTunnelLoop(
 // protocolFallback is a wrapper around backoffHandler that will try fallback option when backoff reaches
 // max retries
 type protocolFallback struct {
-	BackoffHandler
+	retry.BackoffHandler
 	protocol   connection.Protocol
 	inFallback bool
 }
 
 func (pf *protocolFallback) reset() {
-	pf.resetNow()
+	pf.ResetNow()
 	pf.inFallback = false
 }
 
 func (pf *protocolFallback) fallback(fallback connection.Protocol) {
-	pf.resetNow()
+	pf.ResetNow()
 	pf.protocol = fallback
 	pf.inFallback = true
 }
@@ -281,7 +282,7 @@ func ServeTunnel(
 	}
 
 	if protocol == connection.HTTP2 {
-		connOptions := config.ConnectionOptions(edgeConn.LocalAddr().String(), uint8(backoff.retries))
+		connOptions := config.ConnectionOptions(edgeConn.LocalAddr().String(), uint8(backoff.Retries()))
 		err = ServeHTTP2(
 			ctx,
 			connLog,
@@ -382,7 +383,7 @@ func ServeH2mux(
 
 	errGroup.Go(func() error {
 		if config.NamedTunnel != nil {
-			connOptions := config.ConnectionOptions(edgeConn.LocalAddr().String(), uint8(connectedFuse.backoff.retries))
+			connOptions := config.ConnectionOptions(edgeConn.LocalAddr().String(), uint8(connectedFuse.backoff.Retries()))
 			return handler.ServeNamedTunnel(serveCtx, config.NamedTunnel, connOptions, connectedFuse)
 		}
 		registrationOptions := config.RegistrationOptions(connIndex, edgeConn.LocalAddr().String(), cloudflaredUUID)

origin/tunnel_test.go

@@ -4,10 +4,11 @@ import (
 	"testing"
 	"time"
 
-	"github.com/cloudflare/cloudflared/connection"
-
 	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/assert"
+
+	"github.com/cloudflare/cloudflared/connection"
+	"github.com/cloudflare/cloudflared/retry"
 )
 
 type dynamicMockFetcher struct {
@@ -26,7 +27,7 @@ func (dmf *dynamicMockFetcher) fetch() connection.PercentageFetcher {
 
 func TestWaitForBackoffFallback(t *testing.T) {
 	maxRetries := uint(3)
-	backoff := BackoffHandler{
+	backoff := retry.BackoffHandler{
 		MaxRetries: maxRetries,
 		BaseTime:   time.Millisecond * 10,
 	}

retry/backoffhandler.go

@@ -1,4 +1,4 @@
-package origin
+package retry
 
 import (
 	"context"
@@ -7,10 +7,15 @@ import (
 )
 
 // Redeclare time functions so they can be overridden in tests.
-var (
-	timeNow   = time.Now
-	timeAfter = time.After
-)
+type clock struct {
+	Now   func() time.Time
+	After func(d time.Duration) <-chan time.Time
+}
+
+var Clock = clock{
+	Now:   time.Now,
+	After: time.After,
+}
 
 // BackoffHandler manages exponential backoff and limits the maximum number of retries.
 // The base time period is 1 second, doubling with each retry.
@@ -39,7 +44,7 @@ func (b BackoffHandler) GetMaxBackoffDuration(ctx context.Context) (time.Duratio
 		return time.Duration(0), false
 	default:
 	}
-	if !b.resetDeadline.IsZero() && timeNow().After(b.resetDeadline) {
+	if !b.resetDeadline.IsZero() && Clock.Now().After(b.resetDeadline) {
 		// b.retries would be set to 0 at this point
 		return time.Second, true
 	}
@@ -53,7 +58,7 @@ func (b BackoffHandler) GetMaxBackoffDuration(ctx context.Context) (time.Duratio
 // BackoffTimer returns a channel that sends the current time when the exponential backoff timeout expires.
 // Returns nil if the maximum number of retries have been used.
 func (b *BackoffHandler) BackoffTimer() <-chan time.Time {
-	if !b.resetDeadline.IsZero() && timeNow().After(b.resetDeadline) {
+	if !b.resetDeadline.IsZero() && Clock.Now().After(b.resetDeadline) {
 		b.retries = 0
 		b.resetDeadline = time.Time{}
 	}
@@ -66,7 +71,7 @@ func (b *BackoffHandler) BackoffTimer() <-chan time.Time {
 	}
 	maxTimeToWait := time.Duration(b.GetBaseTime() * 1 << (b.retries))
 	timeToWait := time.Duration(rand.Int63n(maxTimeToWait.Nanoseconds()))
-	return timeAfter(timeToWait)
+	return Clock.After(timeToWait)
 }
 
 // Backoff is used to wait according to exponential backoff. Returns false if the
@@ -89,7 +94,7 @@ func (b *BackoffHandler) Backoff(ctx context.Context) bool {
 func (b *BackoffHandler) SetGracePeriod() {
 	maxTimeToWait := b.GetBaseTime() * 2 << (b.retries + 1)
 	timeToWait := time.Duration(rand.Int63n(maxTimeToWait.Nanoseconds()))
-	b.resetDeadline = timeNow().Add(timeToWait)
+	b.resetDeadline = Clock.Now().Add(timeToWait)
 }
 
 func (b BackoffHandler) GetBaseTime() time.Duration {
@@ -108,6 +113,6 @@ func (b *BackoffHandler) ReachedMaxRetries() bool {
 	return b.retries == b.MaxRetries
 }
 
-func (b *BackoffHandler) resetNow() {
+func (b *BackoffHandler) ResetNow() {
 	b.resetDeadline = time.Now()
 }

retry/backoffhandler_test.go

@@ -1,4 +1,4 @@
-package origin
+package retry
 
 import (
 	"context"
@@ -14,7 +14,7 @@ func immediateTimeAfter(time.Duration) <-chan time.Time {
 
 func TestBackoffRetries(t *testing.T) {
 	// make backoff return immediately
-	timeAfter = immediateTimeAfter
+	Clock.After = immediateTimeAfter
 	ctx := context.Background()
 	backoff := BackoffHandler{MaxRetries: 3}
 	if !backoff.Backoff(ctx) {
@@ -33,7 +33,7 @@ func TestBackoffRetries(t *testing.T) {
 
 func TestBackoffCancel(t *testing.T) {
 	// prevent backoff from returning normally
-	timeAfter = func(time.Duration) <-chan time.Time { return make(chan time.Time) }
+	Clock.After = func(time.Duration) <-chan time.Time { return make(chan time.Time) }
 	ctx, cancelFunc := context.WithCancel(context.Background())
 	backoff := BackoffHandler{MaxRetries: 3}
 	cancelFunc()
@@ -47,10 +47,10 @@ func TestBackoffCancel(t *testing.T) {
 
 func TestBackoffGracePeriod(t *testing.T) {
 	currentTime := time.Now()
-	// make timeNow return whatever we like
-	timeNow = func() time.Time { return currentTime }
+	// make Clock.Now return whatever we like
+	Clock.Now = func() time.Time { return currentTime }
 	// make backoff return immediately
-	timeAfter = immediateTimeAfter
+	Clock.After = immediateTimeAfter
 	ctx := context.Background()
 	backoff := BackoffHandler{MaxRetries: 1}
 	if !backoff.Backoff(ctx) {
@@ -71,7 +71,7 @@ func TestBackoffGracePeriod(t *testing.T) {
 
 func TestGetMaxBackoffDurationRetries(t *testing.T) {
 	// make backoff return immediately
-	timeAfter = immediateTimeAfter
+	Clock.After = immediateTimeAfter
 	ctx := context.Background()
 	backoff := BackoffHandler{MaxRetries: 3}
 	if _, ok := backoff.GetMaxBackoffDuration(ctx); !ok {
@@ -96,7 +96,7 @@ func TestGetMaxBackoffDurationRetries(t *testing.T) {
 
 func TestGetMaxBackoffDuration(t *testing.T) {
 	// make backoff return immediately
-	timeAfter = immediateTimeAfter
+	Clock.After = immediateTimeAfter
 	ctx := context.Background()
 	backoff := BackoffHandler{MaxRetries: 3}
 	if duration, ok := backoff.GetMaxBackoffDuration(ctx); !ok || duration > time.Second*2 {
@@ -118,7 +118,7 @@ func TestGetMaxBackoffDuration(t *testing.T) {
 
 func TestBackoffRetryForever(t *testing.T) {
 	// make backoff return immediately
-	timeAfter = immediateTimeAfter
+	Clock.After = immediateTimeAfter
 	ctx := context.Background()
 	backoff := BackoffHandler{MaxRetries: 3, RetryForever: true}
 	if duration, ok := backoff.GetMaxBackoffDuration(ctx); !ok || duration > time.Second*2 {

socks/request_handler.go

@@ -6,8 +6,9 @@ import (
 	"net"
 	"strings"
 
-	"github.com/cloudflare/cloudflared/ipaccess"
 	"github.com/rs/zerolog"
+
+	"github.com/cloudflare/cloudflared/ipaccess"
 )
 
 // RequestHandler is the functions needed to handle a SOCKS5 command

socks/request_handler_test.go

@@ -4,8 +4,9 @@ import (
 	"bytes"
 	"testing"
 
-	"github.com/cloudflare/cloudflared/ipaccess"
 	"github.com/stretchr/testify/assert"
+
+	"github.com/cloudflare/cloudflared/ipaccess"
 )
 
 func TestUnsupportedBind(t *testing.T) {

sshgen/sshgen.go

@@ -12,6 +12,7 @@ import (
 	"io"
 	"io/ioutil"
 	"net/http"
+	"net/url"
 	"time"
 
 	"github.com/coreos/go-oidc/jose"
@@ -51,8 +52,8 @@ type errorResponse struct {
 var mockRequest func(url, contentType string, body io.Reader) (*http.Response, error) = nil
 
 // GenerateShortLivedCertificate generates and stores a keypair for short lived certs
-func GenerateShortLivedCertificate(appInfo *cfpath.AppInfo, token string) error {
-	fullName, err := cfpath.GenerateAppTokenFilePathFromURL(appInfo.AppDomain, appInfo.AppAUD, keyName)
+func GenerateShortLivedCertificate(appURL *url.URL, token string) error {
+	fullName, err := cfpath.GenerateSSHCertFilePathFromURL(appURL, keyName)
 	if err != nil {
 		return err
 	}

sshgen/sshgen_test.go

@@ -9,7 +9,9 @@ import (
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
+	"net/url"
 	"os"
+	"strings"
 	"testing"
 	"time"
 
@@ -32,11 +34,12 @@ type signingArguments struct {
 }
 
 func TestCertGenSuccess(t *testing.T) {
-	appInfo := &cfpath.AppInfo{AppAUD: "abcd1234", AppDomain: "mySite.com"}
+	url, _ := url.Parse("https://cf-test-access.com/testpath")
 	token := tokenGenerator()
 
-	fullName, err := cfpath.GenerateAppTokenFilePathFromURL(appInfo.AppDomain, appInfo.AppAUD, keyName)
+	fullName, err := cfpath.GenerateSSHCertFilePathFromURL(url, keyName)
 	assert.NoError(t, err)
+	assert.True(t, strings.HasSuffix(fullName, "/cf-test-access.com-testpath-cf_key"))
 
 	pubKeyName := fullName + ".pub"
 	certKeyName := fullName + "-cert.pub"
@@ -65,7 +68,7 @@ func TestCertGenSuccess(t *testing.T) {
 		return w.Result(), nil
 	}
 
-	err = GenerateShortLivedCertificate(appInfo, token)
+	err = GenerateShortLivedCertificate(url, token)
 	assert.NoError(t, err)
 
 	exist, err := config.FileExists(fullName)

token/path.go

@@ -2,15 +2,26 @@ package token
 
 import (
 	"fmt"
+	"net/url"
 	"os"
 	"path/filepath"
 	"strings"
 
-	"github.com/mitchellh/go-homedir"
+	homedir "github.com/mitchellh/go-homedir"
 
 	"github.com/cloudflare/cloudflared/config"
 )
 
+// GenerateSSHCertFilePathFromURL will return a file path for creating short lived certificates
+func GenerateSSHCertFilePathFromURL(url *url.URL, suffix string) (string, error) {
+	configPath, err := getConfigPath()
+	if err != nil {
+		return "", err
+	}
+	name := strings.Replace(fmt.Sprintf("%s%s-%s", url.Hostname(), url.EscapedPath(), suffix), "/", "-", -1)
+	return filepath.Join(configPath, name), nil
+}
+
 // GenerateAppTokenFilePathFromURL will return a filepath for given Access org token
 func GenerateAppTokenFilePathFromURL(appDomain, aud string, suffix string) (string, error) {
 	configPath, err := getConfigPath()