Release 2021.3.2

free tunnels

.gitignore

@@ -1,21 +1,17 @@
-.GOPATH/
-bin/
-tmp/
-guide/public
-/.GOPATH
+/tmp
 /bin
 .idea
 .build
 .vscode
 \#*\#
 cscope.*
-cloudflared
-cloudflared.pkg
-cloudflared.exe
-cloudflared.msi
-cloudflared-x86-64*
-!cmd/cloudflared/
+/cloudflared
+/cloudflared.pkg
+/cloudflared.exe
+/cloudflared.msi
+/cloudflared-x86-64*
+/packaging
 .DS_Store
 *-session.log
 ssh_server_tests/.env
-.cover
+/.cover

CHANGES.md

@@ -1,29 +1,92 @@
 **Experimental**: This is a new format for release notes. The format and availability is subject to change.
 
+## 2021.3.2
+
+### New Features
+
+- It is now possible to obtain more detailed information about the cloudflared connectors to Cloudflare Edge via
+  `cloudflared tunnel info <name/uuid>`. It is possible to sort the output as well as output in different formats,
+  such as: `cloudflared tunnel info --sort-by version --invert-sort --output json <name/uuid>`.
+  You can obtain more information via `cloudflared tunnel info --help`.
+
+### Bug Fixes
+
+- Don't look for configuration file in default paths when `--config FILE` flag is present after `tunnel` subcommand.
+- cloudflared access token command now functions correctly with the new token-per-app change from 2021.3.0.
+
+
+## 2021.3.0
+
+### New Features
+
+- [Cloudflare One Routing](https://developers.cloudflare.com/cloudflare-one/tutorials/warp-to-tunnel) specific commands
+  now show up in the `cloudflared tunnel route --help` output.
+- There is a new ingress type that allows cloudflared to proxy SOCKS5 as a bastion. You can use it with an ingress
+  rule by adding `service: socks-proxy`. Traffic is routed to any destination specified by the SOCKS5 packet but only
+  if allowed by a rule. In the following example we allow proxying to a certain CIDR but explicitly forbid one address
+  within it:
+```
+ingress:
+  - hostname: socks.example.com
+    service: socks-proxy
+    originRequest:
+      ipRules:
+        - prefix: 192.168.1.8/32
+          allow: false
+        - prefix: 192.168.1.0/24
+          ports: [80, 443]
+          allow: true
+```
+  
+
+### Improvements
+
+- Nested commands, such as `cloudflared tunnel run`, now consider CLI arguments even if they appear earlier on the
+  command. For instance, `cloudflared --config config.yaml tunnel run` will now behave the same as
+  `cloudflared tunnel --config config.yaml run`
+- Warnings are now shown in the output logs whenever cloudflared is running without the most recent version and
+  `no-autoupdate` is `true`.
+- Access tokens are now stored per Access App instead of per request path. This decreases the number of times that the
+  user is required to authenticate with an Access policy redundantly.
+
+### Bug Fixes
+
+- GitHub [PR #317](https://github.com/cloudflare/cloudflared/issues/317) was broken in 2021.2.5 and is now fixed again.
+
 ## 2021.2.5
 
 ### New Features
-- We introduce [Cloudflare One Routing](https://developers.cloudflare.com/cloudflare-one/tutorials/warp-to-tunnel) in beta mode, where users can create private networks with RFC 1918 IP addresses on Cloudflare's network. Clients running Cloudflare WARP client in the same organization can then connect to services made available by Argo Tunnel. Please share your feedback in the GitHub issue opened.
 
+- We introduce [Cloudflare One Routing](https://developers.cloudflare.com/cloudflare-one/tutorials/warp-to-tunnel) in
+  beta mode. Cloudflare customer can now connect users and private networks with RFC 1918 IP addresses via the
+  Cloudflare edge network. Users running Cloudflare WARP client in the same organization can connect to the services
+  made available by Argo Tunnel IP routes. Please share your feedback in the GitHub issue tracker.
 
 ## 2021.2.4
 
 ### Bug Fixes
-- Reverts the Improvement released in 2021.2.3 for CLI arguments as it introduced a regression where cloudflared failed to read URLs in configuration files.
-- Cloudflared now logs the reason for failed connections if the error is recoverable.
 
+- Reverts the Improvement released in 2021.2.3 for CLI arguments as it introduced a regression where cloudflared failed
+  to read URLs in configuration files.
+- cloudflared now logs the reason for failed connections if the error is recoverable.
 
 ## 2021.2.3
- 
-### Backward Incompatible Changes
-- Removes db-connect. The Cloudflare Workers product will continue to support db-connect implementations with versions of cloudflared that predate this release and include support for db-connect.
- 
-### New Features
-- Introduces support for proxy configurations with websockets in arbitrary TCP connections (#318).
- 
-### Improvements
-- Nested command line arguments (such as `cloudflared tunnel run`) now consider CLI arguments even if they appear earlier in the command. E.g. `cloudflared --config config.yaml tunnel run` will now behave similarly to `cloudflared tunnel --config config.yaml run`
- 
-### Bug Fixes
-- The maximum number of upstream connections is now limited by default which should fix reported issues of cloudflared exhausting CPU usage when faced with connectivity issues.
+
+### Backward Incompatible Changes
+
+- Removes db-connect. The Cloudflare Workers product will continue to support db-connect implementations with versions
+  of cloudflared that predate this release and include support for db-connect.
+
+### New Features
+
+- Introduces support for proxy configurations with websockets in arbitrary TCP connections (#318).
+
+### Improvements
+
+- (reverted) Nested command line argument handling.
+
+### Bug Fixes
+
+- The maximum number of upstream connections is now limited by default which should fix reported issues of cloudflared
+  exhausting CPU usage when faced with connectivity issues.
 

Makefile

@@ -34,6 +34,8 @@ ifneq ($(GOARCH),)
     TARGET_ARCH ?= $(GOARCH)
 else ifeq ($(LOCAL_ARCH),x86_64)
     TARGET_ARCH ?= amd64
+else ifeq ($(LOCAL_ARCH),amd64)
+    TARGET_ARCH ?= amd64
 else ifeq ($(LOCAL_ARCH),i686)
     TARGET_ARCH ?= amd64
 else ifeq ($(shell echo $(LOCAL_ARCH) | head -c 5),armv8)
@@ -80,8 +82,18 @@ clean:
 
 .PHONY: cloudflared
 cloudflared: tunnel-deps
+ifeq ($(FIPS), true)
+	$(info Building cloudflared with go-fips)
+	-test -f fips/fips.go && mv fips/fips.go fips/fips.go.linux-amd64
+	mv fips/fips.go.linux-amd64 fips/fips.go
+endif
+
 	GOOS=$(TARGET_OS) GOARCH=$(TARGET_ARCH) go build -v -mod=vendor $(GO_BUILD_TAGS) $(VERSION_FLAGS) $(IMPORT_PATH)/cmd/cloudflared
 
+ifeq ($(FIPS), true)
+	mv fips/fips.go fips/fips.go.linux-amd64
+endif
+
 .PHONY: container
 container:
 	docker build --build-arg=TARGET_ARCH=$(TARGET_ARCH) --build-arg=TARGET_OS=$(TARGET_OS) -t cloudflare/cloudflared-$(TARGET_OS)-$(TARGET_ARCH):"$(VERSION)" .
@@ -101,10 +113,10 @@ test-ssh-server:
 	docker-compose -f ssh_server_tests/docker-compose.yml up
 
 define publish_package
+	chmod 664 cloudflared*.$(1); \
 	for HOST in $(CF_PKG_HOSTS); do \
 		ssh-keyscan -t rsa $$HOST >> ~/.ssh/known_hosts; \
-		scp -4 cloudflared*.$(1) cfsync@$$HOST:/state/cf-pkg/staging/$(2)/$(TARGET_PUBLIC_REPO)/cloudflared/; \
-		ssh cfsync@$$HOST 'chmod g+w /state/cf-pkg/staging/$(2)/$(TARGET_PUBLIC_REPO)/cloudflared/*.$(1)'; \
+		scp -p -4 cloudflared*.$(1) cfsync@$$HOST:/state/cf-pkg/staging/$(2)/$(TARGET_PUBLIC_REPO)/cloudflared/; \
 	done
 endef
 
@@ -236,7 +248,7 @@ tunnelrpc/tunnelrpc.capnp.go: tunnelrpc/tunnelrpc.capnp
 .PHONY: vet
 vet:
 	go vet -mod=vendor ./...
-	which go-sumtype  # go get github.com/BurntSushi/go-sumtype
+	which go-sumtype  # go get github.com/BurntSushi/go-sumtype (don't do this in build directory or this will cause vendor issues)
 	go-sumtype $$(go list -mod=vendor ./...)
 
 .PHONY: msi

RELEASE_NOTES

@@ -1,3 +1,53 @@
+2021.3.2
+- 2021-03-23 TUN-4042: Capture cloudflared output to debug component tests
+- 2021-03-23 Publish changelog for 2021.3.2
+- 2021-03-16 TUN-4089: Address flakiness in component tests for termination
+- 2021-03-16 TUN-4060: Fix Go Vet warnings (new with go 1.16) where t.Fatalf is called from a test goroutine
+- 2021-03-16 TUN-4091: Use flaky decorator to rerun reconnect component tests when they fail
+- 2021-03-12 TUN-4081: Update log severities to use Zerolog's levels
+- 2021-03-16 TUN-4094: Don't read configuration file for access commands
+- 2021-03-15 TUN-3993: New `cloudflared tunnel info` to obtain details about the active connectors for a tunnel
+- 2021-03-17 TUN-3715: Apply input source to the correct context
+- 2021-03-17 AUTH-3394: Ensure scheme on token command
+- 2021-03-18 TUN-4096: Reduce tunnel not connected assertion backoff to address flaky termination tests
+- 2021-03-19 TUN-3998: Allow to cleanup the connections of a tunnel limited to a single client
+- 2021-02-04 TUN-3715: Only read config file once, right before invoking the command
+
+2021.3.1
+- 2021-03-11 TUN-4051: Add component-tests to test graceful shutdown
+- 2021-03-12 TUN-4052: Add component tests to assert service mode behavior
+
+2021.3.0
+- 2021-03-10 TUN-4075: Dedup test should not compare order of list
+- 2021-03-10 Revert "AUTH-3394: Creates a token per app instead of per path"
+- 2021-03-11 TUN-4066: Remove unnecessary chmod during package publish to CF_PKG_HOSTS
+- 2021-03-11 TUN-4066: Set permissions in build agent before 'scp'-ing to machine hosting package repo
+- 2021-03-11 TUN-4050: Add component tests to assert reconnect behavior
+- 2021-03-10 AUTH-3394: Creates a token per app instead of per path - with fix for free tunnels
+- 2021-03-15 Publish change log for 2021.3.0
+- 2021-03-01 Issue #285 - Makefile does not detect TARGET_ARCH correctly on FreeBSD  (#325)
+- 2021-03-01 TUN-3988: Log why it cannot check if origin cert exists
+- 2021-03-02 TUN-3995: Optional --features flag for tunnel run.
+- 2021-03-02 TUN-3994: Log client_id when running a named tunnel
+- 2021-03-04 TUN-4026: Fix regression where HTTP2 edge transport was no longer propagating control plane errors
+- 2021-03-05 TUN-4055: Skeleton for component tests
+- 2021-03-08 TUN-4047: Add cfsetup target to run component test
+- 2021-03-08 TUN-4016: Delegate decision to update for Worker service
+- 2021-03-02 TUN-3905: Cannot run go mod vendor in cloudflared due to fips
+- 2021-03-08 TUN-4063: Cleanup dependencies between packages.
+- 2021-03-09 Allow partial reads from a GorillaConn; add SetDeadline (from net.Conn) (#330)
+- 2021-03-09 TUN-4069: Fix regression on support for websocket over proxy
+- 2021-03-02 AUTH-3394: Creates a token per app instead of per path
+- 2021-03-01 TUN-4017: Add support for using cloudflared as a full socks proxy.
+- 2021-03-08 TUN-4062: Read component tests config from yaml file
+- 2021-03-08 TUN-4049: Add component tests to assert logging behavior when running from terminal
+- 2021-02-23 TUN-3963: Repoint urfave/cli/v2 library at patched branch at github.com/ipostelnik/cli/v2@fixed which correctly handles reading flags declared at multiple levels of subcommands.
+- 2021-02-25 TUN-3970: Route ip show has alias route ip list
+- 2021-02-26 TUN-3978: Unhide teamnet commands and improve their help
+- 2021-02-26 TUN-3983: Renew CA certs in cloudflared
+- 2021-02-28 TUN-3989: Check in with Updater service in more situations and convey messages to user
+- 2021-02-11 TUN-3819: Remove client-side check that deleted tunnels have no connections
+
 2021.2.5
 - 2021-02-23 Publish change notes for 2021.2.5
 - 2021-02-11 TUN-3838: ResponseWriter no longer reads and origin error tests

carrier/carrier.go

@@ -11,8 +11,8 @@ import (
 	"os"
 	"strings"
 
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/token"
 	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/token"
 
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
@@ -21,6 +21,7 @@ import (
 const LogFieldOriginURL = "originURL"
 
 type StartOptions struct {
+	AppInfo         *token.AppInfo
 	OriginURL       string
 	Headers         http.Header
 	Host            string
@@ -123,7 +124,7 @@ func IsAccessResponse(resp *http.Response) bool {
 	if err != nil || location == nil {
 		return false
 	}
-	if strings.HasPrefix(location.Path, "/cdn-cgi/access/login") {
+	if strings.HasPrefix(location.Path, token.AccessLoginWorkerPath) {
 		return true
 	}
 
@@ -137,7 +138,7 @@ func BuildAccessRequest(options *StartOptions, log *zerolog.Logger) (*http.Reque
 		return nil, err
 	}
 
-	token, err := token.FetchTokenWithRedirect(req.URL, log)
+	token, err := token.FetchTokenWithRedirect(req.URL, options.AppInfo, log)
 	if err != nil {
 		return nil, err
 	}

carrier/carrier_test.go

@@ -81,7 +81,8 @@ func TestStartServer(t *testing.T) {
 	go func() {
 		err := Serve(wsConn, listener, shutdownC, options)
 		if err != nil {
-			t.Fatalf("Error running server: %v", err)
+			t.Errorf("Error running server: %v", err)
+			return
 		}
 	}()
 

carrier/websocket.go

@@ -7,9 +7,9 @@ import (
 	"net/http"
 	"net/http/httputil"
 
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/token"
 	"github.com/cloudflare/cloudflared/ingress"
 	"github.com/cloudflare/cloudflared/socks"
+	"github.com/cloudflare/cloudflared/token"
 	cfwebsocket "github.com/cloudflare/cloudflared/websocket"
 
 	"github.com/gorilla/websocket"
@@ -82,11 +82,24 @@ func createWebsocketStream(options *StartOptions, log *zerolog.Logger) (*cfwebso
 
 	dialer := &websocket.Dialer{
 		TLSClientConfig: options.TLSClientConfig,
+		Proxy:           http.ProxyFromEnvironment,
 	}
 	wsConn, resp, err := cfwebsocket.ClientConnect(req, dialer)
 	defer closeRespBody(resp)
 
 	if err != nil && IsAccessResponse(resp) {
+		// Only get Access app info if we know the origin is protected by Access
+		originReq, err := http.NewRequest(http.MethodGet, options.OriginURL, nil)
+		if err != nil {
+			return nil, err
+		}
+
+		appInfo, err := token.GetAppInfo(originReq.URL)
+		if err != nil {
+			return nil, err
+		}
+		options.AppInfo = appInfo
+
 		wsConn, err = createAccessAuthenticatedStream(options, log)
 		if err != nil {
 			return nil, err
@@ -115,11 +128,7 @@ func createAccessAuthenticatedStream(options *StartOptions, log *zerolog.Logger)
 	}
 
 	// Access Token is invalid for some reason. Go through regen flow
-	originReq, err := http.NewRequest(http.MethodGet, options.OriginURL, nil)
-	if err != nil {
-		return nil, err
-	}
-	if err := token.RemoveTokenIfExists(originReq.URL); err != nil {
+	if err := token.RemoveTokenIfExists(options.AppInfo); err != nil {
 		return nil, err
 	}
 	wsConn, resp, err = createAccessWebSocketStream(options, log)

cfsetup.yaml

@@ -203,6 +203,24 @@ stretch: &stretch
       - (cd / && go get github.com/BurntSushi/go-sumtype)
       - export PATH="$HOME/go/bin:$PATH"
       - make test | gotest-to-teamcity
+  component-test:
+    build_dir: *build_dir
+    builddeps:
+      - *pinned_go_fips
+      - python3.7
+      - python3-pip
+      - python3-setuptools
+      # procps installs the ps command which is needed in test_sysv_service because the init script
+      # uses ps pid to determine if the agent is running
+      - procps
+    pre-cache-copy-paths:
+      - component-tests/requirements.txt
+    pre-cache:
+      - sudo pip3 install --upgrade -r component-tests/requirements.txt
+    post-cache:
+      # Constructs config file from env vars
+      - python3 component-tests/config.py
+      - pytest component-tests
   update-homebrew:
     builddeps:
       - openssh-client

cmd/cloudflared/access/carrier.go

@@ -7,7 +7,7 @@ import (
 	"strings"
 
 	"github.com/cloudflare/cloudflared/carrier"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/h2mux"
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/validation"

cmd/cloudflared/access/cmd.go

@@ -2,20 +2,21 @@ package access
 
 import (
 	"fmt"
+	"io"
 	"net/http"
 	"net/url"
 	"os"
+	"os/exec"
 	"strings"
 	"text/template"
 	"time"
 
 	"github.com/cloudflare/cloudflared/carrier"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/shell"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/token"
 	"github.com/cloudflare/cloudflared/h2mux"
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/sshgen"
+	"github.com/cloudflare/cloudflared/token"
 	"github.com/cloudflare/cloudflared/validation"
 
 	"github.com/getsentry/raven-go"
@@ -39,7 +40,7 @@ Add to your {{.Home}}/.ssh/config:
 
 Host {{.Hostname}}
 {{- if .ShortLivedCerts}}
-  ProxyCommand bash -c '{{.Cloudflared}} access ssh-gen --hostname %h; ssh -tt %r@cfpipe-{{.Hostname}} >&2 <&1' 
+  ProxyCommand bash -c '{{.Cloudflared}} access ssh-gen --hostname %h; ssh -tt %r@cfpipe-{{.Hostname}} >&2 <&1'
 
 Host cfpipe-{{.Hostname}}
   HostName {{.Hostname}}
@@ -76,30 +77,24 @@ func Commands() []*cli.Command {
 			Aliases:  []string{"forward"},
 			Category: "Access",
 			Usage:    "access <subcommand>",
-			Description: `Cloudflare Access protects internal resources by securing, authenticating and monitoring access 
-			per-user and by application. With Cloudflare Access, only authenticated users with the required permissions are 
-			able to reach sensitive resources. The commands provided here allow you to interact with Access protected 
+			Description: `Cloudflare Access protects internal resources by securing, authenticating and monitoring access
+			per-user and by application. With Cloudflare Access, only authenticated users with the required permissions are
+			able to reach sensitive resources. The commands provided here allow you to interact with Access protected
 			applications from the command line.`,
 			Subcommands: []*cli.Command{
 				{
 					Name:   "login",
-					Action: cliutil.ErrorHandler(login),
+					Action: cliutil.Action(login),
 					Usage:  "login <url of access application>",
 					Description: `The login subcommand initiates an authentication flow with your identity provider.
 					The subcommand will launch a browser. For headless systems, a url is provided.
 					Once authenticated with your identity provider, the login command will generate a JSON Web Token (JWT)
-					scoped to your identity, the application you intend to reach, and valid for a session duration set by your 
+					scoped to your identity, the application you intend to reach, and valid for a session duration set by your
 					administrator. cloudflared stores the token in local storage.`,
-					Flags: []cli.Flag{
-						&cli.StringFlag{
-							Name:   "url",
-							Hidden: true,
-						},
-					},
 				},
 				{
 					Name:   "curl",
-					Action: cliutil.ErrorHandler(curl),
+					Action: cliutil.Action(curl),
 					Usage:  "curl [--allow-request, -ar] <url> [<curl args>...]",
 					Description: `The curl subcommand wraps curl and automatically injects the JWT into a cf-access-token
 					header when using curl to reach an application behind Access.`,
@@ -108,7 +103,7 @@ func Commands() []*cli.Command {
 				},
 				{
 					Name:        "token",
-					Action:      cliutil.ErrorHandler(generateToken),
+					Action:      cliutil.Action(generateToken),
 					Usage:       "token -app=<url of access application>",
 					ArgsUsage:   "url of Access application",
 					Description: `The token subcommand produces a JWT which can be used to authenticate requests.`,
@@ -120,7 +115,7 @@ func Commands() []*cli.Command {
 				},
 				{
 					Name:        "tcp",
-					Action:      cliutil.ErrorHandler(ssh),
+					Action:      cliutil.Action(ssh),
 					Aliases:     []string{"rdp", "ssh", "smb"},
 					Usage:       "",
 					ArgsUsage:   "",
@@ -163,18 +158,18 @@ func Commands() []*cli.Command {
 						&cli.StringFlag{
 							Name:    logger.LogSSHLevelFlag,
 							Aliases: []string{"loglevel"}, //added to match the tunnel side
-							Usage:   "Application logging level {fatal, error, info, debug}. ",
+							Usage:   "Application logging level {debug, info, warn, error, fatal}. ",
 						},
 						&cli.StringFlag{
-							Name:  sshConnectTo,
+							Name:   sshConnectTo,
 							Hidden: true,
-							Usage: "Connect to alternate location for testing, value is host, host:port, or sni:port:host",
+							Usage:  "Connect to alternate location for testing, value is host, host:port, or sni:port:host",
 						},
 					},
 				},
 				{
 					Name:        "ssh-config",
-					Action:      cliutil.ErrorHandler(sshConfig),
+					Action:      cliutil.Action(sshConfig),
 					Usage:       "",
 					Description: `Prints an example configuration ~/.ssh/config`,
 					Flags: []cli.Flag{
@@ -190,7 +185,7 @@ func Commands() []*cli.Command {
 				},
 				{
 					Name:        "ssh-gen",
-					Action:      cliutil.ErrorHandler(sshGen),
+					Action:      cliutil.Action(sshGen),
 					Usage:       "",
 					Description: `Generates a short lived certificate for given hostname`,
 					Flags: []cli.Flag{
@@ -220,12 +215,18 @@ func login(c *cli.Context) error {
 		log.Error().Msg("Please provide the url of the Access application")
 		return err
 	}
-	if err := verifyTokenAtEdge(appURL, c, log); err != nil {
+
+	appInfo, err := token.GetAppInfo(appURL)
+	if err != nil {
+		return err
+	}
+
+	if err := verifyTokenAtEdge(appURL, appInfo, c, log); err != nil {
 		log.Err(err).Msg("Could not verify token")
 		return err
 	}
 
-	cfdToken, err := token.GetAppTokenIfExists(appURL)
+	cfdToken, err := token.GetAppTokenIfExists(appInfo)
 	if err != nil {
 		fmt.Fprintln(os.Stderr, "Unable to find token for provided application.")
 		return err
@@ -267,13 +268,17 @@ func curl(c *cli.Context) error {
 		return err
 	}
 
-	tok, err := token.GetAppTokenIfExists(appURL)
+	appInfo, err := token.GetAppInfo(appURL)
+	if err != nil {
+		return err
+	}
+	tok, err := token.GetAppTokenIfExists(appInfo)
 	if err != nil || tok == "" {
 		if allowRequest {
 			log.Info().Msg("You don't have an Access token set. Please run access token <access application> to fetch one.")
-			return shell.Run("curl", cmdArgs...)
+			return run("curl", cmdArgs...)
 		}
-		tok, err = token.FetchToken(appURL, log)
+		tok, err = token.FetchToken(appURL, appInfo, log)
 		if err != nil {
 			log.Err(err).Msg("Failed to refresh token")
 			return err
@@ -282,7 +287,28 @@ func curl(c *cli.Context) error {
 
 	cmdArgs = append(cmdArgs, "-H")
 	cmdArgs = append(cmdArgs, fmt.Sprintf("%s: %s", h2mux.CFAccessTokenHeader, tok))
-	return shell.Run("curl", cmdArgs...)
+	return run("curl", cmdArgs...)
+}
+
+// run kicks off a shell task and pipe the results to the respective std pipes
+func run(cmd string, args ...string) error {
+	c := exec.Command(cmd, args...)
+	stderr, err := c.StderrPipe()
+	if err != nil {
+		return err
+	}
+	go func() {
+		io.Copy(os.Stderr, stderr)
+	}()
+
+	stdout, err := c.StdoutPipe()
+	if err != nil {
+		return err
+	}
+	go func() {
+		io.Copy(os.Stdout, stdout)
+	}()
+	return c.Run()
 }
 
 // token dumps provided token to stdout
@@ -290,12 +316,17 @@ func generateToken(c *cli.Context) error {
 	if err := raven.SetDSN(sentryDSN); err != nil {
 		return err
 	}
-	appURL, err := url.Parse(c.String("app"))
+	appURL, err := url.Parse(ensureURLScheme(c.String("app")))
 	if err != nil || c.NumFlags() < 1 {
 		fmt.Fprintln(os.Stderr, "Please provide a url.")
 		return err
 	}
-	tok, err := token.GetAppTokenIfExists(appURL)
+
+	appInfo, err := token.GetAppInfo(appURL)
+	if err != nil {
+		return err
+	}
+	tok, err := token.GetAppTokenIfExists(appInfo)
 	if err != nil || tok == "" {
 		fmt.Fprintln(os.Stderr, "Unable to find token for provided application. Please run login command to generate token.")
 		return err
@@ -346,19 +377,24 @@ func sshGen(c *cli.Context) error {
 	// this fetchToken function mutates the appURL param. We should refactor that
 	fetchTokenURL := &url.URL{}
 	*fetchTokenURL = *originURL
-	cfdToken, err := token.FetchTokenWithRedirect(fetchTokenURL, log)
+
+	appInfo, err := token.GetAppInfo(fetchTokenURL)
+	if err != nil {
+		return err
+	}
+	cfdToken, err := token.FetchTokenWithRedirect(fetchTokenURL, appInfo, log)
 	if err != nil {
 		return err
 	}
 
-	if err := sshgen.GenerateShortLivedCertificate(originURL, cfdToken); err != nil {
+	if err := sshgen.GenerateShortLivedCertificate(appInfo, cfdToken); err != nil {
 		return err
 	}
 
 	return nil
 }
 
-// getAppURL will pull the appURL needed for fetching a user's Access token
+// getAppURL will pull the request URL needed for fetching a user's Access token
 func getAppURL(cmdArgs []string, log *zerolog.Logger) (*url.URL, error) {
 	if len(cmdArgs) < 1 {
 		log.Error().Msg("Please provide a valid URL as the first argument to curl.")
@@ -433,7 +469,7 @@ func isFileThere(candidate string) bool {
 // verifyTokenAtEdge checks for a token on disk, or generates a new one.
 // Then makes a request to to the origin with the token to ensure it is valid.
 // Returns nil if token is valid.
-func verifyTokenAtEdge(appUrl *url.URL, c *cli.Context, log *zerolog.Logger) error {
+func verifyTokenAtEdge(appUrl *url.URL, appInfo *token.AppInfo, c *cli.Context, log *zerolog.Logger) error {
 	headers := buildRequestHeaders(c.StringSlice(sshHeaderFlag))
 	if c.IsSet(sshTokenIDFlag) {
 		headers.Add(h2mux.CFAccessClientIDHeader, c.String(sshTokenIDFlag))
@@ -441,7 +477,7 @@ func verifyTokenAtEdge(appUrl *url.URL, c *cli.Context, log *zerolog.Logger) err
 	if c.IsSet(sshTokenSecretFlag) {
 		headers.Add(h2mux.CFAccessClientSecretHeader, c.String(sshTokenSecretFlag))
 	}
-	options := &carrier.StartOptions{OriginURL: appUrl.String(), Headers: headers}
+	options := &carrier.StartOptions{AppInfo: appInfo, OriginURL: appUrl.String(), Headers: headers}
 
 	if valid, err := isTokenValid(options, log); err != nil {
 		return err
@@ -449,7 +485,7 @@ func verifyTokenAtEdge(appUrl *url.URL, c *cli.Context, log *zerolog.Logger) err
 		return nil
 	}
 
-	if err := token.RemoveTokenIfExists(appUrl); err != nil {
+	if err := token.RemoveTokenIfExists(appInfo); err != nil {
 		return err
 	}
 

cmd/cloudflared/app_forward_service.go

@@ -2,7 +2,7 @@ package main
 
 import (
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/access"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/config"
 
 	"github.com/rs/zerolog"
 )

cmd/cloudflared/app_resolver_service.go

@@ -1,7 +1,7 @@
 package main
 
 import (
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/tunneldns"
 
 	"github.com/rs/zerolog"

cmd/cloudflared/app_service.go

@@ -1,7 +1,7 @@
 package main
 
 import (
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/overwatch"
 
 	"github.com/rs/zerolog"

cmd/cloudflared/buildinfo/build_info.go

@@ -2,6 +2,7 @@ package buildinfo
 
 import (
 	"github.com/rs/zerolog"
+	"fmt"
 	"runtime"
 )
 
@@ -25,3 +26,7 @@ func (bi *BuildInfo) Log(log *zerolog.Logger) {
 	log.Info().Msgf("Version %s", bi.CloudflaredVersion)
 	log.Info().Msgf("GOOS: %s, GOVersion: %s, GoArch: %s", bi.GoOS, bi.GoVersion, bi.GoArch)
 }
+
+func (bi *BuildInfo) OSArch() string {
+	return fmt.Sprintf("%s_%s", bi.GoOS, bi.GoArch)
+}

cmd/cloudflared/cliutil/deprecated.go

@@ -9,12 +9,12 @@ import (
 func RemovedCommand(name string) *cli.Command {
 	return &cli.Command{
 		Name: name,
-		Action: ErrorHandler(func(context *cli.Context) error {
+		Action: func(context *cli.Context) error {
 			return cli.Exit(
 				fmt.Sprintf("%s command is no longer supported by cloudflared. Consult Argo Tunnel documentation for possible alternative solutions.", name),
 				-1,
 			)
-		}),
+		},
 		Description: fmt.Sprintf("%s is deprecated", name),
 		Hidden: true,
 	}

cmd/cloudflared/cliutil/errors.go

@@ -2,6 +2,7 @@ package cliutil
 
 import (
 	"fmt"
+
 	"github.com/urfave/cli/v2"
 )
 
@@ -21,7 +22,7 @@ func UsageError(format string, args ...interface{}) error {
 }
 
 // Ensures exit with error code if actionFunc returns an error
-func ErrorHandler(actionFunc cli.ActionFunc) cli.ActionFunc {
+func WithErrorHandler(actionFunc cli.ActionFunc) cli.ActionFunc {
 	return func(ctx *cli.Context) error {
 		err := actionFunc(ctx)
 		if err != nil {

cmd/cloudflared/cliutil/handler.go

@@ -0,0 +1,57 @@
+package cliutil
+
+import (
+	"github.com/urfave/cli/v2"
+	"github.com/urfave/cli/v2/altsrc"
+
+	"github.com/cloudflare/cloudflared/config"
+	"github.com/cloudflare/cloudflared/logger"
+)
+
+func Action(actionFunc cli.ActionFunc) cli.ActionFunc {
+	return WithErrorHandler(actionFunc)
+}
+
+func ConfiguredAction(actionFunc cli.ActionFunc) cli.ActionFunc {
+	return WithErrorHandler(func(c *cli.Context) error {
+		if err := setFlagsFromConfigFile(c); err != nil {
+			return err
+		}
+		return actionFunc(c)
+	})
+}
+
+func setFlagsFromConfigFile(c *cli.Context) error {
+	const errorExitCode = 1
+	log := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
+	inputSource, err := config.ReadConfigFile(c, log)
+	if err != nil {
+		if err == config.ErrNoConfigFile {
+			return nil
+		}
+		return cli.Exit(err, errorExitCode)
+	}
+
+	if err := applyConfig(c, inputSource); err != nil {
+		return cli.Exit(err, errorExitCode)
+	}
+	return nil
+}
+
+func applyConfig(c *cli.Context, inputSource altsrc.InputSourceContext) error {
+	for _, context := range c.Lineage() {
+		if context.Command == nil {
+			// we've reached the placeholder root context not associated with the app
+			break
+		}
+		targetFlags := context.Command.Flags
+		if context.Command.Name == "" {
+			// commands that define child subcommands are executed as if they were an app
+			targetFlags = context.App.Flags
+		}
+		if err := altsrc.ApplyInputSourceValues(context, inputSource, targetFlags); err != nil {
+			return err
+		}
+	}
+	return nil
+}

cmd/cloudflared/linux_service.go

@@ -8,8 +8,8 @@ import (
 	"path/filepath"
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/tunnel"
+	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/logger"
 
 	"github.com/rs/zerolog"
@@ -24,7 +24,7 @@ func runApp(app *cli.App, graceShutdownC chan struct{}) {
 			{
 				Name:   "install",
 				Usage:  "Install Argo Tunnel as a system service",
-				Action: cliutil.ErrorHandler(installLinuxService),
+				Action: cliutil.ConfiguredAction(installLinuxService),
 				Flags: []cli.Flag{
 					&cli.BoolFlag{
 						Name:  "legacy",
@@ -35,7 +35,7 @@ func runApp(app *cli.App, graceShutdownC chan struct{}) {
 			{
 				Name:   "uninstall",
 				Usage:  "Uninstall the Argo Tunnel service",
-				Action: cliutil.ErrorHandler(uninstallLinuxService),
+				Action: cliutil.ConfiguredAction(uninstallLinuxService),
 			},
 		},
 	})

cmd/cloudflared/macos_service.go

@@ -25,12 +25,12 @@ func runApp(app *cli.App, graceShutdownC chan struct{}) {
 			{
 				Name:   "install",
 				Usage:  "Install Argo Tunnel as an user launch agent",
-				Action: cliutil.ErrorHandler(installLaunchd),
+				Action: cliutil.ConfiguredAction(installLaunchd),
 			},
 			{
 				Name:   "uninstall",
 				Usage:  "Uninstall the Argo Tunnel launch agent",
-				Action: cliutil.ErrorHandler(uninstallLaunchd),
+				Action: cliutil.ConfiguredAction(uninstallLaunchd),
 			},
 		},
 	})

cmd/cloudflared/main.go

@@ -8,13 +8,13 @@ import (
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/access"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/proxydns"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/tunnel"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/updater"
+	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/metrics"
 	"github.com/cloudflare/cloudflared/overwatch"
-	"github.com/cloudflare/cloudflared/tunneldns"
 	"github.com/cloudflare/cloudflared/watcher"
 
 	"github.com/getsentry/raven-go"
@@ -77,7 +77,6 @@ func main() {
 	See https://developers.cloudflare.com/argo-tunnel/ for more in-depth documentation.`
 	app.Flags = flags()
 	app.Action = action(graceShutdownC)
-	app.Before = tunnel.SetFlagsFromConfigFile
 	app.Commands = commands(cli.ShowVersion)
 
 	tunnel.Init(Version, graceShutdownC) // we need this to support the tunnel sub command...
@@ -90,7 +89,7 @@ func commands(version func(c *cli.Context)) []*cli.Command {
 	cmds := []*cli.Command{
 		{
 			Name:   "update",
-			Action: cliutil.ErrorHandler(updater.Update),
+			Action: cliutil.ConfiguredAction(updater.Update),
 			Usage:  "Update the agent if a new version exists",
 			Flags: []cli.Flag{
 				&cli.BoolFlag{
@@ -130,7 +129,7 @@ To determine if an update happened in a script, check for error code 11.`,
 		},
 	}
 	cmds = append(cmds, tunnel.Commands()...)
-	cmds = append(cmds, tunneldns.Command(false))
+	cmds = append(cmds, proxydns.Command(false))
 	cmds = append(cmds, access.Commands()...)
 	return cmds
 }
@@ -145,7 +144,7 @@ func isEmptyInvocation(c *cli.Context) bool {
 }
 
 func action(graceShutdownC chan struct{}) cli.ActionFunc {
-	return cliutil.ErrorHandler(func(c *cli.Context) (err error) {
+	return cliutil.ConfiguredAction(func(c *cli.Context) (err error) {
 		if isEmptyInvocation(c) {
 			return handleServiceMode(c, graceShutdownC)
 		}

cmd/cloudflared/proxydns/cmd.go

@@ -0,0 +1,114 @@
+package proxydns
+
+import (
+	"net"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"github.com/urfave/cli/v2"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
+	"github.com/cloudflare/cloudflared/logger"
+	"github.com/cloudflare/cloudflared/metrics"
+	"github.com/cloudflare/cloudflared/tunneldns"
+)
+
+func Command(hidden bool) *cli.Command {
+	return &cli.Command{
+		Name:   "proxy-dns",
+		Action: cliutil.ConfiguredAction(Run),
+
+		Usage: "Run a DNS over HTTPS proxy server.",
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:    "metrics",
+				Value:   "localhost:",
+				Usage:   "Listen address for metrics reporting.",
+				EnvVars: []string{"TUNNEL_METRICS"},
+			},
+			&cli.StringFlag{
+				Name:    "address",
+				Usage:   "Listen address for the DNS over HTTPS proxy server.",
+				Value:   "localhost",
+				EnvVars: []string{"TUNNEL_DNS_ADDRESS"},
+			},
+			// Note TUN-3758 , we use Int because UInt is not supported with altsrc
+			&cli.IntFlag{
+				Name:    "port",
+				Usage:   "Listen on given port for the DNS over HTTPS proxy server.",
+				Value:   53,
+				EnvVars: []string{"TUNNEL_DNS_PORT"},
+			},
+			&cli.StringSliceFlag{
+				Name:    "upstream",
+				Usage:   "Upstream endpoint URL, you can specify multiple endpoints for redundancy.",
+				Value:   cli.NewStringSlice("https://1.1.1.1/dns-query", "https://1.0.0.1/dns-query"),
+				EnvVars: []string{"TUNNEL_DNS_UPSTREAM"},
+			},
+			&cli.StringSliceFlag{
+				Name:    "bootstrap",
+				Usage:   "bootstrap endpoint URL, you can specify multiple endpoints for redundancy.",
+				Value:   cli.NewStringSlice("https://162.159.36.1/dns-query", "https://162.159.46.1/dns-query", "https://[2606:4700:4700::1111]/dns-query", "https://[2606:4700:4700::1001]/dns-query"),
+				EnvVars: []string{"TUNNEL_DNS_BOOTSTRAP"},
+			},
+			&cli.IntFlag{
+				Name:    "max-upstream-conns",
+				Usage:   "Maximum concurrent connections to upstream. Setting to 0 means unlimited.",
+				Value:   tunneldns.MaxUpstreamConnsDefault,
+				EnvVars: []string{"TUNNEL_DNS_MAX_UPSTREAM_CONNS"},
+			},
+		},
+		ArgsUsage: " ", // can't be the empty string or we get the default output
+		Hidden:    hidden,
+	}
+}
+
+// Run implements a foreground runner
+func Run(c *cli.Context) error {
+	log := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
+
+	metricsListener, err := net.Listen("tcp", c.String("metrics"))
+	if err != nil {
+		log.Fatal().Err(err).Msg("Failed to open the metrics listener")
+	}
+
+	go metrics.ServeMetrics(metricsListener, nil, nil, log)
+
+	listener, err := tunneldns.CreateListener(
+		c.String("address"),
+		// Note TUN-3758 , we use Int because UInt is not supported with altsrc
+		uint16(c.Int("port")),
+		c.StringSlice("upstream"),
+		c.StringSlice("bootstrap"),
+		c.Int("max-upstream-conns"),
+		log,
+	)
+
+	if err != nil {
+		log.Err(err).Msg("Failed to create the listeners")
+		return err
+	}
+
+	// Try to start the server
+	readySignal := make(chan struct{})
+	err = listener.Start(readySignal)
+	if err != nil {
+		log.Err(err).Msg("Failed to start the listeners")
+		return listener.Stop()
+	}
+	<-readySignal
+
+	// Wait for signal
+	signals := make(chan os.Signal, 10)
+	signal.Notify(signals, syscall.SIGTERM, syscall.SIGINT)
+	defer signal.Stop(signals)
+	<-signals
+
+	// Shut down server
+	err = listener.Stop()
+	if err != nil {
+		log.Err(err).Msg("failed to stop")
+	}
+	return err
+}

cmd/cloudflared/service_template.go

@@ -12,7 +12,7 @@ import (
 
 	"github.com/mitchellh/go-homedir"
 
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/config"
 )
 
 type ServiceTemplate struct {

cmd/cloudflared/shell/shell.go

@@ -1,33 +0,0 @@
-package shell
-
-import (
-	"io"
-	"os"
-	"os/exec"
-)
-
-// OpenBrowser opens the specified URL in the default browser of the user
-func OpenBrowser(url string) error {
-	return getBrowserCmd(url).Start()
-}
-
-// Run will kick off a shell task and pipe the results to the respective std pipes
-func Run(cmd string, args ...string) error {
-	c := exec.Command(cmd, args...)
-	stderr, err := c.StderrPipe()
-	if err != nil {
-		return err
-	}
-	go func() {
-		io.Copy(os.Stderr, stderr)
-	}()
-
-	stdout, err := c.StdoutPipe()
-	if err != nil {
-		return err
-	}
-	go func() {
-		io.Copy(os.Stdout, stdout)
-	}()
-	return c.Run()
-}

cmd/cloudflared/tunnel/cmd.go

@@ -7,7 +7,6 @@ import (
 	"io/ioutil"
 	"net/url"
 	"os"
-	"reflect"
 	"runtime/trace"
 	"strings"
 	"sync"
@@ -15,9 +14,10 @@ import (
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/proxydns"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/ui"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/updater"
+	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/ingress"
 	"github.com/cloudflare/cloudflared/logger"
@@ -74,8 +74,8 @@ const (
 	// uiFlag is to enable launching cloudflared in interactive UI mode
 	uiFlag = "ui"
 
-	debugLevelWarning = "At debug level, request URL, method, protocol, content legnth and header will be logged. " +
-		"Response status, content length and header will also be logged in debug level."
+	debugLevelWarning = "At debug level cloudflared will log request URL, method, protocol, content length, as well as, all request and response headers. " +
+		"This can expose sensitive information in your logs."
 
 	LogFieldCommand             = "command"
 	LogFieldExpandedPath        = "expandedPath"
@@ -100,11 +100,12 @@ func Commands() []*cli.Command {
 		buildRouteCommand(),
 		buildRunCommand(),
 		buildListCommand(),
+		buildInfoCommand(),
 		buildIngressSubcommand(),
 		buildDeleteCommand(),
 		buildCleanupCommand(),
 		// for compatibility, allow following as tunnel subcommands
-		tunneldns.Command(true),
+		proxydns.Command(true),
 		cliutil.RemovedCommand("db-connect"),
 	}
 
@@ -119,8 +120,7 @@ func Commands() []*cli.Command {
 func buildTunnelCommand(subcommands []*cli.Command) *cli.Command {
 	return &cli.Command{
 		Name:      "tunnel",
-		Action:    cliutil.ErrorHandler(TunnelCommand),
-		Before:    SetFlagsFromConfigFile,
+		Action:    cliutil.ConfiguredAction(TunnelCommand),
 		Category:  "Tunnel",
 		Usage:     "Make a locally-running web service accessible over the internet using Argo Tunnel.",
 		ArgsUsage: " ",
@@ -286,16 +286,14 @@ func StartServer(
 	}
 
 	// update needs to be after DNS proxy is up to resolve equinox server address
-	if updater.IsAutoupdateEnabled(c, log) {
-		autoupdateFreq := c.Duration("autoupdate-freq")
-		log.Info().Dur("autoupdateFreq", autoupdateFreq).Msg("Autoupdate frequency is set")
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			autoupdater := updater.NewAutoUpdater(c.Duration("autoupdate-freq"), &listeners, log)
-			errC <- autoupdater.Run(ctx)
-		}()
-	}
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		autoupdater := updater.NewAutoUpdater(
+			c.Bool("no-autoupdate"), c.Duration("autoupdate-freq"), &listeners, log,
+		)
+		errC <- autoupdater.Run(ctx)
+	}()
 
 	// Serve DNS proxy stand-alone if no hostname or tag or app is going to run
 	if dnsProxyStandAlone(c) {
@@ -370,26 +368,6 @@ func StartServer(
 	return waitToShutdown(&wg, cancel, errC, graceShutdownC, c.Duration("grace-period"), log)
 }
 
-func SetFlagsFromConfigFile(c *cli.Context) error {
-	const exitCode = 1
-	log := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
-	inputSource, err := config.ReadConfigFile(c, log)
-	if err != nil {
-		if err == config.ErrNoConfigFile {
-			return nil
-		}
-		return cli.Exit(err, exitCode)
-	}
-	targetFlags := c.Command.Flags
-	if c.Command.Name == "" {
-		targetFlags = c.App.Flags
-	}
-	if err := altsrc.ApplyInputSourceValues(c, inputSource, targetFlags); err != nil {
-		return cli.Exit(err, exitCode)
-	}
-	return nil
-}
-
 func waitToShutdown(wg *sync.WaitGroup,
 	cancelServerContext func(),
 	errC <-chan error,
@@ -478,33 +456,6 @@ func addPortIfMissing(uri *url.URL, port int) string {
 	return fmt.Sprintf("%s:%d", uri.Hostname(), port)
 }
 
-// appendFlags will append extra flags to a slice of flags.
-//
-// The cli package will panic if two flags exist with the same name,
-// so if extraFlags contains a flag that was already defined, modify the
-// original flags to use the extra version.
-func appendFlags(flags []cli.Flag, extraFlags ...cli.Flag) []cli.Flag {
-	for _, extra := range extraFlags {
-		var found bool
-
-		// Check if an extra flag overrides an existing flag.
-		for i, flag := range flags {
-			if reflect.DeepEqual(extra.Names(), flag.Names()) {
-				flags[i] = extra
-				found = true
-				break
-			}
-		}
-
-		// Append the extra flag if it has nothing to override.
-		if !found {
-			flags = append(flags, extra)
-		}
-	}
-
-	return flags
-}
-
 func tunnelFlags(shouldHide bool) []cli.Flag {
 	flags := configureCloudflaredFlags(shouldHide)
 	flags = append(flags, configureProxyFlags(shouldHide)...)
@@ -514,7 +465,7 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 		credentialsFileFlag,
 		altsrc.NewBoolFlag(&cli.BoolFlag{
 			Name:   "is-autoupdated",
-			Usage:  "Signal the new process that Argo Tunnel client has been autoupdated",
+			Usage:  "Signal the new process that Argo Tunnel connector has been autoupdated",
 			Value:  false,
 			Hidden: true,
 		}),
@@ -653,6 +604,7 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			Aliases: []string{"n"},
 			EnvVars: []string{"TUNNEL_NAME"},
 			Usage:   "Stable name to identify the tunnel. Using this flag will create, route and run a tunnel. For production usage, execute each command separately",
+			Hidden:  shouldHide,
 		}),
 		altsrc.NewBoolFlag(&cli.BoolFlag{
 			Name:   uiFlag,
@@ -921,7 +873,7 @@ func configureLoggingFlags(shouldHide bool) []cli.Flag {
 		altsrc.NewStringFlag(&cli.StringFlag{
 			Name:    logger.LogLevelFlag,
 			Value:   "info",
-			Usage:   "Application logging level {fatal, error, info, debug}. " + debugLevelWarning,
+			Usage:   "Application logging level {debug, info, warn, error, fatal}. " + debugLevelWarning,
 			EnvVars: []string{"TUNNEL_LOGLEVEL"},
 			Hidden:  shouldHide,
 		}),
@@ -929,7 +881,7 @@ func configureLoggingFlags(shouldHide bool) []cli.Flag {
 			Name:    logger.LogTransportLevelFlag,
 			Aliases: []string{"proto-loglevel"}, // This flag used to be called proto-loglevel
 			Value:   "info",
-			Usage:   "Transport logging level(previously called protocol logging level) {fatal, error, info, debug}",
+			Usage:   "Transport logging level(previously called protocol logging level) {debug, info, warn, error, fatal}",
 			EnvVars: []string{"TUNNEL_PROTO_LOGLEVEL", "TUNNEL_TRANSPORT_LOGLEVEL"},
 			Hidden:  shouldHide,
 		}),

cmd/cloudflared/tunnel/config_test.go

@@ -0,0 +1,13 @@
+package tunnel
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestDedup(t *testing.T) {
+	expected := []string{"a", "b"}
+	actual := dedup([]string{"a", "b", "a"})
+	require.ElementsMatch(t, expected, actual)
+}

cmd/cloudflared/tunnel/configuration.go

@@ -9,7 +9,7 @@ import (
 	"strings"
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/edgediscovery"
 	"github.com/cloudflare/cloudflared/h2mux"
@@ -109,7 +109,7 @@ func findOriginCert(originCertPath string, log *zerolog.Logger) (string, error)
 	// Check that the user has acquired a certificate using the login command
 	ok, err := config.FileExists(originCertPath)
 	if err != nil {
-		log.Error().Msgf("Cannot check if origin cert exists at path %s", originCertPath)
+		log.Error().Err(err).Msgf("Cannot check if origin cert exists at path %s", originCertPath)
 		return "", fmt.Errorf("cannot check if origin cert exists at path %s", originCertPath)
 	}
 	if !ok {
@@ -199,13 +199,15 @@ func prepareTunnelConfig(
 	if isNamedTunnel {
 		clientUUID, err := uuid.NewRandom()
 		if err != nil {
-			return nil, ingress.Ingress{}, errors.Wrap(err, "can't generate clientUUID")
+			return nil, ingress.Ingress{}, errors.Wrap(err, "can't generate connector UUID")
 		}
+		log.Info().Msgf("Generated Connector ID: %s", clientUUID)
+		features := append(c.StringSlice("features"), origin.FeatureSerializedHeaders)
 		namedTunnel.Client = tunnelpogs.ClientInfo{
 			ClientID: clientUUID[:],
-			Features: []string{origin.FeatureSerializedHeaders},
+			Features: dedup(features),
 			Version:  version,
-			Arch:     fmt.Sprintf("%s_%s", buildInfo.GoOS, buildInfo.GoArch),
+			Arch:     buildInfo.OSArch(),
 		}
 		ingressRules, err = ingress.ParseIngress(cfg)
 		if err != nil && err != ingress.ErrNoIngressRules {
@@ -270,7 +272,7 @@ func prepareTunnelConfig(
 
 	return &origin.TunnelConfig{
 		ConnectionConfig: connectionConfig,
-		BuildInfo:        buildInfo,
+		OSArch:           buildInfo.OSArch(),
 		ClientID:         clientID,
 		EdgeAddrs:        c.StringSlice("edge"),
 		HAConnections:    c.Int("ha-connections"),
@@ -301,3 +303,22 @@ func isWarpRoutingEnabled(warpConfig config.WarpRoutingConfig, isNamedTunnel boo
 func isRunningFromTerminal() bool {
 	return terminal.IsTerminal(int(os.Stdout.Fd()))
 }
+
+// Remove any duplicates from the slice
+func dedup(slice []string) []string {
+
+	// Convert the slice into a set
+	set := make(map[string]bool, 0)
+	for _, str := range slice {
+		set[str] = true
+	}
+
+	// Convert the set back into a slice
+	keys := make([]string, len(set))
+	i := 0
+	for str := range set {
+		keys[i] = str
+		i++
+	}
+	return keys
+}

cmd/cloudflared/tunnel/credential_finder.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"path/filepath"
 
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/config"
 
 	"github.com/google/uuid"
 	"github.com/rs/zerolog"

cmd/cloudflared/tunnel/info.go

@@ -0,0 +1,15 @@
+package tunnel
+
+import (
+	"time"
+
+	"github.com/cloudflare/cloudflared/tunnelstore"
+	"github.com/google/uuid"
+)
+
+type Info struct {
+	ID         uuid.UUID                   `json:"id"`
+	Name       string                      `json:"name"`
+	CreatedAt  time.Time                   `json:"createdAt"`
+	Connectors []*tunnelstore.ActiveClient `json:"conns"`
+}

cmd/cloudflared/tunnel/ingress_subcommands.go

@@ -5,7 +5,7 @@ import (
 	"net/url"
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/ingress"
 
 	"github.com/pkg/errors"
@@ -45,7 +45,7 @@ func buildIngressSubcommand() *cli.Command {
 func buildValidateIngressCommand() *cli.Command {
 	return &cli.Command{
 		Name:        "validate",
-		Action:      cliutil.ErrorHandler(validateIngressCommand),
+		Action:      cliutil.ConfiguredAction(validateIngressCommand),
 		Usage:       "Validate the ingress configuration ",
 		UsageText:   "cloudflared tunnel [--config FILEPATH] ingress validate",
 		Description: "Validates the configuration file, ensuring your ingress rules are OK.",
@@ -55,7 +55,7 @@ func buildValidateIngressCommand() *cli.Command {
 func buildTestURLCommand() *cli.Command {
 	return &cli.Command{
 		Name:      "rule",
-		Action:    cliutil.ErrorHandler(testURLCommand),
+		Action:    cliutil.ConfiguredAction(testURLCommand),
 		Usage:     "Check which ingress rule matches a given request URL",
 		UsageText: "cloudflared tunnel [--config FILEPATH] ingress rule URL",
 		ArgsUsage: "URL",

cmd/cloudflared/tunnel/login.go

@@ -13,9 +13,9 @@ import (
 	"github.com/urfave/cli/v2"
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/transfer"
+	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/logger"
+	"github.com/cloudflare/cloudflared/token"
 )
 
 const (
@@ -26,16 +26,10 @@ const (
 func buildLoginSubcommand(hidden bool) *cli.Command {
 	return &cli.Command{
 		Name:      "login",
-		Action:    cliutil.ErrorHandler(login),
+		Action:    cliutil.ConfiguredAction(login),
 		Usage:     "Generate a configuration file with your login details",
 		ArgsUsage: " ",
-		Flags: []cli.Flag{
-			&cli.StringFlag{
-				Name:   "url",
-				Hidden: true,
-			},
-		},
-		Hidden: hidden,
+		Hidden:    hidden,
 	}
 }
 
@@ -56,7 +50,7 @@ func login(c *cli.Context) error {
 		return err
 	}
 
-	resourceData, err := transfer.Run(
+	resourceData, err := token.RunTransfer(
 		loginURL,
 		"cert",
 		"callback",

cmd/cloudflared/tunnel/subcommand_context.go

@@ -223,13 +223,8 @@ func (sc *subcommandContext) delete(tunnelIDs []uuid.UUID) error {
 		if !tunnel.DeletedAt.IsZero() {
 			return fmt.Errorf("Tunnel %s has already been deleted", tunnel.ID)
 		}
-		// Check if tunnel has existing connections and if force flag is set, cleanup connections
-		if len(tunnel.Connections) > 0 {
-			if !forceFlagSet {
-				return fmt.Errorf("You can not delete tunnel %s because it has active connections. To see connections run the 'list' command. If you believe the tunnel is not active, you can use a -f / --force flag with this command.", id)
-			}
-
-			if err := client.CleanupConnections(tunnel.ID); err != nil {
+		if forceFlagSet {
+			if err := client.CleanupConnections(tunnel.ID, tunnelstore.NewCleanupParams()); err != nil {
 				return errors.Wrapf(err, "Error cleaning up connections for tunnel %s", tunnel.ID)
 			}
 		}
@@ -281,13 +276,24 @@ func (sc *subcommandContext) run(tunnelID uuid.UUID) error {
 }
 
 func (sc *subcommandContext) cleanupConnections(tunnelIDs []uuid.UUID) error {
+	params := tunnelstore.NewCleanupParams()
+	extraLog := ""
+	if connector := sc.c.String("connector-id"); connector != "" {
+		connectorID, err := uuid.Parse(connector)
+		if err != nil {
+			return errors.Wrapf(err, "%s is not a valid client ID (must be a UUID)", connector)
+		}
+		params.ForClient(connectorID)
+		extraLog = fmt.Sprintf(" for connector-id %s", connectorID.String())
+	}
+
 	client, err := sc.client()
 	if err != nil {
 		return err
 	}
 	for _, tunnelID := range tunnelIDs {
-		sc.log.Info().Msgf("Cleanup connection for tunnel %s", tunnelID)
-		if err := client.CleanupConnections(tunnelID); err != nil {
+		sc.log.Info().Msgf("Cleanup connection for tunnel %s%s", tunnelID, extraLog)
+		if err := client.CleanupConnections(tunnelID, params); err != nil {
 			sc.log.Error().Msgf("Error cleaning up connections for tunnel %v, error :%v", tunnelID, err)
 		}
 	}
@@ -348,6 +354,12 @@ func (sc *subcommandContext) findID(input string) (uuid.UUID, error) {
 // one Tunnelstore API call.
 func (sc *subcommandContext) findIDs(inputs []string) ([]uuid.UUID, error) {
 
+	// Shortcut without Tunnelstore call if we find that all inputs are already UUIDs.
+	uuids, err := convertNamesToUuids(inputs, make(map[string]uuid.UUID))
+	if err == nil {
+		return uuids, nil
+	}
+
 	// First, look up all tunnels the user has
 	filter := tunnelstore.NewFilter()
 	filter.NoDeleted()
@@ -367,7 +379,10 @@ func findIDs(tunnels []*tunnelstore.Tunnel, inputs []string) ([]uuid.UUID, error
 		nameToID[tunnel.Name] = tunnel.ID
 	}
 
-	// For each input, try to find the tunnel ID.
+	return convertNamesToUuids(inputs, nameToID)
+}
+
+func convertNamesToUuids(inputs []string, nameToID map[string]uuid.UUID) ([]uuid.UUID, error) {
 	tunnelIDs := make([]uuid.UUID, len(inputs))
 	var badInputs []string
 	for i, input := range inputs {

cmd/cloudflared/tunnel/subcommand_context_test.go

@@ -4,11 +4,12 @@ import (
 	"encoding/base64"
 	"flag"
 	"fmt"
-	"github.com/rs/zerolog"
 	"reflect"
 	"testing"
 	"time"
 
+	"github.com/rs/zerolog"
+
 	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/tunnelstore"
 	"github.com/google/uuid"
@@ -260,7 +261,7 @@ func (d *deleteMockTunnelStore) DeleteTunnel(tunnelID uuid.UUID) error {
 	return nil
 }
 
-func (d *deleteMockTunnelStore) CleanupConnections(tunnelID uuid.UUID) error {
+func (d *deleteMockTunnelStore) CleanupConnections(tunnelID uuid.UUID, _ *tunnelstore.CleanupParams) error {
 	tunnel, ok := d.mockTunnels[tunnelID]
 	if !ok {
 		return fmt.Errorf("Couldn't find tunnel: %v", tunnelID)

cmd/cloudflared/tunnel/subcommands.go

@@ -22,15 +22,17 @@ import (
 	"gopkg.in/yaml.v2"
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/updater"
+	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/tunnelstore"
 )
 
 const (
-	allSortByOptions  = "name, id, createdAt, deletedAt, numConnections"
-	CredFileFlagAlias = "cred-file"
-	CredFileFlag      = "credentials-file"
+	allSortByOptions   = "name, id, createdAt, deletedAt, numConnections"
+	connsSortByOptions = "id, startedAt, numConnections, version"
+	CredFileFlagAlias  = "cred-file"
+	CredFileFlag       = "credentials-file"
 
 	LogFieldTunnelID = "tunnelID"
 )
@@ -63,11 +65,11 @@ var (
 		Aliases: []string{"rd"},
 		Usage:   "Include connections that have recently disconnected in the list",
 	}
-	outputFormatFlag = altsrc.NewStringFlag(&cli.StringFlag{
+	outputFormatFlag = &cli.StringFlag{
 		Name:    "output",
 		Aliases: []string{"o"},
 		Usage:   "Render output using given `FORMAT`. Valid options are 'json' or 'yaml'",
-	})
+	}
 	sortByFlag = &cli.StringFlag{
 		Name:    "sort-by",
 		Value:   "name",
@@ -87,6 +89,11 @@ var (
 			"overwrite the previous tunnel. If you want to use a single hostname with multiple " +
 			"tunnels, you can do so with Cloudflare's Load Balancer product.",
 	})
+	featuresFlag = altsrc.NewStringSliceFlag(&cli.StringSliceFlag{
+		Name:    "features",
+		Aliases: []string{"F"},
+		Usage:   "Opt into various features that are still being developed or tested.",
+	})
 	credentialsFileFlag = altsrc.NewStringFlag(&cli.StringFlag{
 		Name:    CredFileFlag,
 		Aliases: []string{CredFileFlagAlias},
@@ -96,7 +103,8 @@ var (
 	forceDeleteFlag = &cli.BoolFlag{
 		Name:    "force",
 		Aliases: []string{"f"},
-		Usage:   "Allows you to delete a tunnel, even if it has active connections.",
+		Usage: "Cleans up any stale connections before the tunnel is deleted. cloudflared will not " +
+			"delete a tunnel with connections without this flag.",
 		EnvVars: []string{"TUNNEL_RUN_FORCE_OVERWRITE"},
 	}
 	selectProtocolFlag = altsrc.NewStringFlag(&cli.StringFlag{
@@ -107,12 +115,29 @@ var (
 		EnvVars: []string{"TUNNEL_TRANSPORT_PROTOCOL"},
 		Hidden:  true,
 	})
+	sortInfoByFlag = &cli.StringFlag{
+		Name:    "sort-by",
+		Value:   "createdAt",
+		Usage:   fmt.Sprintf("Sorts the list of connections of a tunnel by the given field. Valid options are {%s}", connsSortByOptions),
+		EnvVars: []string{"TUNNEL_INFO_SORT_BY"},
+	}
+	invertInfoSortFlag = &cli.BoolFlag{
+		Name:    "invert-sort",
+		Usage:   "Inverts the sort order of the tunnel info.",
+		EnvVars: []string{"TUNNEL_INFO_INVERT_SORT"},
+	}
+	cleanupClientFlag = &cli.StringFlag{
+		Name:    "connector-id",
+		Aliases: []string{"c"},
+		Usage:   `Constraints the cleanup to stop the connections of a single Connector (by its ID). You can find the various Connectors (and their IDs) currently connected to your tunnel via 'cloudflared tunnel info <name>'.`,
+		EnvVars: []string{"TUNNEL_CLEANUP_CONNECTOR"},
+	}
 )
 
 func buildCreateCommand() *cli.Command {
 	return &cli.Command{
 		Name:      "create",
-		Action:    cliutil.ErrorHandler(createCommand),
+		Action:    cliutil.ConfiguredAction(createCommand),
 		Usage:     "Create a new tunnel with given name",
 		UsageText: "cloudflared tunnel [tunnel command options] create [subcommand options] NAME",
 		Description: `Creates a tunnel, registers it with Cloudflare edge and generates credential file used to run this tunnel.
@@ -144,6 +169,9 @@ func createCommand(c *cli.Context) error {
 	}
 	name := c.Args().First()
 
+	warningChecker := updater.StartWarningCheck(c)
+	defer warningChecker.LogWarningIfAny(sc.log)
+
 	_, err = sc.create(name, c.String(CredFileFlag))
 	return errors.Wrap(err, "failed to create tunnel")
 }
@@ -180,7 +208,7 @@ func writeTunnelCredentials(
 func buildListCommand() *cli.Command {
 	return &cli.Command{
 		Name:        "list",
-		Action:      cliutil.ErrorHandler(listCommand),
+		Action:      cliutil.ConfiguredAction(listCommand),
 		Usage:       "List existing tunnels",
 		UsageText:   "cloudflared tunnel [tunnel command options] list [subcommand options]",
 		Description: "cloudflared tunnel list will display all active tunnels, their created time and associated connections. Use -d flag to include deleted tunnels. See the list of options to filter the list",
@@ -204,6 +232,9 @@ func listCommand(c *cli.Context) error {
 		return err
 	}
 
+	warningChecker := updater.StartWarningCheck(c)
+	defer warningChecker.LogWarningIfAny(sc.log)
+
 	filter := tunnelstore.NewFilter()
 	if !c.Bool("show-deleted") {
 		filter.NoDeleted()
@@ -266,21 +297,16 @@ func listCommand(c *cli.Context) error {
 	} else {
 		fmt.Println("You have no tunnels, use 'cloudflared tunnel create' to define a new tunnel")
 	}
+
 	return nil
 }
 
 func formatAndPrintTunnelList(tunnels []*tunnelstore.Tunnel, showRecentlyDisconnected bool) {
-	const (
-		minWidth = 0
-		tabWidth = 8
-		padding  = 1
-		padChar  = ' '
-		flags    = 0
-	)
-
-	writer := tabwriter.NewWriter(os.Stdout, minWidth, tabWidth, padding, padChar, flags)
+	writer := tabWriter()
 	defer writer.Flush()
 
+	_, _ = fmt.Fprintln(writer, "You can obtain more detailed information for each tunnel with `cloudflared tunnel info <name/uuid>`")
+
 	// Print column headers with tabbed columns
 	_, _ = fmt.Fprintln(writer, "ID\tNAME\tCREATED\tCONNECTIONS\t")
 
@@ -322,10 +348,156 @@ func fmtConnections(connections []tunnelstore.Connection, showRecentlyDisconnect
 	return strings.Join(output, ", ")
 }
 
+func buildInfoCommand() *cli.Command {
+	return &cli.Command{
+		Name:        "info",
+		Action:      cliutil.ConfiguredAction(tunnelInfo),
+		Usage:       "List details about the active connectors for a tunnel",
+		UsageText:   "cloudflared tunnel [tunnel command options] info [subcommand options] [TUNNEL]",
+		Description: "cloudflared tunnel info displays details about the active connectors for a given tunnel (identified by name or uuid).",
+		Flags: []cli.Flag{
+			outputFormatFlag,
+			showRecentlyDisconnected,
+			sortInfoByFlag,
+			invertInfoSortFlag,
+		},
+		CustomHelpTemplate: commandHelpTemplate(),
+	}
+}
+
+func tunnelInfo(c *cli.Context) error {
+	sc, err := newSubcommandContext(c)
+	if err != nil {
+		return err
+	}
+
+	warningChecker := updater.StartWarningCheck(c)
+	defer warningChecker.LogWarningIfAny(sc.log)
+
+	if c.NArg() > 1 {
+		return cliutil.UsageError(`"cloudflared tunnel info" accepts only one argument, the ID or name of the tunnel to run.`)
+	}
+	tunnelID, err := sc.findID(c.Args().First())
+	if err != nil {
+		return errors.Wrap(err, "error parsing tunnel ID")
+	}
+
+	client, err := sc.client()
+	if err != nil {
+		return err
+	}
+
+	clients, err := client.ListActiveClients(tunnelID)
+	if err != nil {
+		return err
+	}
+
+	sortBy := c.String("sort-by")
+	invalidSortField := false
+	sort.Slice(clients, func(i, j int) bool {
+		cmp := func() bool {
+			switch sortBy {
+			case "id":
+				return clients[i].ID.String() < clients[j].ID.String()
+			case "createdAt":
+				return clients[i].RunAt.Unix() < clients[j].RunAt.Unix()
+			case "numConnections":
+				return len(clients[i].Connections) < len(clients[j].Connections)
+			case "version":
+				return clients[i].Version < clients[j].Version
+			default:
+				invalidSortField = true
+				return clients[i].RunAt.Unix() < clients[j].RunAt.Unix()
+			}
+		}()
+		if c.Bool("invert-sort") {
+			return !cmp
+		}
+		return cmp
+	})
+	if invalidSortField {
+		sc.log.Error().Msgf("%s is not a valid sort field. Valid sort fields are %s. Defaulting to 'name'.", sortBy, connsSortByOptions)
+	}
+
+	tunnel, err := getTunnel(sc, tunnelID)
+	if err != nil {
+		return err
+	}
+	info := Info{
+		tunnel.ID,
+		tunnel.Name,
+		tunnel.CreatedAt,
+		clients,
+	}
+
+	if outputFormat := c.String(outputFormatFlag.Name); outputFormat != "" {
+		return renderOutput(outputFormat, info)
+	}
+
+	if len(clients) > 0 {
+		formatAndPrintConnectionsList(info, c.Bool("show-recently-disconnected"))
+	} else {
+		fmt.Printf("Your tunnel %s does not have any active connection.\n", tunnelID)
+	}
+
+	return nil
+}
+
+func getTunnel(sc *subcommandContext, tunnelID uuid.UUID) (*tunnelstore.Tunnel, error) {
+	filter := tunnelstore.NewFilter()
+	filter.ByTunnelID(tunnelID)
+	tunnels, err := sc.list(filter)
+	if err != nil {
+		return nil, err
+	}
+	if len(tunnels) != 1 {
+		return nil, errors.Errorf("Expected to find a single tunnel with uuid %v but found %d tunnels.", tunnelID, len(tunnels))
+	}
+	return tunnels[0], nil
+}
+
+func formatAndPrintConnectionsList(tunnelInfo Info, showRecentlyDisconnected bool) {
+	writer := tabWriter()
+	defer writer.Flush()
+
+	_, _ = fmt.Fprintf(writer, "NAME:     %s\nID:       %s\nCREATED:  %s\n\n", tunnelInfo.Name, tunnelInfo.ID, tunnelInfo.CreatedAt)
+
+	_, _ = fmt.Fprintln(writer, "CONNECTOR ID\tCREATED\tARCHITECTURE\tVERSION\tORIGIN IP\tEDGE\t")
+	for _, c := range tunnelInfo.Connectors {
+		var originIp = ""
+		if len(c.Connections) > 0 {
+			originIp = c.Connections[0].OriginIP.String()
+		}
+		formattedStr := fmt.Sprintf(
+			"%s\t%s\t%s\t%s\t%s\t%s\t",
+			c.ID,
+			c.RunAt.Format(time.RFC3339),
+			c.Arch,
+			c.Version,
+			originIp,
+			fmtConnections(c.Connections, showRecentlyDisconnected),
+		)
+		_, _ = fmt.Fprintln(writer, formattedStr)
+	}
+}
+
+func tabWriter() *tabwriter.Writer {
+	const (
+		minWidth = 0
+		tabWidth = 8
+		padding  = 1
+		padChar  = ' '
+		flags    = 0
+	)
+
+	writer := tabwriter.NewWriter(os.Stdout, minWidth, tabWidth, padding, padChar, flags)
+	return writer
+}
+
 func buildDeleteCommand() *cli.Command {
 	return &cli.Command{
 		Name:               "delete",
-		Action:             cliutil.ErrorHandler(deleteCommand),
+		Action:             cliutil.ConfiguredAction(deleteCommand),
 		Usage:              "Delete existing tunnel by UUID or name",
 		UsageText:          "cloudflared tunnel [tunnel command options] delete [subcommand options] TUNNEL",
 		Description:        "cloudflared tunnel delete will delete tunnels with the given tunnel UUIDs or names. A tunnel cannot be deleted if it has active connections. To delete the tunnel unconditionally, use -f flag.",
@@ -344,6 +516,9 @@ func deleteCommand(c *cli.Context) error {
 		return cliutil.UsageError(`"cloudflared tunnel delete" requires at least 1 argument, the ID or name of the tunnel to delete.`)
 	}
 
+	warningChecker := updater.StartWarningCheck(c)
+	defer warningChecker.LogWarningIfAny(sc.log)
+
 	tunnelIDs, err := sc.findIDs(c.Args().Slice())
 	if err != nil {
 		return err
@@ -370,12 +545,12 @@ func buildRunCommand() *cli.Command {
 		forceFlag,
 		credentialsFileFlag,
 		selectProtocolFlag,
+		featuresFlag,
 	}
 	flags = append(flags, configureProxyFlags(false)...)
 	return &cli.Command{
 		Name:      "run",
-		Action:    cliutil.ErrorHandler(runCommand),
-		Before:    SetFlagsFromConfigFile,
+		Action:    cliutil.ConfiguredAction(runCommand),
 		Usage:     "Proxy a local web server by running the given tunnel",
 		UsageText: "cloudflared tunnel [tunnel command options] run [subcommand options] [TUNNEL]",
 		Description: `Runs the tunnel identified by name or UUUD, creating highly available connections
@@ -427,10 +602,11 @@ func runNamedTunnel(sc *subcommandContext, tunnelRef string) error {
 func buildCleanupCommand() *cli.Command {
 	return &cli.Command{
 		Name:               "cleanup",
-		Action:             cliutil.ErrorHandler(cleanupCommand),
+		Action:             cliutil.ConfiguredAction(cleanupCommand),
 		Usage:              "Cleanup tunnel connections",
 		UsageText:          "cloudflared tunnel [tunnel command options] cleanup [subcommand options] TUNNEL",
 		Description:        "Delete connections for tunnels with the given UUIDs or names.",
+		Flags:              []cli.Flag{cleanupClientFlag},
 		CustomHelpTemplate: commandHelpTemplate(),
 	}
 }
@@ -456,15 +632,24 @@ func cleanupCommand(c *cli.Context) error {
 func buildRouteCommand() *cli.Command {
 	return &cli.Command{
 		Name:      "route",
-		Action:    cliutil.ErrorHandler(routeCommand),
-		Usage:     "Define what hostname or load balancer can route to this tunnel",
-		UsageText: "cloudflared tunnel [tunnel command options] route [subcommand options] dns|lb TUNNEL HOSTNAME [LB-POOL]",
-		Description: `The route defines what hostname or load balancer will proxy requests to this tunnel.
+		Action:    cliutil.ConfiguredAction(routeCommand),
+		Usage:     "Define which traffic routed from Cloudflare edge to this tunnel: requests to a DNS hostname, to a Cloudflare Load Balancer, or traffic originating from Cloudflare WARP clients",
+		UsageText: "cloudflared tunnel [tunnel command options] route [subcommand options] [dns TUNNEL HOSTNAME]|[lb TUNNEL HOSTNAME LB-POOL]|[ip NETWORK TUNNEL]",
+		Description: `The route command defines how Cloudflare will proxy requests to this tunnel.
 
-   To route a hostname by creating a CNAME to tunnel's address:
-      cloudflared tunnel route dns <tunnel ID> <hostname>
-   To use this tunnel as a load balancer origin, creating pool and load balancer if necessary:
-      cloudflared tunnel route lb <tunnel ID> <load balancer name> <load balancer pool>`,
+To route a hostname by creating a DNS CNAME record to a tunnel:
+   cloudflared tunnel route dns <tunnel ID or name> <hostname>
+You can read more at: https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/routing-to-tunnel/dns
+
+To use this tunnel as a load balancer origin, creating pool and load balancer if necessary:
+   cloudflared tunnel route lb <tunnel ID or name> <hostname> <load balancer pool>
+You can read more at: https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/routing-to-tunnel/lb
+
+For Cloudflare WARP traffic to be routed to your private network, reachable from this tunnel as origins, use:
+   cloudflared tunnel route ip <network CIDR> <tunnel ID or name>
+Further information about managing Cloudflare WARP traffic to your tunnel is available at:
+   cloudflared tunnel route ip --help
+`,
 		CustomHelpTemplate: commandHelpTemplate(),
 		Subcommands: []*cli.Command{
 			buildRouteIPSubcommand(),

cmd/cloudflared/tunnel/teamnet_subcommands.go

@@ -6,54 +6,57 @@ import (
 	"os"
 	"text/tabwriter"
 
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
-	"github.com/cloudflare/cloudflared/teamnet"
 	"github.com/pkg/errors"
 
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/updater"
+	"github.com/cloudflare/cloudflared/teamnet"
+
 	"github.com/urfave/cli/v2"
 )
 
 func buildRouteIPSubcommand() *cli.Command {
 	return &cli.Command{
 		Name:      "ip",
-		Category:  "Tunnel",
-		Usage:     "Configure and query private routes",
+		Usage:     "Configure and query Cloudflare WARP routing to services or private networks available through this tunnel.",
 		UsageText: "cloudflared tunnel [--config FILEPATH] route COMMAND [arguments...]",
-		Hidden:    true,
-		Description: `cloudflared can provision private routes from your private IP space to origins
-		in your corporate network. Users enrolled in your Cloudflare for Teams organization can reach
-		those routes through the Cloudflare Warp client. You can also build rules to determine who
-		can reach certain routes.
-		`,
+		Description: `cloudflared can provision private routes from any IP space to origins in your corporate network.
+Users enrolled in your Cloudflare for Teams organization can reach those routes through the
+Cloudflare WARP client. You can also build rules to determine who can reach certain routes.`,
 		Subcommands: []*cli.Command{
 			{
 				Name:      "add",
-				Action:    cliutil.ErrorHandler(addRouteCommand),
-				Usage:     "Add a new Teamnet route to the table",
+				Action:    cliutil.ConfiguredAction(addRouteCommand),
+				Usage:     "Add any new network to the routing table reachable via the tunnel",
 				UsageText: "cloudflared tunnel [--config FILEPATH] route ip add [CIDR] [TUNNEL] [COMMENT?]",
-				Description: `Adds a private route to a CIDR in your private IP space. Requests will
-				be sent through the Cloudflare Warp client running on a user's machine, proxied
-				through the specified tunnel, and reach an IP in the given CIDR.`,
+				Description: `Adds any network route space (represented as a CIDR) to your routing table.
+That network space becomes reachable for requests egressing from a user's machine
+as long as it is using Cloudflare WARP client and is enrolled in the same account
+that is running the tunnel chosen here. Further, those requests will be proxied to
+the specified tunnel, and reach an IP in the given CIDR, as long as that IP is
+reachable from the tunnel.`,
 			},
 			{
 				Name:        "show",
-				Action:      cliutil.ErrorHandler(showRoutesCommand),
+				Aliases:     []string{"list"},
+				Action:      cliutil.ConfiguredAction(showRoutesCommand),
 				Usage:       "Show the routing table",
 				UsageText:   "cloudflared tunnel [--config FILEPATH] route ip show [flags]",
-				Description: `Shows your organization's private route table. You can use flags to filter the results.`,
+				Description: `Shows your organization private routing table. You can use flags to filter the results.`,
 				Flags:       showRoutesFlags(),
 			},
 			{
-				Name:        "delete",
-				Action:      cliutil.ErrorHandler(deleteRouteCommand),
-				Usage:       "Delete a row from your organization's private routing table",
-				UsageText:   "cloudflared tunnel [--config FILEPATH] route ip delete [CIDR]",
-				Description: `Deletes the row for a given CIDR from your routing table`,
+				Name:      "delete",
+				Action:    cliutil.ConfiguredAction(deleteRouteCommand),
+				Usage:     "Delete a row from your organization's private routing table",
+				UsageText: "cloudflared tunnel [--config FILEPATH] route ip delete [CIDR]",
+				Description: `Deletes the row for a given CIDR from your routing table. That portion
+of your network will no longer be reachable by the WARP clients.`,
 			},
 			{
 				Name:      "get",
-				Action:    cliutil.ErrorHandler(getRouteByIPCommand),
-				Usage:     "Check which row of the routing table matches a given IP",
+				Action:    cliutil.ConfiguredAction(getRouteByIPCommand),
+				Usage:     "Check which row of the routing table matches a given IP.",
 				UsageText: "cloudflared tunnel [--config FILEPATH] route ip get [IP]",
 				Description: `Checks which row of the routing table will be used to proxy a given IP.
 				This helps check and validate your config.`,
@@ -80,6 +83,9 @@ func showRoutesCommand(c *cli.Context) error {
 		return errors.Wrap(err, "invalid config for routing filters")
 	}
 
+	warningChecker := updater.StartWarningCheck(c)
+	defer warningChecker.LogWarningIfAny(sc.log)
+
 	routes, err := sc.listRoutes(filter)
 	if err != nil {
 		return err
@@ -94,6 +100,7 @@ func showRoutesCommand(c *cli.Context) error {
 	} else {
 		fmt.Println("You have no routes, use 'cloudflared tunnel route ip add' to add a route")
 	}
+
 	return nil
 }
 

cmd/cloudflared/updater/check.go

@@ -0,0 +1,49 @@
+package updater
+
+import (
+	"github.com/rs/zerolog"
+	"github.com/urfave/cli/v2"
+)
+
+type VersionWarningChecker struct {
+	warningChan chan string
+}
+
+func StartWarningCheck(c *cli.Context) VersionWarningChecker {
+	checker := VersionWarningChecker{
+		warningChan: make(chan string),
+	}
+
+	go func() {
+		options := updateOptions{
+			updateDisabled:  true,
+			isBeta:          c.Bool("beta"),
+			isStaging:       c.Bool("staging"),
+			isForced:        false,
+			intendedVersion: "",
+		}
+		checkResult, err := CheckForUpdate(options)
+		if err == nil {
+			checker.warningChan <- checkResult.UserMessage()
+		}
+		close(checker.warningChan)
+	}()
+
+	return checker
+}
+
+func (checker VersionWarningChecker) getWarning() string {
+	select {
+	case message := <-checker.warningChan:
+		return message
+	default:
+		// No feedback on time, we don't wait for it, since this is best-effort.
+		return ""
+	}
+}
+
+func (checker VersionWarningChecker) LogWarningIfAny(log *zerolog.Logger) {
+	if warning := checker.getWarning(); warning != "" {
+		log.Warn().Msg(warning)
+	}
+}

cmd/cloudflared/updater/service.go

@@ -1,14 +1,15 @@
 package updater
 
-// Version is the functions needed to perform an update
-type Version interface {
+// CheckResult is the behaviour resulting from checking in with the Update Service
+type CheckResult interface {
 	Apply() error
-	String() string
+	Version() string
+	UserMessage() string
 }
 
 // Service is the functions to get check for new updates
 type Service interface {
-	Check() (Version, error)
+	Check() (CheckResult, error)
 }
 
 const (
@@ -23,4 +24,7 @@ const (
 
 	// VersionKeyName is the url parameter key to send to the checkin API to specific what version to upgrade or downgrade to
 	VersionKeyName = "version"
+
+	// ClientVersionName is the url parameter key to send the version that this cloudflared is currently running with
+	ClientVersionName = "clientVersion"
 )

cmd/cloudflared/updater/update.go

@@ -3,18 +3,18 @@ package updater
 import (
 	"context"
 	"fmt"
-	"github.com/rs/zerolog"
 	"os"
 	"path/filepath"
 	"runtime"
 	"time"
 
+	"github.com/facebookgo/grace/gracenet"
+	"github.com/rs/zerolog"
 	"github.com/urfave/cli/v2"
 	"golang.org/x/crypto/ssh/terminal"
 
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/logger"
-	"github.com/facebookgo/grace/gracenet"
 )
 
 const (
@@ -61,16 +61,18 @@ func (e *statusErr) ExitCode() int {
 }
 
 type updateOptions struct {
-	isBeta    bool
-	isStaging bool
-	isForced  bool
-	version   string
+	updateDisabled  bool
+	isBeta          bool
+	isStaging       bool
+	isForced        bool
+	intendedVersion string
 }
 
 type UpdateOutcome struct {
-	Updated bool
-	Version string
-	Error   error
+	Updated     bool
+	Version     string
+	UserMessage string
+	Error       error
 }
 
 func (uo *UpdateOutcome) noUpdate() bool {
@@ -81,10 +83,10 @@ func Init(v string) {
 	version = v
 }
 
-func checkForUpdateAndApply(options updateOptions) UpdateOutcome {
+func CheckForUpdate(options updateOptions) (CheckResult, error) {
 	cfdPath, err := os.Executable()
 	if err != nil {
-		return UpdateOutcome{Error: err}
+		return nil, err
 	}
 
 	url := UpdateURL
@@ -93,24 +95,22 @@ func checkForUpdateAndApply(options updateOptions) UpdateOutcome {
 	}
 
 	s := NewWorkersService(version, url, cfdPath, Options{IsBeta: options.isBeta,
-		IsForced: options.isForced, RequestedVersion: options.version})
+		IsForced: options.isForced, RequestedVersion: options.intendedVersion})
 
-	v, err := s.Check()
+	return s.Check()
+}
+
+func applyUpdate(options updateOptions, update CheckResult) UpdateOutcome {
+	if update.Version() == "" || options.updateDisabled {
+		return UpdateOutcome{UserMessage: update.UserMessage()}
+	}
+
+	err := update.Apply()
 	if err != nil {
 		return UpdateOutcome{Error: err}
 	}
 
-	//already on the latest version
-	if v == nil {
-		return UpdateOutcome{}
-	}
-
-	err = v.Apply()
-	if err != nil {
-		return UpdateOutcome{Error: err}
-	}
-
-	return UpdateOutcome{Updated: true, Version: v.String()}
+	return UpdateOutcome{Updated: true, Version: update.Version(), UserMessage: update.UserMessage()}
 }
 
 // Update is the handler for the update command from the command line
@@ -137,7 +137,13 @@ func Update(c *cli.Context) error {
 		log.Info().Msg("cloudflared is set to upgrade to the latest publish version regardless of the current version")
 	}
 
-	updateOutcome := loggedUpdate(log, updateOptions{isBeta: isBeta, isStaging: isStaging, isForced: isForced, version: c.String("version")})
+	updateOutcome := loggedUpdate(log, updateOptions{
+		updateDisabled:  false,
+		isBeta:          isBeta,
+		isStaging:       isStaging,
+		isForced:        isForced,
+		intendedVersion: c.String("version"),
+	})
 	if updateOutcome.Error != nil {
 		return &statusErr{updateOutcome.Error}
 	}
@@ -152,12 +158,18 @@ func Update(c *cli.Context) error {
 
 // Checks for an update and applies it if one is available
 func loggedUpdate(log *zerolog.Logger, options updateOptions) UpdateOutcome {
-	updateOutcome := checkForUpdateAndApply(options)
+	checkResult, err := CheckForUpdate(options)
+	if err != nil {
+		log.Err(err).Msg("update check failed")
+		return UpdateOutcome{Error: err}
+	}
+
+	updateOutcome := applyUpdate(options, checkResult)
 	if updateOutcome.Updated {
 		log.Info().Str(LogFieldVersion, updateOutcome.Version).Msg("cloudflared has been updated")
 	}
 	if updateOutcome.Error != nil {
-		log.Err(updateOutcome.Error).Msg("update check failed: %s")
+		log.Err(updateOutcome.Error).Msg("update failed to apply")
 	}
 
 	return updateOutcome
@@ -177,44 +189,53 @@ type configurable struct {
 	freq    time.Duration
 }
 
-func NewAutoUpdater(freq time.Duration, listeners *gracenet.Net, log *zerolog.Logger) *AutoUpdater {
-	updaterConfigurable := &configurable{
-		enabled: true,
-		freq:    freq,
-	}
-	if freq == 0 {
-		updaterConfigurable.enabled = false
-		updaterConfigurable.freq = DefaultCheckUpdateFreq
-	}
+func NewAutoUpdater(updateDisabled bool, freq time.Duration, listeners *gracenet.Net, log *zerolog.Logger) *AutoUpdater {
 	return &AutoUpdater{
-		configurable:     updaterConfigurable,
+		configurable:     createUpdateConfig(updateDisabled, freq, log),
 		listeners:        listeners,
 		updateConfigChan: make(chan *configurable),
 		log:              log,
 	}
 }
 
+func createUpdateConfig(updateDisabled bool, freq time.Duration, log *zerolog.Logger) *configurable {
+	if isAutoupdateEnabled(log, updateDisabled, freq) {
+		log.Info().Dur("autoupdateFreq", freq).Msg("Autoupdate frequency is set")
+		return &configurable{
+			enabled: true,
+			freq:    freq,
+		}
+	} else {
+		return &configurable{
+			enabled: false,
+			freq:    DefaultCheckUpdateFreq,
+		}
+	}
+}
+
 func (a *AutoUpdater) Run(ctx context.Context) error {
 	ticker := time.NewTicker(a.configurable.freq)
 	for {
-		if a.configurable.enabled {
-			updateOutcome := loggedUpdate(a.log, updateOptions{})
-			if updateOutcome.Updated {
-				if IsSysV() {
-					// SysV doesn't have a mechanism to keep service alive, we have to restart the process
-					a.log.Info().Msg("Restarting service managed by SysV...")
-					pid, err := a.listeners.StartProcess()
-					if err != nil {
-						a.log.Err(err).Msg("Unable to restart server automatically")
-						return &statusErr{err: err}
-					}
-					// stop old process after autoupdate. Otherwise we create a new process
-					// after each update
-					a.log.Info().Msgf("PID of the new process is %d", pid)
+		updateOutcome := loggedUpdate(a.log, updateOptions{updateDisabled: !a.configurable.enabled})
+		if updateOutcome.Updated {
+			Init(updateOutcome.Version)
+			if IsSysV() {
+				// SysV doesn't have a mechanism to keep service alive, we have to restart the process
+				a.log.Info().Msg("Restarting service managed by SysV...")
+				pid, err := a.listeners.StartProcess()
+				if err != nil {
+					a.log.Err(err).Msg("Unable to restart server automatically")
+					return &statusErr{err: err}
 				}
-				return &statusSuccess{newVersion: updateOutcome.Version}
+				// stop old process after autoupdate. Otherwise we create a new process
+				// after each update
+				a.log.Info().Msgf("PID of the new process is %d", pid)
 			}
+			return &statusSuccess{newVersion: updateOutcome.Version}
+		} else if updateOutcome.UserMessage != "" {
+			a.log.Warn().Msg(updateOutcome.UserMessage)
 		}
+
 		select {
 		case <-ctx.Done():
 			return ctx.Err()
@@ -229,27 +250,18 @@ func (a *AutoUpdater) Run(ctx context.Context) error {
 }
 
 // Update is the method to pass new AutoUpdaterConfigurable to a running AutoUpdater. It is safe to be called concurrently
-func (a *AutoUpdater) Update(newFreq time.Duration) {
-	newConfigurable := &configurable{
-		enabled: true,
-		freq:    newFreq,
-	}
-	// A ero duration means autoupdate is disabled
-	if newFreq == 0 {
-		newConfigurable.enabled = false
-		newConfigurable.freq = DefaultCheckUpdateFreq
-	}
-	a.updateConfigChan <- newConfigurable
+func (a *AutoUpdater) Update(updateDisabled bool, newFreq time.Duration) {
+	a.updateConfigChan <- createUpdateConfig(updateDisabled, newFreq, a.log)
 }
 
-func IsAutoupdateEnabled(c *cli.Context, log *zerolog.Logger) bool {
-	if !SupportAutoUpdate(log) {
+func isAutoupdateEnabled(log *zerolog.Logger, updateDisabled bool, updateFreq time.Duration) bool {
+	if !supportAutoUpdate(log) {
 		return false
 	}
-	return !c.Bool("no-autoupdate") && c.Duration("autoupdate-freq") != 0
+	return !updateDisabled && updateFreq != 0
 }
 
-func SupportAutoUpdate(log *zerolog.Logger) bool {
+func supportAutoUpdate(log *zerolog.Logger) bool {
 	if runtime.GOOS == "windows" {
 		log.Info().Msg(noUpdateOnWindowsMessage)
 		return false

cmd/cloudflared/updater/update_test.go

@@ -2,17 +2,19 @@ package updater
 
 import (
 	"context"
+	"flag"
 	"testing"
 
 	"github.com/facebookgo/grace/gracenet"
 	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/assert"
+	"github.com/urfave/cli/v2"
 )
 
 func TestDisabledAutoUpdater(t *testing.T) {
 	listeners := &gracenet.Net{}
 	log := zerolog.Nop()
-	autoupdater := NewAutoUpdater(0, listeners, &log)
+	autoupdater := NewAutoUpdater(false, 0, listeners, &log)
 	ctx, cancel := context.WithCancel(context.Background())
 	errC := make(chan error)
 	go func() {
@@ -26,3 +28,13 @@ func TestDisabledAutoUpdater(t *testing.T) {
 	// Make sure that autoupdater terminates after canceling the context
 	assert.Equal(t, context.Canceled, <-errC)
 }
+
+func TestCheckInWithUpdater(t *testing.T) {
+	flagSet := flag.NewFlagSet(t.Name(), flag.PanicOnError)
+	cliCtx := cli.NewContext(cli.NewApp(), flagSet, nil)
+
+	warningChecker := StartWarningCheck(cliCtx)
+	warning := warningChecker.getWarning()
+	// Assuming this runs either on a release or development version, then the Worker will never have anything to tell us.
+	assert.Empty(t, warning)
+}

cmd/cloudflared/updater/workers_service.go

@@ -5,8 +5,6 @@ import (
 	"errors"
 	"net/http"
 	"runtime"
-	"strconv"
-	"strings"
 )
 
 // Options are the update options supported by the
@@ -27,6 +25,8 @@ type VersionResponse struct {
 	Version      string `json:"version"`
 	Checksum     string `json:"checksum"`
 	IsCompressed bool   `json:"compressed"`
+	UserMessage  string `json:"userMessage"`
+	ShouldUpdate bool   `json:"shouldUpdate"`
 	Error        string `json:"error"`
 }
 
@@ -50,7 +50,7 @@ func NewWorkersService(currentVersion, url, targetPath string, opts Options) Ser
 }
 
 // Check does a check in with the Workers API to get a new version update
-func (s *WorkersService) Check() (Version, error) {
+func (s *WorkersService) Check() (CheckResult, error) {
 	client := &http.Client{
 		Timeout: clientTimeout,
 	}
@@ -59,6 +59,7 @@ func (s *WorkersService) Check() (Version, error) {
 	q := req.URL.Query()
 	q.Add(OSKeyName, runtime.GOOS)
 	q.Add(ArchitectureKeyName, runtime.GOARCH)
+	q.Add(ClientVersionName, s.currentVersion)
 
 	if s.opts.IsBeta {
 		q.Add(BetaKeyName, "true")
@@ -84,70 +85,10 @@ func (s *WorkersService) Check() (Version, error) {
 		return nil, errors.New(v.Error)
 	}
 
-	if !s.opts.IsForced && !IsNewerVersion(s.currentVersion, v.Version) {
-		return nil, nil
+	versionToUpdate := ""
+	if v.ShouldUpdate {
+		versionToUpdate = v.Version
 	}
 
-	return NewWorkersVersion(v.URL, v.Version, v.Checksum, s.targetPath, v.IsCompressed), nil
-}
-
-// IsNewerVersion checks semantic versioning for the latest version
-// cloudflared tagging is more of a date than a semantic version,
-// but the same comparision logic still holds for major.minor.patch
-// e.g. 2020.8.2 is newer than 2020.8.1.
-func IsNewerVersion(current string, check string) bool {
-	if strings.Contains(strings.ToLower(current), "dev") {
-		return false // dev builds shouldn't update
-	}
-
-	cMajor, cMinor, cPatch, err := SemanticParts(current)
-	if err != nil {
-		return false
-	}
-
-	nMajor, nMinor, nPatch, err := SemanticParts(check)
-	if err != nil {
-		return false
-	}
-
-	if nMajor > cMajor {
-		return true
-	}
-
-	if nMajor == cMajor && nMinor > cMinor {
-		return true
-	}
-
-	if nMajor == cMajor && nMinor == cMinor && nPatch > cPatch {
-		return true
-	}
-	return false
-}
-
-// SemanticParts gets the major, minor, and patch version of a semantic version string
-// e.g. 3.1.2 would return 3, 1, 2, nil
-func SemanticParts(version string) (major int, minor int, patch int, err error) {
-	major = 0
-	minor = 0
-	patch = 0
-	parts := strings.Split(version, ".")
-	if len(parts) != 3 {
-		err = errors.New("invalid version")
-		return
-	}
-	major, err = strconv.Atoi(parts[0])
-	if err != nil {
-		return
-	}
-
-	minor, err = strconv.Atoi(parts[1])
-	if err != nil {
-		return
-	}
-
-	patch, err = strconv.Atoi(parts[2])
-	if err != nil {
-		return
-	}
-	return
+	return NewWorkersVersion(v.URL, versionToUpdate, v.Checksum, s.targetPath, v.UserMessage, v.IsCompressed), nil
 }

cmd/cloudflared/updater/workers_service_test.go

@@ -6,6 +6,7 @@ import (
 	"compress/gzip"
 	"crypto/sha256"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"log"
@@ -13,6 +14,8 @@ import (
 	"net/http/httptest"
 	"os"
 	"runtime"
+	"strconv"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -32,15 +35,20 @@ func respondWithData(w http.ResponseWriter, b []byte, status int) {
 	w.Write(b)
 }
 
+const mostRecentVersion = "2021.2.5"
+const mostRecentBetaVersion = "2021.3.0"
+const knownBuggyVersion = "2020.12.0"
+const expectedUserMsg = "This message is expected when running a known buggy version"
+
 func updateHandler(w http.ResponseWriter, r *http.Request) {
-	version := "2020.08.05"
+	version := mostRecentVersion
 	host := fmt.Sprintf("http://%s", r.Host)
 	url := host + "/download"
 
 	query := r.URL.Query()
 
 	if query.Get(BetaKeyName) == "true" {
-		version = "2020.08.06"
+		version = mostRecentBetaVersion
 		url = host + "/beta"
 	}
 
@@ -59,7 +67,15 @@ func updateHandler(w http.ResponseWriter, r *http.Request) {
 	fmt.Fprint(h, version)
 	checksum := fmt.Sprintf("%x", h.Sum(nil))
 
-	v := VersionResponse{URL: url, Version: version, Checksum: checksum}
+	var userMessage = ""
+	if query.Get(ClientVersionName) == knownBuggyVersion {
+		userMessage = expectedUserMsg
+	}
+	shouldUpdate := requestedVersion != "" || IsNewerVersion(query.Get(ClientVersionName), version)
+
+	v := VersionResponse{
+		URL: url, Version: version, Checksum: checksum, UserMessage: userMessage, ShouldUpdate: shouldUpdate,
+	}
 	respondWithJSON(w, v, http.StatusOK)
 }
 
@@ -71,7 +87,7 @@ func gzipUpdateHandler(w http.ResponseWriter, r *http.Request) {
 	checksum := fmt.Sprintf("%x", h.Sum(nil))
 
 	url := fmt.Sprintf("http://%s/gzip-download.tgz", r.Host)
-	v := VersionResponse{URL: url, Version: version, Checksum: checksum}
+	v := VersionResponse{URL: url, Version: version, Checksum: checksum, ShouldUpdate: true}
 	respondWithJSON(w, v, http.StatusOK)
 }
 
@@ -96,7 +112,7 @@ func compressedDownloadHandler(w http.ResponseWriter, r *http.Request) {
 }
 
 func downloadHandler(w http.ResponseWriter, r *http.Request) {
-	version := "2020.08.05"
+	version := mostRecentVersion
 	requestedVersion := r.URL.Query().Get(VersionKeyName)
 	if requestedVersion != "" {
 		version = requestedVersion
@@ -105,13 +121,71 @@ func downloadHandler(w http.ResponseWriter, r *http.Request) {
 }
 
 func betaHandler(w http.ResponseWriter, r *http.Request) {
-	respondWithData(w, []byte("2020.08.06"), http.StatusOK)
+	respondWithData(w, []byte(mostRecentBetaVersion), http.StatusOK)
 }
 
 func failureHandler(w http.ResponseWriter, r *http.Request) {
 	respondWithJSON(w, VersionResponse{Error: "unsupported os and architecture"}, http.StatusBadRequest)
 }
 
+func IsNewerVersion(current string, check string) bool {
+	if current == "" || check == "" {
+		return false
+	}
+	if strings.Contains(strings.ToLower(current), "dev") {
+		return false // dev builds shouldn't update
+	}
+
+	cMajor, cMinor, cPatch, err := SemanticParts(current)
+	if err != nil {
+		return false
+	}
+
+	nMajor, nMinor, nPatch, err := SemanticParts(check)
+	if err != nil {
+		return false
+	}
+
+	if nMajor > cMajor {
+		return true
+	}
+
+	if nMajor == cMajor && nMinor > cMinor {
+		return true
+	}
+
+	if nMajor == cMajor && nMinor == cMinor && nPatch > cPatch {
+		return true
+	}
+	return false
+}
+
+func SemanticParts(version string) (major int, minor int, patch int, err error) {
+	major = 0
+	minor = 0
+	patch = 0
+	parts := strings.Split(version, ".")
+	if len(parts) != 3 {
+		err = errors.New("invalid version")
+		return
+	}
+	major, err = strconv.Atoi(parts[0])
+	if err != nil {
+		return
+	}
+
+	minor, err = strconv.Atoi(parts[1])
+	if err != nil {
+		return
+	}
+
+	patch, err = strconv.Atoi(parts[2])
+	if err != nil {
+		return
+	}
+	return
+}
+
 func createServer() *httptest.Server {
 	mux := http.NewServeMux()
 	mux.HandleFunc("/updater", updateHandler)
@@ -124,7 +198,7 @@ func createServer() *httptest.Server {
 }
 
 func createTestFile(t *testing.T, path string) {
-	f, err := os.Create("tmpfile")
+	f, err := os.Create(path)
 	require.NoError(t, err)
 	fmt.Fprint(f, "2020.08.04")
 	f.Close()
@@ -142,13 +216,13 @@ func TestUpdateService(t *testing.T) {
 	s := NewWorkersService("2020.8.2", fmt.Sprintf("%s/updater", ts.URL), testFilePath, Options{})
 	v, err := s.Check()
 	require.NoError(t, err)
-	require.Equal(t, v.String(), "2020.08.05")
+	require.Equal(t, v.Version(), mostRecentVersion)
 
 	require.NoError(t, v.Apply())
 	dat, err := ioutil.ReadFile(testFilePath)
 	require.NoError(t, err)
 
-	require.Equal(t, string(dat), "2020.08.05")
+	require.Equal(t, string(dat), mostRecentVersion)
 }
 
 func TestBetaUpdateService(t *testing.T) {
@@ -162,13 +236,13 @@ func TestBetaUpdateService(t *testing.T) {
 	s := NewWorkersService("2020.8.2", fmt.Sprintf("%s/updater", ts.URL), testFilePath, Options{IsBeta: true})
 	v, err := s.Check()
 	require.NoError(t, err)
-	require.Equal(t, v.String(), "2020.08.06")
+	require.Equal(t, v.Version(), mostRecentBetaVersion)
 
 	require.NoError(t, v.Apply())
 	dat, err := ioutil.ReadFile(testFilePath)
 	require.NoError(t, err)
 
-	require.Equal(t, string(dat), "2020.08.06")
+	require.Equal(t, string(dat), mostRecentBetaVersion)
 }
 
 func TestFailUpdateService(t *testing.T) {
@@ -193,10 +267,11 @@ func TestNoUpdateService(t *testing.T) {
 	createTestFile(t, testFilePath)
 	defer os.Remove(testFilePath)
 
-	s := NewWorkersService("2020.8.5", fmt.Sprintf("%s/updater", ts.URL), testFilePath, Options{})
+	s := NewWorkersService(mostRecentVersion, fmt.Sprintf("%s/updater", ts.URL), testFilePath, Options{})
 	v, err := s.Check()
 	require.NoError(t, err)
-	require.Nil(t, v)
+	require.NotNil(t, v)
+	require.Empty(t, v.Version())
 }
 
 func TestForcedUpdateService(t *testing.T) {
@@ -210,13 +285,13 @@ func TestForcedUpdateService(t *testing.T) {
 	s := NewWorkersService("2020.8.5", fmt.Sprintf("%s/updater", ts.URL), testFilePath, Options{IsForced: true})
 	v, err := s.Check()
 	require.NoError(t, err)
-	require.Equal(t, v.String(), "2020.08.05")
+	require.Equal(t, v.Version(), mostRecentVersion)
 
 	require.NoError(t, v.Apply())
 	dat, err := ioutil.ReadFile(testFilePath)
 	require.NoError(t, err)
 
-	require.Equal(t, string(dat), "2020.08.05")
+	require.Equal(t, string(dat), mostRecentVersion)
 }
 
 func TestUpdateSpecificVersionService(t *testing.T) {
@@ -231,7 +306,7 @@ func TestUpdateSpecificVersionService(t *testing.T) {
 	s := NewWorkersService("2020.8.2", fmt.Sprintf("%s/updater", ts.URL), testFilePath, Options{RequestedVersion: reqVersion})
 	v, err := s.Check()
 	require.NoError(t, err)
-	require.Equal(t, reqVersion, v.String())
+	require.Equal(t, reqVersion, v.Version())
 
 	require.NoError(t, v.Apply())
 	dat, err := ioutil.ReadFile(testFilePath)
@@ -251,7 +326,7 @@ func TestCompressedUpdateService(t *testing.T) {
 	s := NewWorkersService("2020.8.2", fmt.Sprintf("%s/compressed", ts.URL), testFilePath, Options{})
 	v, err := s.Check()
 	require.NoError(t, err)
-	require.Equal(t, "2020.09.02", v.String())
+	require.Equal(t, "2020.09.02", v.Version())
 
 	require.NoError(t, v.Apply())
 	dat, err := ioutil.ReadFile(testFilePath)
@@ -260,14 +335,17 @@ func TestCompressedUpdateService(t *testing.T) {
 	require.Equal(t, "2020.09.02", string(dat))
 }
 
-func TestVersionParsing(t *testing.T) {
-	require.False(t, IsNewerVersion("2020.8.2", "2020.8.2"))
-	require.True(t, IsNewerVersion("2020.8.2", "2020.8.3"))
-	require.True(t, IsNewerVersion("2020.8.2", "2021.1.2"))
-	require.True(t, IsNewerVersion("2020.8.2", "2020.9.1"))
-	require.True(t, IsNewerVersion("2020.8.2", "2020.12.45"))
-	require.False(t, IsNewerVersion("2020.8.2", "2020.6.3"))
-	require.False(t, IsNewerVersion("DEV", "2020.8.5"))
-	require.False(t, IsNewerVersion("2020.8.2", "asdlkfjasdf"))
-	require.True(t, IsNewerVersion("3.0.1", "4.2.1"))
+func TestUpdateWhenRunningKnownBuggyVersion(t *testing.T) {
+	ts := createServer()
+	defer ts.Close()
+
+	testFilePath := "tmpfile"
+	createTestFile(t, testFilePath)
+	defer os.Remove(testFilePath)
+
+	s := NewWorkersService(knownBuggyVersion, fmt.Sprintf("%s/updater", ts.URL), testFilePath, Options{})
+	v, err := s.Check()
+	require.NoError(t, err)
+	require.Equal(t, v.Version(), mostRecentVersion)
+	require.Equal(t, v.UserMessage(), expectedUserMsg)
 }

cmd/cloudflared/updater/workers_update.go

@@ -54,6 +54,7 @@ type WorkersVersion struct {
 	version      string
 	targetPath   string
 	isCompressed bool
+	userMessage  string
 }
 
 // NewWorkersVersion creates a new Version object. This is normally created by a WorkersService JSON checkin response
@@ -61,8 +62,17 @@ type WorkersVersion struct {
 // version is the version of this update
 // checksum is the expected checksum of the downloaded file
 // target path is where the file should be replace. Normally this the running cloudflared's path
-func NewWorkersVersion(url, version, checksum, targetPath string, isCompressed bool) Version {
-	return &WorkersVersion{downloadURL: url, version: version, checksum: checksum, targetPath: targetPath, isCompressed: isCompressed}
+// userMessage is a possible message to convey back to the user after having checked in with the Updater Service
+// isCompressed tells whether the asset to update cloudflared is compressed or not
+func NewWorkersVersion(url, version, checksum, targetPath, userMessage string, isCompressed bool) CheckResult {
+	return &WorkersVersion{
+		downloadURL:  url,
+		version:      version,
+		checksum:     checksum,
+		targetPath:   targetPath,
+		isCompressed: isCompressed,
+		userMessage:  userMessage,
+	}
 }
 
 // Apply does the actual verification and update logic.
@@ -114,10 +124,16 @@ func (v *WorkersVersion) Apply() error {
 }
 
 // String returns the version number of this update/release (e.g. 2020.08.05)
-func (v *WorkersVersion) String() string {
+func (v *WorkersVersion) Version() string {
 	return v.version
 }
 
+// String returns a possible message to convey back to user after having checked in with the Updater Service. E.g.
+// it can warn about the need to update the version currently running.
+func (v *WorkersVersion) UserMessage() string {
+	return v.userMessage
+}
+
 // download the file from the link in the json
 func download(url, filepath string, isCompressed bool) error {
 	client := &http.Client{

cmd/cloudflared/windows_service.go

@@ -12,6 +12,9 @@ import (
 	"time"
 	"unsafe"
 
+	"github.com/pkg/errors"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
 	"github.com/cloudflare/cloudflared/logger"
 
 	"github.com/urfave/cli/v2"
@@ -48,13 +51,13 @@ func runApp(app *cli.App, graceShutdownC chan struct{}) {
 			{
 				Name:   "install",
 				Usage:  "Install Argo Tunnel as a Windows service",
-				Action: installWindowsService,
-						},
+				Action: cliutil.ConfiguredAction(installWindowsService),
+			},
 			{
 				Name:   "uninstall",
 				Usage:  "Uninstall the Argo Tunnel service",
-				Action: uninstallWindowsService,
-						},
+				Action: cliutil.ConfiguredAction(uninstallWindowsService),
+			},
 		},
 	})
 
@@ -177,35 +180,30 @@ func installWindowsService(c *cli.Context) error {
 	zeroLogger.Info().Msg("Installing Argo Tunnel Windows service")
 	exepath, err := os.Executable()
 	if err != nil {
-		zeroLogger.Err(err).Msg("Cannot find path name that start the process")
-		return err
+		return errors.Wrap(err, "Cannot find path name that start the process")
 	}
 	m, err := mgr.Connect()
 	if err != nil {
-		zeroLogger.Err(err).Msg("Cannot establish a connection to the service control manager")
-		return err
+		return errors.Wrap(err, "Cannot establish a connection to the service control manager")
 	}
 	defer m.Disconnect()
 	s, err := m.OpenService(windowsServiceName)
 	log := zeroLogger.With().Str(LogFieldWindowsServiceName, windowsServiceName).Logger()
 	if err == nil {
 		s.Close()
-		log.Err(err).Msg("service already exists")
-		return fmt.Errorf("service %s already exists", windowsServiceName)
+		return fmt.Errorf("Service %s already exists", windowsServiceName)
 	}
 	config := mgr.Config{StartType: mgr.StartAutomatic, DisplayName: windowsServiceDescription}
 	s, err = m.CreateService(windowsServiceName, exepath, config)
 	if err != nil {
-		log.Error().Msg("Cannot install service")
-		return err
+		return errors.Wrap(err, "Cannot install service")
 	}
 	defer s.Close()
 	log.Info().Msg("Argo Tunnel agent service is installed")
 	err = eventlog.InstallAsEventCreate(windowsServiceName, eventlog.Error|eventlog.Warning|eventlog.Info)
 	if err != nil {
 		s.Delete()
-		log.Err(err).Msg("Cannot install event logger")
-		return fmt.Errorf("SetupEventLogSource() failed: %s", err)
+		return errors.Wrap(err, "Cannot install event logger")
 	}
 	err = configRecoveryOption(s.Handle)
 	if err != nil {
@@ -223,26 +221,22 @@ func uninstallWindowsService(c *cli.Context) error {
 	log.Info().Msg("Uninstalling Argo Tunnel Windows Service")
 	m, err := mgr.Connect()
 	if err != nil {
-		log.Error().Msg("Cannot establish a connection to the service control manager")
-		return err
+		return errors.Wrap(err, "Cannot establish a connection to the service control manager")
 	}
 	defer m.Disconnect()
 	s, err := m.OpenService(windowsServiceName)
 	if err != nil {
-		log.Error().Msg("service is not installed")
-		return fmt.Errorf("service %s is not installed", windowsServiceName)
+		return fmt.Errorf("Service %s is not installed", windowsServiceName)
 	}
 	defer s.Close()
 	err = s.Delete()
 	if err != nil {
-		log.Error().Msg("Cannot delete service")
-		return err
+		return errors.Wrap(err, "Cannot delete service")
 	}
 	log.Info().Msg("Argo Tunnel agent service is uninstalled")
 	err = eventlog.Remove(windowsServiceName)
 	if err != nil {
-		log.Error().Msg("Cannot remove event logger")
-		return fmt.Errorf("RemoveEventLogSource() failed: %s", err)
+		return errors.Wrap(err, "Cannot remove event logger")
 	}
 	return nil
 }

component-tests/.gitignore

@@ -0,0 +1,2 @@
+__pycache__
+.pytest_cache

component-tests/README.md

@@ -0,0 +1,52 @@
+# Requirements
+1. Python 3.7 or later with packages in the given `requirements.txt`
+   - E.g. with conda:
+   - `conda create -n component-tests python=3.7`  
+   - `conda activate component-tests`
+   - `pip3 install -r requirements.txt`
+
+2. Create a config yaml file, for example:
+```
+cloudflared_binary: "cloudflared"
+tunnel: "3d539f97-cd3a-4d8e-c33b-65e9099c7a8d"
+credentials_file: "/Users/tunnel/.cloudflared/3d539f97-cd3a-4d8e-c33b-65e9099c7a8d.json"
+classic_hostname: "classic-tunnel-component-tests.example.com"
+origincert: "/Users/tunnel/.cloudflared/cert.pem"
+ingress:
+- hostname: named-tunnel-component-tests.example.com
+  service: http_status:200
+- service: http_status:404
+```
+
+3. Route hostname to the tunnel. For the example config above, we can do that via
+```
+   cloudflared tunnel route dns 3d539f97-cd3a-4d8e-c33b-65e9099c7a8d named-tunnel-component-tests.example.com
+```
+
+4. Turn on linter
+If you are using Visual Studio, follow https://code.visualstudio.com/docs/python/linting to turn on linter.
+
+5. Turn on formatter
+If you are using Visual Studio, follow https://code.visualstudio.com/docs/python/editing#_formatting
+to turn on formatter and https://marketplace.visualstudio.com/items?itemName=cbrevik.toggle-format-on-save
+to turn on format on save.
+
+6. If you have cloudflared running as a service on your machine, you can either stop the service or ignore the service tests
+via `--ignore test_service.py`
+
+# How to run
+Specify path to config file via env var `COMPONENT_TESTS_CONFIG`. This is required.
+## All tests
+Run `pytest` inside this(component-tests) folder
+
+## Specific files
+Run `pytest <file 1 name>.py <file 2 name>.py`
+
+## Specific tests
+Run `pytest file.py -k <test 1 name> -k <test 2 name>`
+
+## Live Logging
+Running with `-o log_cli=true` outputs logging to CLI as the tests are. By default the log level is WARN.
+`--log-cli-level` control logging level.
+For example, to log at info level, run `pytest -o log_cli=true --log-cli-level=INFO`.
+See https://docs.pytest.org/en/latest/logging.html#live-logs for more documentation on logging.

component-tests/config.py

@@ -0,0 +1,132 @@
+#!/usr/bin/env python
+import base64
+import copy
+import os
+import yaml
+
+from dataclasses import dataclass, InitVar
+
+from constants import METRICS_PORT
+
+from util import LOGGER
+
+# frozen=True raises exception when assigning to fields. This emulates immutability
+
+
+@dataclass(frozen=True)
+class BaseConfig:
+    cloudflared_binary: str
+    no_autoupdate: bool = True
+    metrics: str = f'localhost:{METRICS_PORT}'
+
+    def merge_config(self, additional):
+        config = copy.copy(additional)
+        config['no-autoupdate'] = self.no_autoupdate
+        config['metrics'] = self.metrics
+        return config
+
+
+@dataclass(frozen=True)
+class NamedTunnelBaseConfig(BaseConfig):
+    # The attributes of the parent class are ordered before attributes in this class,
+    # so we have to use default values here and check if they are set in __post_init__
+    tunnel: str = None
+    credentials_file: str = None
+    ingress: list = None
+
+    def __post_init__(self):
+        if self.tunnel is None:
+            raise TypeError("Field tunnel is not set")
+        if self.credentials_file is None:
+            raise TypeError("Field credentials_file is not set")
+        if self.ingress is None:
+            raise TypeError("Field ingress is not set")
+
+    def merge_config(self, additional):
+        config = super(NamedTunnelBaseConfig, self).merge_config(additional)
+        config['tunnel'] = self.tunnel
+        config['credentials-file'] = self.credentials_file
+        # In some cases we want to override default ingress, such as in config tests
+        if 'ingress' not in config:
+            config['ingress'] = self.ingress
+        return config
+
+
+@dataclass(frozen=True)
+class NamedTunnelConfig(NamedTunnelBaseConfig):
+    full_config: dict = None
+    additional_config: InitVar[dict] = {}
+
+    def __post_init__(self, additional_config):
+        # Cannot call set self.full_config because the class is frozen, instead, we can use __setattr__
+        # https://docs.python.org/3/library/dataclasses.html#frozen-instances
+        object.__setattr__(self, 'full_config',
+                           self.merge_config(additional_config))
+
+    def get_url(self):
+        return "https://" + self.ingress[0]['hostname']
+
+
+@dataclass(frozen=True)
+class ClassicTunnelBaseConfig(BaseConfig):
+    hostname: str = None
+    origincert: str = None
+
+    def __post_init__(self):
+        if self.hostname is None:
+            raise TypeError("Field tunnel is not set")
+        if self.origincert is None:
+            raise TypeError("Field credentials_file is not set")
+
+    def merge_config(self, additional):
+        config = super(ClassicTunnelBaseConfig, self).merge_config(additional)
+        config['hostname'] = self.hostname
+        config['origincert'] = self.origincert
+        return config
+
+
+@dataclass(frozen=True)
+class ClassicTunnelConfig(ClassicTunnelBaseConfig):
+    full_config: dict = None
+    additional_config: InitVar[dict] = {}
+
+    def __post_init__(self, additional_config):
+        # Cannot call set self.full_config because the class is frozen, instead, we can use __setattr__
+        # https://docs.python.org/3/library/dataclasses.html#frozen-instances
+        object.__setattr__(self, 'full_config',
+                           self.merge_config(additional_config))
+
+    def get_url(self):
+        return "https://" + self.hostname
+
+
+def build_config_from_env():
+    config_path = get_env("COMPONENT_TESTS_CONFIG")
+    config_content = base64.b64decode(
+        get_env("COMPONENT_TESTS_CONFIG_CONTENT")).decode('utf-8')
+    config_yaml = yaml.safe_load(config_content)
+
+    credentials_file = get_env("COMPONENT_TESTS_CREDENTIALS_FILE")
+    write_file(credentials_file, config_yaml["credentials_file"])
+
+    origincert = get_env("COMPONENT_TESTS_ORIGINCERT")
+    write_file(origincert, config_yaml["origincert"])
+
+    write_file(config_content, config_path)
+
+
+def write_file(content, path):
+    with open(path, 'w') as outfile:
+        outfile.write(content)
+        outfile.close
+
+
+def get_env(env_name):
+    val = os.getenv(env_name)
+    if val is None:
+        raise Exception(f"{env_name} is not set")
+    return val
+
+
+if __name__ == '__main__':
+    build_config_from_env()

component-tests/conftest.py

@@ -0,0 +1,35 @@
+import os
+import pytest
+import yaml
+
+from time import sleep
+
+from config import NamedTunnelConfig, ClassicTunnelConfig
+from constants import BACKOFF_SECS
+from util import LOGGER
+
+
+@pytest.fixture(scope="session")
+def component_tests_config():
+    config_file = os.getenv("COMPONENT_TESTS_CONFIG")
+    if config_file is None:
+        raise Exception(
+            "Need to provide path to config file in COMPONENT_TESTS_CONFIG")
+    with open(config_file, 'r') as stream:
+        config = yaml.safe_load(stream)
+        LOGGER.info(f"component tests base config {config}")
+
+        def _component_tests_config(additional_config={}, named_tunnel=True):
+            if named_tunnel:
+                return NamedTunnelConfig(additional_config=additional_config,
+                                         cloudflared_binary=config['cloudflared_binary'], tunnel=config['tunnel'], credentials_file=config['credentials_file'],  ingress=config['ingress'])
+            return ClassicTunnelConfig(
+                additional_config=additional_config, cloudflared_binary=config['cloudflared_binary'], hostname=config['classic_hostname'], origincert=config['origincert'])
+
+        return _component_tests_config
+
+
+# This fixture is automatically called before each tests to make sure the previous cloudflared has been shutdown
+@pytest.fixture(autouse=True)
+def wait_previous_cloudflared():
+    sleep(BACKOFF_SECS)

component-tests/constants.py

@@ -0,0 +1,3 @@
+METRICS_PORT = 51000
+MAX_RETRIES = 5
+BACKOFF_SECS = 7

component-tests/requirements.txt

@@ -0,0 +1,5 @@
+flaky==3.7.0
+pytest==6.2.2
+pyyaml==5.4.1
+requests==2.25.1
+retrying==1.3.3

component-tests/test_config.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+from util import start_cloudflared
+
+
+class TestConfig:
+    # tmp_path is a fixture provides a temporary directory unique to the test invocation
+    def test_validate_ingress_rules(self, tmp_path, component_tests_config):
+        extra_config = {
+            'ingress': [
+                {
+                    "hostname": "example.com",
+                    "service": "https://localhost:8000",
+                    "originRequest": {
+                        "originServerName": "test.example.com",
+                        "caPool": "/etc/certs/ca.pem"
+                    },
+                },
+                {
+                    "hostname": "api.example.com",
+                    "path": "login",
+                    "service": "https://localhost:9000",
+                },
+                {
+                    "hostname": "wss.example.com",
+                    "service": "wss://localhost:8000",
+                },
+                {
+                    "hostname": "ssh.example.com",
+                    "service": "ssh://localhost:8000",
+                },
+                {"service": "http_status:404"}
+            ],
+        }
+        config = component_tests_config(extra_config)
+        validate_args = ["ingress", "validate"]
+        _ = start_cloudflared(tmp_path, config, validate_args)
+
+        self.match_rule(tmp_path, config,
+                        "http://example.com/index.html", 1)
+        self.match_rule(tmp_path, config,
+                        "https://example.com/index.html", 1)
+        self.match_rule(tmp_path, config,
+                        "https://api.example.com/login", 2)
+        self.match_rule(tmp_path, config,
+                        "https://wss.example.com", 3)
+        self.match_rule(tmp_path, config,
+                        "https://ssh.example.com", 4)
+        self.match_rule(tmp_path, config,
+                        "https://api.example.com", 5)
+
+    # This is used to check that the command tunnel ingress url <url> matches rule number <rule_num>. Note that rule number uses 1-based indexing
+
+    def match_rule(self, tmp_path, config, url, rule_num):
+        args = ["ingress", "rule", url]
+        match_rule = start_cloudflared(tmp_path, config, args)
+
+        assert f"Matched rule #{rule_num}" .encode() in match_rule.stdout

component-tests/test_logging.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python
+import json
+import os
+from util import start_cloudflared, wait_tunnel_ready, send_requests, LOGGER
+
+
+class TestLogging:
+    # TODO: Test logging when running as a service https://jira.cfops.it/browse/TUN-4082
+    # Rolling logger rotate log files after 1 MB
+    rotate_after_size = 1000 * 1000
+    default_log_file = "cloudflared.log"
+    expect_message = "Starting tunnel"
+
+    def test_logging_to_terminal(self, tmp_path, component_tests_config):
+        config = component_tests_config()
+        with start_cloudflared(tmp_path, config, new_process=True) as cloudflared:
+            wait_tunnel_ready(tunnel_url=config.get_url())
+            self.assert_log_to_terminal(cloudflared)
+
+    def test_logging_to_file(self, tmp_path, component_tests_config):
+        log_file = tmp_path / self.default_log_file
+        extra_config = {
+            # Convert from pathlib.Path to str
+            "logfile": str(log_file),
+        }
+        config = component_tests_config(extra_config)
+        with start_cloudflared(tmp_path, config, new_process=True, capture_output=False):
+            wait_tunnel_ready(tunnel_url=config.get_url())
+            self.assert_log_in_file(log_file)
+            self.assert_json_log(log_file)
+
+    def test_logging_to_dir(self, tmp_path, component_tests_config):
+        log_dir = tmp_path / "logs"
+        extra_config = {
+            "loglevel": "debug",
+            # Convert from pathlib.Path to str
+            "log-directory": str(log_dir),
+        }
+        config = component_tests_config(extra_config)
+        with start_cloudflared(tmp_path, config, new_process=True, capture_output=False):
+            wait_tunnel_ready(tunnel_url=config.get_url())
+            self.assert_log_to_dir(config, log_dir)
+
+    def assert_log_to_terminal(self, cloudflared):
+        stderr = cloudflared.stderr.read(200)
+        # cloudflared logs the following when it first starts
+        # 2021-03-10T12:30:39Z INF Starting tunnel tunnelID=<tunnel ID>
+        assert self.expect_message.encode(
+        ) in stderr, f"{stderr} doesn't contain {self.expect_message}"
+
+    def assert_log_in_file(self, file, expect_message=""):
+        with open(file, "r") as f:
+            log = f.read(200)
+            # cloudflared logs the following when it first starts
+            # {"level":"info","tunnelID":"<tunnel ID>","time":"2021-03-10T12:21:13Z","message":"Starting tunnel"}
+            assert self.expect_message in log, f"{log} doesn't contain {self.expect_message}"
+
+    def assert_json_log(self, file):
+        with open(file, "r") as f:
+            line = f.readline()
+            json_log = json.loads(line)
+            self.assert_in_json(json_log, "level")
+            self.assert_in_json(json_log, "time")
+            self.assert_in_json(json_log, "message")
+
+    def assert_in_json(self, j, key):
+        assert key in j, f"{key} is not in j"
+
+    def assert_log_to_dir(self, config, log_dir):
+        max_batches = 3
+        batch_requests = 1000
+        for _ in range(max_batches):
+            send_requests(config.get_url(),
+                          batch_requests, require_ok=False)
+            files = os.listdir(log_dir)
+            if len(files) == 2:
+                current_log_file_index = files.index(self.default_log_file)
+                current_file = log_dir / files[current_log_file_index]
+                stats = os.stat(current_file)
+                assert stats.st_size > 0
+                self.assert_json_log(current_file)
+
+                # One file is the current log file, the other is the rotated log file
+                rotated_log_file_index = 0 if current_log_file_index == 1 else 1
+                rotated_file = log_dir / files[rotated_log_file_index]
+                stats = os.stat(rotated_file)
+                assert stats.st_size > self.rotate_after_size
+                self.assert_log_in_file(rotated_file)
+                self.assert_json_log(current_file)
+                return
+
+        raise Exception(
+            f"Log file isn't rotated after sending {max_batches * batch_requests} requests")

component-tests/test_reconnect.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python
+import copy
+from flaky import flaky
+from retrying import retry
+from time import sleep
+
+from util import start_cloudflared, wait_tunnel_ready, check_tunnel_not_connected, send_requests
+
+
+@flaky(max_runs=3, min_passes=1)
+class TestReconnect():
+    default_ha_conns = 4
+    default_reconnect_secs = 5
+    extra_config = {
+        "stdin-control": True,
+    }
+
+    def test_named_reconnect(self, tmp_path, component_tests_config):
+        config = component_tests_config(self.extra_config)
+        with start_cloudflared(tmp_path, config, new_process=True, allow_input=True) as cloudflared:
+            # Repeat the test multiple times because some issues only occur after multiple reconnects
+            self.assert_reconnect(config, cloudflared, 5)
+
+    def test_classic_reconnect(self, tmp_path, component_tests_config):
+        extra_config = copy.copy(self.extra_config)
+        extra_config["hello-world"] = True
+        config = component_tests_config(
+            additional_config=extra_config, named_tunnel=False)
+        with start_cloudflared(tmp_path, config, cfd_args=[], new_process=True, allow_input=True) as cloudflared:
+            self.assert_reconnect(config, cloudflared, 1)
+
+    def send_reconnect(self, cloudflared, secs):
+        # Although it is recommended to use the Popen.communicate method, we cannot
+        # use it because it blocks on reading stdout and stderr until EOF is reached
+        cloudflared.stdin.write(f"reconnect {secs}s\n".encode())
+        cloudflared.stdin.flush()
+
+    def assert_reconnect(self, config, cloudflared, repeat):
+        wait_tunnel_ready(tunnel_url=config.get_url())
+        for _ in range(repeat):
+            for i in range(self.default_ha_conns):
+                self.send_reconnect(cloudflared, self.default_reconnect_secs)
+                expect_connections = self.default_ha_conns-i-1
+                if expect_connections > 0:
+                    # Don't check if tunnel returns 200 here because there is a race condition between wait_tunnel_ready
+                    # retrying to get 200 response and reconnecting
+                    wait_tunnel_ready(
+                        require_min_connections=expect_connections)
+                else:
+                    check_tunnel_not_connected()
+
+            sleep(self.default_reconnect_secs + 10)
+            wait_tunnel_ready(tunnel_url=config.get_url(),
+                              require_min_connections=self.default_ha_conns)

component-tests/test_service.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python
+from contextlib import contextmanager
+import os
+from pathlib import Path
+import platform
+import pytest
+import subprocess
+
+from util import start_cloudflared, cloudflared_cmd, wait_tunnel_ready, LOGGER
+
+
+def select_platform(plat):
+    return pytest.mark.skipif(
+        platform.system() != plat, reason=f"Only runs on {plat}")
+
+
+def default_config_dir():
+    return os.path.join(Path.home(), ".cloudflared")
+
+
+def default_config_file():
+    return os.path.join(default_config_dir(), "config.yml")
+
+
+class TestServiceMode():
+    @select_platform("Darwin")
+    @pytest.mark.skipif(os.path.exists(default_config_file()), reason=f"There is already a config file in default path")
+    def test_launchd_service(self, component_tests_config):
+        # On Darwin cloudflared service defaults to run classic tunnel command
+        additional_config = {
+            "hello-world": True,
+        }
+        config = component_tests_config(
+            additional_config=additional_config, named_tunnel=False)
+        with self.run_service(Path(default_config_dir()), config):
+            self.launchctl_cmd("list")
+            self.launchctl_cmd("start")
+            wait_tunnel_ready(tunnel_url=config.get_url())
+            self.launchctl_cmd("stop")
+
+        os.remove(default_config_file())
+        self.launchctl_cmd("list", success=False)
+
+    @select_platform("Linux")
+    @pytest.mark.skipif(os.path.exists("/etc/cloudflared/config.yml"), reason=f"There is already a config file in default path")
+    def test_sysv_service(self, tmp_path, component_tests_config):
+        config = component_tests_config()
+        with self.run_service(tmp_path, config, root=True):
+            self.sysv_cmd("start")
+            self.sysv_cmd("status")
+            wait_tunnel_ready(tunnel_url=config.get_url())
+            self.sysv_cmd("stop")
+        # Service install copies config file to /etc/cloudflared/config.yml
+        subprocess.run(["sudo", "rm", "/etc/cloudflared/config.yml"])
+        self.sysv_cmd("status", success=False)
+
+    @contextmanager
+    def run_service(self, tmp_path, config, root=False):
+        try:
+            service = start_cloudflared(
+                tmp_path, config, cfd_args=["service", "install"], cfd_pre_args=[], capture_output=False, root=root)
+            yield service
+        finally:
+            start_cloudflared(
+                tmp_path, config, cfd_args=["service", "uninstall"], cfd_pre_args=[], capture_output=False, root=root)
+
+    def launchctl_cmd(self, action, success=True):
+        cmd = subprocess.run(
+            ["launchctl", action, "com.cloudflare.cloudflared"], check=success)
+        if not success:
+            assert cmd.returncode != 0, f"Expect {cmd.args} to fail, but it succeed"
+
+    def sysv_cmd(self, action, success=True):
+        cmd = subprocess.run(
+            ["sudo", "service", "cloudflared", action], check=success)
+        if not success:
+            assert cmd.returncode != 0, f"Expect {cmd.args} to fail, but it succeed"

component-tests/test_termination.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python
+from contextlib import contextmanager
+import requests
+import signal
+import threading
+import time
+
+from util import start_cloudflared, wait_tunnel_ready, check_tunnel_not_connected, LOGGER
+
+
+class TestTermination():
+    grace_period = 5
+    timeout = 10
+    extra_config = {
+        "grace-period": f"{grace_period}s",
+    }
+    signals = [signal.SIGTERM, signal.SIGINT]
+    sse_endpoint = "/sse?freq=1s"
+
+    def test_graceful_shutdown(self, tmp_path, component_tests_config):
+        config = component_tests_config(self.extra_config)
+        for sig in self.signals:
+            with start_cloudflared(
+                    tmp_path, config, new_process=True, capture_output=False) as cloudflared:
+                wait_tunnel_ready(tunnel_url=config.get_url())
+
+                connected = threading.Condition()
+                in_flight_req = threading.Thread(
+                    target=self.stream_request, args=(config, connected,))
+                in_flight_req.start()
+
+                with connected:
+                    connected.wait(self.timeout)
+                # Send signal after the SSE connection is established
+                self.terminate_by_signal(cloudflared, sig)
+                self.wait_eyeball_thread(
+                    in_flight_req, self.grace_period + self.timeout)
+
+    # test cloudflared terminates before grace period expires when all eyeball
+    # connections are drained
+    def test_shutdown_once_no_connection(self, tmp_path, component_tests_config):
+        config = component_tests_config(self.extra_config)
+        for sig in self.signals:
+            with start_cloudflared(
+                    tmp_path, config, new_process=True, capture_output=False) as cloudflared:
+                wait_tunnel_ready(tunnel_url=config.get_url())
+
+                connected = threading.Condition()
+                in_flight_req = threading.Thread(
+                    target=self.stream_request, args=(config, connected, True, ))
+                in_flight_req.start()
+
+                with connected:
+                    connected.wait(self.timeout)
+                with self.within_grace_period():
+                    # Send signal after the SSE connection is established
+                    self.terminate_by_signal(cloudflared, sig)
+                    self.wait_eyeball_thread(in_flight_req, self.grace_period)
+
+    def test_no_connection_shutdown(self, tmp_path, component_tests_config):
+        config = component_tests_config(self.extra_config)
+        for sig in self.signals:
+            with start_cloudflared(
+                    tmp_path, config, new_process=True, capture_output=False) as cloudflared:
+                wait_tunnel_ready(tunnel_url=config.get_url())
+                with self.within_grace_period():
+                    self.terminate_by_signal(cloudflared, sig)
+
+    def terminate_by_signal(self, cloudflared, sig):
+        cloudflared.send_signal(sig)
+        check_tunnel_not_connected()
+        cloudflared.wait()
+
+    def wait_eyeball_thread(self, thread, timeout):
+        thread.join(timeout)
+        assert thread.is_alive() == False, "eyeball thread is still alive"
+
+    # Using this context asserts logic within the context is executed within grace period
+    @contextmanager
+    def within_grace_period(self):
+        try:
+            start = time.time()
+            yield
+        finally:
+            duration = time.time() - start
+            assert duration < self.grace_period
+
+    def stream_request(self, config, connected, early_terminate):
+        expected_terminate_message = "502 Bad Gateway"
+        url = config.get_url() + self.sse_endpoint
+
+        with requests.get(url, timeout=5, stream=True) as resp:
+            with connected:
+                connected.notifyAll()
+            lines = 0
+            for line in resp.iter_lines():
+                if expected_terminate_message.encode() == line:
+                    break
+                lines += 1
+                if early_terminate and lines == 2:
+                    return
+            # /sse returns count followed by 2 new lines
+            assert lines >= (self.grace_period * 2)

component-tests/util.py

@@ -0,0 +1,100 @@
+from contextlib import contextmanager
+import logging
+import requests
+from retrying import retry
+import subprocess
+import yaml
+
+from time import sleep
+
+from constants import METRICS_PORT, MAX_RETRIES, BACKOFF_SECS
+
+LOGGER = logging.getLogger(__name__)
+
+
+def write_config(directory, config):
+    config_path = directory / "config.yml"
+    with open(config_path, 'w') as outfile:
+        yaml.dump(config, outfile)
+    return config_path
+
+
+def start_cloudflared(directory, config, cfd_args=["run"], cfd_pre_args=["tunnel"], new_process=False, allow_input=False, capture_output=True, root=False):
+    config_path = write_config(directory, config.full_config)
+    cmd = cloudflared_cmd(config, config_path, cfd_args, cfd_pre_args, root)
+    if new_process:
+        return run_cloudflared_background(cmd, allow_input, capture_output)
+    # By setting check=True, it will raise an exception if the process exits with non-zero exit code
+    return subprocess.run(cmd, check=True, capture_output=capture_output)
+
+
+def cloudflared_cmd(config, config_path, cfd_args, cfd_pre_args, root):
+    cmd = []
+    if root:
+        cmd += ["sudo"]
+    cmd += [config.cloudflared_binary]
+    cmd += cfd_pre_args
+    cmd += ["--config", config_path]
+    cmd += cfd_args
+    LOGGER.info(f"Run cmd {cmd} with config {config}")
+    return cmd
+
+
+@contextmanager
+def run_cloudflared_background(cmd, allow_input, capture_output):
+    output = subprocess.PIPE if capture_output else subprocess.DEVNULL
+    stdin = subprocess.PIPE if allow_input else None
+    try:
+        cfd = subprocess.Popen(cmd, stdin=stdin, stdout=output, stderr=output)
+        yield cfd
+    finally:
+        cfd.terminate()
+        if capture_output:
+            LOGGER.info(
+                f"cloudflared log: {cfd.stderr.read()}")
+
+
+@retry(stop_max_attempt_number=MAX_RETRIES, wait_fixed=BACKOFF_SECS * 1000)
+def wait_tunnel_ready(tunnel_url=None, require_min_connections=1):
+    metrics_url = f'http://localhost:{METRICS_PORT}/ready'
+
+    with requests.Session() as s:
+        resp = send_request(s, metrics_url, True)
+        assert resp.json()[
+            "readyConnections"] >= require_min_connections, f"Ready endpoint returned {resp.json()} but we expect at least {require_min_connections} connections"
+        if tunnel_url is not None:
+            send_request(s, tunnel_url, True)
+
+
+@retry(stop_max_attempt_number=MAX_RETRIES * BACKOFF_SECS, wait_fixed=1000)
+def check_tunnel_not_connected():
+    url = f'http://localhost:{METRICS_PORT}/ready'
+
+    try:
+        resp = requests.get(url, timeout=1)
+        assert resp.status_code == 503, f"Expect {url} returns 503, got {resp.status_code}"
+    # cloudflared might already terminate
+    except requests.exceptions.ConnectionError as e:
+        LOGGER.warning(f"Failed to connect to {url}, error: {e}")
+
+
+# In some cases we don't need to check response status, such as when sending batch requests to generate logs
+def send_requests(url, count, require_ok=True):
+    errors = 0
+    with requests.Session() as s:
+        for _ in range(count):
+            resp = send_request(s, url, require_ok)
+            if resp is None:
+                errors += 1
+            sleep(0.01)
+    if errors > 0:
+        LOGGER.warning(
+            f"{errors} out of {count} requests to {url} return non-200 status")
+
+
+@retry(stop_max_attempt_number=MAX_RETRIES, wait_fixed=BACKOFF_SECS * 1000)
+def send_request(session, url, require_ok):
+    resp = session.get(url, timeout=BACKOFF_SECS)
+    if require_ok:
+        assert resp.status_code == 200, f"{url} returned {resp}"
+    return resp if resp.status_code == 200 else None

config/configuration.go

@@ -11,11 +11,11 @@ import (
 
 	"github.com/mitchellh/go-homedir"
 	"github.com/pkg/errors"
+	"github.com/rs/zerolog"
 	"github.com/urfave/cli/v2"
 	"gopkg.in/yaml.v2"
 
 	"github.com/cloudflare/cloudflared/validation"
-	"github.com/rs/zerolog"
 )
 
 var (
@@ -221,6 +221,14 @@ type OriginRequestConfig struct {
 	ProxyPort *uint `yaml:"proxyPort"`
 	// Valid options are 'socks' or empty.
 	ProxyType *string `yaml:"proxyType"`
+	// IP rules for the proxy service
+	IPRules []IngressIPRule `yaml:"ipRules"`
+}
+
+type IngressIPRule struct {
+	Prefix *string `yaml:"prefix"`
+	Ports  []int   `yaml:"ports"`
+	Allow  bool    `yaml:"allow"`
 }
 
 type Configuration struct {

connection/http2.go

@@ -108,6 +108,7 @@ func (c *http2Connection) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	switch connType {
 	case TypeControlStream:
 		proxyErr = c.serveControlStream(r.Context(), respWriter)
+		c.controlStreamErr = proxyErr
 	case TypeWebsocket:
 		stripWebsocketUpgradeHeader(r)
 		proxyErr = c.config.OriginProxy.Proxy(respWriter, r, TypeWebsocket)

connection/http2_test.go

@@ -114,6 +114,7 @@ func TestServeHTTP(t *testing.T) {
 }
 
 type mockNamedTunnelRPCClient struct {
+	shouldFail   error
 	registered   chan struct{}
 	unregistered chan struct{}
 }
@@ -125,6 +126,9 @@ func (mc mockNamedTunnelRPCClient) RegisterConnection(
 	connIndex uint8,
 	observer *Observer,
 ) error {
+	if mc.shouldFail != nil {
+		return mc.shouldFail
+	}
 	close(mc.registered)
 	return nil
 }
@@ -136,12 +140,14 @@ func (mc mockNamedTunnelRPCClient) GracefulShutdown(ctx context.Context, gracePe
 func (mockNamedTunnelRPCClient) Close() {}
 
 type mockRPCClientFactory struct {
+	shouldFail   error
 	registered   chan struct{}
 	unregistered chan struct{}
 }
 
 func (mf *mockRPCClientFactory) newMockRPCClient(context.Context, io.ReadWriteCloser, *zerolog.Logger) NamedTunnelRPCClient {
 	return mockNamedTunnelRPCClient{
+		shouldFail:   mf.shouldFail,
 		registered:   mf.registered,
 		unregistered: mf.unregistered,
 	}
@@ -249,6 +255,39 @@ func TestServeControlStream(t *testing.T) {
 	wg.Wait()
 }
 
+func TestFailRegistration(t *testing.T) {
+	http2Conn, edgeConn := newTestHTTP2Connection()
+
+	rpcClientFactory := mockRPCClientFactory{
+		shouldFail:   errDuplicationConnection,
+		registered:   make(chan struct{}),
+		unregistered: make(chan struct{}),
+	}
+	http2Conn.newRPCClientFunc = rpcClientFactory.newMockRPCClient
+
+	ctx, cancel := context.WithCancel(context.Background())
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		http2Conn.Serve(ctx)
+	}()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://localhost:8080/", nil)
+	require.NoError(t, err)
+	req.Header.Set(internalUpgradeHeader, controlStreamUpgrade)
+
+	edgeHTTP2Conn, err := testTransport.NewClientConn(edgeConn)
+	require.NoError(t, err)
+	resp, err := edgeHTTP2Conn.RoundTrip(req)
+	require.NoError(t, err)
+	require.Equal(t, http.StatusBadGateway, resp.StatusCode)
+
+	assert.NotNil(t, http2Conn.controlStreamErr)
+	cancel()
+	wg.Wait()
+}
+
 func TestGracefulShutdownHTTP2(t *testing.T) {
 	http2Conn, edgeConn := newTestHTTP2Connection()
 

go.mod

@@ -57,3 +57,5 @@ require (
 	gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 // indirect
 	zombiezen.com/go/capnproto2 v2.18.0+incompatible
 )
+
+replace github.com/urfave/cli/v2 => github.com/ipostelnik/cli/v2 v2.3.1-0.20210223162147-c8b4f97735e4

go.sum

@@ -62,7 +62,6 @@ github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbt
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
-github.com/DATA-DOG/go-sqlmock v1.3.3 h1:CWUqKXe0s8A2z6qCgkP4Kru7wC11YoAnoupUKFDnH08=
 github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
 github.com/DataDog/datadog-go v3.5.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
 github.com/DataDog/zstd v1.3.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
@@ -82,7 +81,6 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
-github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E=
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
 github.com/aliyun/alibaba-cloud-sdk-go v0.0.0-20190808125512-07798873deee/go.mod h1:myCDvQSzCW+wB1WAlocEru4wMGJxy+vlxHdhegi1CDQ=
 github.com/aliyun/aliyun-oss-go-sdk v0.0.0-20190307165228-86c17b95fcd5/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8=
@@ -219,7 +217,6 @@ github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1
 github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc=
 github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I=
 github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
-github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/gobwas/httphead v0.0.0-20200921212729-da3d93bc3c58 h1:YyrUZvJaU8Q0QsoVo+xLFBgWDTam29PKea6GYmwvSiQ=
@@ -350,9 +347,10 @@ github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJ
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo=
 github.com/infobloxopen/go-trees v0.0.0-20190313150506-2af4e13f9062/go.mod h1:PcNJqIlcX/dj3DTG/+QQnRvSgTMG6CLpRMjWcv4+J6w=
+github.com/ipostelnik/cli/v2 v2.3.1-0.20210223162147-c8b4f97735e4 h1:U2OxGKZHC5puhKZnGo6wWWjuWusJCM6J/S1eF7Vt+Uk=
+github.com/ipostelnik/cli/v2 v2.3.1-0.20210223162147-c8b4f97735e4/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
 github.com/jimstudt/http-authentication v0.0.0-20140401203705-3eca13d6893a/go.mod h1:wK6yTYYcgjHE1Z1QtXACPDjcFJyBskHEdagmnq3vsP8=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
-github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc=
 github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
@@ -378,7 +376,6 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxv
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
-github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
@@ -590,8 +587,6 @@ github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
 github.com/urfave/cli v1.22.1 h1:+mkCCcOFKPnCmVYVcURKps1Xe+3zP90gSYGNfRkjoIY=
 github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
-github.com/urfave/cli/v2 v2.2.0 h1:JTTnM6wKzdA0Jqodd966MVj4vWbbquZykeX1sKbe2C4=
-github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ=
 github.com/vultr/govultr v0.1.4/go.mod h1:9H008Uxr/C4vFNGLqKx232C206GL0PBHzOP0809bGNA=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
 github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
@@ -783,7 +778,6 @@ golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 h1:myAQVi0cGEoqQVR5POX+8RR2mrocKqNN1hmeMqhX27k=
 golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -978,6 +972,7 @@ gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRN
 gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

h2mux/h2mux_test.go

@@ -663,7 +663,8 @@ func AssertIfPipeReadable(t *testing.T, pipe io.ReadCloser) {
 		b := []byte{0}
 		n, err := pipe.Read(b)
 		if n > 0 {
-			t.Fatalf("read pipe was not empty")
+			t.Errorf("read pipe was not empty")
+			return
 		}
 		errC <- err
 	}()

h2mux/shared_buffer_test.go

@@ -81,7 +81,8 @@ func TestSharedBufferConcurrentReadWrite(t *testing.T) {
 				expectedResult.Write(block[:blockSize])
 				n, err := b.Write(block[:blockSize])
 				if n != blockSize || err != nil {
-					t.Fatalf("write error: %d %s", n, err)
+					t.Errorf("write error: %d %s", n, err)
+					return
 				}
 			}
 		}
@@ -94,7 +95,8 @@ func TestSharedBufferConcurrentReadWrite(t *testing.T) {
 			for i := 0; i < 256; i++ {
 				n, err := io.ReadFull(b, block[:blockSize])
 				if n != blockSize || err != nil {
-					t.Fatalf("read error: %d %s", n, err)
+					t.Errorf("read error: %d %s", n, err)
+					return
 				}
 				actualResult.Write(block[:blockSize])
 			}

ingress/ingress.go

@@ -9,7 +9,8 @@ import (
 	"strings"
 	"sync"
 
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/config"
+	"github.com/cloudflare/cloudflared/ipaccess"
 
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
@@ -26,6 +27,7 @@ var (
 
 const (
 	ServiceBastion     = "bastion"
+	ServiceSocksProxy  = "socks-proxy"
 	ServiceWarpRouting = "warp-routing"
 )
 
@@ -175,6 +177,23 @@ func validate(ingress []config.UnvalidatedIngressRule, defaults OriginRequestCon
 			service = &srv
 		} else if r.Service == "hello_world" || r.Service == "hello-world" || r.Service == "helloworld" {
 			service = new(helloWorld)
+		} else if r.Service == ServiceSocksProxy {
+			rules := make([]ipaccess.Rule, len(r.OriginRequest.IPRules))
+
+			for i, ipRule := range r.OriginRequest.IPRules {
+				rule, err := ipaccess.NewRuleByCIDR(ipRule.Prefix, ipRule.Ports, ipRule.Allow)
+				if err != nil {
+					return Ingress{}, fmt.Errorf("unable to create ip rule for %s: %s", r.Service, err)
+				}
+				rules[i] = rule
+			}
+
+			accessPolicy, err := ipaccess.NewPolicy(false, rules)
+			if err != nil {
+				return Ingress{}, fmt.Errorf("unable to create ip access policy for %s: %s", r.Service, err)
+			}
+
+			service = newSocksProxyOverWSService(accessPolicy)
 		} else if r.Service == ServiceBastion || cfg.BastionMode {
 			// Bastion mode will always start a Websocket proxy server, which will
 			// overwrite the localService.URL field when `start` is called. So,

ingress/ingress_test.go

@@ -14,7 +14,8 @@ import (
 	"github.com/urfave/cli/v2"
 	"gopkg.in/yaml.v2"
 
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/config"
+	"github.com/cloudflare/cloudflared/ipaccess"
 	"github.com/cloudflare/cloudflared/tlsconfig"
 )
 
@@ -304,6 +305,33 @@ ingress:
 				},
 			},
 		},
+		{
+			name: "SOCKS services",
+			args: args{rawYAML: `
+ingress:
+- hostname: socks.foo.com
+  service: socks-proxy
+  originRequest:
+    ipRules:
+      - prefix: 1.1.1.0/24
+        ports: [80, 443]
+        allow: true
+      - prefix: 0.0.0.0/0
+        allow: false
+- service: http_status:404
+`},
+			want: []Rule{
+				{
+					Hostname: "socks.foo.com",
+					Service:  newSocksProxyOverWSService(accessPolicy()),
+					Config:   defaultConfig,
+				},
+				{
+					Service: &fourOhFour,
+					Config:  defaultConfig,
+				},
+			},
+		},
 		{
 			name: "URL isn't necessary if using bastion",
 			args: args{rawYAML: `
@@ -548,6 +576,16 @@ func MustParseURL(t *testing.T, rawURL string) *url.URL {
 	return u
 }
 
+func accessPolicy() *ipaccess.Policy {
+	cidr1 := "1.1.1.0/24"
+	cidr2 := "0.0.0.0/0"
+	rule1, _ := ipaccess.NewRuleByCIDR(&cidr1, []int{80, 443}, true)
+	rule2, _ := ipaccess.NewRuleByCIDR(&cidr2, nil, false)
+	rules := []ipaccess.Rule{rule1, rule2}
+	accessPolicy, _ := ipaccess.NewPolicy(false, rules)
+	return accessPolicy
+}
+
 func BenchmarkFindMatch(b *testing.B) {
 	rulesYAML := `
 ingress:

ingress/origin_connection.go

@@ -7,6 +7,8 @@ import (
 	"net"
 	"net/http"
 
+	"github.com/cloudflare/cloudflared/ipaccess"
+	"github.com/cloudflare/cloudflared/socks"
 	"github.com/cloudflare/cloudflared/websocket"
 	gws "github.com/gorilla/websocket"
 	"github.com/rs/zerolog"
@@ -107,3 +109,17 @@ func newWSConnection(clientTLSConfig *tls.Config, r *http.Request) (OriginConnec
 		resp,
 	}, resp, nil
 }
+
+// socksProxyOverWSConnection is an OriginConnection that streams SOCKS connections over WS.
+// The connection to the origin happens inside the SOCKS code as the client specifies the origin
+// details in the packet.
+type socksProxyOverWSConnection struct {
+	accessPolicy *ipaccess.Policy
+}
+
+func (sp *socksProxyOverWSConnection) Stream(ctx context.Context, tunnelConn io.ReadWriter, log *zerolog.Logger) {
+	socks.StreamNetHandler(websocket.NewConn(ctx, tunnelConn, log), sp.accessPolicy, log)
+}
+
+func (sp *socksProxyOverWSConnection) Close() {
+}

ingress/origin_proxy.go

@@ -145,3 +145,14 @@ func (o *tcpOverWSService) bastionDest(r *http.Request) (string, error) {
 func removePath(dest string) string {
 	return strings.SplitN(dest, "/", 2)[0]
 }
+
+func (o *socksProxyOverWSService) EstablishConnection(r *http.Request) (OriginConnection, *http.Response, error) {
+	originConn := o.conn
+	resp := &http.Response{
+		Status:        switchingProtocolText,
+		StatusCode:    http.StatusSwitchingProtocols,
+		Header:        websocket.NewResponseHeader(r),
+		ContentLength: -1,
+	}
+	return originConn, resp, nil
+}

ingress/origin_request_config.go

@@ -3,9 +3,11 @@ package ingress
 import (
 	"time"
 
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
-	"github.com/cloudflare/cloudflared/tlsconfig"
+	"github.com/cloudflare/cloudflared/ipaccess"
 	"github.com/urfave/cli/v2"
+
+	"github.com/cloudflare/cloudflared/config"
+	"github.com/cloudflare/cloudflared/tlsconfig"
 )
 
 const (
@@ -212,6 +214,8 @@ type OriginRequestConfig struct {
 	ProxyPort uint `yaml:"proxyPort"`
 	// What sort of proxy should be started
 	ProxyType string `yaml:"proxyType"`
+	// IP rules for the proxy service
+	IPRules []ipaccess.Rule `yaml:"ipRules"`
 }
 
 func (defaults *OriginRequestConfig) setConnectTimeout(overrides config.OriginRequestConfig) {

ingress/origin_request_config_test.go

@@ -5,7 +5,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/config"
 	"github.com/stretchr/testify/require"
 	"github.com/urfave/cli/v2"
 	"gopkg.in/yaml.v2"

ingress/origin_service.go

@@ -12,6 +12,7 @@ import (
 	"time"
 
 	"github.com/cloudflare/cloudflared/hello"
+	"github.com/cloudflare/cloudflared/ipaccess"
 	"github.com/cloudflare/cloudflared/socks"
 	"github.com/cloudflare/cloudflared/tlsconfig"
 	"github.com/cloudflare/cloudflared/websocket"
@@ -100,6 +101,10 @@ type tcpOverWSService struct {
 	streamHandler streamHandlerFunc
 }
 
+type socksProxyOverWSService struct {
+	conn *socksProxyOverWSConnection
+}
+
 func newTCPOverWSService(url *url.URL) *tcpOverWSService {
 	switch url.Scheme {
 	case "ssh":
@@ -122,6 +127,16 @@ func newBastionService() *tcpOverWSService {
 	}
 }
 
+func newSocksProxyOverWSService(accessPolicy *ipaccess.Policy) *socksProxyOverWSService {
+	proxy := socksProxyOverWSService{
+		conn: &socksProxyOverWSConnection{
+			accessPolicy: accessPolicy,
+		},
+	}
+
+	return &proxy
+}
+
 func addPortIfMissing(uri *url.URL, port int) {
 	if uri.Port() == "" {
 		uri.Host = fmt.Sprintf("%s:%d", uri.Hostname(), port)
@@ -144,6 +159,14 @@ func (o *tcpOverWSService) start(wg *sync.WaitGroup, log *zerolog.Logger, shutdo
 	return nil
 }
 
+func (o *socksProxyOverWSService) start(wg *sync.WaitGroup, log *zerolog.Logger, shutdownC <-chan struct{}, errC chan error, cfg OriginRequestConfig) error {
+	return nil
+}
+
+func (o *socksProxyOverWSService) String() string {
+	return ServiceSocksProxy
+}
+
 // HelloWorld is an OriginService for the built-in Hello World server.
 // Users only use this for testing and experimenting with cloudflared.
 type helloWorld struct {

ipaccess/access.go

@@ -0,0 +1,101 @@
+package ipaccess
+
+import (
+	"fmt"
+	"net"
+	"sort"
+)
+
+type Policy struct {
+	defaultAllow bool
+	rules        []Rule
+}
+
+type Rule struct {
+	ipNet *net.IPNet
+	ports []int
+	allow bool
+}
+
+func NewPolicy(defaultAllow bool, rules []Rule) (*Policy, error) {
+	for _, rule := range rules {
+		if err := rule.Validate(); err != nil {
+			return nil, err
+		}
+	}
+
+	policy := Policy{
+		defaultAllow: defaultAllow,
+		rules:        rules,
+	}
+
+	return &policy, nil
+}
+
+func NewRuleByCIDR(prefix *string, ports []int, allow bool) (Rule, error) {
+	if prefix == nil || len(*prefix) == 0 {
+		return Rule{}, fmt.Errorf("no prefix provided")
+	}
+
+	_, ipnet, err := net.ParseCIDR(*prefix)
+	if err != nil {
+		return Rule{}, fmt.Errorf("unable to parse cidr: %s", *prefix)
+	}
+
+	return NewRule(ipnet, ports, allow)
+}
+
+func NewRule(ipnet *net.IPNet, ports []int, allow bool) (Rule, error) {
+	rule := Rule{
+		ipNet: ipnet,
+		ports: ports,
+		allow: allow,
+	}
+	return rule, rule.Validate()
+}
+
+func (r *Rule) Validate() error {
+	if r.ipNet == nil {
+		return fmt.Errorf("no ipnet set on the rule")
+	}
+
+	if len(r.ports) > 0 {
+		sort.Ints(r.ports)
+		for _, port := range r.ports {
+			if port < 1 || port > 65535 {
+				return fmt.Errorf("invalid port %d, needs to be between 1 and 65535", port)
+			}
+		}
+	}
+
+	return nil
+}
+
+func (h *Policy) Allowed(ip net.IP, port int) (bool, *Rule) {
+	if len(h.rules) == 0 {
+		return h.defaultAllow, nil
+	}
+
+	for _, rule := range h.rules {
+		if rule.ipNet.Contains(ip) {
+			if len(rule.ports) == 0 {
+				return rule.allow, &rule
+			} else if pos := sort.SearchInts(rule.ports, port); pos < len(rule.ports) && rule.ports[pos] == port {
+				return rule.allow, &rule
+			}
+		}
+	}
+
+	return h.defaultAllow, nil
+}
+
+func (ipr *Rule) String() string {
+	return fmt.Sprintf("prefix:%s/port:%s/allow:%t", ipr.ipNet, ipr.PortsString(), ipr.allow)
+}
+
+func (ipr *Rule) PortsString() string {
+	if len(ipr.ports) > 0 {
+		return fmt.Sprint(ipr.ports)
+	}
+	return "all"
+}

ipaccess/access_test.go

@@ -0,0 +1,107 @@
+package ipaccess
+
+import (
+	"bytes"
+	"net"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRuleCreation(t *testing.T) {
+	_, ipnet, _ := net.ParseCIDR("1.1.1.1/24")
+
+	_, err := NewRule(nil, []int{80}, false)
+	assert.Error(t, err, "expected error as no ipnet provided")
+
+	_, err = NewRule(ipnet, []int{65536, 80}, false)
+	assert.Error(t, err, "expected error as port higher than 65535")
+
+	_, err = NewRule(ipnet, []int{80, -1}, false)
+	assert.Error(t, err, "expected error as port less than 0")
+
+	rule, err := NewRule(ipnet, []int{443, 80}, false)
+	assert.NoError(t, err)
+	assert.True(t, ipnet.IP.Equal(rule.ipNet.IP) && bytes.Compare(ipnet.Mask, rule.ipNet.Mask) == 0, "ipnet expected to be %+v, got: %+v", ipnet, rule.ipNet)
+	assert.True(t, len(rule.ports) == 2 && rule.ports[0] == 80 && rule.ports[1] == 443, "expected ports to be sorted")
+}
+
+func TestRuleCreationByCIDR(t *testing.T) {
+	var cidr *string
+	_, err := NewRuleByCIDR(cidr, []int{80}, false)
+	assert.Error(t, err, "expected error as cidr is nil")
+
+	badCidr := "1.1.1.1"
+	cidr = &badCidr
+	_, err = NewRuleByCIDR(cidr, []int{80}, false)
+	assert.Error(t, err, "expected error as the cidr is bad")
+
+	goodCidr := "1.1.1.1/24"
+	_, ipnet, _ := net.ParseCIDR("1.1.1.0/24")
+	cidr = &goodCidr
+	rule, err := NewRuleByCIDR(cidr, []int{80}, false)
+	assert.NoError(t, err)
+	assert.True(t, ipnet.IP.Equal(rule.ipNet.IP) && bytes.Compare(ipnet.Mask, rule.ipNet.Mask) == 0, "ipnet expected to be %+v, got: %+v", ipnet, rule.ipNet)
+}
+
+func TestRulesNoRules(t *testing.T) {
+	ip, _, _ := net.ParseCIDR("1.2.3.4/24")
+
+	policy, _ := NewPolicy(true, []Rule{})
+
+	allowed, rule := policy.Allowed(ip, 80)
+	assert.True(t, allowed, "expected to be allowed as no rules and default allow")
+	assert.Nil(t, rule, "expected to be nil as no rules")
+
+	policy, _ = NewPolicy(false, []Rule{})
+
+	allowed, rule = policy.Allowed(ip, 80)
+	assert.False(t, allowed, "expected to be denied as no rules and default deny")
+	assert.Nil(t, rule, "expected to be nil as no rules")
+}
+
+func TestRulesMatchIPAndPort(t *testing.T) {
+	ip1, ipnet1, _ := net.ParseCIDR("1.2.3.4/24")
+	ip2, _, _ := net.ParseCIDR("2.3.4.5/24")
+
+	rule1, _ := NewRule(ipnet1, []int{80, 443}, true)
+	rules := []Rule{
+		rule1,
+	}
+
+	policy, _ := NewPolicy(false, rules)
+
+	allowed, rule := policy.Allowed(ip1, 80)
+	assert.True(t, allowed, "expected to be allowed as matching rule")
+	assert.True(t, rule.ipNet == ipnet1, "expected to match ipnet1")
+
+	allowed, rule = policy.Allowed(ip2, 80)
+	assert.False(t, allowed, "expected to be denied as no matching rule")
+	assert.Nil(t, rule, "expected to be nil")
+}
+
+func TestRulesMatchIPAndPort2(t *testing.T) {
+	ip1, ipnet1, _ := net.ParseCIDR("1.2.3.4/24")
+	ip2, ipnet2, _ := net.ParseCIDR("2.3.4.5/24")
+
+	rule1, _ := NewRule(ipnet1, []int{53, 80}, false)
+	rule2, _ := NewRule(ipnet2, []int{53, 80}, true)
+	rules := []Rule{
+		rule1,
+		rule2,
+	}
+
+	policy, _ := NewPolicy(false, rules)
+
+	allowed, rule := policy.Allowed(ip1, 80)
+	assert.False(t, allowed, "expected to be denied as matching rule")
+	assert.True(t, rule.ipNet == ipnet1, "expected to match ipnet1")
+
+	allowed, rule = policy.Allowed(ip2, 80)
+	assert.True(t, allowed, "expected to be allowed as matching rule")
+	assert.True(t, rule.ipNet == ipnet2, "expected to match ipnet1")
+
+	allowed, rule = policy.Allowed(ip2, 81)
+	assert.False(t, allowed, "expected to be denied as no matching rule")
+	assert.Nil(t, rule, "expected to be nil")
+}

logger/create.go

@@ -64,6 +64,8 @@ func (t resilientMultiWriter) Write(p []byte) (n int, err error) {
 	return len(p), nil
 }
 
+var levelErrorLogged = false
+
 func newZerolog(loggerConfig *Config) *zerolog.Logger {
 	var writers []io.Writer
 
@@ -91,11 +93,15 @@ func newZerolog(loggerConfig *Config) *zerolog.Logger {
 
 	multi := resilientMultiWriter{writers}
 
-	level, err := zerolog.ParseLevel(loggerConfig.MinLevel)
-	if err != nil {
-		return fallbackLogger(err)
+	level, levelErr := zerolog.ParseLevel(loggerConfig.MinLevel)
+	if levelErr != nil {
+		level = zerolog.InfoLevel
 	}
 	log := zerolog.New(multi).With().Timestamp().Logger().Level(level)
+	if !levelErrorLogged && levelErr != nil {
+		log.Error().Msgf("Failed to parse log level %q, using %q instead", loggerConfig.MinLevel, level)
+		levelErrorLogged = true
+	}
 
 	return &log
 }
@@ -151,8 +157,8 @@ func Create(loggerConfig *Config) *zerolog.Logger {
 func createConsoleLogger(config ConsoleConfig) io.Writer {
 	consoleOut := os.Stderr
 	return zerolog.ConsoleWriter{
-		Out:     colorable.NewColorable(consoleOut),
-		NoColor: config.noColor || !term.IsTerminal(int(consoleOut.Fd())),
+		Out:        colorable.NewColorable(consoleOut),
+		NoColor:    config.noColor || !term.IsTerminal(int(consoleOut.Fd())),
 		TimeFormat: consoleTimeFormat,
 	}
 }

origin/pool.go

@@ -1,17 +1,17 @@
-package buffer
+package origin
 
 import (
 	"sync"
 )
 
-type Pool struct {
-	// A Pool must not be copied after first use.
+type bufferPool struct {
+	// A bufferPool must not be copied after first use.
 	// https://golang.org/pkg/sync/#Pool
 	buffers sync.Pool
 }
 
-func NewPool(bufferSize int) *Pool {
-	return &Pool{
+func newBufferPool(bufferSize int) *bufferPool {
+	return &bufferPool{
 		buffers: sync.Pool{
 			New: func() interface{} {
 				return make([]byte, bufferSize)
@@ -20,10 +20,10 @@ func NewPool(bufferSize int) *Pool {
 	}
 }
 
-func (p *Pool) Get() []byte {
+func (p *bufferPool) Get() []byte {
 	return p.buffers.Get().([]byte)
 }
 
-func (p *Pool) Put(buf []byte) {
+func (p *bufferPool) Put(buf []byte) {
 	p.buffers.Put(buf)
 }

origin/proxy.go

@@ -9,12 +9,12 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/cloudflare/cloudflared/buffer"
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog"
+
 	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/ingress"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
-	"github.com/pkg/errors"
-	"github.com/rs/zerolog"
 )
 
 const (
@@ -26,7 +26,7 @@ type proxy struct {
 	warpRouting  *ingress.WarpRoutingService
 	tags         []tunnelpogs.Tag
 	log          *zerolog.Logger
-	bufferPool   *buffer.Pool
+	bufferPool   *bufferPool
 }
 
 func NewOriginProxy(
@@ -40,7 +40,7 @@ func NewOriginProxy(
 		warpRouting:  warpRouting,
 		tags:         tags,
 		log:          log,
-		bufferPool:   buffer.NewPool(512 * 1024),
+		bufferPool:   newBufferPool(512 * 1024),
 	}
 }
 
@@ -58,7 +58,7 @@ func (p *proxy) Proxy(w connection.ResponseWriter, req *http.Request, sourceConn
 	p.appendTagHeaders(req)
 	if sourceConnectionType == connection.TypeTCP {
 		if p.warpRouting == nil {
-			err := errors.New(`cloudflared received a request from Warp client, but your configuration has disabled ingress from Warp clients. To enable this, set "warp-routing:\n\t enabled: true" in your config.yaml`)
+			err := errors.New(`cloudflared received a request from WARP client, but your configuration has disabled ingress from WARP clients. To enable this, set "warp-routing:\n\t enabled: true" in your config.yaml`)
 			p.log.Error().Msg(err.Error())
 			return err
 		}

origin/proxy_test.go

@@ -15,7 +15,7 @@ import (
 
 	"github.com/cloudflare/cloudflared/logger"
 
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/hello"
 	"github.com/cloudflare/cloudflared/ingress"

origin/supervisor.go

@@ -30,12 +30,6 @@ const (
 	refreshAuthMaxBackoff = 10
 	// Waiting time before retrying a failed 'Authenticate' connection
 	refreshAuthRetryDuration = time.Second * 10
-	// Maximum time to make an Authenticate RPC
-	authTokenTimeout = time.Second * 30
-)
-
-var (
-	errEventDigestUnset = errors.New("event digest unset")
 )
 
 // Supervisor manages non-declarative tunnels. Establishes TCP connections with the edge, and

origin/tunnel.go

@@ -15,7 +15,6 @@ import (
 	"github.com/rs/zerolog"
 	"golang.org/x/sync/errgroup"
 
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
 	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/edgediscovery"
 	"github.com/cloudflare/cloudflared/h2mux"
@@ -40,7 +39,7 @@ const (
 
 type TunnelConfig struct {
 	ConnectionConfig *connection.Config
-	BuildInfo        *buildinfo.BuildInfo
+	OSArch           string
 	ClientID         string
 	CloseConnOnce    *sync.Once // Used to close connectedSignal no more than once
 	EdgeAddrs        []string
@@ -72,7 +71,7 @@ func (c *TunnelConfig) RegistrationOptions(connectionID uint8, OriginLocalIP str
 	return &tunnelpogs.RegistrationOptions{
 		ClientID:             c.ClientID,
 		Version:              c.ReportedVersion,
-		OS:                   fmt.Sprintf("%s_%s", c.BuildInfo.GoOS, c.BuildInfo.GoArch),
+		OS:                   c.OSArch,
 		ExistingTunnelPolicy: policy,
 		PoolName:             c.LBPool,
 		Tags:                 c.Tags,

socks/connection_handler_test.go

@@ -40,7 +40,7 @@ func sendSocksRequest(t *testing.T) []byte {
 
 func startTestServer(t *testing.T, httpHandler func(w http.ResponseWriter, r *http.Request)) {
 	// create a socks server
-	requestHandler := NewRequestHandler(NewNetDialer())
+	requestHandler := NewRequestHandler(NewNetDialer(), nil)
 	socksServer := NewConnectionHandler(requestHandler)
 	listener, err := net.Listen("tcp", "localhost:8086")
 	assert.NoError(t, err)

socks/request_handler.go

@@ -6,6 +6,7 @@ import (
 	"net"
 	"strings"
 
+	"github.com/cloudflare/cloudflared/ipaccess"
 	"github.com/rs/zerolog"
 )
 
@@ -16,14 +17,16 @@ type RequestHandler interface {
 
 // StandardRequestHandler implements the base socks5 command processing
 type StandardRequestHandler struct {
-	dialer Dialer
+	dialer       Dialer
+	accessPolicy *ipaccess.Policy
 }
 
 // NewRequestHandler creates a standard SOCKS5 request handler
 // This handles the SOCKS5 commands and proxies them to their destination
-func NewRequestHandler(dialer Dialer) RequestHandler {
+func NewRequestHandler(dialer Dialer, accessPolicy *ipaccess.Policy) RequestHandler {
 	return &StandardRequestHandler{
-		dialer: dialer,
+		dialer:       dialer,
+		accessPolicy: accessPolicy,
 	}
 }
 
@@ -46,6 +49,25 @@ func (h *StandardRequestHandler) Handle(req *Request, conn io.ReadWriter) error
 
 // handleConnect is used to handle a connect command
 func (h *StandardRequestHandler) handleConnect(conn io.ReadWriter, req *Request) error {
+	if h.accessPolicy != nil {
+		if req.DestAddr.IP == nil {
+			addr, err := net.ResolveIPAddr("ip", req.DestAddr.FQDN)
+			if err != nil {
+				_ = sendReply(conn, ruleFailure, req.DestAddr)
+				return fmt.Errorf("unable to resolve host to confirm acceess")
+			}
+
+			req.DestAddr.IP = addr.IP
+		}
+		if allowed, rule := h.accessPolicy.Allowed(req.DestAddr.IP, req.DestAddr.Port); !allowed {
+			_ = sendReply(conn, ruleFailure, req.DestAddr)
+			if rule != nil {
+				return fmt.Errorf("Connect to %v denied due to iprule: %s", req.DestAddr, rule.String())
+			}
+			return fmt.Errorf("Connect to %v denied", req.DestAddr)
+		}
+	}
+
 	target, localAddr, err := h.dialer.Dial(req.DestAddr.Address())
 	if err != nil {
 		msg := err.Error()
@@ -110,7 +132,17 @@ func (h *StandardRequestHandler) handleAssociate(conn io.ReadWriter, req *Reques
 
 func StreamHandler(tunnelConn io.ReadWriter, originConn net.Conn, log *zerolog.Logger) {
 	dialer := NewConnDialer(originConn)
-	requestHandler := NewRequestHandler(dialer)
+	requestHandler := NewRequestHandler(dialer, nil)
+	socksServer := NewConnectionHandler(requestHandler)
+
+	if err := socksServer.Serve(tunnelConn); err != nil {
+		log.Debug().Err(err).Msg("Socks stream handler error")
+	}
+}
+
+func StreamNetHandler(tunnelConn io.ReadWriter, accessPolicy *ipaccess.Policy, log *zerolog.Logger) {
+	dialer := NewNetDialer()
+	requestHandler := NewRequestHandler(dialer, accessPolicy)
 	socksServer := NewConnectionHandler(requestHandler)
 
 	if err := socksServer.Serve(tunnelConn); err != nil {

socks/request_handler_test.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"testing"
 
+	"github.com/cloudflare/cloudflared/ipaccess"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -11,7 +12,7 @@ func TestUnsupportedBind(t *testing.T) {
 	req := createRequest(t, socks5Version, bindCommand, "2001:db8::68", 1337, false)
 	var b bytes.Buffer
 
-	requestHandler := NewRequestHandler(NewNetDialer())
+	requestHandler := NewRequestHandler(NewNetDialer(), nil)
 	err := requestHandler.Handle(req, &b)
 	assert.NoError(t, err)
 	assert.True(t, b.Bytes()[1] == commandNotSupported, "expected a response")
@@ -21,8 +22,61 @@ func TestUnsupportedAssociate(t *testing.T) {
 	req := createRequest(t, socks5Version, associateCommand, "127.0.0.1", 1337, false)
 	var b bytes.Buffer
 
-	requestHandler := NewRequestHandler(NewNetDialer())
+	requestHandler := NewRequestHandler(NewNetDialer(), nil)
 	err := requestHandler.Handle(req, &b)
 	assert.NoError(t, err)
 	assert.True(t, b.Bytes()[1] == commandNotSupported, "expected a response")
 }
+
+func TestHandleConnect(t *testing.T) {
+	req := createRequest(t, socks5Version, connectCommand, "127.0.0.1", 1337, false)
+	var b bytes.Buffer
+
+	requestHandler := NewRequestHandler(NewNetDialer(), nil)
+	err := requestHandler.Handle(req, &b)
+	assert.Error(t, err)
+	assert.True(t, b.Bytes()[1] == connectionRefused, "expected a response")
+}
+
+func TestHandleConnectIPAccess(t *testing.T) {
+	prefix := "127.0.0.0/24"
+	rule1, _ := ipaccess.NewRuleByCIDR(&prefix, []int{1337}, true)
+	rule2, _ := ipaccess.NewRuleByCIDR(&prefix, []int{1338}, false)
+	rules := []ipaccess.Rule{rule1, rule2}
+	var b bytes.Buffer
+
+	accessPolicy, _ := ipaccess.NewPolicy(false, nil)
+	requestHandler := NewRequestHandler(NewNetDialer(), accessPolicy)
+	req := createRequest(t, socks5Version, connectCommand, "127.0.0.1", 1337, false)
+	err := requestHandler.Handle(req, &b)
+	assert.Error(t, err)
+	assert.True(t, b.Bytes()[1] == ruleFailure, "expected to be denied as no rules and defaultAllow=false")
+
+	b.Reset()
+	accessPolicy, _ = ipaccess.NewPolicy(true, nil)
+	requestHandler = NewRequestHandler(NewNetDialer(), accessPolicy)
+	req = createRequest(t, socks5Version, connectCommand, "127.0.0.1", 1337, false)
+	err = requestHandler.Handle(req, &b)
+	assert.Error(t, err)
+	assert.True(t, b.Bytes()[1] == connectionRefused, "expected to be allowed as no rules and defaultAllow=true")
+
+	b.Reset()
+	accessPolicy, _ = ipaccess.NewPolicy(false, rules)
+	requestHandler = NewRequestHandler(NewNetDialer(), accessPolicy)
+	req = createRequest(t, socks5Version, connectCommand, "127.0.0.1", 1337, false)
+	err = requestHandler.Handle(req, &b)
+	assert.Error(t, err)
+	assert.True(t, b.Bytes()[1] == connectionRefused, "expected to be allowed as matching rule")
+
+	b.Reset()
+	req = createRequest(t, socks5Version, connectCommand, "127.0.0.1", 1338, false)
+	err = requestHandler.Handle(req, &b)
+	assert.Error(t, err)
+	assert.True(t, b.Bytes()[1] == ruleFailure, "expected to be denied as matching rule")
+
+	b.Reset()
+	req = createRequest(t, socks5Version, connectCommand, "127.0.0.1", 1339, false)
+	err = requestHandler.Handle(req, &b)
+	assert.Error(t, err)
+	assert.True(t, b.Bytes()[1] == ruleFailure, "expect to be denied as no matching rule and defaultAllow=false")
+}

sshgen/sshgen.go

@@ -12,15 +12,14 @@ import (
 	"io"
 	"io/ioutil"
 	"net/http"
-	"net/url"
 	"time"
 
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
-	cfpath "github.com/cloudflare/cloudflared/cmd/cloudflared/path"
 	"github.com/coreos/go-oidc/jose"
 	homedir "github.com/mitchellh/go-homedir"
 	"github.com/pkg/errors"
 	gossh "golang.org/x/crypto/ssh"
+	"github.com/cloudflare/cloudflared/config"
+	cfpath "github.com/cloudflare/cloudflared/token"
 )
 
 const (
@@ -51,8 +50,8 @@ type errorResponse struct {
 var mockRequest func(url, contentType string, body io.Reader) (*http.Response, error) = nil
 
 // GenerateShortLivedCertificate generates and stores a keypair for short lived certs
-func GenerateShortLivedCertificate(appURL *url.URL, token string) error {
-	fullName, err := cfpath.GenerateAppTokenFilePathFromURL(appURL, keyName)
+func GenerateShortLivedCertificate(appInfo *cfpath.AppInfo, token string) error {
+	fullName, err := cfpath.GenerateAppTokenFilePathFromURL(appInfo.AppDomain, appInfo.AppAUD, keyName)
 	if err != nil {
 		return err
 	}

sshgen/sshgen_test.go

@@ -9,15 +9,15 @@ import (
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
-	"net/url"
 	"os"
 	"testing"
 	"time"
 
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
-	cfpath "github.com/cloudflare/cloudflared/cmd/cloudflared/path"
 	"github.com/coreos/go-oidc/jose"
 	"github.com/stretchr/testify/assert"
+
+	"github.com/cloudflare/cloudflared/config"
+	cfpath "github.com/cloudflare/cloudflared/token"
 )
 
 const (
@@ -32,10 +32,10 @@ type signingArguments struct {
 }
 
 func TestCertGenSuccess(t *testing.T) {
-	url, _ := url.Parse("https://cf-test-access.com/testpath")
+	appInfo := &cfpath.AppInfo{AppAUD: "abcd1234", AppDomain: "mySite.com"}
 	token := tokenGenerator()
 
-	fullName, err := cfpath.GenerateAppTokenFilePathFromURL(url, keyName)
+	fullName, err := cfpath.GenerateAppTokenFilePathFromURL(appInfo.AppDomain, appInfo.AppAUD, keyName)
 	assert.NoError(t, err)
 
 	pubKeyName := fullName + ".pub"
@@ -65,7 +65,7 @@ func TestCertGenSuccess(t *testing.T) {
 		return w.Result(), nil
 	}
 
-	err = GenerateShortLivedCertificate(url, token)
+	err = GenerateShortLivedCertificate(appInfo, token)
 	assert.NoError(t, err)
 
 	exist, err := config.FileExists(fullName)

tlsconfig/cloudflare_ca.go

@@ -9,11 +9,11 @@ import (
 var cloudflareRootCA = []byte(`
 Issuer: C=US, ST=California, L=San Francisco, O=CloudFlare, Inc., OU=CloudFlare Origin SSL ECC Certificate Authority
 -----BEGIN CERTIFICATE-----
-MIICiDCCAi6gAwIBAgIUXZP3MWb8MKwBE1Qbawsp1sfA/Y4wCgYIKoZIzj0EAwIw
+MIICiTCCAi6gAwIBAgIUXZP3MWb8MKwBE1Qbawsp1sfA/Y4wCgYIKoZIzj0EAwIw
 gY8xCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQHEw1T
 YW4gRnJhbmNpc2NvMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMTgwNgYDVQQL
 Ey9DbG91ZEZsYXJlIE9yaWdpbiBTU0wgRUNDIENlcnRpZmljYXRlIEF1dGhvcml0
-eTAeFw0xNjAyMjIxODI0MDBaFw0yMTAyMjIwMDI0MDBaMIGPMQswCQYDVQQGEwJV
+eTAeFw0xOTA4MjMyMTA4MDBaFw0yOTA4MTUxNzAwMDBaMIGPMQswCQYDVQQGEwJV
 UzETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEZ
 MBcGA1UEChMQQ2xvdWRGbGFyZSwgSW5jLjE4MDYGA1UECxMvQ2xvdWRGbGFyZSBP
 cmlnaW4gU1NMIEVDQyBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkwWTATBgcqhkjOPQIB
@@ -21,8 +21,8 @@ BggqhkjOPQMBBwNCAASR+sGALuaGshnUbcxKry+0LEXZ4NY6JUAtSeA6g87K3jaA
 xpIg9G50PokpfWkhbarLfpcZu0UAoYy2su0EhN7wo2YwZDAOBgNVHQ8BAf8EBAMC
 AQYwEgYDVR0TAQH/BAgwBgEB/wIBAjAdBgNVHQ4EFgQUhTBdOypw1O3VkmcH/es5
 tBoOOKcwHwYDVR0jBBgwFoAUhTBdOypw1O3VkmcH/es5tBoOOKcwCgYIKoZIzj0E
-AwIDSAAwRQIgEiIEHQr5UKma50D1WRMJBUSgjg24U8n8E2mfw/8UPz0CIQCr5V/e
-mcifak4CQsr+DH4pn5SJD7JxtCG3YGswW8QZsw==
+AwIDSQAwRgIhAKilfntP2ILGZjwajktkBtXE1pB4Y/fjAfLkIRUzrI15AiEA5UCL
+XYZZ9m2c3fKwIenMMojL1eqydsgqj/wK4p5kagQ=
 -----END CERTIFICATE-----
 Issuer: C=US, O=CloudFlare, Inc., OU=CloudFlare Origin SSL Certificate Authority, L=San Francisco, ST=California
 -----BEGIN CERTIFICATE-----
@@ -51,39 +51,39 @@ MOz2U0OBSif3FTkhCgZWQKOOLo1P42jHC3ssUZAtVNXrCk3fw9/E15k8NPkBazZ6
 -----END CERTIFICATE-----
 Issuer: C=US, O=CloudFlare, Inc., OU=Origin Pull, L=San Francisco, ST=California, CN=origin-pull.cloudflare.net
 -----BEGIN CERTIFICATE-----
-MIIGBjCCA/CgAwIBAgIIV5G6lVbCLmEwCwYJKoZIhvcNAQENMIGQMQswCQYDVQQG
-EwJVUzEZMBcGA1UEChMQQ2xvdWRGbGFyZSwgSW5jLjEUMBIGA1UECxMLT3JpZ2lu
-IFB1bGwxFjAUBgNVBAcTDVNhbiBGcmFuY2lzY28xEzARBgNVBAgTCkNhbGlmb3Ju
-aWExIzAhBgNVBAMTGm9yaWdpbi1wdWxsLmNsb3VkZmxhcmUubmV0MB4XDTE1MDEx
-MzAyNDc1M1oXDTIwMDExMjAyNTI1M1owgZAxCzAJBgNVBAYTAlVTMRkwFwYDVQQK
-ExBDbG91ZEZsYXJlLCBJbmMuMRQwEgYDVQQLEwtPcmlnaW4gUHVsbDEWMBQGA1UE
-BxMNU2FuIEZyYW5jaXNjbzETMBEGA1UECBMKQ2FsaWZvcm5pYTEjMCEGA1UEAxMa
-b3JpZ2luLXB1bGwuY2xvdWRmbGFyZS5uZXQwggIiMA0GCSqGSIb3DQEBAQUAA4IC
-DwAwggIKAoICAQDdsts6I2H5dGyn4adACQRXlfo0KmwsN7B5rxD8C5qgy6spyONr
-WV0ecvdeGQfWa8Gy/yuTuOnsXfy7oyZ1dm93c3Mea7YkM7KNMc5Y6m520E9tHooc
-f1qxeDpGSsnWc7HWibFgD7qZQx+T+yfNqt63vPI0HYBOYao6hWd3JQhu5caAcIS2
-ms5tzSSZVH83ZPe6Lkb5xRgLl3eXEFcfI2DjnlOtLFqpjHuEB3Tr6agfdWyaGEEi
-lRY1IB3k6TfLTaSiX2/SyJ96bp92wvTSjR7USjDV9ypf7AD6u6vwJZ3bwNisNw5L
-ptph0FBnc1R6nDoHmvQRoyytoe0rl/d801i9Nru/fXa+l5K2nf1koR3IX440Z2i9
-+Z4iVA69NmCbT4MVjm7K3zlOtwfI7i1KYVv+ATo4ycgBuZfY9f/2lBhIv7BHuZal
-b9D+/EK8aMUfjDF4icEGm+RQfExv2nOpkR4BfQppF/dLmkYfjgtO1403X0ihkT6T
-PYQdmYS6Jf53/KpqC3aA+R7zg2birtvprinlR14MNvwOsDOzsK4p8WYsgZOR4Qr2
-gAx+z2aVOs/87+TVOR0r14irQsxbg7uP2X4t+EXx13glHxwG+CnzUVycDLMVGvuG
-aUgF9hukZxlOZnrl6VOf1fg0Caf3uvV8smOkVw6DMsGhBZSJVwao0UQNqQIDAQAB
-o2YwZDAOBgNVHQ8BAf8EBAMCAAYwEgYDVR0TAQH/BAgwBgEB/wIBAjAdBgNVHQ4E
-FgQUQ1lLK2mLgOERM2pXzVc42p59xeswHwYDVR0jBBgwFoAUQ1lLK2mLgOERM2pX
-zVc42p59xeswCwYJKoZIhvcNAQENA4ICAQDKDQM1qPRVP/4Gltz0D6OU6xezFBKr
-LWtDoA1qW2F7pkiYawCP9MrDPDJsHy7dx+xw3bBZxOsK5PA/T7p1dqpEl6i8F692
-g//EuYOifLYw3ySPe3LRNhvPl/1f6Sn862VhPvLa8aQAAwR9e/CZvlY3fj+6G5ik
-3it7fikmKUsVnugNOkjmwI3hZqXfJNc7AtHDFw0mEOV0dSeAPTo95N9cxBbm9PKv
-qAEmTEXp2trQ/RjJ/AomJyfA1BQjsD0j++DI3a9/BbDwWmr1lJciKxiNKaa0BRLB
-dKMrYQD+PkPNCgEuojT+paLKRrMyFUzHSG1doYm46NE9/WARTh3sFUp1B7HZSBqA
-kHleoB/vQ/mDuW9C3/8Jk2uRUdZxR+LoNZItuOjU8oTy6zpN1+GgSj7bHjiy9rfA
-F+ehdrz+IOh80WIiqs763PGoaYUyzxLvVowLWNoxVVoc9G+PqFKqD988XlipHVB6
-Bz+1CD4D/bWrs3cC9+kk/jFmrrAymZlkFX8tDb5aXASSLJjUjcptci9SKqtI2h0J
-wUGkD7+bQAr+7vr8/R+CBmNMe7csE8NeEX6lVMF7Dh0a1YKQa6hUN18bBuYgTMuT
-QzMmZpRpIBB321ZBlcnlxiTJvWxvbCPHKHj20VwwAz7LONF59s84ZsOqfoBv8gKM
-s0s5dsq5zpLeaw==
+MIIGCjCCA/KgAwIBAgIIV5G6lVbCLmEwDQYJKoZIhvcNAQENBQAwgZAxCzAJBgNV
+BAYTAlVTMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMRQwEgYDVQQLEwtPcmln
+aW4gUHVsbDEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzETMBEGA1UECBMKQ2FsaWZv
+cm5pYTEjMCEGA1UEAxMab3JpZ2luLXB1bGwuY2xvdWRmbGFyZS5uZXQwHhcNMTkx
+MDEwMTg0NTAwWhcNMjkxMTAxMTcwMDAwWjCBkDELMAkGA1UEBhMCVVMxGTAXBgNV
+BAoTEENsb3VkRmxhcmUsIEluYy4xFDASBgNVBAsTC09yaWdpbiBQdWxsMRYwFAYD
+VQQHEw1TYW4gRnJhbmNpc2NvMRMwEQYDVQQIEwpDYWxpZm9ybmlhMSMwIQYDVQQD
+ExpvcmlnaW4tcHVsbC5jbG91ZGZsYXJlLm5ldDCCAiIwDQYJKoZIhvcNAQEBBQAD
+ggIPADCCAgoCggIBAN2y2zojYfl0bKfhp0AJBFeV+jQqbCw3sHmvEPwLmqDLqynI
+42tZXR5y914ZB9ZrwbL/K5O46exd/LujJnV2b3dzcx5rtiQzso0xzljqbnbQT20e
+ihx/WrF4OkZKydZzsdaJsWAPuplDH5P7J82q3re88jQdgE5hqjqFZ3clCG7lxoBw
+hLaazm3NJJlUfzdk97ouRvnFGAuXd5cQVx8jYOOeU60sWqmMe4QHdOvpqB91bJoY
+QSKVFjUgHeTpN8tNpKJfb9LIn3pun3bC9NKNHtRKMNX3Kl/sAPq7q/AlndvA2Kw3
+Dkum2mHQUGdzVHqcOgea9BGjLK2h7SuX93zTWL02u799dr6Xkrad/WShHchfjjRn
+aL35niJUDr02YJtPgxWObsrfOU63B8juLUphW/4BOjjJyAG5l9j1//aUGEi/sEe5
+lqVv0P78QrxoxR+MMXiJwQab5FB8TG/ac6mRHgF9CmkX90uaRh+OC07XjTdfSKGR
+PpM9hB2ZhLol/nf8qmoLdoD5HvODZuKu2+muKeVHXgw2/A6wM7OwrinxZiyBk5Hh
+CvaADH7PZpU6z/zv5NU5HSvXiKtCzFuDu4/Zfi34RfHXeCUfHAb4KfNRXJwMsxUa
++4ZpSAX2G6RnGU5meuXpU5/V+DQJp/e69XyyY6RXDoMywaEFlIlXBqjRRA2pAgMB
+AAGjZjBkMA4GA1UdDwEB/wQEAwIBBjASBgNVHRMBAf8ECDAGAQH/AgECMB0GA1Ud
+DgQWBBRDWUsraYuA4REzalfNVzjann3F6zAfBgNVHSMEGDAWgBRDWUsraYuA4REz
+alfNVzjann3F6zANBgkqhkiG9w0BAQ0FAAOCAgEAkQ+T9nqcSlAuW/90DeYmQOW1
+QhqOor5psBEGvxbNGV2hdLJY8h6QUq48BCevcMChg/L1CkznBNI40i3/6heDn3IS
+zVEwXKf34pPFCACWVMZxbQjkNRTiH8iRur9EsaNQ5oXCPJkhwg2+IFyoPAAYURoX
+VcI9SCDUa45clmYHJ/XYwV1icGVI8/9b2JUqklnOTa5tugwIUi5sTfipNcJXHhgz
+6BKYDl0/UP0lLKbsUETXeTGDiDpxZYIgbcFrRDDkHC6BSvdWVEiH5b9mH2BON60z
+0O0j8EEKTwi9jnafVtZQXP/D8yoVowdFDjXcKkOPF/1gIh9qrFR6GdoPVgB3SkLc
+5ulBqZaCHm563jsvWb/kXJnlFxW+1bsO9BDD6DweBcGdNurgmH625wBXksSdD7y/
+fakk8DagjbjKShYlPEFOAqEcliwjF45eabL0t27MJV61O/jHzHL3dknXeE4BDa2j
+bA+JbyJeUMtU7KMsxvx82RmhqBEJJDBCJ3scVptvhDMRrtqDBW5JShxoAOcpFQGm
+iYWicn46nPDjgTU0bX1ZPpTpryXbvciVL5RkVBuyX2ntcOLDPlZWgxZCBp96x07F
+AnOzKgZk4RzZPNAxCXERVxajn/FLcOhglVAKo5H0ac+AitlQ0ip55D2/mf8o72tM
+fVQ6VpyjEXdiIXWUq/o=
 -----END CERTIFICATE-----`)
 
 func GetCloudflareRootCA() ([]*x509.Certificate, error) {

token/encrypt.go

@@ -4,12 +4,12 @@
 // You can read more here https://godoc.org/golang.org/x/crypto/nacl/box.
 //
 //		msg := []byte("super safe message.")
-//		alice, err := New("alice_priv_key.pem", "alice_pub_key.pem")
+//		alice, err := NewEncrypter("alice_priv_key.pem", "alice_pub_key.pem")
 //		if err != nil {
 //			log.Fatal(err)
 //		}
 //
-//		bob, err := New("bob_priv_key.pem", "bob_pub_key.pem")
+//		bob, err := NewEncrypter("bob_priv_key.pem", "bob_pub_key.pem")
 //		if err != nil {
 //			log.Fatal(err)
 //		}
@@ -23,7 +23,7 @@
 //			log.Fatal(err)
 //		}
 //		fmt.Println(string(data))
-package encrypter
+package token
 
 import (
 	"bytes"
@@ -44,8 +44,8 @@ type Encrypter struct {
 	publicKey  *[32]byte
 }
 
-// New returns a new encrypter with initialized keypair
-func New(privateKey, publicKey string) (*Encrypter, error) {
+// NewEncrypter returns a new encrypter with initialized keypair
+func NewEncrypter(privateKey, publicKey string) (*Encrypter, error) {
 	e := &Encrypter{}
 	pubKey, key, err := e.fetchOrGenerateKeys(privateKey, publicKey)
 	if err != nil {

token/launch_browser_darwin.go

@@ -1,6 +1,6 @@
 //+build darwin
 
-package shell
+package token
 
 import (
 	"os/exec"

token/launch_browser_other.go

@@ -1,6 +1,6 @@
 //+build !windows,!darwin,!linux,!netbsd,!freebsd,!openbsd
 
-package shell
+package token
 
 import (
 	"os/exec"

token/launch_browser_unix.go

@@ -1,6 +1,6 @@
 //+build linux freebsd openbsd netbsd
 
-package shell
+package token
 
 import (
 	"os/exec"

token/launch_browser_windows.go

@@ -1,6 +1,6 @@
 //+build windows
 
-package shell
+package token
 
 import (
 	"fmt"

token/path.go

@@ -1,28 +1,29 @@
-package path
+package token
 
 import (
 	"fmt"
-	"net/url"
 	"os"
 	"path/filepath"
 	"strings"
 
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
 	"github.com/mitchellh/go-homedir"
+
+	"github.com/cloudflare/cloudflared/config"
 )
 
 // GenerateAppTokenFilePathFromURL will return a filepath for given Access org token
-func GenerateAppTokenFilePathFromURL(url *url.URL, suffix string) (string, error) {
+func GenerateAppTokenFilePathFromURL(appDomain, aud string, suffix string) (string, error) {
 	configPath, err := getConfigPath()
 	if err != nil {
 		return "", err
 	}
-	name := strings.Replace(fmt.Sprintf("%s%s-%s", url.Hostname(), url.EscapedPath(), suffix), "/", "-", -1)
+	name := fmt.Sprintf("%s-%s-%s", appDomain, aud, suffix)
+	name = strings.Replace(strings.Replace(name, "/", "-", -1), "*", "-", -1)
 	return filepath.Join(configPath, name), nil
 }
 
-// GenerateOrgTokenFilePathFromURL will return a filepath for given Access application token
-func GenerateOrgTokenFilePathFromURL(authDomain string) (string, error) {
+// generateOrgTokenFilePathFromURL will return a filepath for given Access application token
+func generateOrgTokenFilePathFromURL(authDomain string) (string, error) {
 	configPath, err := getConfigPath()
 	if err != nil {
 		return "", err

token/shell.go

@@ -0,0 +1,7 @@
+package token
+
+// OpenBrowser opens the specified URL in the default browser of the user
+func OpenBrowser(url string) error {
+	return getBrowserCmd(url).Start()
+}
+

token/token.go

@@ -13,9 +13,7 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/path"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/transfer"
+	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/origin"
 
 	"github.com/coreos/go-oidc/jose"
@@ -24,10 +22,18 @@ import (
 )
 
 const (
-	keyName     = "token"
-	tokenHeader = "CF_Authorization"
+	keyName               = "token"
+	tokenCookie           = "CF_Authorization"
+	appDomainHeader       = "CF-Access-Domain"
+	AccessLoginWorkerPath = "/cdn-cgi/access/login"
 )
 
+type AppInfo struct {
+	AuthDomain string
+	AppAUD     string
+	AppDomain  string
+}
+
 type lock struct {
 	lockFilePath string
 	backoff      *origin.BackoffHandler
@@ -102,7 +108,7 @@ func (l *lock) Acquire() error {
 		os.Exit(0)
 	})
 
-	// Check for a path.lock file
+	// Check for a lock file
 	// if the lock file exists; start polling
 	// if not, create the lock file and go through the normal flow.
 	// See AUTH-1736 for the reason why we do all this
@@ -144,23 +150,23 @@ func isTokenLocked(lockFilePath string) bool {
 
 // FetchTokenWithRedirect will either load a stored token or generate a new one
 // it appends the full url as the redirect URL to the access cli request if opening the browser
-func FetchTokenWithRedirect(appURL *url.URL, log *zerolog.Logger) (string, error) {
-	return getToken(appURL, false, log)
+func FetchTokenWithRedirect(appURL *url.URL, appInfo *AppInfo, log *zerolog.Logger) (string, error) {
+	return getToken(appURL, appInfo, false, log)
 }
 
 // FetchToken will either load a stored token or generate a new one
 // it appends the host of the appURL as the redirect URL to the access cli request if opening the browser
-func FetchToken(appURL *url.URL, log *zerolog.Logger) (string, error) {
-	return getToken(appURL, true, log)
+func FetchToken(appURL *url.URL, appInfo *AppInfo, log *zerolog.Logger) (string, error) {
+	return getToken(appURL, appInfo, true, log)
 }
 
 // getToken will either load a stored token or generate a new one
-func getToken(appURL *url.URL, useHostOnly bool, log *zerolog.Logger) (string, error) {
-	if token, err := GetAppTokenIfExists(appURL); token != "" && err == nil {
+func getToken(appURL *url.URL, appInfo *AppInfo, useHostOnly bool, log *zerolog.Logger) (string, error) {
+	if token, err := GetAppTokenIfExists(appInfo); token != "" && err == nil {
 		return token, nil
 	}
 
-	appTokenPath, err := path.GenerateAppTokenFilePathFromURL(appURL, keyName)
+	appTokenPath, err := GenerateAppTokenFilePathFromURL(appInfo.AppDomain, appInfo.AppAUD, keyName)
 	if err != nil {
 		return "", errors.Wrap(err, "failed to generate app token file path")
 	}
@@ -172,40 +178,36 @@ func getToken(appURL *url.URL, useHostOnly bool, log *zerolog.Logger) (string, e
 	defer fileLockAppToken.Release()
 
 	// check to see if another process has gotten a token while we waited for the lock
-	if token, err := GetAppTokenIfExists(appURL); token != "" && err == nil {
+	if token, err := GetAppTokenIfExists(appInfo); token != "" && err == nil {
 		return token, nil
 	}
 
 	// If an app token couldnt be found on disk, check for an org token and attempt to exchange it for an app token.
 	var orgTokenPath string
-	// Get auth domain to format into org token file path
-	if authDomain, err := getAuthDomain(appURL); err != nil {
-		log.Error().Msgf("failed to get auth domain: %s", err)
-	} else {
-		orgToken, err := GetOrgTokenIfExists(authDomain)
+	orgToken, err := GetOrgTokenIfExists(appInfo.AuthDomain)
+	if err != nil {
+		orgTokenPath, err = generateOrgTokenFilePathFromURL(appInfo.AuthDomain)
 		if err != nil {
-			orgTokenPath, err = path.GenerateOrgTokenFilePathFromURL(authDomain)
-			if err != nil {
-				return "", errors.Wrap(err, "failed to generate org token file path")
-			}
-
-			fileLockOrgToken := newLock(orgTokenPath)
-			if err = fileLockOrgToken.Acquire(); err != nil {
-				return "", errors.Wrap(err, "failed to acquire org token lock")
-			}
-			defer fileLockOrgToken.Release()
-			// check if an org token has been created since the lock was acquired
-			orgToken, err = GetOrgTokenIfExists(authDomain)
+			return "", errors.Wrap(err, "failed to generate org token file path")
 		}
-		if err == nil {
-			if appToken, err := exchangeOrgToken(appURL, orgToken); err != nil {
-				log.Debug().Msgf("failed to exchange org token for app token: %s", err)
-			} else {
-				if err := ioutil.WriteFile(appTokenPath, []byte(appToken), 0600); err != nil {
-					return "", errors.Wrap(err, "failed to write app token to disk")
-				}
-				return appToken, nil
+
+		fileLockOrgToken := newLock(orgTokenPath)
+		if err = fileLockOrgToken.Acquire(); err != nil {
+			return "", errors.Wrap(err, "failed to acquire org token lock")
+		}
+		defer fileLockOrgToken.Release()
+		// check if an org token has been created since the lock was acquired
+		orgToken, err = GetOrgTokenIfExists(appInfo.AuthDomain)
+	}
+	if err == nil {
+		if appToken, err := exchangeOrgToken(appURL, orgToken); err != nil {
+			log.Debug().Msgf("failed to exchange org token for app token: %s", err)
+		} else {
+			// generate app path
+			if err := ioutil.WriteFile(appTokenPath, []byte(appToken), 0600); err != nil {
+				return "", errors.Wrap(err, "failed to write app token to disk")
 			}
+			return appToken, nil
 		}
 	}
 	return getTokensFromEdge(appURL, appTokenPath, orgTokenPath, useHostOnly, log)
@@ -220,7 +222,7 @@ func getTokensFromEdge(appURL *url.URL, appTokenPath, orgTokenPath string, useHo
 	// this weird parameter is the resource name (token) and the key/value
 	// we want to send to the transfer service. the key is token and the value
 	// is blank (basically just the id generated in the transfer service)
-	resourceData, err := transfer.Run(appURL, keyName, keyName, "", true, useHostOnly, log)
+	resourceData, err := RunTransfer(appURL, keyName, keyName, "", true, useHostOnly, log)
 	if err != nil {
 		return "", errors.Wrap(err, "failed to run transfer service")
 	}
@@ -244,31 +246,46 @@ func getTokensFromEdge(appURL *url.URL, appTokenPath, orgTokenPath string, useHo
 
 }
 
-// getAuthDomain makes a request to the appURL and stops at the first redirect. The 302 location header will contain the
+// GetAppInfo makes a request to the appURL and stops at the first redirect. The 302 location header will contain the
 // auth domain
-func getAuthDomain(appURL *url.URL) (string, error) {
+func GetAppInfo(reqURL *url.URL) (*AppInfo, error) {
 	client := &http.Client{
 		// do not follow redirects
 		CheckRedirect: func(req *http.Request, via []*http.Request) error {
-			return http.ErrUseLastResponse
+			// stop after hitting login endpoint since it will contain app path
+			if strings.Contains(via[len(via)-1].URL.Path, AccessLoginWorkerPath) {
+				return http.ErrUseLastResponse
+			}
+			return nil
 		},
 		Timeout: time.Second * 7,
 	}
 
-	authDomainReq, err := http.NewRequest("HEAD", appURL.String(), nil)
+	appInfoReq, err := http.NewRequest("HEAD", reqURL.String(), nil)
 	if err != nil {
-		return "", errors.Wrap(err, "failed to create auth domain request")
+		return nil, errors.Wrap(err, "failed to create app info request")
 	}
-	resp, err := client.Do(authDomainReq)
+	resp, err := client.Do(appInfoReq)
 	if err != nil {
-		return "", errors.Wrap(err, "failed to get auth domain")
+		return nil, errors.Wrap(err, "failed to get app info")
 	}
 	resp.Body.Close()
-	location, err := resp.Location()
-	if err != nil {
-		return "", fmt.Errorf("failed to get auth domain. Received status code %d from %s", resp.StatusCode, appURL.String())
+	location := resp.Request.URL
+	if !strings.Contains(location.Path, AccessLoginWorkerPath) {
+		return nil, fmt.Errorf("failed to get Access app info for %s", reqURL.String())
 	}
-	return location.Hostname(), nil
+
+	aud := resp.Request.URL.Query().Get("kid")
+	if aud == "" {
+		return nil, errors.New("Empty app aud")
+	}
+
+	domain := resp.Header.Get(appDomainHeader)
+	if domain == "" {
+		return nil, errors.New("Empty app domain")
+	}
+
+	return &AppInfo{location.Hostname(), aud, domain}, nil
 
 }
 
@@ -278,8 +295,8 @@ func exchangeOrgToken(appURL *url.URL, orgToken string) (string, error) {
 	client := &http.Client{
 		CheckRedirect: func(req *http.Request, via []*http.Request) error {
 			// attach org token to login request
-			if strings.Contains(req.URL.Path, "cdn-cgi/access/login") {
-				req.AddCookie(&http.Cookie{Name: tokenHeader, Value: orgToken})
+			if strings.Contains(req.URL.Path, AccessLoginWorkerPath) {
+				req.AddCookie(&http.Cookie{Name: tokenCookie, Value: orgToken})
 			}
 			// stop after hitting authorized endpoint since it will contain the app token
 			if strings.Contains(via[len(via)-1].URL.Path, "cdn-cgi/access/authorized") {
@@ -302,7 +319,7 @@ func exchangeOrgToken(appURL *url.URL, orgToken string) (string, error) {
 	var appToken string
 	for _, c := range resp.Cookies() {
 		//if Org token revoked on exchange, getTokensFromEdge instead
-		validAppToken := c.Name == tokenHeader && time.Now().Before(c.Expires)
+		validAppToken := c.Name == tokenCookie && time.Now().Before(c.Expires)
 		if validAppToken {
 			appToken = c.Value
 			break
@@ -316,7 +333,7 @@ func exchangeOrgToken(appURL *url.URL, orgToken string) (string, error) {
 }
 
 func GetOrgTokenIfExists(authDomain string) (string, error) {
-	path, err := path.GenerateOrgTokenFilePathFromURL(authDomain)
+	path, err := generateOrgTokenFilePathFromURL(authDomain)
 	if err != nil {
 		return "", err
 	}
@@ -337,8 +354,8 @@ func GetOrgTokenIfExists(authDomain string) (string, error) {
 	return token.Encode(), nil
 }
 
-func GetAppTokenIfExists(url *url.URL) (string, error) {
-	path, err := path.GenerateAppTokenFilePathFromURL(url, keyName)
+func GetAppTokenIfExists(appInfo *AppInfo) (string, error) {
+	path, err := GenerateAppTokenFilePathFromURL(appInfo.AppDomain, appInfo.AppAUD, keyName)
 	if err != nil {
 		return "", err
 	}
@@ -375,8 +392,8 @@ func getTokenIfExists(path string) (*jose.JWT, error) {
 }
 
 // RemoveTokenIfExists removes the a token from local storage if it exists
-func RemoveTokenIfExists(url *url.URL) error {
-	path, err := path.GenerateAppTokenFilePathFromURL(url, keyName)
+func RemoveTokenIfExists(appInfo *AppInfo) error {
+	path, err := GenerateAppTokenFilePathFromURL(appInfo.AppDomain, appInfo.AppAUD, keyName)
 	if err != nil {
 		return err
 	}

token/token_test.go

@@ -1,3 +1,5 @@
+// +build linux
+
 package token
 
 import (

token/transfer.go

@@ -1,4 +1,4 @@
-package transfer
+package token
 
 import (
 	"bytes"
@@ -10,8 +10,6 @@ import (
 	"os"
 	"time"
 
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/encrypter"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/shell"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
 )
@@ -21,14 +19,14 @@ const (
 	clientTimeout = time.Second * 60
 )
 
-// Run does the transfer "dance" with the end result downloading the supported resource.
+// RunTransfer does the transfer "dance" with the end result downloading the supported resource.
 // The expanded description is run is encapsulation of shared business logic needed
 // to request a resource (token/cert/etc) from the transfer service (loginhelper).
 // The "dance" we refer to is building a HTTP request, opening that in a browser waiting for
 // the user to complete an action, while it long polls in the background waiting for an
 // action to be completed to download the resource.
-func Run(transferURL *url.URL, resourceName, key, value string, shouldEncrypt bool, useHostOnly bool, log *zerolog.Logger) ([]byte, error) {
-	encrypterClient, err := encrypter.New("cloudflared_priv.pem", "cloudflared_pub.pem")
+func RunTransfer(transferURL *url.URL, resourceName, key, value string, shouldEncrypt bool, useHostOnly bool, log *zerolog.Logger) ([]byte, error) {
+	encrypterClient, err := NewEncrypter("cloudflared_priv.pem", "cloudflared_pub.pem")
 	if err != nil {
 		return nil, err
 	}
@@ -38,7 +36,7 @@ func Run(transferURL *url.URL, resourceName, key, value string, shouldEncrypt bo
 	}
 
 	// See AUTH-1423 for why we use stderr (the way git wraps ssh)
-	err = shell.OpenBrowser(requestURL)
+	err = OpenBrowser(requestURL)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "Please open the following URL and log in with your Cloudflare account:\n\n%s\n\nLeave cloudflared running to download the %s automatically.\n", requestURL, resourceName)
 	} else {

tunneldns/tunnel.go

@@ -2,22 +2,14 @@ package tunneldns
 
 import (
 	"net"
-	"os"
-	"os/signal"
 	"strconv"
 	"sync"
-	"syscall"
-
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
-	"github.com/cloudflare/cloudflared/logger"
-	"github.com/cloudflare/cloudflared/metrics"
 
 	"github.com/coredns/coredns/core/dnsserver"
 	"github.com/coredns/coredns/plugin"
 	"github.com/coredns/coredns/plugin/cache"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
-	"github.com/urfave/cli/v2"
 )
 
 const (
@@ -33,104 +25,6 @@ type Listener struct {
 	log    *zerolog.Logger
 }
 
-func Command(hidden bool) *cli.Command {
-	return &cli.Command{
-		Name:   "proxy-dns",
-		Action: cliutil.ErrorHandler(Run),
-		Usage:  "Run a DNS over HTTPS proxy server.",
-		Flags: []cli.Flag{
-			&cli.StringFlag{
-				Name:    "metrics",
-				Value:   "localhost:",
-				Usage:   "Listen address for metrics reporting.",
-				EnvVars: []string{"TUNNEL_METRICS"},
-			},
-			&cli.StringFlag{
-				Name:    "address",
-				Usage:   "Listen address for the DNS over HTTPS proxy server.",
-				Value:   "localhost",
-				EnvVars: []string{"TUNNEL_DNS_ADDRESS"},
-			},
-			// Note TUN-3758 , we use Int because UInt is not supported with altsrc
-			&cli.IntFlag{
-				Name:    "port",
-				Usage:   "Listen on given port for the DNS over HTTPS proxy server.",
-				Value:   53,
-				EnvVars: []string{"TUNNEL_DNS_PORT"},
-			},
-			&cli.StringSliceFlag{
-				Name:    "upstream",
-				Usage:   "Upstream endpoint URL, you can specify multiple endpoints for redundancy.",
-				Value:   cli.NewStringSlice("https://1.1.1.1/dns-query", "https://1.0.0.1/dns-query"),
-				EnvVars: []string{"TUNNEL_DNS_UPSTREAM"},
-			},
-			&cli.StringSliceFlag{
-				Name:    "bootstrap",
-				Usage:   "bootstrap endpoint URL, you can specify multiple endpoints for redundancy.",
-				Value:   cli.NewStringSlice("https://162.159.36.1/dns-query", "https://162.159.46.1/dns-query", "https://[2606:4700:4700::1111]/dns-query", "https://[2606:4700:4700::1001]/dns-query"),
-				EnvVars: []string{"TUNNEL_DNS_BOOTSTRAP"},
-			},
-			&cli.IntFlag{
-				Name:    "max-upstream-conns",
-				Usage:   "Maximum concurrent connections to upstream. Setting to 0 means unlimited.",
-				Value:   MaxUpstreamConnsDefault,
-				EnvVars: []string{"TUNNEL_DNS_MAX_UPSTREAM_CONNS"},
-			},
-		},
-		ArgsUsage: " ", // can't be the empty string or we get the default output
-		Hidden:    hidden,
-	}
-}
-
-// Run implements a foreground runner
-func Run(c *cli.Context) error {
-	log := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
-
-	metricsListener, err := net.Listen("tcp", c.String("metrics"))
-	if err != nil {
-		log.Fatal().Err(err).Msg("Failed to open the metrics listener")
-	}
-
-	go metrics.ServeMetrics(metricsListener, nil, nil, log)
-
-	listener, err := CreateListener(
-		c.String("address"),
-		// Note TUN-3758 , we use Int because UInt is not supported with altsrc
-		uint16(c.Int("port")),
-		c.StringSlice("upstream"),
-		c.StringSlice("bootstrap"),
-		c.Int("max-upstream-conns"),
-		log,
-	)
-
-	if err != nil {
-		log.Err(err).Msg("Failed to create the listeners")
-		return err
-	}
-
-	// Try to start the server
-	readySignal := make(chan struct{})
-	err = listener.Start(readySignal)
-	if err != nil {
-		log.Err(err).Msg("Failed to start the listeners")
-		return listener.Stop()
-	}
-	<-readySignal
-
-	// Wait for signal
-	signals := make(chan os.Signal, 10)
-	signal.Notify(signals, syscall.SIGTERM, syscall.SIGINT)
-	defer signal.Stop(signals)
-	<-signals
-
-	// Shut down server
-	err = listener.Stop()
-	if err != nil {
-		log.Err(err).Msg("failed to stop")
-	}
-	return err
-}
-
 // Create a CoreDNS server plugin from configuration
 func createConfig(address string, port uint16, p plugin.Handler) *dnsserver.Config {
 	c := &dnsserver.Config{

tunnelstore/cleanup_params.go

@@ -0,0 +1,25 @@
+package tunnelstore
+
+import (
+	"net/url"
+
+	"github.com/google/uuid"
+)
+
+type CleanupParams struct {
+	queryParams url.Values
+}
+
+func NewCleanupParams() *CleanupParams {
+	return &CleanupParams{
+		queryParams: url.Values{},
+	}
+}
+
+func (cp *CleanupParams) ForClient(clientID uuid.UUID) {
+	cp.queryParams.Set("client_id", clientID.String())
+}
+
+func (cp CleanupParams) encode() string {
+	return cp.queryParams.Encode()
+}

tunnelstore/client.go

@@ -43,6 +43,17 @@ type Connection struct {
 	ColoName           string    `json:"colo_name"`
 	ID                 uuid.UUID `json:"id"`
 	IsPendingReconnect bool      `json:"is_pending_reconnect"`
+	OriginIP           net.IP    `json:"origin_ip"`
+	OpenedAt           time.Time `json:"opened_at"`
+}
+
+type ActiveClient struct {
+	ID          uuid.UUID    `json:"id"`
+	Features    []string     `json:"features"`
+	Version     string       `json:"version"`
+	Arch        string       `json:"arch"`
+	RunAt       time.Time    `json:"run_at"`
+	Connections []Connection `json:"conns"`
 }
 
 type Change = string
@@ -192,7 +203,8 @@ type Client interface {
 	GetTunnel(tunnelID uuid.UUID) (*Tunnel, error)
 	DeleteTunnel(tunnelID uuid.UUID) error
 	ListTunnels(filter *Filter) ([]*Tunnel, error)
-	CleanupConnections(tunnelID uuid.UUID) error
+	ListActiveClients(tunnelID uuid.UUID) ([]*ActiveClient, error)
+	CleanupConnections(tunnelID uuid.UUID, params *CleanupParams) error
 	RouteTunnel(tunnelID uuid.UUID, route Route) (RouteResult, error)
 
 	// Teamnet endpoints
@@ -336,9 +348,32 @@ func parseListTunnels(body io.ReadCloser) ([]*Tunnel, error) {
 	return tunnels, err
 }
 
-func (r *RESTClient) CleanupConnections(tunnelID uuid.UUID) error {
+func (r *RESTClient) ListActiveClients(tunnelID uuid.UUID) ([]*ActiveClient, error) {
 	endpoint := r.baseEndpoints.accountLevel
 	endpoint.Path = path.Join(endpoint.Path, fmt.Sprintf("%v/connections", tunnelID))
+	resp, err := r.sendRequest("GET", endpoint, nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "REST request failed")
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode == http.StatusOK {
+		return parseConnectionsDetails(resp.Body)
+	}
+
+	return nil, r.statusCodeToError("list connection details", resp)
+}
+
+func parseConnectionsDetails(reader io.Reader) ([]*ActiveClient, error) {
+	var clients []*ActiveClient
+	err := parseResponse(reader, &clients)
+	return clients, err
+}
+
+func (r *RESTClient) CleanupConnections(tunnelID uuid.UUID, params *CleanupParams) error {
+	endpoint := r.baseEndpoints.accountLevel
+	endpoint.RawQuery = params.encode()
+	endpoint.Path = path.Join(endpoint.Path, fmt.Sprintf("%v/connections", tunnelID))
 	resp, err := r.sendRequest("DELETE", endpoint, nil)
 	if err != nil {
 		return errors.Wrap(err, "REST request failed")