Release 2021.2.2

Also changed the socks test code so that it binds to localhost, so that
we don't get popups saying "would you like to allow socks.test to use
the network"

.dockerignore

@@ -0,0 +1 @@
+.git

.gitignore

@@ -18,3 +18,4 @@ cloudflared-x86-64*
 .DS_Store
 *-session.log
 ssh_server_tests/.env
+.cover

Dockerfile

@@ -1,7 +1,7 @@
 # use a builder image for building cloudflare
 ARG TARGET_GOOS
 ARG TARGET_GOARCH
-FROM golang:1.15.6 as builder
+FROM golang:1.15.7 as builder
 ENV GO111MODULE=on \
     CGO_ENABLED=0 \
     TARGET_GOOS=${TARGET_GOOS} \

Makefile

@@ -1,10 +1,19 @@
 VERSION       := $(shell git describe --tags --always --dirty="-dev" --match "[0-9][0-9][0-9][0-9].*.*")
-DATE          := $(shell date -u '+%Y-%m-%d-%H%M UTC')
-VERSION_FLAGS := -ldflags='-X "main.Version=$(VERSION)" -X "main.BuildTime=$(DATE)"'
 MSI_VERSION   := $(shell git tag -l --sort=v:refname | grep "w" | tail -1 | cut -c2-)
 #MSI_VERSION expects the format of the tag to be: (wX.X.X). Starts with the w character to not break cfsetup.
 #e.g. w3.0.1 or w4.2.10. It trims off the w character when creating the MSI.
 
+ifeq ($(FIPS), true)
+	GO_BUILD_TAGS := $(GO_BUILD_TAGS) fips
+endif
+
+ifneq ($(GO_BUILD_TAGS),)
+	GO_BUILD_TAGS := -tags $(GO_BUILD_TAGS)
+endif
+
+DATE          := $(shell date -u '+%Y-%m-%d-%H%M UTC')
+VERSION_FLAGS := -ldflags='-X "main.Version=$(VERSION)" -X "main.BuildTime=$(DATE)"'
+
 IMPORT_PATH   := github.com/cloudflare/cloudflared
 PACKAGE_DIR   := $(CURDIR)/packaging
 INSTALL_BINDIR := /usr/bin/
@@ -71,7 +80,7 @@ clean:
 
 .PHONY: cloudflared
 cloudflared: tunnel-deps
-	GOOS=$(TARGET_OS) GOARCH=$(TARGET_ARCH) go build -v -mod=vendor $(VERSION_FLAGS) $(IMPORT_PATH)/cmd/cloudflared
+	GOOS=$(TARGET_OS) GOARCH=$(TARGET_ARCH) go build -v -mod=vendor $(GO_BUILD_TAGS) $(VERSION_FLAGS) $(IMPORT_PATH)/cmd/cloudflared
 
 .PHONY: container
 container:
@@ -79,7 +88,13 @@ container:
 
 .PHONY: test
 test: vet
+ifndef CI
 	go test -v -mod=vendor -race $(VERSION_FLAGS) ./...
+else
+	@mkdir -p .cover
+	go test -v -mod=vendor -race $(VERSION_FLAGS) -coverprofile=".cover/c.out" ./...
+	go tool cover -html ".cover/c.out" -o .cover/all.html
+endif
 
 .PHONY: test-ssh-server
 test-ssh-server:

RELEASE_NOTES

@@ -1,3 +1,51 @@
+2021.2.2
+- 2021-02-04 TUN-3864: Users can choose where credentials file is written after creating a tunnel
+- 2021-02-04 TUN-3869: Improve reliability of graceful shutdown.
+- 2021-02-07 TUN-3878: Do not supply -tags when none are specified
+- 2021-02-04 TUN-3635: Send event when unregistering tunnel for gracful shutdown so /ready endpoint reports down status befoe connections finish handling pending requests.
+- 2021-02-08 TUN-3890: Code coverage for cloudflared in CI
+- 2021-02-09 AUTH-3375 exchangeOrgToken deleted cookie fix
+- 2020-11-18 Update error message to use login command
+
+2021.2.1
+- 2021-02-04 TUN-3858: Do not suffix cloudflared version with -fips
+
+2021.2.0
+- 2021-02-01 TUN-3837: Remove automation_email from cloudflared status page test
+- 2021-02-03 TUN-3848: Use transport logger for h2mux
+- 2021-02-03 TUN-3854: cloudflared tunnel list flags to sort output
+- 2021-01-21 TUN-3195: Don't colorize console logs when stderr is not a terminal
+- 2021-01-20 Fixed connection error handling by removing duplicated errors, standardizing on non-pointer error types
+- 2021-01-20 TUN-3118: Changed graceful shutdown to immediately unregister tunnel from the edge, keep the connection open until the edge drops it or grace period expires
+- 2021-01-25 TUN-3165: Add reference to Argo Tunnel documentation in the help output
+- 2021-01-25 TUN-3806: Use a .dockerignore
+- 2021-01-21 TUN-3795: Use RFC-3339 style date format for logs, produce timestamp in UTC
+- 2021-01-26 TUN-3795: Removed errant test
+- 2021-01-25 TUN-3792: Handle graceful shutdown correctly when running as a windows service. Only expose one shutdown channel globally, which now triggers the graceful shutdown sequence across all modes. Removed separate handling of zero-duration grace period, instead it's checked only when we need to wait for exit.
+- 2021-01-27 TUN-3811: Better error reporting on http2 connection termination. Registration errors from control loop are now propagated out of the connection server code. Unified error handling between h2mux and http2 connections so we log and retry errors the same way, regardless of underlying transport.
+- 2021-01-28 TUN-3830: Use Go 1.15.7
+- 2021-01-28 TUN-3826: Use go-fips when building cloudflared for linux/amd64
+- 2021-01-19 TUN-3777: Fix /ready endpoint for classic tunnels
+- 2021-01-19 TUN-3773: Add back pprof endpoints
+
+2021.1.5
+- 2021-01-15 TUN-3594: Log ingress response at debug level
+- 2021-01-15 TUN-3765: Fix doubly nested log output by `logfile` option
+- 2021-01-16 TUN-3767: Tolerate logging errors
+- 2021-01-17 TUN-3768: Reuse file loggers
+- 2021-01-14 TUN-3738: Refactor observer to avoid potential of blocking on tunnel notifications
+- 2021-01-15 TUN-3766: Print flags defined at all levels of command hierarchy, not just locally defined flags for a command. This fixes output of overriden settings for  subcommand.
+
+2021.1.4
+- 2021-01-14 TUN-3759: Single file logging output should always append
+
+2021.1.3
+- 2021-01-14 TUN-3756: File logging output must consider the directory
+- 2021-01-14 TUN-3757: Fix legacy Uint flags that are incorrectly handled by ufarve library
+
+2021.1.2
+- 2021-01-13 TUN-3747: Fix logging in Windows
+
 2021.1.1
 - 2021-01-13 TUN-3744: Fix compilation error in windows service
 

cfsetup.yaml

@@ -1,26 +1,30 @@
-pinned_go: &pinned_go go=1.15.6-1
+pinned_go: &pinned_go go=1.15.7-1
+pinned_go_fips: &pinned_go_fips go-fips=1.15.5-3
+
 build_dir: &build_dir /cfsetup_build
 default-flavor: buster
 stretch: &stretch
   build:
     build_dir: *build_dir
     builddeps:
-      - *pinned_go
+      - *pinned_go_fips
       - build-essential
     post-cache:
       - export GOOS=linux
       - export GOARCH=amd64
+      - export FIPS=true
       - make cloudflared
   build-deb:
     build_dir: *build_dir
     builddeps:
-      - *pinned_go
+      - *pinned_go_fips
       - build-essential
       - fakeroot
       - rubygem-fpm
     post-cache:
       - export GOOS=linux
       - export GOARCH=amd64
+      - export FIPS=true
       - make cloudflared-deb
   build-deb-arm64:
     build_dir: *build_dir
@@ -36,7 +40,7 @@ stretch: &stretch
   publish-deb:
     build_dir: *build_dir
     builddeps:
-      - *pinned_go
+      - *pinned_go_fips
       - build-essential
       - fakeroot
       - rubygem-fpm
@@ -44,20 +48,22 @@ stretch: &stretch
     post-cache:
       - export GOOS=linux
       - export GOARCH=amd64
+      - export FIPS=true
       - make publish-deb
   release-linux-amd64:
     build_dir: *build_dir
     builddeps:
-      - *pinned_go
+      - *pinned_go_fips
       - build-essential
     post-cache:
       - export GOOS=linux
       - export GOARCH=amd64
+      - export FIPS=true
       - make release
   github-release-linux-amd64:
     build_dir: *build_dir
     builddeps:
-      - *pinned_go
+      - *pinned_go_fips
       - build-essential
       - python3-setuptools
       - python3-pip
@@ -66,6 +72,7 @@ stretch: &stretch
     post-cache:
       - export GOOS=linux
       - export GOARCH=amd64
+      - export FIPS=true
       - make github-release
   release-linux-armv6:
     build_dir: *build_dir
@@ -185,11 +192,12 @@ stretch: &stretch
   test:
     build_dir: *build_dir
     builddeps:
-      - *pinned_go
+      - *pinned_go_fips
       - build-essential
     post-cache:
       - export GOOS=linux
       - export GOARCH=amd64
+      - export FIPS=true
       # cd to a non-module directory: https://github.com/golang/go/issues/24250
       - (cd / && go get github.com/BurntSushi/go-sumtype)
       - export PATH="$HOME/go/bin:$PATH"
@@ -248,8 +256,8 @@ centos-7:
       - https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
     pre-cache:
       - yum install -y fakeroot
-      - wget https://golang.org/dl/go1.15.6.linux-amd64.tar.gz -P /tmp/
-      - tar -C /usr/local -xzf /tmp/go1.15.6.linux-amd64.tar.gz
+      - wget https://golang.org/dl/go1.15.7.linux-amd64.tar.gz -P /tmp/
+      - tar -C /usr/local -xzf /tmp/go1.15.7.linux-amd64.tar.gz
     post-cache:
       - export PATH=$PATH:/usr/local/go/bin
       - export GOOS=linux
@@ -260,8 +268,8 @@ centos-7:
     builddeps: *el7_builddeps
     pre-cache:
       - yum install -y fakeroot
-      - wget https://golang.org/dl/go1.15.6.linux-amd64.tar.gz -P /tmp/
-      - tar -C /usr/local -xzf /tmp/go1.15.6.linux-amd64.tar.gz
+      - wget https://golang.org/dl/go1.15.7.linux-amd64.tar.gz -P /tmp/
+      - tar -C /usr/local -xzf /tmp/go1.15.7.linux-amd64.tar.gz
     post-cache:
       - export PATH=$PATH:/usr/local/go/bin
       - export GOOS=linux

cmd/cloudflared/access/cmd.go

@@ -55,12 +55,11 @@ const sentryDSN = "https://56a9c9fa5c364ab28f34b14f35ea0f1b@sentry.io/189878"
 
 var (
 	shutdownC      chan struct{}
-	graceShutdownC chan struct{}
 )
 
 // Init will initialize and store vars from the main program
-func Init(s, g chan struct{}) {
-	shutdownC, graceShutdownC = s, g
+func Init(shutdown chan struct{}) {
+	shutdownC = shutdown
 }
 
 // Flags return the global flags for Access related commands (hopefully none)
@@ -292,7 +291,7 @@ func generateToken(c *cli.Context) error {
 	}
 	tok, err := token.GetAppTokenIfExists(appURL)
 	if err != nil || tok == "" {
-		fmt.Fprintln(os.Stderr, "Unable to find token for provided application. Please run token command to generate token.")
+		fmt.Fprintln(os.Stderr, "Unable to find token for provided application. Please run login command to generate token.")
 		return err
 	}
 

cmd/cloudflared/fips.go

@@ -0,0 +1,5 @@
+// +build fips
+
+package main
+
+import _ "crypto/tls/fipsonly"

cmd/cloudflared/linux_service.go

@@ -16,7 +16,7 @@ import (
 	"github.com/urfave/cli/v2"
 )
 
-func runApp(app *cli.App, shutdownC, graceShutdownC chan struct{}) {
+func runApp(app *cli.App, graceShutdownC chan struct{}) {
 	app.Commands = append(app.Commands, &cli.Command{
 		Name:  "service",
 		Usage: "Manages the Argo Tunnel system service",

cmd/cloudflared/macos_service.go

@@ -17,7 +17,7 @@ const (
 	launchdIdentifier = "com.cloudflare.cloudflared"
 )
 
-func runApp(app *cli.App, shutdownC, graceShutdownC chan struct{}) {
+func runApp(app *cli.App, graceShutdownC chan struct{}) {
 	app.Commands = append(app.Commands, &cli.Command{
 		Name:  "service",
 		Usage: "Manages the Argo Tunnel launch agent",

cmd/cloudflared/main.go

@@ -46,10 +46,7 @@ func main() {
 	metrics.RegisterBuildInfo(BuildTime, Version)
 	raven.SetRelease(Version)
 
-	// Force shutdown channel used by the app. When closed, app must terminate.
-	// Windows service manager closes this channel when it receives shutdown command.
-	shutdownC := make(chan struct{})
-	// Graceful shutdown channel used by the app. When closed, app must terminate.
+	// Graceful shutdown channel used by the app. When closed, app must terminate gracefully.
 	// Windows service manager closes this channel when it receives stop command.
 	graceShutdownC := make(chan struct{})
 
@@ -73,16 +70,18 @@ func main() {
 	app.Version = fmt.Sprintf("%s (built %s)", Version, BuildTime)
 	app.Description = `cloudflared connects your machine or user identity to Cloudflare's global network.
 	You can use it to authenticate a session to reach an API behind Access, route web traffic to this machine,
-	and configure access control.`
+	and configure access control.
+
+	See https://developers.cloudflare.com/argo-tunnel/ for more in-depth documentation.`
 	app.Flags = flags()
-	app.Action = action(Version, shutdownC, graceShutdownC)
+	app.Action = action(graceShutdownC)
 	app.Before = tunnel.SetFlagsFromConfigFile
 	app.Commands = commands(cli.ShowVersion)
 
-	tunnel.Init(Version, shutdownC, graceShutdownC) // we need this to support the tunnel sub command...
-	access.Init(shutdownC, graceShutdownC)
+	tunnel.Init(Version, graceShutdownC) // we need this to support the tunnel sub command...
+	access.Init(graceShutdownC)
 	updater.Init(Version)
-	runApp(app, shutdownC, graceShutdownC)
+	runApp(app, graceShutdownC)
 }
 
 func commands(version func(c *cli.Context)) []*cli.Command {
@@ -143,10 +142,10 @@ func isEmptyInvocation(c *cli.Context) bool {
 	return c.NArg() == 0 && c.NumFlags() == 0
 }
 
-func action(version string, shutdownC, graceShutdownC chan struct{}) cli.ActionFunc {
+func action(graceShutdownC chan struct{}) cli.ActionFunc {
 	return cliutil.ErrorHandler(func(c *cli.Context) (err error) {
 		if isEmptyInvocation(c) {
-			return handleServiceMode(c, shutdownC)
+			return handleServiceMode(c, graceShutdownC)
 		}
 		tags := make(map[string]string)
 		tags["hostname"] = c.String("hostname")

cmd/cloudflared/token/token.go

@@ -301,7 +301,9 @@ func exchangeOrgToken(appURL *url.URL, orgToken string) (string, error) {
 	resp.Body.Close()
 	var appToken string
 	for _, c := range resp.Cookies() {
-		if c.Name == tokenHeader {
+		//if Org token revoked on exchange, getTokensFromEdge instead
+		validAppToken := c.Name == tokenHeader && time.Now().Before(c.Expires)
+		if validAppToken {
 			appToken = c.Value
 			break
 		}

cmd/cloudflared/tunnel/cmd.go

@@ -32,7 +32,6 @@ import (
 	"github.com/coreos/go-systemd/daemon"
 	"github.com/facebookgo/grace/gracenet"
 	"github.com/getsentry/raven-go"
-	"github.com/google/uuid"
 	"github.com/mitchellh/go-homedir"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
@@ -87,7 +86,6 @@ const (
 )
 
 var (
-	shutdownC      chan struct{}
 	graceShutdownC chan struct{}
 	version        string
 )
@@ -156,7 +154,7 @@ func TunnelCommand(c *cli.Context) error {
 		return err
 	}
 	if name := c.String("name"); name != "" { // Start a named tunnel
-		return runAdhocNamedTunnel(sc, name)
+		return runAdhocNamedTunnel(sc, name, c.String(CredFileFlag))
 	}
 	if ref := config.GetConfiguration().TunnelID; ref != "" {
 		return fmt.Errorf("Use `cloudflared tunnel run` to start tunnel %s", ref)
@@ -166,15 +164,15 @@ func TunnelCommand(c *cli.Context) error {
 	return runClassicTunnel(sc)
 }
 
-func Init(v string, s, g chan struct{}) {
-	version, shutdownC, graceShutdownC = v, s, g
+func Init(ver string, gracefulShutdown chan struct{}) {
+	version, graceShutdownC = ver, gracefulShutdown
 }
 
 // runAdhocNamedTunnel create, route and run a named tunnel in one command
-func runAdhocNamedTunnel(sc *subcommandContext, name string) error {
+func runAdhocNamedTunnel(sc *subcommandContext, name, credentialsOutputPath string) error {
 	tunnel, ok, err := sc.tunnelActive(name)
 	if err != nil || !ok {
-		tunnel, err = sc.create(name)
+		tunnel, err = sc.create(name, credentialsOutputPath)
 		if err != nil {
 			return errors.Wrap(err, "failed to create tunnel")
 		}
@@ -199,7 +197,7 @@ func runAdhocNamedTunnel(sc *subcommandContext, name string) error {
 
 // runClassicTunnel creates a "classic" non-named tunnel
 func runClassicTunnel(sc *subcommandContext) error {
-	return StartServer(sc.c, version, shutdownC, graceShutdownC, nil, sc.log, sc.isUIEnabled)
+	return StartServer(sc.c, version, nil, sc.log, sc.isUIEnabled)
 }
 
 func routeFromFlag(c *cli.Context) (tunnelstore.Route, bool) {
@@ -215,8 +213,6 @@ func routeFromFlag(c *cli.Context) (tunnelstore.Route, bool) {
 func StartServer(
 	c *cli.Context,
 	version string,
-	shutdownC,
-	graceShutdownC chan struct{},
 	namedTunnel *connection.NamedTunnelConfig,
 	log *zerolog.Logger,
 	isUIEnabled bool,
@@ -225,8 +221,6 @@ func StartServer(
 	var wg sync.WaitGroup
 	listeners := gracenet.Net{}
 	errC := make(chan error)
-	connectedSignal := signal.New(make(chan struct{}))
-	dnsReadySignal := make(chan struct{})
 
 	if config.GetConfiguration().Source() == "" {
 		log.Info().Msg(config.ErrNoConfigFile.Error())
@@ -269,36 +263,29 @@ func StartServer(
 	buildInfo.Log(log)
 	logClientOptions(c, log)
 
+	// this context drives the server, when it's cancelled tunnel and all other components (origins, dns, etc...) should stop
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	go waitForSignal(graceShutdownC, log)
+
 	if c.IsSet("proxy-dns") {
+		dnsReadySignal := make(chan struct{})
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
-			errC <- runDNSProxyServer(c, dnsReadySignal, shutdownC, log)
+			errC <- runDNSProxyServer(c, dnsReadySignal, ctx.Done(), log)
 		}()
-	} else {
-		close(dnsReadySignal)
+		// Wait for proxy-dns to come up (if used)
+		<-dnsReadySignal
 	}
 
-	// Wait for proxy-dns to come up (if used)
-	<-dnsReadySignal
-
+	connectedSignal := signal.New(make(chan struct{}))
 	go notifySystemd(connectedSignal)
 	if c.IsSet("pidfile") {
 		go writePidFile(connectedSignal, c.String("pidfile"), log)
 	}
 
-	cloudflaredID, err := uuid.NewRandom()
-	if err != nil {
-		log.Err(err).Msg("Cannot generate cloudflared ID")
-		return err
-	}
-
-	ctx, cancel := context.WithCancel(context.Background())
-	go func() {
-		<-shutdownC
-		cancel()
-	}()
-
 	// update needs to be after DNS proxy is up to resolve equinox server address
 	if updater.IsAutoupdateEnabled(c, log) {
 		autoupdateFreq := c.Duration("autoupdate-freq")
@@ -315,7 +302,7 @@ func StartServer(
 	if dnsProxyStandAlone(c) {
 		connectedSignal.Notify()
 		// no grace period, handle SIGINT/SIGTERM immediately
-		return waitToShutdown(&wg, errC, shutdownC, graceShutdownC, 0, log)
+		return waitToShutdown(&wg, cancel, errC, graceShutdownC, 0, log)
 	}
 
 	url := c.String("url")
@@ -326,15 +313,11 @@ func StartServer(
 		return fmt.Errorf(errText)
 	}
 
-	transportLog := logger.CreateTransportLoggerFromContext(c, isUIEnabled)
+	logTransport := logger.CreateTransportLoggerFromContext(c, isUIEnabled)
 
-	readinessCh := make(chan connection.Event, 16)
-	uiCh := make(chan connection.Event, 16)
-	eventChannels := []chan connection.Event{
-		readinessCh,
-		uiCh,
-	}
-	tunnelConfig, ingressRules, err := prepareTunnelConfig(c, buildInfo, version, log, transportLog, namedTunnel, isUIEnabled, eventChannels)
+	observer := connection.NewObserver(log, logTransport, isUIEnabled)
+
+	tunnelConfig, ingressRules, err := prepareTunnelConfig(c, buildInfo, version, log, logTransport, observer, namedTunnel)
 	if err != nil {
 		log.Err(err).Msg("Couldn't start tunnel")
 		return err
@@ -349,10 +332,12 @@ func StartServer(
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
-		errC <- metrics.ServeMetrics(metricsListener, shutdownC, readinessCh, log)
+		readinessServer := metrics.NewReadyServer(log)
+		observer.RegisterSink(readinessServer)
+		errC <- metrics.ServeMetrics(metricsListener, ctx.Done(), readinessServer, log)
 	}()
 
-	if err := ingressRules.StartOrigins(&wg, log, shutdownC, errC); err != nil {
+	if err := ingressRules.StartOrigins(&wg, log, ctx.Done(), errC); err != nil {
 		return err
 	}
 
@@ -364,28 +349,26 @@ func StartServer(
 
 	wg.Add(1)
 	go func() {
-		defer wg.Done()
-		errC <- origin.StartTunnelDaemon(ctx, tunnelConfig, connectedSignal, cloudflaredID, reconnectCh)
+		defer func() {
+			wg.Done()
+			log.Info().Msg("Tunnel server stopped")
+		}()
+		errC <- origin.StartTunnelDaemon(ctx, tunnelConfig, connectedSignal, reconnectCh, graceShutdownC)
 	}()
 
 	if isUIEnabled {
-		tunnelInfo := ui.NewUIModel(
+		tunnelUI := ui.NewUIModel(
 			version,
 			hostname,
 			metricsListener.Addr().String(),
 			&ingressRules,
 			tunnelConfig.HAConnections,
 		)
-		tunnelInfo.LaunchUI(ctx, log, transportLog, uiCh)
-	} else {
-		go func() {
-			for range uiCh {
-				// Consume UI events into a noop
-			}
-		}()
+		app := tunnelUI.Launch(ctx, log, logTransport)
+		observer.RegisterSink(app)
 	}
 
-	return waitToShutdown(&wg, errC, shutdownC, graceShutdownC, c.Duration("grace-period"), log)
+	return waitToShutdown(&wg, cancel, errC, graceShutdownC, c.Duration("grace-period"), log)
 }
 
 func SetFlagsFromConfigFile(c *cli.Context) error {
@@ -409,30 +392,44 @@ func SetFlagsFromConfigFile(c *cli.Context) error {
 }
 
 func waitToShutdown(wg *sync.WaitGroup,
-	errC chan error,
-	shutdownC, graceShutdownC chan struct{},
+	cancelServerContext func(),
+	errC <-chan error,
+	graceShutdownC <-chan struct{},
 	gracePeriod time.Duration,
 	log *zerolog.Logger,
 ) error {
 	var err error
-	if gracePeriod > 0 {
-		err = waitForSignalWithGraceShutdown(errC, shutdownC, graceShutdownC, gracePeriod, log)
-	} else {
-		err = waitForSignal(errC, shutdownC, log)
-		close(graceShutdownC)
+	select {
+	case err = <-errC:
+		log.Error().Err(err).Msg("Initiating shutdown")
+	case <-graceShutdownC:
+		log.Debug().Msg("Graceful shutdown signalled")
+		if gracePeriod > 0 {
+			// wait for either grace period or service termination
+			select {
+			case <-time.Tick(gracePeriod):
+			case <-errC:
+			}
+		}
 	}
 
-	if err != nil {
-		log.Err(err).Msg("Quitting due to error")
-	} else {
-		log.Info().Msg("Quitting...")
-	}
-	// Wait for clean exit, discarding all errors
+	// stop server context
+	cancelServerContext()
+
+	// Wait for clean exit, discarding all errors while we wait
+	stopDiscarding := make(chan struct{})
 	go func() {
-		for range errC {
+		for {
+			select {
+			case <-errC: // ignore
+			case <-stopDiscarding:
+				return
+			}
 		}
 	}()
 	wg.Wait()
+	close(stopDiscarding)
+
 	return err
 }
 
@@ -542,6 +539,7 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 	flags = append(flags, configureLoggingFlags(shouldHide)...)
 	flags = append(flags, configureProxyDNSFlags(shouldHide)...)
 	flags = append(flags, []cli.Flag{
+		credentialsFileFlag,
 		altsrc.NewBoolFlag(&cli.BoolFlag{
 			Name:   "is-autoupdated",
 			Usage:  "Signal the new process that Argo Tunnel client has been autoupdated",
@@ -622,13 +620,15 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			Value:  time.Second * 5,
 			Hidden: true,
 		}),
-		altsrc.NewUint64Flag(&cli.Uint64Flag{
+		// Note TUN-3758 , we use Int because UInt is not supported with altsrc
+		altsrc.NewIntFlag(&cli.IntFlag{
 			Name:   "heartbeat-count",
 			Usage:  "Minimum number of unacked heartbeats to send before closing the connection.",
 			Value:  5,
 			Hidden: true,
 		}),
-		altsrc.NewUintFlag(&cli.UintFlag{
+		// Note TUN-3758 , we use Int because UInt is not supported with altsrc
+		altsrc.NewIntFlag(&cli.IntFlag{
 			Name:    "retries",
 			Value:   5,
 			Usage:   "Maximum number of retries for connection/protocol errors.",
@@ -647,7 +647,8 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			EnvVars: []string{"TUNNEL_GRACE_PERIOD"},
 			Hidden:  true,
 		}),
-		altsrc.NewUintFlag(&cli.UintFlag{
+		// Note TUN-3758 , we use Int because UInt is not supported with altsrc
+		altsrc.NewIntFlag(&cli.IntFlag{
 			Name:    "compression-quality",
 			Value:   0,
 			Usage:   "(beta) Use cross-stream compression instead HTTP compression. 0-off, 1-low, 2-medium, >=3-high.",
@@ -932,6 +933,7 @@ func sshFlags(shouldHide bool) []cli.Flag {
 			EnvVars: []string{"TUNNEL_PROXY_ADDRESS"},
 			Hidden:  shouldHide,
 		}),
+		// Note TUN-3758 , we use Int because UInt is not supported with altsrc
 		altsrc.NewIntFlag(&cli.IntFlag{
 			Name:    ingress.ProxyPortFlag,
 			Usage:   "Listen port for the proxy.",
@@ -1043,7 +1045,7 @@ func stdinControl(reconnectCh chan origin.ReconnectSignal, log *zerolog.Logger)
 						continue
 					}
 				}
-				log.Info().Msgf("Sending reconnect signal %+v", reconnect)
+				log.Info().Msgf("Sending %+v", reconnect)
 				reconnectCh <- reconnect
 			default:
 				log.Info().Str(LogFieldCommand, command).Msg("Unknown command")

cmd/cloudflared/tunnel/configuration.go

@@ -61,19 +61,12 @@ func generateRandomClientID(log *zerolog.Logger) (string, error) {
 
 func logClientOptions(c *cli.Context, log *zerolog.Logger) {
 	flags := make(map[string]interface{})
-	for _, flag := range c.LocalFlagNames() {
+	for _, flag := range c.FlagNames() {
 		flags[flag] = c.Generic(flag)
 	}
 
-	sliceFlags := []string{"header", "tag", "proxy-dns-upstream", "upstream", "edge"}
-	for _, sliceFlag := range sliceFlags {
-		if len(c.StringSlice(sliceFlag)) > 0 {
-			flags[sliceFlag] = strings.Join(c.StringSlice(sliceFlag), ", ")
-		}
-	}
-
 	if len(flags) > 0 {
-		log.Info().Msgf("Environment variables %v", flags)
+		log.Info().Msgf("Settings: %v", flags)
 	}
 
 	envs := make(map[string]string)
@@ -156,11 +149,9 @@ func prepareTunnelConfig(
 	c *cli.Context,
 	buildInfo *buildinfo.BuildInfo,
 	version string,
-	log *zerolog.Logger,
-	transportLogger *zerolog.Logger,
+	log, logTransport *zerolog.Logger,
+	observer *connection.Observer,
 	namedTunnel *connection.NamedTunnelConfig,
-	isUIEnabled bool,
-	eventChans []chan connection.Event,
 ) (*origin.TunnelConfig, ingress.Ingress, error) {
 	isNamedTunnel := namedTunnel != nil
 
@@ -261,9 +252,11 @@ func prepareTunnelConfig(
 		ReplaceExisting: c.Bool("force"),
 	}
 	muxerConfig := &connection.MuxerConfig{
-		HeartbeatInterval:  c.Duration("heartbeat-interval"),
-		MaxHeartbeats:      c.Uint64("heartbeat-count"),
-		CompressionSetting: h2mux.CompressionSetting(c.Uint64("compression-quality")),
+		HeartbeatInterval: c.Duration("heartbeat-interval"),
+		// Note TUN-3758 , we use Int because UInt is not supported with altsrc
+		MaxHeartbeats: uint64(c.Int("heartbeat-count")),
+		// Note TUN-3758 , we use Int because UInt is not supported with altsrc
+		CompressionSetting: h2mux.CompressionSetting(uint64(c.Int("compression-quality"))),
 		MetricsUpdateFreq:  c.Duration("metrics-update-freq"),
 	}
 
@@ -279,14 +272,15 @@ func prepareTunnelConfig(
 		LBPool:           c.String("lb-pool"),
 		Tags:             tags,
 		Log:              log,
-		Observer:         connection.NewObserver(transportLogger, eventChans, isUIEnabled),
+		LogTransport:     logTransport,
+		Observer:         observer,
 		ReportedVersion:  version,
-		Retries:          c.Uint("retries"),
+		// Note TUN-3758 , we use Int because UInt is not supported with altsrc
+		Retries:          uint(c.Int("retries")),
 		RunFromTerminal:  isRunningFromTerminal(),
 		NamedTunnel:      namedTunnel,
 		ClassicTunnel:    classicTunnel,
 		MuxerConfig:      muxerConfig,
-		TunnelEventChans: eventChans,
 		ProtocolSelector: protocolSelector,
 		EdgeTLSConfigs:   edgeTLSConfigs,
 	}, ingressRules, nil

cmd/cloudflared/tunnel/server.go

@@ -8,7 +8,7 @@ import (
 	"github.com/urfave/cli/v2"
 )
 
-func runDNSProxyServer(c *cli.Context, dnsReadySignal, shutdownC chan struct{}, log *zerolog.Logger) error {
+func runDNSProxyServer(c *cli.Context, dnsReadySignal chan struct{}, shutdownC <-chan struct{}, log *zerolog.Logger) error {
 	port := c.Int("proxy-dns-port")
 	if port <= 0 || port > 65535 {
 		return errors.New("The 'proxy-dns-port' must be a valid port number in <1, 65535> range.")
@@ -26,5 +26,6 @@ func runDNSProxyServer(c *cli.Context, dnsReadySignal, shutdownC chan struct{},
 	}
 	<-shutdownC
 	_ = listener.Stop()
+	log.Info().Msg("DNS server stopped")
 	return nil
 }

cmd/cloudflared/tunnel/signal.go

@@ -4,84 +4,20 @@ import (
 	"os"
 	"os/signal"
 	"syscall"
-	"time"
 
 	"github.com/rs/zerolog"
 )
 
-const LogFieldSignal = "signal"
-
-// waitForSignal notifies all routines to shutdownC immediately by closing the
-// shutdownC when one of the routines in main exits, or when this process receives
-// SIGTERM/SIGINT
-func waitForSignal(errC chan error, shutdownC chan struct{}, log *zerolog.Logger) error {
+// waitForSignal closes graceShutdownC to indicate that we should start graceful shutdown sequence
+func waitForSignal(graceShutdownC chan struct{}, logger *zerolog.Logger) {
 	signals := make(chan os.Signal, 10)
 	signal.Notify(signals, syscall.SIGTERM, syscall.SIGINT)
 	defer signal.Stop(signals)
 
 	select {
-	case err := <-errC:
-		log.Err(err).Msg("terminating due to error")
-		close(shutdownC)
-		return err
-	case s := <-signals:
-		log.Info().Str(LogFieldSignal, s.String()).Msg("terminating due to signal")
-		close(shutdownC)
-	case <-shutdownC:
-	}
-	return nil
-}
-
-// waitForSignalWithGraceShutdown notifies all routines to shutdown immediately
-// by closing the shutdownC when one of the routines in main exits.
-// When this process recieves SIGTERM/SIGINT, it closes the graceShutdownC to
-// notify certain routines to start graceful shutdown. When grace period is over,
-// or when some routine exits, it notifies the rest of the routines to shutdown
-// immediately by closing shutdownC.
-// In the case of handling commands from Windows Service Manager, closing graceShutdownC
-// initiate graceful shutdown.
-func waitForSignalWithGraceShutdown(errC chan error,
-	shutdownC, graceShutdownC chan struct{},
-	gracePeriod time.Duration,
-	logger *zerolog.Logger,
-) error {
-	signals := make(chan os.Signal, 10)
-	signal.Notify(signals, syscall.SIGTERM, syscall.SIGINT)
-	defer signal.Stop(signals)
-
-	select {
-	case err := <-errC:
-		logger.Info().Msgf("Initiating graceful shutdown due to %v ...", err)
-		close(graceShutdownC)
-		close(shutdownC)
-		return err
 	case s := <-signals:
 		logger.Info().Msgf("Initiating graceful shutdown due to signal %s ...", s)
 		close(graceShutdownC)
-		waitForGracePeriod(signals, errC, shutdownC, gracePeriod)
 	case <-graceShutdownC:
-		waitForGracePeriod(signals, errC, shutdownC, gracePeriod)
-	case <-shutdownC:
-		close(graceShutdownC)
 	}
-
-	return nil
-}
-
-func waitForGracePeriod(signals chan os.Signal,
-	errC chan error,
-	shutdownC chan struct{},
-	gracePeriod time.Duration,
-) {
-	// Unregister signal handler early, so the client can send a second SIGTERM/SIGINT
-	// to force shutdown cloudflared
-	signal.Stop(signals)
-	graceTimerTick := time.Tick(gracePeriod)
-	// send close signal via shutdownC when grace period expires or when an
-	// error is encountered.
-	select {
-	case <-graceTimerTick:
-	case <-errC:
-	}
-	close(shutdownC)
-}
+}

cmd/cloudflared/tunnel/signal_test.go

@@ -2,11 +2,12 @@ package tunnel
 
 import (
 	"fmt"
-	"github.com/rs/zerolog"
+	"sync"
 	"syscall"
 	"testing"
 	"time"
 
+	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -18,40 +19,21 @@ var (
 	graceShutdownErr = fmt.Errorf("receive grace shutdown")
 )
 
-func testChannelClosed(t *testing.T, c chan struct{}) {
+func channelClosed(c chan struct{}) bool {
 	select {
 	case <-c:
-		return
+		return true
 	default:
-		t.Fatal("Channel should be closed")
+		return false
 	}
 }
 
-func TestWaitForSignal(t *testing.T) {
+func TestSignalShutdown(t *testing.T) {
 	log := zerolog.Nop()
 
-	// Test handling server error
-	errC := make(chan error)
-	shutdownC := make(chan struct{})
-
-	go func() {
-		errC <- serverErr
-	}()
-
-	// received error, shutdownC should be closed
-	err := waitForSignal(errC, shutdownC, &log)
-	assert.Equal(t, serverErr, err)
-	testChannelClosed(t, shutdownC)
-
 	// Test handling SIGTERM & SIGINT
 	for _, sig := range []syscall.Signal{syscall.SIGTERM, syscall.SIGINT} {
-		errC = make(chan error)
-		shutdownC = make(chan struct{})
-
-		go func(shutdownC chan struct{}) {
-			<-shutdownC
-			errC <- shutdownErr
-		}(shutdownC)
+		graceShutdownC := make(chan struct{})
 
 		go func(sig syscall.Signal) {
 			// sleep for a tick to prevent sending signal before calling waitForSignal
@@ -59,99 +41,64 @@ func TestWaitForSignal(t *testing.T) {
 			_ = syscall.Kill(syscall.Getpid(), sig)
 		}(sig)
 
-		err = waitForSignal(errC, shutdownC, &log)
-		assert.Equal(t, nil, err)
-		assert.Equal(t, shutdownErr, <-errC)
-		testChannelClosed(t, shutdownC)
+		time.AfterFunc(time.Second, func() {
+			select {
+			case <-graceShutdownC:
+			default:
+				close(graceShutdownC)
+				t.Fatal("waitForSignal timed out")
+			}
+		})
+
+		waitForSignal(graceShutdownC, &log)
+		assert.True(t, channelClosed(graceShutdownC))
 	}
 }
 
-func TestWaitForSignalWithGraceShutdown(t *testing.T) {
-	// Test server returning error
-	errC := make(chan error)
-	shutdownC := make(chan struct{})
-	graceshutdownC := make(chan struct{})
+func TestWaitForShutdown(t *testing.T) {
+	log := zerolog.Nop()
 
+	errC := make(chan error)
+	graceShutdownC := make(chan struct{})
+	const gracePeriod = 5 * time.Second
+
+	contextCancelled := false
+	cancel := func() {
+		contextCancelled = true
+	}
+	var wg sync.WaitGroup
+
+	// on, error stop immediately
+	contextCancelled = false
+	startTime := time.Now()
 	go func() {
 		errC <- serverErr
 	}()
-
-	log := zerolog.Nop()
-
-	// received error, both shutdownC and graceshutdownC should be closed
-	err := waitForSignalWithGraceShutdown(errC, shutdownC, graceshutdownC, tick, &log)
+	err := waitToShutdown(&wg, cancel, errC, graceShutdownC, gracePeriod, &log)
 	assert.Equal(t, serverErr, err)
-	testChannelClosed(t, shutdownC)
-	testChannelClosed(t, graceshutdownC)
+	assert.True(t, contextCancelled)
+	assert.False(t, channelClosed(graceShutdownC))
+	assert.True(t, time.Now().Sub(startTime) < time.Second) // check that wait ended early
 
-	// shutdownC closed, graceshutdownC should also be closed and no error
-	errC = make(chan error)
-	shutdownC = make(chan struct{})
-	graceshutdownC = make(chan struct{})
-	close(shutdownC)
-	err = waitForSignalWithGraceShutdown(errC, shutdownC, graceshutdownC, tick, &log)
-	assert.NoError(t, err)
-	testChannelClosed(t, shutdownC)
-	testChannelClosed(t, graceshutdownC)
+	// on graceful shutdown, ignore error but stop as soon as an error arrives
+	contextCancelled = false
+	startTime = time.Now()
+	go func() {
+		close(graceShutdownC)
+		time.Sleep(tick)
+		errC <- serverErr
+	}()
+	err = waitToShutdown(&wg, cancel, errC, graceShutdownC, gracePeriod, &log)
+	assert.Nil(t, err)
+	assert.True(t, contextCancelled)
+	assert.True(t, time.Now().Sub(startTime) < time.Second) // check that wait ended early
 
-	// graceshutdownC closed, shutdownC should also be closed and no error
-	errC = make(chan error)
-	shutdownC = make(chan struct{})
-	graceshutdownC = make(chan struct{})
-	close(graceshutdownC)
-	err = waitForSignalWithGraceShutdown(errC, shutdownC, graceshutdownC, tick, &log)
-	assert.NoError(t, err)
-	testChannelClosed(t, shutdownC)
-	testChannelClosed(t, graceshutdownC)
-
-	// Test handling SIGTERM & SIGINT
-	for _, sig := range []syscall.Signal{syscall.SIGTERM, syscall.SIGINT} {
-		errC := make(chan error)
-		shutdownC = make(chan struct{})
-		graceshutdownC = make(chan struct{})
-
-		go func(shutdownC, graceshutdownC chan struct{}) {
-			<-graceshutdownC
-			<-shutdownC
-			errC <- graceShutdownErr
-		}(shutdownC, graceshutdownC)
-
-		go func(sig syscall.Signal) {
-			// sleep for a tick to prevent sending signal before calling waitForSignalWithGraceShutdown
-			time.Sleep(tick)
-			_ = syscall.Kill(syscall.Getpid(), sig)
-		}(sig)
-
-		err = waitForSignalWithGraceShutdown(errC, shutdownC, graceshutdownC, tick, &log)
-		assert.Equal(t, nil, err)
-		assert.Equal(t, graceShutdownErr, <-errC)
-		testChannelClosed(t, shutdownC)
-		testChannelClosed(t, graceshutdownC)
-	}
-
-	// Test handling SIGTERM & SIGINT, server send error before end of grace period
-	for _, sig := range []syscall.Signal{syscall.SIGTERM, syscall.SIGINT} {
-		errC := make(chan error)
-		shutdownC = make(chan struct{})
-		graceshutdownC = make(chan struct{})
-
-		go func(shutdownC, graceshutdownC chan struct{}) {
-			<-graceshutdownC
-			errC <- graceShutdownErr
-			<-shutdownC
-			errC <- shutdownErr
-		}(shutdownC, graceshutdownC)
-
-		go func(sig syscall.Signal) {
-			// sleep for a tick to prevent sending signal before calling waitForSignalWithGraceShutdown
-			time.Sleep(tick)
-			_ = syscall.Kill(syscall.Getpid(), sig)
-		}(sig)
-
-		err = waitForSignalWithGraceShutdown(errC, shutdownC, graceshutdownC, tick, &log)
-		assert.Equal(t, nil, err)
-		assert.Equal(t, shutdownErr, <-errC)
-		testChannelClosed(t, shutdownC)
-		testChannelClosed(t, graceshutdownC)
-	}
+	// with graceShutdownC closed stop right away without grace period
+	contextCancelled = false
+	startTime = time.Now()
+	err = waitToShutdown(&wg, cancel, errC, graceShutdownC, 0, &log)
+	assert.Nil(t, err)
+	assert.True(t, contextCancelled)
+	assert.True(t, time.Now().Sub(startTime) < time.Second) // check that wait ended early
 }
+

cmd/cloudflared/tunnel/subcommand_context.go

@@ -147,7 +147,7 @@ func (sc *subcommandContext) readTunnelCredentials(credFinder CredFinder) (conne
 	return credentials, nil
 }
 
-func (sc *subcommandContext) create(name string) (*tunnelstore.Tunnel, error) {
+func (sc *subcommandContext) create(name string, credentialsOutputPath string) (*tunnelstore.Tunnel, error) {
 	client, err := sc.client()
 	if err != nil {
 		return nil, errors.Wrap(err, "couldn't create client to talk to Argo Tunnel backend")
@@ -173,7 +173,7 @@ func (sc *subcommandContext) create(name string) (*tunnelstore.Tunnel, error) {
 		TunnelID:     tunnel.ID,
 		TunnelName:   name,
 	}
-	filePath, writeFileErr := writeTunnelCredentials(credential.certPath, &tunnelCredentials)
+	filePath, writeFileErr := writeTunnelCredentials(credential.certPath, credentialsOutputPath, &tunnelCredentials)
 	if writeFileErr != nil {
 		var errorLines []string
 		errorLines = append(errorLines, fmt.Sprintf("Your tunnel '%v' was created with ID %v. However, cloudflared couldn't write to the tunnel credentials file at %v.json.", tunnel.Name, tunnel.ID, tunnel.ID))
@@ -274,8 +274,6 @@ func (sc *subcommandContext) run(tunnelID uuid.UUID) error {
 	return StartServer(
 		sc.c,
 		version,
-		shutdownC,
-		graceShutdownC,
 		&connection.NamedTunnelConfig{Credentials: credentials},
 		sc.log,
 		sc.isUIEnabled,

cmd/cloudflared/tunnel/subcommands.go

@@ -28,6 +28,7 @@ import (
 )
 
 const (
+	allSortByOptions  = "name, id, createdAt, deletedAt, numConnections"
 	CredFileFlagAlias = "cred-file"
 	CredFileFlag      = "credentials-file"
 
@@ -67,6 +68,17 @@ var (
 		Aliases: []string{"o"},
 		Usage:   "Render output using given `FORMAT`. Valid options are 'json' or 'yaml'",
 	})
+	sortByFlag = &cli.StringFlag{
+		Name:    "sort-by",
+		Value:   "name",
+		Usage:   fmt.Sprintf("Sorts the list of tunnels by the given field. Valid options are {%s}", allSortByOptions),
+		EnvVars: []string{"TUNNEL_LIST_SORT_BY"},
+	}
+	invertSortFlag = &cli.BoolFlag{
+		Name:    "invert-sort",
+		Usage:   "Inverts the sort order of the tunnel list.",
+		EnvVars: []string{"TUNNEL_LIST_INVERT_SORT"},
+	}
 	forceFlag = altsrc.NewBoolFlag(&cli.BoolFlag{
 		Name:    "force",
 		Aliases: []string{"f"},
@@ -78,7 +90,7 @@ var (
 	credentialsFileFlag = altsrc.NewStringFlag(&cli.StringFlag{
 		Name:    CredFileFlag,
 		Aliases: []string{CredFileFlagAlias},
-		Usage:   "File path of tunnel credentials",
+		Usage:   "Filepath at which to read/write the tunnel credentials",
 		EnvVars: []string{"TUNNEL_CRED_FILE"},
 	})
 	forceDeleteFlag = &cli.BoolFlag{
@@ -109,7 +121,7 @@ func buildCreateCommand() *cli.Command {
   For example, to create a tunnel named 'my-tunnel' run:
 
   $ cloudflared tunnel create my-tunnel`,
-		Flags:              []cli.Flag{outputFormatFlag},
+		Flags:              []cli.Flag{outputFormatFlag, credentialsFileFlag},
 		CustomHelpTemplate: commandHelpTemplate(),
 	}
 }
@@ -132,7 +144,7 @@ func createCommand(c *cli.Context) error {
 	}
 	name := c.Args().First()
 
-	_, err = sc.create(name)
+	_, err = sc.create(name, c.String(CredFileFlag))
 	return errors.Wrap(err, "failed to create tunnel")
 }
 
@@ -142,12 +154,18 @@ func tunnelFilePath(tunnelID uuid.UUID, directory string) (string, error) {
 	return homedir.Expand(filePath)
 }
 
+// If an `outputFile` is given, write the credentials there.
+// Otherwise, write it to the same directory as the originCert,
+// with the filename `<tunnel id>.json`.
 func writeTunnelCredentials(
-	originCertPath string,
+	originCertPath, outputFile string,
 	credentials *connection.Credentials,
 ) (filePath string, err error) {
-	originCertDir := filepath.Dir(originCertPath)
-	filePath, err = tunnelFilePath(credentials.TunnelID, originCertDir)
+	filePath = outputFile
+	if outputFile == "" {
+		originCertDir := filepath.Dir(originCertPath)
+		filePath, err = tunnelFilePath(credentials.TunnelID, originCertDir)
+	}
 	if err != nil {
 		return "", err
 	}
@@ -161,12 +179,21 @@ func writeTunnelCredentials(
 
 func buildListCommand() *cli.Command {
 	return &cli.Command{
-		Name:               "list",
-		Action:             cliutil.ErrorHandler(listCommand),
-		Usage:              "List existing tunnels",
-		UsageText:          "cloudflared tunnel [tunnel command options] list [subcommand options]",
-		Description:        "cloudflared tunnel list will display all active tunnels, their created time and associated connections. Use -d flag to include deleted tunnels. See the list of options to filter the list",
-		Flags:              []cli.Flag{outputFormatFlag, showDeletedFlag, listNameFlag, listExistedAtFlag, listIDFlag, showRecentlyDisconnected},
+		Name:        "list",
+		Action:      cliutil.ErrorHandler(listCommand),
+		Usage:       "List existing tunnels",
+		UsageText:   "cloudflared tunnel [tunnel command options] list [subcommand options]",
+		Description: "cloudflared tunnel list will display all active tunnels, their created time and associated connections. Use -d flag to include deleted tunnels. See the list of options to filter the list",
+		Flags: []cli.Flag{
+			outputFormatFlag,
+			showDeletedFlag,
+			listNameFlag,
+			listExistedAtFlag,
+			listIDFlag,
+			showRecentlyDisconnected,
+			sortByFlag,
+			invertSortFlag,
+		},
 		CustomHelpTemplate: commandHelpTemplate(),
 	}
 }
@@ -200,6 +227,36 @@ func listCommand(c *cli.Context) error {
 		return err
 	}
 
+	// Sort the tunnels
+	sortBy := c.String("sort-by")
+	invalidSortField := false
+	sort.Slice(tunnels, func(i, j int) bool {
+		cmp := func() bool {
+			switch sortBy {
+			case "name":
+				return tunnels[i].Name < tunnels[j].Name
+			case "id":
+				return tunnels[i].ID.String() < tunnels[j].ID.String()
+			case "createdAt":
+				return tunnels[i].CreatedAt.Unix() < tunnels[j].CreatedAt.Unix()
+			case "deletedAt":
+				return tunnels[i].DeletedAt.Unix() < tunnels[j].DeletedAt.Unix()
+			case "numConnections":
+				return len(tunnels[i].Connections) < len(tunnels[j].Connections)
+			default:
+				invalidSortField = true
+				return tunnels[i].Name < tunnels[j].Name
+			}
+		}()
+		if c.Bool("invert-sort") {
+			return !cmp
+		}
+		return cmp
+	})
+	if invalidSortField {
+		sc.log.Error().Msgf("%s is not a valid sort field. Valid sort fields are %s. Defaulting to 'name'.", sortBy, allSortByOptions)
+	}
+
 	if outputFormat := c.String(outputFormatFlag.Name); outputFormat != "" {
 		return renderOutput(outputFormat, tunnels)
 	}

cmd/cloudflared/ui/launch_ui.go

@@ -15,7 +15,6 @@ import (
 
 type connState struct {
 	location string
-	state    connection.Status
 }
 
 type uiModel struct {
@@ -32,6 +31,7 @@ type palette struct {
 	defaultText  string
 	disconnected string
 	reconnecting string
+	unregistered string
 }
 
 func NewUIModel(version, hostname, metricsURL string, ing *ingress.Ingress, haConnections int) *uiModel {
@@ -48,11 +48,10 @@ func NewUIModel(version, hostname, metricsURL string, ing *ingress.Ingress, haCo
 	}
 }
 
-func (data *uiModel) LaunchUI(
+func (data *uiModel) Launch(
 	ctx context.Context,
 	log, transportLog *zerolog.Logger,
-	tunnelEventChan <-chan connection.Event,
-) {
+) connection.EventSink {
 	// Configure the logger to stream logs into the textview
 
 	// Add TextView as a group to write output to
@@ -68,6 +67,7 @@ func (data *uiModel) LaunchUI(
 		defaultText:  "white",
 		disconnected: "red",
 		reconnecting: "orange",
+		unregistered: "orange",
 	}
 
 	app := tview.NewApplication()
@@ -114,28 +114,9 @@ func (data *uiModel) LaunchUI(
 	grid.AddItem(logFrame, 4, 0, 5, 2, 0, 0, false)
 
 	go func() {
-		for {
-			select {
-			case <-ctx.Done():
-				app.Stop()
-				return
-			case event := <-tunnelEventChan:
-				switch event.EventType {
-				case connection.Connected:
-					data.setConnTableCell(event, connTable, palette)
-				case connection.Disconnected, connection.Reconnecting:
-					data.changeConnStatus(event, connTable, log, palette)
-				case connection.SetURL:
-					tunnelHostText.SetText(event.URL)
-					data.edgeURL = event.URL
-				case connection.RegisteringTunnel:
-					if data.edgeURL == "" {
-						tunnelHostText.SetText("Registering tunnel...")
-					}
-				}
-			}
-			app.Draw()
-		}
+		<-ctx.Done()
+		app.Stop()
+		return
 	}()
 
 	go func() {
@@ -143,6 +124,23 @@ func (data *uiModel) LaunchUI(
 			log.Error().Msgf("Error launching UI: %s", err)
 		}
 	}()
+
+	return connection.EventSinkFunc(func(event connection.Event) {
+		switch event.EventType {
+		case connection.Connected:
+			data.setConnTableCell(event, connTable, palette)
+		case connection.Disconnected, connection.Reconnecting, connection.Unregistering:
+			data.changeConnStatus(event, connTable, log, palette)
+		case connection.SetURL:
+			tunnelHostText.SetText(event.URL)
+			data.edgeURL = event.URL
+		case connection.RegisteringTunnel:
+			if data.edgeURL == "" {
+				tunnelHostText.SetText(fmt.Sprintf("Registering tunnel connection %d...", event.Index))
+			}
+		}
+		app.Draw()
+	})
 }
 
 func NewDynamicColorTextView() *tview.TextView {
@@ -170,10 +168,7 @@ func (data *uiModel) changeConnStatus(event connection.Event, table *tview.Table
 
 	locationState := event.Location
 
-	if event.EventType == connection.Disconnected {
-		connState.state = connection.Disconnected
-	} else if event.EventType == connection.Reconnecting {
-		connState.state = connection.Reconnecting
+	if event.EventType == connection.Reconnecting {
 		locationState = "Reconnecting..."
 	}
 
@@ -199,7 +194,6 @@ func (data *uiModel) setConnTableCell(event connection.Event, table *tview.Table
 	connectionNum := index + 1
 
 	// Update slice to keep track of connection location and state in UI table
-	data.connections[index].state = connection.Connected
 	data.connections[index].location = event.Location
 
 	// Update text in table cell to show disconnected state
@@ -221,6 +215,8 @@ func newCellText(palette palette, connectionNum int, location string, connectedS
 		dotColor = palette.disconnected
 	case connection.Reconnecting:
 		dotColor = palette.reconnecting
+	case connection.Unregistering:
+		dotColor = palette.unregistered
 	}
 
 	return fmt.Sprintf(connFmtString, dotColor, palette.defaultText, connectionNum, location)

cmd/cloudflared/windows_service.go

@@ -40,21 +40,21 @@ const (
 	LogFieldWindowsServiceName = "windowsServiceName"
 )
 
-func runApp(app *cli.App, shutdownC, graceShutdownC chan struct{}) {
+func runApp(app *cli.App, graceShutdownC chan struct{}) {
 	app.Commands = append(app.Commands, &cli.Command{
 		Name:  "service",
 		Usage: "Manages the Argo Tunnel Windows service",
 		Subcommands: []*cli.Command{
-			&cli.Command{
+			{
 				Name:   "install",
 				Usage:  "Install Argo Tunnel as a Windows service",
 				Action: installWindowsService,
-			},
-			&cli.Command{
+						},
+			{
 				Name:   "uninstall",
 				Usage:  "Uninstall the Argo Tunnel service",
 				Action: uninstallWindowsService,
-			},
+						},
 		},
 	})
 
@@ -82,7 +82,7 @@ func runApp(app *cli.App, shutdownC, graceShutdownC chan struct{}) {
 	// Run executes service name by calling windowsService which is a Handler
 	// interface that implements Execute method.
 	// It will set service status to stop after Execute returns
-	err = svc.Run(windowsServiceName, &windowsService{app: app, shutdownC: shutdownC, graceShutdownC: graceShutdownC})
+	err = svc.Run(windowsServiceName, &windowsService{app: app, graceShutdownC: graceShutdownC})
 	if err != nil {
 		if errno, ok := err.(syscall.Errno); ok && int(errno) == serviceControllerConnectionFailure {
 			// Hack: assume this is a false negative from the IsAnInteractiveSession() check above.
@@ -96,11 +96,11 @@ func runApp(app *cli.App, shutdownC, graceShutdownC chan struct{}) {
 
 type windowsService struct {
 	app            *cli.App
-	shutdownC      chan struct{}
 	graceShutdownC chan struct{}
 }
 
-// called by the package code at the start of the service
+// Execute is called by the service manager when service starts, the state
+// of the service will be set to Stopped when this function returns.
 func (s *windowsService) Execute(serviceArgs []string, r <-chan svc.ChangeRequest, statusChan chan<- svc.Status) (ssec bool, errno uint32) {
 	log := logger.Create(nil)
 	elog, err := eventlog.Open(windowsServiceName)
@@ -128,13 +128,12 @@ func (s *windowsService) Execute(serviceArgs []string, r <-chan svc.ChangeReques
 	}
 	elog.Info(1, fmt.Sprintf("%s service arguments: %v", windowsServiceName, args))
 
-	const cmdsAccepted = svc.AcceptStop | svc.AcceptShutdown
 	statusChan <- svc.Status{State: svc.StartPending}
 	errC := make(chan error)
 	go func() {
 		errC <- s.app.Run(args)
 	}()
-	statusChan <- svc.Status{State: svc.Running, Accepts: cmdsAccepted}
+	statusChan <- svc.Status{State: svc.Running, Accepts: svc.AcceptStop | svc.AcceptShutdown}
 
 	for {
 		select {
@@ -143,17 +142,25 @@ func (s *windowsService) Execute(serviceArgs []string, r <-chan svc.ChangeReques
 			case svc.Interrogate:
 				statusChan <- c.CurrentStatus
 			case svc.Stop, svc.Shutdown:
-				close(s.graceShutdownC)
-				statusChan <- svc.Status{State: svc.Stopped, Accepts: cmdsAccepted}
+				if s.graceShutdownC != nil {
+					// start graceful shutdown
+					elog.Info(1, "cloudflared starting graceful shutdown")
+					close(s.graceShutdownC)
+					s.graceShutdownC = nil
+					statusChan <- svc.Status{State: svc.StopPending}
+					continue
+				}
+				// repeated attempts at graceful shutdown forces immediate stop
+				elog.Info(1, "cloudflared terminating immediately")
 				statusChan <- svc.Status{State: svc.StopPending}
-				return
+				return false, 0
 			default:
 				elog.Error(1, fmt.Sprintf("unexpected control request #%d", c))
 			}
 		case err := <-errC:
-			ssec = true
 			if err != nil {
 				elog.Error(1, fmt.Sprintf("cloudflared terminated with error %v", err))
+				ssec = true
 				errno = 1
 			} else {
 				elog.Info(1, "cloudflared terminated without error")

connection/connection_test.go

@@ -27,13 +27,6 @@ var (
 		Scheme: "https",
 		Host:   "connectiontest.argotunnel.com",
 	}
-	testTunnelEventChan = make(chan Event)
-	testObserver        = &Observer{
-		&log,
-		m,
-		[]chan Event{testTunnelEventChan},
-		false,
-	}
 	testLargeResp = make([]byte, largeFileSize)
 )
 

connection/errors.go

@@ -4,55 +4,40 @@ import (
 	"github.com/cloudflare/cloudflared/edgediscovery"
 	"github.com/cloudflare/cloudflared/h2mux"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
-	"github.com/prometheus/client_golang/prometheus"
 )
 
 const (
 	DuplicateConnectionError = "EDUPCONN"
 )
 
-// RegisterTunnel error from client
-type clientRegisterTunnelError struct {
-	cause error
-}
-
-func newRPCError(cause error, counter *prometheus.CounterVec, name rpcName) clientRegisterTunnelError {
-	counter.WithLabelValues(cause.Error(), string(name)).Inc()
-	return clientRegisterTunnelError{cause: cause}
-}
-
-func (e clientRegisterTunnelError) Error() string {
-	return e.cause.Error()
-}
-
 type DupConnRegisterTunnelError struct{}
 
-var errDuplicationConnection = &DupConnRegisterTunnelError{}
+var errDuplicationConnection = DupConnRegisterTunnelError{}
 
 func (e DupConnRegisterTunnelError) Error() string {
 	return "already connected to this server, trying another address"
 }
 
 // RegisterTunnel error from server
-type serverRegisterTunnelError struct {
-	cause     error
-	permanent bool
+type ServerRegisterTunnelError struct {
+	Cause     error
+	Permanent bool
 }
 
-func (e serverRegisterTunnelError) Error() string {
-	return e.cause.Error()
+func (e ServerRegisterTunnelError) Error() string {
+	return e.Cause.Error()
 }
 
-func serverRegistrationErrorFromRPC(err error) *serverRegisterTunnelError {
+func serverRegistrationErrorFromRPC(err error) ServerRegisterTunnelError {
 	if retryable, ok := err.(*tunnelpogs.RetryableError); ok {
-		return &serverRegisterTunnelError{
-			cause:     retryable.Unwrap(),
-			permanent: false,
+		return ServerRegisterTunnelError{
+			Cause:     retryable.Unwrap(),
+			Permanent: false,
 		}
 	}
-	return &serverRegisterTunnelError{
-		cause:     err,
-		permanent: true,
+	return ServerRegisterTunnelError{
+		Cause:     err,
+		Permanent: true,
 	}
 }
 
@@ -62,6 +47,8 @@ func (e muxerShutdownError) Error() string {
 	return "muxer shutdown"
 }
 
+var errMuxerStopped = muxerShutdownError{}
+
 func isHandshakeErrRecoverable(err error, connIndex uint8, observer *Observer) bool {
 	log := observer.log.With().
 		Uint8(LogFieldConnIndex, connIndex).

connection/event.go

@@ -22,4 +22,6 @@ const (
 	SetURL
 	// RegisteringTunnel means the non-named tunnel is registering its connection.
 	RegisteringTunnel
+	// We're unregistering tunnel from the edge in preparation for a disconnect
+	Unregistering
 )

connection/h2mux.go

@@ -2,6 +2,7 @@ package connection
 
 import (
 	"context"
+	"io"
 	"net"
 	"net/http"
 	"time"
@@ -28,7 +29,12 @@ type h2muxConnection struct {
 	connIndexStr string
 	connIndex    uint8
 
-	observer *Observer
+	observer          *Observer
+	gracefulShutdownC <-chan struct{}
+	stoppedGracefully bool
+
+	// newRPCClientFunc allows us to mock RPCs during testing
+	newRPCClientFunc func(context.Context, io.ReadWriteCloser, *zerolog.Logger) NamedTunnelRPCClient
 }
 
 type MuxerConfig struct {
@@ -57,18 +63,21 @@ func NewH2muxConnection(
 	edgeConn net.Conn,
 	connIndex uint8,
 	observer *Observer,
+	gracefulShutdownC <-chan struct{},
 ) (*h2muxConnection, error, bool) {
 	h := &h2muxConnection{
-		config:       config,
-		muxerConfig:  muxerConfig,
-		connIndexStr: uint8ToString(connIndex),
-		connIndex:    connIndex,
-		observer:     observer,
+		config:            config,
+		muxerConfig:       muxerConfig,
+		connIndexStr:      uint8ToString(connIndex),
+		connIndex:         connIndex,
+		observer:          observer,
+		gracefulShutdownC: gracefulShutdownC,
+		newRPCClientFunc:  newRegistrationRPCClient,
 	}
 
 	// Establish a muxed connection with the edge
 	// Client mux handshake with agent server
-	muxer, err := h2mux.Handshake(edgeConn, edgeConn, *muxerConfig.H2MuxerConfig(h, observer.log), h2mux.ActiveStreams)
+	muxer, err := h2mux.Handshake(edgeConn, edgeConn, *muxerConfig.H2MuxerConfig(h, observer.logTransport), h2mux.ActiveStreams)
 	if err != nil {
 		recoverable := isHandshakeErrRecoverable(err, connIndex, observer)
 		return nil, err, recoverable
@@ -77,21 +86,14 @@ func NewH2muxConnection(
 	return h, nil, false
 }
 
-func (h *h2muxConnection) ServeNamedTunnel(ctx context.Context, namedTunnel *NamedTunnelConfig, credentialManager CredentialManager, connOptions *tunnelpogs.ConnectionOptions, connectedFuse ConnectedFuse) error {
+func (h *h2muxConnection) ServeNamedTunnel(ctx context.Context, namedTunnel *NamedTunnelConfig, connOptions *tunnelpogs.ConnectionOptions, connectedFuse ConnectedFuse) error {
 	errGroup, serveCtx := errgroup.WithContext(ctx)
 	errGroup.Go(func() error {
 		return h.serveMuxer(serveCtx)
 	})
 
 	errGroup.Go(func() error {
-		stream, err := h.newRPCStream(serveCtx, register)
-		if err != nil {
-			return err
-		}
-		rpcClient := newRegistrationRPCClient(ctx, stream, h.observer.log)
-		defer rpcClient.Close()
-
-		if err = rpcClient.RegisterConnection(serveCtx, namedTunnel, connOptions, h.connIndex, h.observer); err != nil {
+		if err := h.registerNamedTunnel(serveCtx, namedTunnel, connOptions); err != nil {
 			return err
 		}
 		connectedFuse.Connected()
@@ -102,7 +104,15 @@ func (h *h2muxConnection) ServeNamedTunnel(ctx context.Context, namedTunnel *Nam
 		h.controlLoop(serveCtx, connectedFuse, true)
 		return nil
 	})
-	return errGroup.Wait()
+
+	err := errGroup.Wait()
+	if err == errMuxerStopped {
+		if h.stoppedGracefully {
+			return nil
+		}
+		h.observer.log.Info().Uint8(LogFieldConnIndex, h.connIndex).Msg("Unexpected muxer shutdown")
+	}
+	return err
 }
 
 func (h *h2muxConnection) ServeClassicTunnel(ctx context.Context, classicTunnel *ClassicTunnelConfig, credentialManager CredentialManager, registrationOptions *tunnelpogs.RegistrationOptions, connectedFuse ConnectedFuse) error {
@@ -134,7 +144,15 @@ func (h *h2muxConnection) ServeClassicTunnel(ctx context.Context, classicTunnel
 		h.controlLoop(serveCtx, connectedFuse, false)
 		return nil
 	})
-	return errGroup.Wait()
+
+	err := errGroup.Wait()
+	if err == errMuxerStopped {
+		if h.stoppedGracefully {
+			return nil
+		}
+		h.observer.log.Info().Uint8(LogFieldConnIndex, h.connIndex).Msg("Unexpected muxer shutdown")
+	}
+	return err
 }
 
 func (h *h2muxConnection) serveMuxer(ctx context.Context) error {
@@ -143,22 +161,36 @@ func (h *h2muxConnection) serveMuxer(ctx context.Context) error {
 	// here to notify other routines to stop
 	err := h.muxer.Serve(ctx)
 	if err == nil {
-		return muxerShutdownError{}
+		return errMuxerStopped
 	}
 	return err
 }
 
 func (h *h2muxConnection) controlLoop(ctx context.Context, connectedFuse ConnectedFuse, isNamedTunnel bool) {
 	updateMetricsTickC := time.Tick(h.muxerConfig.MetricsUpdateFreq)
+	var shutdownCompleted <-chan struct{}
 	for {
 		select {
-		case <-ctx.Done():
-			// UnregisterTunnel blocks until the RPC call returns
+		case <-h.gracefulShutdownC:
 			if connectedFuse.IsConnected() {
 				h.unregister(isNamedTunnel)
 			}
-			h.muxer.Shutdown()
+			h.stoppedGracefully = true
+			h.gracefulShutdownC = nil
+			shutdownCompleted = h.muxer.Shutdown()
+
+		case <-shutdownCompleted:
 			return
+
+		case <-ctx.Done():
+			// UnregisterTunnel blocks until the RPC call returns
+			if !h.stoppedGracefully && connectedFuse.IsConnected() {
+				h.unregister(isNamedTunnel)
+			}
+			h.muxer.Shutdown()
+			// don't wait for shutdown to finish when context is closed, this is the hard termination path
+			return
+
 		case <-updateMetricsTickC:
 			h.observer.metrics.updateMuxerMetrics(h.connIndexStr, h.muxer.Metrics())
 		}

connection/h2mux_test.go

@@ -11,10 +11,12 @@ import (
 	"testing"
 	"time"
 
-	"github.com/cloudflare/cloudflared/h2mux"
 	"github.com/gobwas/ws/wsutil"
+	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+
+	"github.com/cloudflare/cloudflared/h2mux"
 )
 
 var (
@@ -31,14 +33,22 @@ func newH2MuxConnection(t require.TestingT) (*h2muxConnection, *h2mux.Muxer) {
 	edgeMuxChan := make(chan *h2mux.Muxer)
 	go func() {
 		edgeMuxConfig := h2mux.MuxerConfig{
-			Log: testObserver.log,
+			Log: &log,
+			Handler: h2mux.MuxedStreamFunc(func(stream *h2mux.MuxedStream) error {
+				// we only expect RPC traffic in client->edge direction, provide minimal support for mocking
+				require.True(t, stream.IsRPCStream())
+				return stream.WriteHeaders([]h2mux.Header{
+					{Name:  ":status", Value: "200"},
+				})
+			}),
 		}
 		edgeMux, err := h2mux.Handshake(edgeConn, edgeConn, edgeMuxConfig, h2mux.ActiveStreams)
 		require.NoError(t, err)
 		edgeMuxChan <- edgeMux
 	}()
 	var connIndex = uint8(0)
-	h2muxConn, err, _ := NewH2muxConnection(testConfig, testMuxerConfig, originConn, connIndex, testObserver)
+	testObserver := NewObserver(&log, &log, false)
+	h2muxConn, err, _ := NewH2muxConnection(testConfig, testMuxerConfig, originConn, connIndex, testObserver, nil)
 	require.NoError(t, err)
 	return h2muxConn, <-edgeMuxChan
 }
@@ -168,6 +178,55 @@ func TestServeStreamWS(t *testing.T) {
 	wg.Wait()
 }
 
+func TestGracefulShutdownH2Mux(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	h2muxConn, edgeMux := newH2MuxConnection(t)
+
+	shutdownC := make(chan struct{})
+	unregisteredC := make(chan struct{})
+	h2muxConn.gracefulShutdownC = shutdownC
+	h2muxConn.newRPCClientFunc = func(_ context.Context, _ io.ReadWriteCloser, _ *zerolog.Logger) NamedTunnelRPCClient {
+		return &mockNamedTunnelRPCClient{
+			registered:   nil,
+			unregistered: unregisteredC,
+		}
+	}
+
+	var wg sync.WaitGroup
+	wg.Add(3)
+	go func() {
+		defer wg.Done()
+		_ = edgeMux.Serve(ctx)
+	}()
+	go func() {
+		defer wg.Done()
+		_ = h2muxConn.serveMuxer(ctx)
+	}()
+
+	go func() {
+		defer wg.Done()
+		h2muxConn.controlLoop(ctx, &mockConnectedFuse{}, true)
+	}()
+
+	time.Sleep(100 * time.Millisecond)
+	close(shutdownC)
+
+	select {
+	case <-unregisteredC:
+		break // ok
+	case <-time.Tick(time.Second):
+		assert.Fail(t, "timed out waiting for control loop to unregister")
+	}
+
+	cancel()
+	wg.Wait()
+
+	assert.True(t, h2muxConn.stoppedGracefully)
+	assert.Nil(t, h2muxConn.gracefulShutdownC)
+}
+
 func hasHeader(stream *h2mux.MuxedStream, name, val string) bool {
 	for _, header := range stream.Headers {
 		if header.Name == name && header.Value == val {

connection/http2.go

@@ -2,6 +2,7 @@ package connection
 
 import (
 	"context"
+	"fmt"
 	"io"
 	"math"
 	"net"
@@ -22,6 +23,8 @@ const (
 	controlStreamUpgrade  = "control-stream"
 )
 
+var errEdgeConnectionClosed = fmt.Errorf("connection with edge closed")
+
 type http2Connection struct {
 	conn         net.Conn
 	server       *http2.Server
@@ -31,10 +34,14 @@ type http2Connection struct {
 	observer     *Observer
 	connIndexStr string
 	connIndex    uint8
-	wg           *sync.WaitGroup
 	// newRPCClientFunc allows us to mock RPCs during testing
 	newRPCClientFunc func(context.Context, io.ReadWriteCloser, *zerolog.Logger) NamedTunnelRPCClient
-	connectedFuse    ConnectedFuse
+
+	activeRequestsWG  sync.WaitGroup
+	connectedFuse     ConnectedFuse
+	gracefulShutdownC <-chan struct{}
+	stoppedGracefully bool
+	controlStreamErr  error // result of running control stream handler
 }
 
 func NewHTTP2Connection(
@@ -45,25 +52,26 @@ func NewHTTP2Connection(
 	observer *Observer,
 	connIndex uint8,
 	connectedFuse ConnectedFuse,
+	gracefulShutdownC <-chan struct{},
 ) *http2Connection {
 	return &http2Connection{
 		conn: conn,
 		server: &http2.Server{
 			MaxConcurrentStreams: math.MaxUint32,
 		},
-		config:           config,
-		namedTunnel:      namedTunnelConfig,
-		connOptions:      connOptions,
-		observer:         observer,
-		connIndexStr:     uint8ToString(connIndex),
-		connIndex:        connIndex,
-		wg:               &sync.WaitGroup{},
-		newRPCClientFunc: newRegistrationRPCClient,
-		connectedFuse:    connectedFuse,
+		config:            config,
+		namedTunnel:       namedTunnelConfig,
+		connOptions:       connOptions,
+		observer:          observer,
+		connIndexStr:      uint8ToString(connIndex),
+		connIndex:         connIndex,
+		newRPCClientFunc:  newRegistrationRPCClient,
+		connectedFuse:     connectedFuse,
+		gracefulShutdownC: gracefulShutdownC,
 	}
 }
 
-func (c *http2Connection) Serve(ctx context.Context) {
+func (c *http2Connection) Serve(ctx context.Context) error {
 	go func() {
 		<-ctx.Done()
 		c.close()
@@ -72,11 +80,21 @@ func (c *http2Connection) Serve(ctx context.Context) {
 		Context: ctx,
 		Handler: c,
 	})
+
+	switch {
+	case c.stoppedGracefully:
+		return nil
+	case c.controlStreamErr != nil:
+		return c.controlStreamErr
+	default:
+		c.observer.log.Info().Uint8(LogFieldConnIndex, c.connIndex).Msg("Lost connection with the edge")
+		return errEdgeConnectionClosed
+	}
 }
 
 func (c *http2Connection) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	c.wg.Add(1)
-	defer c.wg.Done()
+	c.activeRequestsWG.Add(1)
+	defer c.activeRequestsWG.Done()
 
 	respWriter := &http2RespWriter{
 		r: r.Body,
@@ -93,6 +111,7 @@ func (c *http2Connection) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	if isControlStreamUpgrade(r) {
 		respWriter.shouldFlush = true
 		err = c.serveControlStream(r.Context(), respWriter)
+		c.controlStreamErr = err
 	} else if isWebsocketUpgrade(r) {
 		respWriter.shouldFlush = true
 		stripWebsocketUpgradeHeader(r)
@@ -115,14 +134,23 @@ func (c *http2Connection) serveControlStream(ctx context.Context, respWriter *ht
 	}
 	c.connectedFuse.Connected()
 
-	<-ctx.Done()
+	// wait for connection termination or start of graceful shutdown
+	select {
+	case <-ctx.Done():
+		break
+	case <-c.gracefulShutdownC:
+		c.stoppedGracefully = true
+	}
+
+	c.observer.sendUnregisteringEvent(c.connIndex)
 	rpcClient.GracefulShutdown(ctx, c.config.GracePeriod)
+	c.observer.log.Info().Uint8(LogFieldConnIndex, c.connIndex).Msg("Unregistered tunnel connection")
 	return nil
 }
 
 func (c *http2Connection) close() {
 	// Wait for all serve HTTP handlers to return
-	c.wg.Wait()
+	c.activeRequestsWG.Wait()
 	c.conn.Close()
 }
 

connection/http2_test.go

@@ -12,6 +12,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/assert"
+
 	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 
@@ -33,9 +35,10 @@ func newTestHTTP2Connection() (*http2Connection, net.Conn) {
 		testConfig,
 		&NamedTunnelConfig{},
 		&pogs.ConnectionOptions{},
-		testObserver,
+		NewObserver(&log, &log, false),
 		connIndex,
 		mockConnectedFuse{},
+		nil,
 	), edgeConn
 }
 
@@ -241,10 +244,72 @@ func TestServeControlStream(t *testing.T) {
 	<-rpcClientFactory.registered
 	cancel()
 	<-rpcClientFactory.unregistered
+	assert.False(t, http2Conn.stoppedGracefully)
 
 	wg.Wait()
 }
 
+func TestGracefulShutdownHTTP2(t *testing.T) {
+	http2Conn, edgeConn := newTestHTTP2Connection()
+
+	rpcClientFactory := mockRPCClientFactory{
+		registered:   make(chan struct{}),
+		unregistered: make(chan struct{}),
+	}
+	events := &eventCollectorSink{}
+	http2Conn.newRPCClientFunc = rpcClientFactory.newMockRPCClient
+	http2Conn.observer.RegisterSink(events)
+	shutdownC := make(chan struct{})
+	http2Conn.gracefulShutdownC = shutdownC
+
+	ctx, cancel := context.WithCancel(context.Background())
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		http2Conn.Serve(ctx)
+	}()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://localhost:8080/", nil)
+	require.NoError(t, err)
+	req.Header.Set(internalUpgradeHeader, controlStreamUpgrade)
+
+	edgeHTTP2Conn, err := testTransport.NewClientConn(edgeConn)
+	require.NoError(t, err)
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		_, _ = edgeHTTP2Conn.RoundTrip(req)
+	}()
+
+	select {
+	case <-rpcClientFactory.registered:
+		break //ok
+	case <-time.Tick(time.Second):
+		t.Fatal("timeout out waiting for registration")
+	}
+
+	// signal graceful shutdown
+	close(shutdownC)
+
+	select {
+	case <-rpcClientFactory.unregistered:
+		break //ok
+	case <-time.Tick(time.Second):
+		t.Fatal("timeout out waiting for unregistered signal")
+	}
+	assert.True(t, http2Conn.stoppedGracefully)
+
+	cancel()
+	wg.Wait()
+
+	events.assertSawEvent(t, Event{
+		Index:     http2Conn.connIndex,
+		EventType: Unregistering,
+	})
+}
+
 func benchmarkServeHTTP(b *testing.B, test testRequest) {
 	http2Conn, edgeConn := newTestHTTP2Connection()
 
@@ -281,6 +346,7 @@ func benchmarkServeHTTP(b *testing.B, test testRequest) {
 	cancel()
 	wg.Wait()
 }
+
 func BenchmarkServeHTTPSimple(b *testing.B) {
 	test := testRequest{
 		name:           "ok",

connection/metrics.go

@@ -299,7 +299,7 @@ func convertRTTMilliSec(t time.Duration) float64 {
 }
 
 // Metrics that can be collected without asking the edge
-func newTunnelMetrics() *tunnelMetrics {
+func initTunnelMetrics() *tunnelMetrics {
 	maxConcurrentRequestsPerTunnel := prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Namespace: MetricsNamespace,
@@ -403,3 +403,15 @@ func (t *tunnelMetrics) registerServerLocation(connectionID, loc string) {
 	t.serverLocations.WithLabelValues(connectionID, loc).Inc()
 	t.oldServerLocations[connectionID] = loc
 }
+
+var tunnelMetricsInternal struct {
+	sync.Once
+	metrics *tunnelMetrics
+}
+
+func newTunnelMetrics() *tunnelMetrics {
+	tunnelMetricsInternal.Do(func() {
+		tunnelMetricsInternal.metrics = initTunnelMetrics()
+	})
+	return tunnelMetricsInternal.metrics
+}

connection/observer.go

@@ -10,22 +10,39 @@ import (
 	"github.com/rs/zerolog"
 )
 
-const LogFieldLocation = "location"
+const (
+	LogFieldLocation          = "location"
+	observerChannelBufferSize = 16
+)
 
 type Observer struct {
-	log              *zerolog.Logger
-	metrics          *tunnelMetrics
-	tunnelEventChans []chan Event
-	uiEnabled        bool
+	log             *zerolog.Logger
+	logTransport    *zerolog.Logger
+	metrics         *tunnelMetrics
+	tunnelEventChan chan Event
+	uiEnabled       bool
+	addSinkChan     chan EventSink
 }
 
-func NewObserver(log *zerolog.Logger, tunnelEventChans []chan Event, uiEnabled bool) *Observer {
-	return &Observer{
-		log,
-		newTunnelMetrics(),
-		tunnelEventChans,
-		uiEnabled,
+type EventSink interface {
+	OnTunnelEvent(event Event)
+}
+
+func NewObserver(log, logTransport *zerolog.Logger, uiEnabled bool) *Observer {
+	o := &Observer{
+		log:             log,
+		logTransport:    logTransport,
+		metrics:         newTunnelMetrics(),
+		uiEnabled:       uiEnabled,
+		tunnelEventChan: make(chan Event, observerChannelBufferSize),
+		addSinkChan:     make(chan EventSink, observerChannelBufferSize),
 	}
+	go o.dispatchEvents()
+	return o
+}
+
+func (o *Observer) RegisterSink(sink EventSink) {
+	o.addSinkChan <- sink
 }
 
 func (o *Observer) logServerInfo(connIndex uint8, location, msg string) {
@@ -84,8 +101,8 @@ func trialZoneMsg(url string) []string {
 	}
 }
 
-func (o *Observer) sendRegisteringEvent() {
-	o.sendEvent(Event{EventType: RegisteringTunnel})
+func (o *Observer) sendRegisteringEvent(connIndex uint8) {
+	o.sendEvent(Event{Index: connIndex, EventType: RegisteringTunnel})
 }
 
 func (o *Observer) sendConnectedEvent(connIndex uint8, location string) {
@@ -100,12 +117,39 @@ func (o *Observer) SendReconnect(connIndex uint8) {
 	o.sendEvent(Event{Index: connIndex, EventType: Reconnecting})
 }
 
+func (o *Observer) sendUnregisteringEvent(connIndex uint8) {
+	o.sendEvent(Event{Index: connIndex, EventType: Unregistering})
+}
+
 func (o *Observer) SendDisconnect(connIndex uint8) {
 	o.sendEvent(Event{Index: connIndex, EventType: Disconnected})
 }
 
 func (o *Observer) sendEvent(e Event) {
-	for _, ch := range o.tunnelEventChans {
-		ch <- e
+	select {
+	case o.tunnelEventChan <- e:
+		break
+	default:
+		o.log.Warn().Msg("observer channel buffer is full")
 	}
 }
+
+func (o *Observer) dispatchEvents() {
+	var sinks []EventSink
+	for {
+		select {
+		case sink := <-o.addSinkChan:
+			sinks = append(sinks, sink)
+		case evt := <-o.tunnelEventChan:
+			for _, sink := range sinks {
+				sink.OnTunnelEvent(evt)
+			}
+		}
+	}
+}
+
+type EventSinkFunc func(event Event)
+
+func (f EventSinkFunc) OnTunnelEvent(event Event) {
+	f(event)
+}

connection/observer_test.go

@@ -4,14 +4,13 @@ import (
 	"strconv"
 	"sync"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 )
 
-// can only be called once
-var m = newTunnelMetrics()
-
 func TestRegisterServerLocation(t *testing.T) {
+	m := newTunnelMetrics()
 	tunnels := 20
 	var wg sync.WaitGroup
 	wg.Add(tunnels)
@@ -43,3 +42,45 @@ func TestRegisterServerLocation(t *testing.T) {
 	}
 
 }
+
+func TestObserverEventsDontBlock(t *testing.T) {
+	observer := NewObserver(&log, &log, false)
+	var mu sync.Mutex
+	observer.RegisterSink(EventSinkFunc(func(_ Event) {
+		// callback will block if lock is already held
+		mu.Lock()
+		mu.Unlock()
+	}))
+
+	timeout := time.AfterFunc(5*time.Second, func() {
+		mu.Unlock() // release the callback on timer expiration
+		t.Fatal("observer is blocked")
+	})
+
+	mu.Lock() // block the callback
+	for i := 0; i < 2*observerChannelBufferSize; i++ {
+		observer.sendRegisteringEvent(0)
+	}
+	if pending := timeout.Stop(); pending {
+		// release the callback if timer hasn't expired yet
+		mu.Unlock()
+	}
+}
+
+
+type eventCollectorSink struct {
+	observedEvents []Event
+	mu sync.Mutex
+}
+
+func (s *eventCollectorSink) OnTunnelEvent(event Event) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.observedEvents = append(s.observedEvents, event)
+}
+
+func (s *eventCollectorSink) assertSawEvent(t *testing.T, event Event) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	assert.Contains(t, s.observedEvents, event)
+}

connection/rpc.go

@@ -138,7 +138,7 @@ const (
 )
 
 func (h *h2muxConnection) registerTunnel(ctx context.Context, credentialSetter CredentialManager, classicTunnel *ClassicTunnelConfig, registrationOptions *tunnelpogs.RegistrationOptions) error {
-	h.observer.sendRegisteringEvent()
+	h.observer.sendRegisteringEvent(registrationOptions.ConnectionID)
 
 	stream, err := h.newRPCStream(ctx, register)
 	if err != nil {
@@ -209,9 +209,9 @@ func (h *h2muxConnection) processRegisterTunnelError(err tunnelpogs.TunnelRegist
 		return errDuplicationConnection
 	}
 	h.observer.metrics.regFail.WithLabelValues("server_error", string(name)).Inc()
-	return serverRegisterTunnelError{
-		cause:     err,
-		permanent: err.IsPermanent(),
+	return ServerRegisterTunnelError{
+		Cause:     err,
+		Permanent: err.IsPermanent(),
 	}
 }
 
@@ -272,17 +272,38 @@ func (h *h2muxConnection) logServerInfo(ctx context.Context, rpcClient *tunnelSe
 	return nil
 }
 
+func (h *h2muxConnection) registerNamedTunnel(
+	ctx context.Context,
+	namedTunnel *NamedTunnelConfig,
+	connOptions *tunnelpogs.ConnectionOptions,
+) error {
+	stream, err := h.newRPCStream(ctx, register)
+	if err != nil {
+		return err
+	}
+	rpcClient := h.newRPCClientFunc(ctx, stream, h.observer.log)
+	defer rpcClient.Close()
+
+	if err = rpcClient.RegisterConnection(ctx, namedTunnel, connOptions, h.connIndex, h.observer); err != nil {
+		return err
+	}
+	return nil
+}
+
 func (h *h2muxConnection) unregister(isNamedTunnel bool) {
+	h.observer.sendUnregisteringEvent(h.connIndex)
+
 	unregisterCtx, cancel := context.WithTimeout(context.Background(), h.config.GracePeriod)
 	defer cancel()
 
-	stream, err := h.newRPCStream(unregisterCtx, register)
+	stream, err := h.newRPCStream(unregisterCtx, unregister)
 	if err != nil {
 		return
 	}
+	defer stream.Close()
 
 	if isNamedTunnel {
-		rpcClient := newRegistrationRPCClient(unregisterCtx, stream, h.observer.log)
+		rpcClient := h.newRPCClientFunc(unregisterCtx, stream, h.observer.log)
 		defer rpcClient.Close()
 
 		rpcClient.GracefulShutdown(unregisterCtx, h.config.GracePeriod)
@@ -293,4 +314,6 @@ func (h *h2muxConnection) unregister(isNamedTunnel bool) {
 		// gracePeriod is encoded in int64 using capnproto
 		_ = rpcClient.client.UnregisterTunnel(unregisterCtx, h.config.GracePeriod.Nanoseconds())
 	}
+
+	h.observer.log.Info().Uint8(LogFieldConnIndex, h.connIndex).Msg("Unregistered tunnel connection")
 }

dev.Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.15.6 as builder
+FROM golang:1.15.7 as builder
 ENV GO111MODULE=on \
     CGO_ENABLED=0
 WORKDIR /go/src/github.com/cloudflare/cloudflared/

go.mod

@@ -40,6 +40,7 @@ require (
 	github.com/kshvakov/clickhouse v1.3.11
 	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/lib/pq v1.2.0
+	github.com/mattn/go-colorable v0.1.8
 	github.com/mattn/go-sqlite3 v1.11.0
 	github.com/miekg/dns v1.1.31
 	github.com/mitchellh/go-homedir v1.1.0
@@ -59,7 +60,8 @@ require (
 	golang.org/x/net v0.0.0-20200904194848-62affa334b73
 	golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43 // indirect
 	golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208
-	golang.org/x/sys v0.0.0-20200909081042-eff7692f9009
+	golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4
+	golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf
 	google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d // indirect
 	google.golang.org/grpc v1.32.0 // indirect
 	gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect

go.sum

@@ -427,8 +427,12 @@ github.com/marten-seemann/chacha20 v0.2.0/go.mod h1:HSdjFau7GzYRj+ahFNwsO3ouVJr1
 github.com/marten-seemann/qpack v0.1.0/go.mod h1:LFt1NU/Ptjip0C2CPkhimBz5CGE3WGDAUWqna+CNTrI=
 github.com/marten-seemann/qtls v0.4.1/go.mod h1:pxVXcHHw1pNIt8Qo0pwSYQEoZ8yYOOPXTCZLQQunvRc=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
+github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8=
+github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
+github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/mattn/go-runewidth v0.0.8 h1:3tS41NlGYSmhhe/8fhGRzc+z3AYCw1Fe1WAyLuujKs0=
@@ -795,6 +799,7 @@ golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -812,6 +817,12 @@ golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200909081042-eff7692f9009 h1:W0lCpv29Hv0UaM1LXb9QlBHLNP8UFfcKjblhVCWftOM=
 golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 h1:myAQVi0cGEoqQVR5POX+8RR2mrocKqNN1hmeMqhX27k=
+golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf h1:MZ2shdL+ZM/XzY3ZGOnh4Nlpnxz5GSOhOmtHo3iPU6M=
+golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

h2mux/muxmetrics.go

@@ -142,7 +142,7 @@ func (updater *muxMetricsUpdaterImpl) run(log *zerolog.Logger) error {
 	for {
 		select {
 		case <-updater.abortChan:
-			log.Info().Msgf("mux - metrics: Stopping mux metrics updater")
+			log.Debug().Msgf("mux - metrics: Stopping mux metrics updater")
 			return nil
 		case roundTripMeasurement := <-updater.updateRTTChan:
 			go updater.rttData.update(roundTripMeasurement)

hello/hello_test.go

@@ -27,7 +27,7 @@ func TestCreateTLSListenerOnlyHostSuccess(t *testing.T) {
 }
 
 func TestCreateTLSListenerOnlyPortSuccess(t *testing.T) {
-	listener, err := CreateTLSListener(":8888")
+	listener, err := CreateTLSListener("localhost:8888")
 	if err != nil {
 		t.Fatal(err)
 	}

ingress/origin_request_config.go

@@ -92,7 +92,8 @@ func originRequestFromSingeRule(c *cli.Context) OriginRequestConfig {
 		proxyAddress = c.String(flag)
 	}
 	if flag := ProxyPortFlag; c.IsSet(flag) {
-		proxyPort = c.Uint(flag)
+		// Note TUN-3758 , we use Int because UInt is not supported with altsrc
+		proxyPort = uint(c.Int(flag))
 	}
 	if c.IsSet(Socks5Flag) {
 		proxyType = socksProxy

logger/configuration.go

@@ -1,6 +1,8 @@
 package logger
 
-import "path/filepath"
+import (
+	"path/filepath"
+)
 
 var defaultConfig = createDefaultConfig()
 

logger/create.go

@@ -5,10 +5,15 @@ import (
 	"io"
 	"os"
 	"path"
+	"path/filepath"
+	"sync"
+	"time"
 
+	"github.com/mattn/go-colorable"
 	"github.com/rs/zerolog"
 	fallbacklog "github.com/rs/zerolog/log"
 	"github.com/urfave/cli/v2"
+	"golang.org/x/term"
 	"gopkg.in/natefinch/lumberjack.v2"
 )
 
@@ -26,8 +31,18 @@ const (
 
 	dirPermMode  = 0744 // rwxr--r--
 	filePermMode = 0644 // rw-r--r--
+
+	consoleTimeFormat = time.RFC3339
 )
 
+func init() {
+	zerolog.TimestampFunc = utcNow
+}
+
+func utcNow() time.Time {
+	return time.Now().UTC()
+}
+
 func fallbackLogger(err error) *zerolog.Logger {
 	failLog := fallbacklog.With().Logger()
 	fallbacklog.Error().Msgf("Falling back to a default logger due to logger setup failure: %s", err)
@@ -35,6 +50,20 @@ func fallbackLogger(err error) *zerolog.Logger {
 	return &failLog
 }
 
+type resilientMultiWriter struct {
+	writers []io.Writer
+}
+
+// This custom resilientMultiWriter is an alternative to zerolog's so that we can make it resilient to individual
+// writer's errors. E.g., when running as a Windows service, the console writer fails, but we don't want to
+// allow that to prevent all logging to fail due to breaking the for loop upon an error.
+func (t resilientMultiWriter) Write(p []byte) (n int, err error) {
+	for _, w := range t.writers {
+		_, _ = w.Write(p)
+	}
+	return len(p), nil
+}
+
 func newZerolog(loggerConfig *Config) *zerolog.Logger {
 	var writers []io.Writer
 
@@ -43,7 +72,7 @@ func newZerolog(loggerConfig *Config) *zerolog.Logger {
 	}
 
 	if loggerConfig.FileConfig != nil {
-		fileLogger, err := createFileLogger(*loggerConfig.FileConfig)
+		fileLogger, err := createFileWriter(*loggerConfig.FileConfig)
 		if err != nil {
 			return fallbackLogger(err)
 		}
@@ -60,7 +89,7 @@ func newZerolog(loggerConfig *Config) *zerolog.Logger {
 		writers = append(writers, rollingLogger)
 	}
 
-	multi := zerolog.MultiLevelWriter(writers...)
+	multi := resilientMultiWriter{writers}
 
 	level, err := zerolog.ParseLevel(loggerConfig.MinLevel)
 	if err != nil {
@@ -109,41 +138,62 @@ func createFromContext(
 
 func Create(loggerConfig *Config) *zerolog.Logger {
 	if loggerConfig == nil {
-		loggerConfig = &defaultConfig
+		loggerConfig = &Config{
+			defaultConfig.ConsoleConfig,
+			nil,
+			nil,
+			defaultConfig.MinLevel,
+		}
 	}
-
 	return newZerolog(loggerConfig)
 }
 
 func createConsoleLogger(config ConsoleConfig) io.Writer {
+	consoleOut := os.Stderr
 	return zerolog.ConsoleWriter{
-		Out:     os.Stderr,
-		NoColor: config.noColor,
+		Out:     colorable.NewColorable(consoleOut),
+		NoColor: config.noColor || !term.IsTerminal(int(consoleOut.Fd())),
+		TimeFormat: consoleTimeFormat,
 	}
 }
 
-func createFileLogger(config FileConfig) (io.Writer, error) {
-	var logFile io.Writer
-	fullpath := config.Fullpath()
+type fileInitializer struct {
+	once          sync.Once
+	writer        io.Writer
+	creationError error
+}
 
-	// Try to open the existing file
-	logFile, err := os.OpenFile(fullpath, os.O_APPEND|os.O_WRONLY, filePermMode)
-	if err != nil {
-		// If the existing file wasn't found, or couldn't be opened, just ignore
-		// it and recreate a new one.
-		logFile, err = createLogFile(config)
-		// If creating a new logfile fails, then we have no choice but to error out.
+var (
+	singleFileInit   fileInitializer
+	rotatingFileInit fileInitializer
+)
+
+func createFileWriter(config FileConfig) (io.Writer, error) {
+	singleFileInit.once.Do(func() {
+
+		var logFile io.Writer
+		fullpath := config.Fullpath()
+
+		// Try to open the existing file
+		logFile, err := os.OpenFile(fullpath, os.O_APPEND|os.O_WRONLY, filePermMode)
 		if err != nil {
-			return nil, err
+			// If the existing file wasn't found, or couldn't be opened, just ignore
+			// it and recreate a new one.
+			logFile, err = createDirFile(config)
+			// If creating a new logfile fails, then we have no choice but to error out.
+			if err != nil {
+				singleFileInit.creationError = err
+				return
+			}
 		}
-	}
 
-	fileLogger := zerolog.New(logFile).With().Logger()
+		singleFileInit.writer = logFile
+	})
 
-	return fileLogger, nil
+	return singleFileInit.writer, singleFileInit.creationError
 }
 
-func createLogFile(config FileConfig) (io.Writer, error) {
+func createDirFile(config FileConfig) (io.Writer, error) {
 	if config.Dirname != "" {
 		err := os.MkdirAll(config.Dirname, dirPermMode)
 
@@ -154,7 +204,8 @@ func createLogFile(config FileConfig) (io.Writer, error) {
 
 	mode := os.FileMode(filePermMode)
 
-	logFile, err := os.OpenFile(config.Filename, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, mode)
+	fullPath := filepath.Join(config.Dirname, config.Filename)
+	logFile, err := os.OpenFile(fullPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, mode)
 	if err != nil {
 		return nil, fmt.Errorf("unable to create a new logfile: %s", err)
 	}
@@ -163,14 +214,19 @@ func createLogFile(config FileConfig) (io.Writer, error) {
 }
 
 func createRollingLogger(config RollingConfig) (io.Writer, error) {
-	if err := os.MkdirAll(config.Dirname, dirPermMode); err != nil {
-		return nil, err
-	}
+	rotatingFileInit.once.Do(func() {
+		if err := os.MkdirAll(config.Dirname, dirPermMode); err != nil {
+			rotatingFileInit.creationError = err
+			return
+		}
 
-	return &lumberjack.Logger{
-		Filename:   path.Join(config.Dirname, config.Filename),
-		MaxBackups: config.maxBackups,
-		MaxSize:    config.maxSize,
-		MaxAge:     config.maxAge,
-	}, nil
+		rotatingFileInit.writer = &lumberjack.Logger{
+			Filename:   path.Join(config.Dirname, config.Filename),
+			MaxBackups: config.maxBackups,
+			MaxSize:    config.maxSize,
+			MaxAge:     config.maxAge,
+		}
+	})
+
+	return rotatingFileInit.writer, rotatingFileInit.creationError
 }

logger/create_test.go

@@ -0,0 +1,89 @@
+package logger
+
+import (
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog"
+	"github.com/stretchr/testify/assert"
+	"io"
+	"testing"
+)
+
+var writeCalls int
+
+type mockedWriter struct {
+	wantErr bool
+}
+
+func (c mockedWriter) Write(p []byte) (int, error) {
+	writeCalls++
+
+	if c.wantErr {
+		return -1, errors.New("Expected error")
+	}
+
+	return len(p), nil
+}
+
+// Tests that a new writer is only used if it actually works.
+func TestResilientMultiWriter(t *testing.T) {
+	tests := []struct {
+		name    string
+		writers []io.Writer
+	}{
+		{
+			name:    "All valid writers",
+			writers: []io.Writer{
+				mockedWriter {
+					wantErr: false,
+				},
+				mockedWriter {
+					wantErr: false,
+				},
+			},
+		},
+		{
+			name:    "All invalid writers",
+			writers: []io.Writer{
+				mockedWriter {
+					wantErr: true,
+				},
+				mockedWriter {
+					wantErr: true,
+				},
+			},
+		},
+		{
+			name:    "First invalid writer",
+			writers: []io.Writer{
+				mockedWriter {
+					wantErr: true,
+				},
+				mockedWriter {
+					wantErr: false,
+				},
+			},
+		},
+		{
+			name:    "First valid writer",
+			writers: []io.Writer{
+				mockedWriter {
+					wantErr: false,
+				},
+				mockedWriter {
+					wantErr: true,
+				},
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		writers := tt.writers
+		multiWriter := resilientMultiWriter{writers}
+
+		logger := zerolog.New(multiWriter).With().Timestamp().Logger().Level(zerolog.InfoLevel)
+		logger.Info().Msg("Test msg")
+
+		assert.Equal(t, len(writers), writeCalls)
+		writeCalls = 0
+	}
+}

metrics/metrics.go

@@ -10,8 +10,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/cloudflare/cloudflared/connection"
-
+	"github.com/gorilla/mux"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/rs/zerolog"
@@ -23,21 +22,25 @@ const (
 	startupTime     = time.Millisecond * 500
 )
 
-func newMetricsHandler(connectionEvents <-chan connection.Event, log *zerolog.Logger) *http.ServeMux {
-	readyServer := NewReadyServer(connectionEvents, log)
-	mux := http.NewServeMux()
-	mux.Handle("/metrics", promhttp.Handler())
-	mux.HandleFunc("/healthcheck", func(w http.ResponseWriter, r *http.Request) {
+func newMetricsHandler(readyServer *ReadyServer) *mux.Router {
+	router := mux.NewRouter()
+	router.PathPrefix("/debug/").Handler(http.DefaultServeMux)
+
+	router.Handle("/metrics", promhttp.Handler())
+	router.HandleFunc("/healthcheck", func(w http.ResponseWriter, r *http.Request) {
 		_, _ = fmt.Fprintf(w, "OK\n")
 	})
-	mux.Handle("/ready", readyServer)
-	return mux
+	if readyServer != nil {
+		router.Handle("/ready", readyServer)
+	}
+
+	return router
 }
 
 func ServeMetrics(
 	l net.Listener,
 	shutdownC <-chan struct{},
-	connectionEvents <-chan connection.Event,
+	readyServer *ReadyServer,
 	log *zerolog.Logger,
 ) (err error) {
 	var wg sync.WaitGroup
@@ -45,7 +48,7 @@ func ServeMetrics(
 	trace.AuthRequest = func(*http.Request) (bool, bool) { return true, true }
 	// TODO: parameterize ReadTimeout and WriteTimeout. The maximum time we can
 	// profile CPU usage depends on WriteTimeout
-	h := newMetricsHandler(connectionEvents, log)
+	h := newMetricsHandler(readyServer)
 	server := &http.Server{
 		ReadTimeout:  10 * time.Second,
 		WriteTimeout: 10 * time.Second,

metrics/readiness.go

@@ -19,30 +19,28 @@ type ReadyServer struct {
 }
 
 // NewReadyServer initializes a ReadyServer and starts listening for dis/connection events.
-func NewReadyServer(connectionEvents <-chan conn.Event, log *zerolog.Logger) *ReadyServer {
-	rs := ReadyServer{
+func NewReadyServer(log *zerolog.Logger) *ReadyServer {
+	return &ReadyServer{
 		isConnected: make(map[int]bool, 0),
 		log:         log,
 	}
-	go func() {
-		for c := range connectionEvents {
-			switch c.EventType {
-			case conn.Connected:
-				rs.Lock()
-				rs.isConnected[int(c.Index)] = true
-				rs.Unlock()
-			case conn.Disconnected, conn.Reconnecting, conn.RegisteringTunnel:
-				rs.Lock()
-				rs.isConnected[int(c.Index)] = false
-				rs.Unlock()
-			case conn.SetURL:
-				continue
-			default:
-				rs.log.Error().Msgf("Unknown connection event case %v", c)
-			}
-		}
-	}()
-	return &rs
+}
+
+func (rs *ReadyServer) OnTunnelEvent(c conn.Event) {
+	switch c.EventType {
+	case conn.Connected:
+		rs.Lock()
+		rs.isConnected[int(c.Index)] = true
+		rs.Unlock()
+	case conn.Disconnected, conn.Reconnecting, conn.RegisteringTunnel, conn.Unregistering:
+		rs.Lock()
+		rs.isConnected[int(c.Index)] = false
+		rs.Unlock()
+	case conn.SetURL:
+		break
+	default:
+		rs.log.Error().Msgf("Unknown connection event case %v", c)
+	}
 }
 
 type body struct {

metrics/readiness_test.go

@@ -3,6 +3,11 @@ package metrics
 import (
 	"net/http"
 	"testing"
+
+	"github.com/rs/zerolog"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/cloudflare/cloudflared/connection"
 )
 
 func TestReadyServer_makeResponse(t *testing.T) {
@@ -56,3 +61,74 @@ func TestReadyServer_makeResponse(t *testing.T) {
 		})
 	}
 }
+
+func TestReadinessEventHandling(t *testing.T) {
+	nopLogger := zerolog.Nop()
+	rs := NewReadyServer(&nopLogger)
+
+	// start not ok
+	code, ready := rs.makeResponse()
+	assert.NotEqualValues(t, http.StatusOK, code)
+	assert.Zero(t, ready)
+
+	// one connected => ok
+	rs.OnTunnelEvent(connection.Event{
+		Index:     1,
+		EventType: connection.Connected,
+	})
+	code, ready = rs.makeResponse()
+	assert.EqualValues(t, http.StatusOK, code)
+	assert.EqualValues(t, 1, ready)
+
+	// another connected => still ok
+	rs.OnTunnelEvent(connection.Event{
+		Index:     2,
+		EventType: connection.Connected,
+	})
+	code, ready = rs.makeResponse()
+	assert.EqualValues(t, http.StatusOK, code)
+	assert.EqualValues(t, 2, ready)
+
+	// one reconnecting => still ok
+	rs.OnTunnelEvent(connection.Event{
+		Index:     2,
+		EventType: connection.Reconnecting,
+	})
+	code, ready = rs.makeResponse()
+	assert.EqualValues(t, http.StatusOK, code)
+	assert.EqualValues(t, 1, ready)
+
+	// Regression test for TUN-3777
+	rs.OnTunnelEvent(connection.Event{
+		Index:     1,
+		EventType: connection.RegisteringTunnel,
+	})
+	code, ready = rs.makeResponse()
+	assert.NotEqualValues(t, http.StatusOK, code)
+	assert.Zero(t, ready)
+
+	// other connected then unregistered  => not ok
+	rs.OnTunnelEvent(connection.Event{
+		Index:     1,
+		EventType: connection.Connected,
+	})
+	code, ready = rs.makeResponse()
+	assert.EqualValues(t, http.StatusOK, code)
+	assert.EqualValues(t, 1, ready)
+	rs.OnTunnelEvent(connection.Event{
+		Index:     1,
+		EventType: connection.Unregistering,
+	})
+	code, ready = rs.makeResponse()
+	assert.NotEqualValues(t, http.StatusOK, code)
+	assert.Zero(t, ready)
+
+	// other disconnected => not ok
+	rs.OnTunnelEvent(connection.Event{
+		Index:     1,
+		EventType: connection.Disconnected,
+	})
+	code, ready = rs.makeResponse()
+	assert.NotEqualValues(t, http.StatusOK, code)
+	assert.Zero(t, ready)
+}

origin/cloudflare_status_page_test.go

@@ -137,8 +137,7 @@ func TestParseStatusPage(t *testing.T) {
 								"page_id": "yh6f0r4529hb",
 								"group_id": "1km35smx8p41",
 								"group": false,
-								"only_show_if_degraded": false,
-								"automation_email": "component+g4tb35rs9yw7@notifications.statuspage.io"
+								"only_show_if_degraded": false
 							}
 						],
 						"impact": "minor"
@@ -208,8 +207,7 @@ func TestParseStatusPage(t *testing.T) {
 								"page_id": "yh6f0r4529hb",
 								"group_id": "1km35smx8p41",
 								"group": false,
-								"only_show_if_degraded": false,
-								"automation_email": "component+4c231tkdlpcl@notifications.statuspage.io"
+								"only_show_if_degraded": false
 							}
 						],
 						"impact": "minor"

origin/external_control.go

@@ -10,11 +10,11 @@ type ReconnectSignal struct {
 }
 
 // Error allows us to use ReconnectSignal as a special error to force connection abort
-func (r *ReconnectSignal) Error() string {
+func (r ReconnectSignal) Error() string {
 	return "reconnect signal"
 }
 
-func (r *ReconnectSignal) DelayBeforeReconnect() {
+func (r ReconnectSignal) DelayBeforeReconnect() {
 	if r.Delay > 0 {
 		time.Sleep(r.Delay)
 	}

origin/proxy.go

@@ -191,7 +191,7 @@ func (c *client) logRequest(r *http.Request, cfRay string, lbProbe bool, ruleNum
 func (c *client) logOriginResponse(r *http.Response, cfRay string, lbProbe bool, ruleNum int) {
 	responseByCode.WithLabelValues(strconv.Itoa(r.StatusCode)).Inc()
 	if cfRay != "" {
-		c.log.Info().Msgf("CF-RAY: %s Status: %s served by ingress %d", cfRay, r.Status, ruleNum)
+		c.log.Debug().Msgf("CF-RAY: %s Status: %s served by ingress %d", cfRay, r.Status, ruleNum)
 	} else if lbProbe {
 		c.log.Debug().Msgf("Response to Load Balancer health check %s", r.Status)
 	} else {

origin/supervisor.go

@@ -3,6 +3,7 @@ package origin
 import (
 	"context"
 	"errors"
+	"fmt"
 	"net"
 	"time"
 
@@ -50,23 +51,31 @@ type Supervisor struct {
 	nextConnectedIndex  int
 	nextConnectedSignal chan struct{}
 
-	log *zerolog.Logger
+	log          *zerolog.Logger
+	logTransport *zerolog.Logger
 
 	reconnectCredentialManager *reconnectCredentialManager
 	useReconnectToken          bool
+
+	reconnectCh       chan ReconnectSignal
+	gracefulShutdownC <-chan struct{}
 }
 
+var errEarlyShutdown = errors.New("shutdown started")
+
 type tunnelError struct {
 	index int
 	addr  *net.TCPAddr
 	err   error
 }
 
-func NewSupervisor(config *TunnelConfig, cloudflaredUUID uuid.UUID) (*Supervisor, error) {
-	var (
-		edgeIPs *edgediscovery.Edge
-		err     error
-	)
+func NewSupervisor(config *TunnelConfig, reconnectCh chan ReconnectSignal, gracefulShutdownC <-chan struct{}) (*Supervisor, error) {
+	cloudflaredUUID, err := uuid.NewRandom()
+	if err != nil {
+		return nil, fmt.Errorf("failed to generate cloudflared instance ID: %w", err)
+	}
+
+	var edgeIPs *edgediscovery.Edge
 	if len(config.EdgeAddrs) > 0 {
 		edgeIPs, err = edgediscovery.StaticEdge(config.Log, config.EdgeAddrs)
 	} else {
@@ -88,13 +97,22 @@ func NewSupervisor(config *TunnelConfig, cloudflaredUUID uuid.UUID) (*Supervisor
 		tunnelErrors:               make(chan tunnelError),
 		tunnelsConnecting:          map[int]chan struct{}{},
 		log:                        config.Log,
+		logTransport:               config.LogTransport,
 		reconnectCredentialManager: newReconnectCredentialManager(connection.MetricsNamespace, connection.TunnelSubsystem, config.HAConnections),
 		useReconnectToken:          useReconnectToken,
+		reconnectCh:                reconnectCh,
+		gracefulShutdownC:          gracefulShutdownC,
 	}, nil
 }
 
-func (s *Supervisor) Run(ctx context.Context, connectedSignal *signal.Signal, reconnectCh chan ReconnectSignal) error {
-	if err := s.initialize(ctx, connectedSignal, reconnectCh); err != nil {
+func (s *Supervisor) Run(
+	ctx context.Context,
+	connectedSignal *signal.Signal,
+) error {
+	if err := s.initialize(ctx, connectedSignal); err != nil {
+		if err == errEarlyShutdown {
+			return nil
+		}
 		return err
 	}
 	var tunnelsWaiting []int
@@ -117,6 +135,7 @@ func (s *Supervisor) Run(ctx context.Context, connectedSignal *signal.Signal, re
 		}
 	}
 
+	shuttingDown := false
 	for {
 		select {
 		// Context cancelled
@@ -130,8 +149,8 @@ func (s *Supervisor) Run(ctx context.Context, connectedSignal *signal.Signal, re
 		// (note that this may also be caused by context cancellation)
 		case tunnelError := <-s.tunnelErrors:
 			tunnelsActive--
-			if tunnelError.err != nil {
-				s.log.Err(tunnelError.err).Msg("supervisor: Tunnel disconnected")
+			if tunnelError.err != nil && !shuttingDown {
+				s.log.Err(tunnelError.err).Int(connection.LogFieldConnIndex, tunnelError.index).Msg("Connection terminated")
 				tunnelsWaiting = append(tunnelsWaiting, tunnelError.index)
 				s.waitForNextTunnel(tunnelError.index)
 
@@ -139,14 +158,16 @@ func (s *Supervisor) Run(ctx context.Context, connectedSignal *signal.Signal, re
 					backoffTimer = backoff.BackoffTimer()
 				}
 
-				// Previously we'd mark the edge address as bad here, but now we'll just silently use
-				// another.
+				// Previously we'd mark the edge address as bad here, but now we'll just silently use another.
+			} else if tunnelsActive == 0 {
+				// all connected tunnels exited gracefully, no more work to do
+				return nil
 			}
 		// Backoff was set and its timer expired
 		case <-backoffTimer:
 			backoffTimer = nil
 			for _, index := range tunnelsWaiting {
-				go s.startTunnel(ctx, index, s.newConnectedTunnelSignal(index), reconnectCh)
+				go s.startTunnel(ctx, index, s.newConnectedTunnelSignal(index))
 			}
 			tunnelsActive += len(tunnelsWaiting)
 			tunnelsWaiting = nil
@@ -166,31 +187,38 @@ func (s *Supervisor) Run(ctx context.Context, connectedSignal *signal.Signal, re
 				// No more tunnels outstanding, clear backoff timer
 				backoff.SetGracePeriod()
 			}
+		case <-s.gracefulShutdownC:
+			shuttingDown = true
 		}
 	}
 }
 
 // Returns nil if initialization succeeded, else the initialization error.
-func (s *Supervisor) initialize(ctx context.Context, connectedSignal *signal.Signal, reconnectCh chan ReconnectSignal) error {
-	availableAddrs := int(s.edgeIPs.AvailableAddrs())
+func (s *Supervisor) initialize(
+	ctx context.Context,
+	connectedSignal *signal.Signal,
+) error {
+	availableAddrs := s.edgeIPs.AvailableAddrs()
 	if s.config.HAConnections > availableAddrs {
 		s.log.Info().Msgf("You requested %d HA connections but I can give you at most %d.", s.config.HAConnections, availableAddrs)
 		s.config.HAConnections = availableAddrs
 	}
 
-	go s.startFirstTunnel(ctx, connectedSignal, reconnectCh)
+	go s.startFirstTunnel(ctx, connectedSignal)
 	select {
 	case <-ctx.Done():
 		<-s.tunnelErrors
 		return ctx.Err()
 	case tunnelError := <-s.tunnelErrors:
 		return tunnelError.err
+	case <-s.gracefulShutdownC:
+		return errEarlyShutdown
 	case <-connectedSignal.Wait():
 	}
 	// At least one successful connection, so start the rest
 	for i := 1; i < s.config.HAConnections; i++ {
 		ch := signal.New(make(chan struct{}))
-		go s.startTunnel(ctx, i, ch, reconnectCh)
+		go s.startTunnel(ctx, i, ch)
 		time.Sleep(registrationInterval)
 	}
 	return nil
@@ -198,7 +226,10 @@ func (s *Supervisor) initialize(ctx context.Context, connectedSignal *signal.Sig
 
 // startTunnel starts the first tunnel connection. The resulting error will be sent on
 // s.tunnelErrors. It will send a signal via connectedSignal if registration succeed
-func (s *Supervisor) startFirstTunnel(ctx context.Context, connectedSignal *signal.Signal, reconnectCh chan ReconnectSignal) {
+func (s *Supervisor) startFirstTunnel(
+	ctx context.Context,
+	connectedSignal *signal.Signal,
+) {
 	var (
 		addr *net.TCPAddr
 		err  error
@@ -221,7 +252,8 @@ func (s *Supervisor) startFirstTunnel(ctx context.Context, connectedSignal *sign
 		firstConnIndex,
 		connectedSignal,
 		s.cloudflaredUUID,
-		reconnectCh,
+		s.reconnectCh,
+		s.gracefulShutdownC,
 	)
 	// If the first tunnel disconnects, keep restarting it.
 	edgeErrors := 0
@@ -253,14 +285,19 @@ func (s *Supervisor) startFirstTunnel(ctx context.Context, connectedSignal *sign
 			firstConnIndex,
 			connectedSignal,
 			s.cloudflaredUUID,
-			reconnectCh,
+			s.reconnectCh,
+			s.gracefulShutdownC,
 		)
 	}
 }
 
 // startTunnel starts a new tunnel connection. The resulting error will be sent on
 // s.tunnelErrors.
-func (s *Supervisor) startTunnel(ctx context.Context, index int, connectedSignal *signal.Signal, reconnectCh chan ReconnectSignal) {
+func (s *Supervisor) startTunnel(
+	ctx context.Context,
+	index int,
+	connectedSignal *signal.Signal,
+) {
 	var (
 		addr *net.TCPAddr
 		err  error
@@ -281,7 +318,8 @@ func (s *Supervisor) startTunnel(ctx context.Context, index int, connectedSignal
 		uint8(index),
 		connectedSignal,
 		s.cloudflaredUUID,
-		reconnectCh,
+		s.reconnectCh,
+		s.gracefulShutdownC,
 	)
 }
 
@@ -324,7 +362,7 @@ func (s *Supervisor) authenticate(ctx context.Context, numPreviousAttempts int)
 		// This callback is invoked by h2mux when the edge initiates a stream.
 		return nil // noop
 	})
-	muxerConfig := s.config.MuxerConfig.H2MuxerConfig(handler, s.log)
+	muxerConfig := s.config.MuxerConfig.H2MuxerConfig(handler, s.logTransport)
 	muxer, err := h2mux.Handshake(edgeConn, edgeConn, *muxerConfig, h2mux.ActiveStreams)
 	if err != nil {
 		return nil, err

origin/tunnel.go

@@ -12,7 +12,6 @@ import (
 
 	"github.com/google/uuid"
 	"github.com/pkg/errors"
-	"github.com/prometheus/client_golang/prometheus"
 	"github.com/rs/zerolog"
 	"golang.org/x/sync/errgroup"
 
@@ -27,9 +26,7 @@ import (
 
 const (
 	dialTimeout              = 15 * time.Second
-	muxerTimeout             = 5 * time.Second
 	lbProbeUserAgentPrefix   = "Mozilla/5.0 (compatible; Cloudflare-Traffic-Manager/1.0; +https://www.cloudflare.com/traffic-manager/;"
-	DuplicateConnectionError = "EDUPCONN"
 	FeatureSerializedHeaders = "serialized_headers"
 	FeatureQuickReconnects   = "quick_reconnects"
 )
@@ -37,9 +34,7 @@ const (
 type rpcName string
 
 const (
-	register     rpcName = "register"
 	reconnect    rpcName = "reconnect"
-	unregister   rpcName = "unregister"
 	authenticate rpcName = " authenticate"
 )
 
@@ -56,6 +51,7 @@ type TunnelConfig struct {
 	LBPool           string
 	Tags             []tunnelpogs.Tag
 	Log              *zerolog.Logger
+	LogTransport     *zerolog.Logger
 	Observer         *connection.Observer
 	ReportedVersion  string
 	Retries          uint
@@ -64,41 +60,10 @@ type TunnelConfig struct {
 	NamedTunnel      *connection.NamedTunnelConfig
 	ClassicTunnel    *connection.ClassicTunnelConfig
 	MuxerConfig      *connection.MuxerConfig
-	TunnelEventChans []chan connection.Event
 	ProtocolSelector connection.ProtocolSelector
 	EdgeTLSConfigs   map[connection.Protocol]*tls.Config
 }
 
-type muxerShutdownError struct{}
-
-func (e muxerShutdownError) Error() string {
-	return "muxer shutdown"
-}
-
-// RegisterTunnel error from server
-type serverRegisterTunnelError struct {
-	cause     error
-	permanent bool
-}
-
-func (e serverRegisterTunnelError) Error() string {
-	return e.cause.Error()
-}
-
-// RegisterTunnel error from client
-type clientRegisterTunnelError struct {
-	cause error
-}
-
-func newRPCError(cause error, counter *prometheus.CounterVec, name rpcName) clientRegisterTunnelError {
-	counter.WithLabelValues(cause.Error(), string(name)).Inc()
-	return clientRegisterTunnelError{cause: cause}
-}
-
-func (e clientRegisterTunnelError) Error() string {
-	return e.cause.Error()
-}
-
 func (c *TunnelConfig) RegistrationOptions(connectionID uint8, OriginLocalIP string, uuid uuid.UUID) *tunnelpogs.RegistrationOptions {
 	policy := tunnelrpc.ExistingTunnelPolicy_balance
 	if c.HAConnections <= 1 && c.LBPool == "" {
@@ -143,12 +108,18 @@ func (c *TunnelConfig) SupportedFeatures() []string {
 	return features
 }
 
-func StartTunnelDaemon(ctx context.Context, config *TunnelConfig, connectedSignal *signal.Signal, cloudflaredID uuid.UUID, reconnectCh chan ReconnectSignal) error {
-	s, err := NewSupervisor(config, cloudflaredID)
+func StartTunnelDaemon(
+	ctx context.Context,
+	config *TunnelConfig,
+	connectedSignal *signal.Signal,
+	reconnectCh chan ReconnectSignal,
+	graceShutdownC <-chan struct{},
+) error {
+	s, err := NewSupervisor(config, reconnectCh, graceShutdownC)
 	if err != nil {
 		return err
 	}
-	return s.Run(ctx, connectedSignal, reconnectCh)
+	return s.Run(ctx, connectedSignal)
 }
 
 func ServeTunnelLoop(
@@ -160,13 +131,14 @@ func ServeTunnelLoop(
 	connectedSignal *signal.Signal,
 	cloudflaredUUID uuid.UUID,
 	reconnectCh chan ReconnectSignal,
+	gracefulShutdownC <-chan struct{},
 ) error {
 	haConnections.Inc()
 	defer haConnections.Dec()
 
 	connLog := config.Log.With().Uint8(connection.LogFieldConnIndex, connIndex).Logger()
 
-	protocallFallback := &protocallFallback{
+	protocolFallback := &protocolFallback{
 		BackoffHandler{MaxRetries: config.Retries},
 		config.ProtocolSelector.Current(),
 		false,
@@ -190,94 +162,99 @@ func ServeTunnelLoop(
 			addr,
 			connIndex,
 			connectedFuse,
-			protocallFallback,
+			protocolFallback,
 			cloudflaredUUID,
 			reconnectCh,
-			protocallFallback.protocol,
+			protocolFallback.protocol,
+			gracefulShutdownC,
 		)
 		if !recoverable {
 			return err
 		}
 
-		err = waitForBackoff(ctx, &connLog, protocallFallback, config, connIndex, err)
-		if err != nil {
+		config.Observer.SendReconnect(connIndex)
+
+		duration, ok := protocolFallback.GetBackoffDuration(ctx)
+		if !ok {
 			return err
 		}
+		connLog.Info().Msgf("Retrying connection in %s seconds", duration)
+
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-gracefulShutdownC:
+			return nil
+		case <-protocolFallback.BackoffTimer():
+			if !selectNextProtocol(&connLog, protocolFallback, config.ProtocolSelector) {
+				return err
+			}
+		}
 	}
 }
 
-// protocallFallback is a wrapper around backoffHandler that will try fallback option when backoff reaches
+// protocolFallback is a wrapper around backoffHandler that will try fallback option when backoff reaches
 // max retries
-type protocallFallback struct {
+type protocolFallback struct {
 	BackoffHandler
 	protocol   connection.Protocol
 	inFallback bool
 }
 
-func (pf *protocallFallback) reset() {
+func (pf *protocolFallback) reset() {
 	pf.resetNow()
 	pf.inFallback = false
 }
 
-func (pf *protocallFallback) fallback(fallback connection.Protocol) {
+func (pf *protocolFallback) fallback(fallback connection.Protocol) {
 	pf.resetNow()
 	pf.protocol = fallback
 	pf.inFallback = true
 }
 
-// Expect err to always be non nil
-func waitForBackoff(
-	ctx context.Context,
-	log *zerolog.Logger,
-	protobackoff *protocallFallback,
-	config *TunnelConfig,
-	connIndex uint8,
-	err error,
-) error {
-	duration, ok := protobackoff.GetBackoffDuration(ctx)
-	if !ok {
-		return err
-	}
-
-	config.Observer.SendReconnect(connIndex)
-	log.Info().
-		Err(err).
-		Msgf("Retrying connection in %s seconds", duration)
-	protobackoff.Backoff(ctx)
-
-	if protobackoff.ReachedMaxRetries() {
-		fallback, hasFallback := config.ProtocolSelector.Fallback()
+// selectNextProtocol picks connection protocol for the next retry iteration,
+// returns true if it was able to pick the protocol, false if we are out of options and should stop retrying
+func selectNextProtocol(
+	connLog *zerolog.Logger,
+	protocolBackoff *protocolFallback,
+	selector connection.ProtocolSelector,
+) bool {
+	if protocolBackoff.ReachedMaxRetries() {
+		fallback, hasFallback := selector.Fallback()
 		if !hasFallback {
-			return err
+			return false
 		}
 		// Already using fallback protocol, no point to retry
-		if protobackoff.protocol == fallback {
-			return err
+		if protocolBackoff.protocol == fallback {
+			return false
 		}
-		log.Info().Msgf("Fallback to use %s", fallback)
-		protobackoff.fallback(fallback)
-	} else if !protobackoff.inFallback {
-		current := config.ProtocolSelector.Current()
-		if protobackoff.protocol != current {
-			protobackoff.protocol = current
-			config.Log.Info().Msgf("Change protocol to %s", current)
+		connLog.Info().Msgf("Switching to fallback protocol %s", fallback)
+		protocolBackoff.fallback(fallback)
+	} else if !protocolBackoff.inFallback {
+		current := selector.Current()
+		if protocolBackoff.protocol != current {
+			protocolBackoff.protocol = current
+			connLog.Info().Msgf("Changing protocol to %s", current)
 		}
 	}
-	return nil
+	return true
 }
 
+// ServeTunnel runs a single tunnel connection, returns nil on graceful shutdown,
+// on error returns a flag indicating if error can be retried
 func ServeTunnel(
 	ctx context.Context,
-	log *zerolog.Logger,
+	connLong *zerolog.Logger,
 	credentialManager *reconnectCredentialManager,
 	config *TunnelConfig,
 	addr *net.TCPAddr,
 	connIndex uint8,
 	fuse *h2mux.BooleanFuse,
-	backoff *protocallFallback,
+	backoff *protocolFallback,
 	cloudflaredUUID uuid.UUID,
 	reconnectCh chan ReconnectSignal,
 	protocol connection.Protocol,
+	gracefulShutdownC <-chan struct{},
 ) (err error, recoverable bool) {
 	// Treat panics as recoverable errors
 	defer func() {
@@ -302,26 +279,78 @@ func ServeTunnel(
 		fuse:    fuse,
 		backoff: backoff,
 	}
+
 	if protocol == connection.HTTP2 {
 		connOptions := config.ConnectionOptions(edgeConn.LocalAddr().String(), uint8(backoff.retries))
-		return ServeHTTP2(ctx, log, config, edgeConn, connOptions, connIndex, connectedFuse, reconnectCh)
+		err = ServeHTTP2(
+			ctx,
+			connLong,
+			config,
+			edgeConn,
+			connOptions,
+			connIndex,
+			connectedFuse,
+			reconnectCh,
+			gracefulShutdownC,
+		)
+	} else {
+		err = ServeH2mux(
+			ctx,
+			connLong,
+			credentialManager,
+			config,
+			edgeConn,
+			connIndex,
+			connectedFuse,
+			cloudflaredUUID,
+			reconnectCh,
+			gracefulShutdownC,
+		)
 	}
-	return ServeH2mux(
-		ctx,
-		log,
-		credentialManager,
-		config,
-		edgeConn,
-		connIndex,
-		connectedFuse,
-		cloudflaredUUID,
-		reconnectCh,
-	)
+
+	if err != nil {
+		switch err := err.(type) {
+		case connection.DupConnRegisterTunnelError:
+			// don't retry this connection anymore, let supervisor pick a new address
+			return err, false
+		case connection.ServerRegisterTunnelError:
+			connLong.Err(err).Msg("Register tunnel error from server side")
+			// Don't send registration error return from server to Sentry. They are
+			// logged on server side
+			if incidents := config.IncidentLookup.ActiveIncidents(); len(incidents) > 0 {
+				connLong.Error().Msg(activeIncidentsMsg(incidents))
+			}
+			return err.Cause, !err.Permanent
+		case ReconnectSignal:
+			connLong.Info().
+				Uint8(connection.LogFieldConnIndex, connIndex).
+				Msgf("Restarting connection due to reconnect signal in %s", err.Delay)
+			err.DelayBeforeReconnect()
+			return err, true
+		default:
+			if err == context.Canceled {
+				connLong.Debug().Err(err).Msgf("Serve tunnel error")
+				return err, false
+			}
+			connLong.Err(err).Msgf("Serve tunnel error")
+			_, permanent := err.(unrecoverableError)
+			return err, !permanent
+		}
+	}
+	return nil, false
+}
+
+type unrecoverableError struct {
+	err error
+}
+
+func (r unrecoverableError) Error() string {
+	return r.err.Error()
 }
 
 func ServeH2mux(
 	ctx context.Context,
-	log *zerolog.Logger,
+	connLog *zerolog.Logger,
 	credentialManager *reconnectCredentialManager,
 	config *TunnelConfig,
 	edgeConn net.Conn,
@@ -329,8 +358,9 @@ func ServeH2mux(
 	connectedFuse *connectedFuse,
 	cloudflaredUUID uuid.UUID,
 	reconnectCh chan ReconnectSignal,
-) (err error, recoverable bool) {
-	config.Log.Debug().Msgf("Connecting via h2mux")
+	gracefulShutdownC <-chan struct{},
+) error {
+	connLog.Debug().Msgf("Connecting via h2mux")
 	// Returns error from parsing the origin URL or handshake errors
 	handler, err, recoverable := connection.NewH2muxConnection(
 		config.ConnectionConfig,
@@ -338,74 +368,46 @@ func ServeH2mux(
 		edgeConn,
 		connIndex,
 		config.Observer,
+		gracefulShutdownC,
 	)
 	if err != nil {
-		return err, recoverable
+		if !recoverable {
+			return unrecoverableError{err}
+		}
+		return err
 	}
 
 	errGroup, serveCtx := errgroup.WithContext(ctx)
 
-	errGroup.Go(func() (err error) {
+	errGroup.Go(func() error {
 		if config.NamedTunnel != nil {
 			connOptions := config.ConnectionOptions(edgeConn.LocalAddr().String(), uint8(connectedFuse.backoff.retries))
-			return handler.ServeNamedTunnel(serveCtx, config.NamedTunnel, credentialManager, connOptions, connectedFuse)
+			return handler.ServeNamedTunnel(serveCtx, config.NamedTunnel, connOptions, connectedFuse)
 		}
 		registrationOptions := config.RegistrationOptions(connIndex, edgeConn.LocalAddr().String(), cloudflaredUUID)
 		return handler.ServeClassicTunnel(serveCtx, config.ClassicTunnel, credentialManager, registrationOptions, connectedFuse)
 	})
 
-	errGroup.Go(listenReconnect(serveCtx, reconnectCh))
+	errGroup.Go(func() error {
+		return listenReconnect(serveCtx, reconnectCh, gracefulShutdownC)
+	})
 
-	err = errGroup.Wait()
-	if err != nil {
-		switch err := err.(type) {
-		case *connection.DupConnRegisterTunnelError:
-			// don't retry this connection anymore, let supervisor pick new a address
-			return err, false
-		case *serverRegisterTunnelError:
-			log.Err(err).Msg("Register tunnel error from server side")
-			// Don't send registration error return from server to Sentry. They are
-			// logged on server side
-			if incidents := config.IncidentLookup.ActiveIncidents(); len(incidents) > 0 {
-				log.Error().Msg(activeIncidentsMsg(incidents))
-			}
-			return err.cause, !err.permanent
-		case *clientRegisterTunnelError:
-			log.Err(err).Msg("Register tunnel error on client side")
-			return err, true
-		case *muxerShutdownError:
-			log.Info().Msg("Muxer shutdown")
-			return err, true
-		case *ReconnectSignal:
-			log.Info().
-				Uint8(connection.LogFieldConnIndex, connIndex).
-				Msgf("Restarting connection due to reconnect signal in %s", err.Delay)
-			err.DelayBeforeReconnect()
-			return err, true
-		default:
-			if err == context.Canceled {
-				log.Debug().Err(err).Msgf("Serve tunnel error")
-				return err, false
-			}
-			log.Err(err).Msgf("Serve tunnel error")
-			return err, true
-		}
-	}
-	return nil, true
+	return errGroup.Wait()
 }
 
 func ServeHTTP2(
 	ctx context.Context,
-	log *zerolog.Logger,
+	connLog *zerolog.Logger,
 	config *TunnelConfig,
 	tlsServerConn net.Conn,
 	connOptions *tunnelpogs.ConnectionOptions,
 	connIndex uint8,
 	connectedFuse connection.ConnectedFuse,
 	reconnectCh chan ReconnectSignal,
-) (err error, recoverable bool) {
-	log.Debug().Msgf("Connecting via http2")
-	server := connection.NewHTTP2Connection(
+	gracefulShutdownC <-chan struct{},
+) error {
+	connLog.Debug().Msgf("Connecting via http2")
+	h2conn := connection.NewHTTP2Connection(
 		tlsServerConn,
 		config.ConnectionConfig,
 		config.NamedTunnel,
@@ -413,37 +415,40 @@ func ServeHTTP2(
 		config.Observer,
 		connIndex,
 		connectedFuse,
+		gracefulShutdownC,
 	)
 
 	errGroup, serveCtx := errgroup.WithContext(ctx)
 	errGroup.Go(func() error {
-		server.Serve(serveCtx)
-		return fmt.Errorf("connection with edge closed")
+		return h2conn.Serve(serveCtx)
 	})
 
-	errGroup.Go(listenReconnect(serveCtx, reconnectCh))
+	errGroup.Go(func() error {
+		err := listenReconnect(serveCtx, reconnectCh, gracefulShutdownC)
+		if err != nil {
+			// forcefully break the connection (this is only used for testing)
+			_ = tlsServerConn.Close()
+		}
+		return err
+	})
 
-	err = errGroup.Wait()
-	if err != nil {
-		return err, true
-	}
-	return nil, false
+	return errGroup.Wait()
 }
 
-func listenReconnect(ctx context.Context, reconnectCh <-chan ReconnectSignal) func() error {
-	return func() error {
-		select {
-		case reconnect := <-reconnectCh:
-			return &reconnect
-		case <-ctx.Done():
-			return nil
-		}
+func listenReconnect(ctx context.Context, reconnectCh <-chan ReconnectSignal, gracefulShutdownCh <-chan struct{}) error {
+	select {
+	case reconnect := <-reconnectCh:
+		return reconnect
+	case <-gracefulShutdownCh:
+		return nil
+	case <-ctx.Done():
+		return nil
 	}
 }
 
 type connectedFuse struct {
 	fuse    *h2mux.BooleanFuse
-	backoff *protocallFallback
+	backoff *protocolFallback
 }
 
 func (cf *connectedFuse) Connected() {
@@ -466,5 +471,4 @@ func activeIncidentsMsg(incidents []Incident) string {
 		incidentStrings = append(incidentStrings, incidentString)
 	}
 	return preamble + " " + strings.Join(incidentStrings, "; ")
-
 }

origin/tunnel_test.go

@@ -1,8 +1,6 @@
 package origin
 
 import (
-	"context"
-	"fmt"
 	"testing"
 	"time"
 
@@ -25,13 +23,13 @@ func (dmf *dynamicMockFetcher) fetch() connection.PercentageFetcher {
 		return dmf.percentage, nil
 	}
 }
+
 func TestWaitForBackoffFallback(t *testing.T) {
 	maxRetries := uint(3)
 	backoff := BackoffHandler{
 		MaxRetries: maxRetries,
 		BaseTime:   time.Millisecond * 10,
 	}
-	ctx := context.Background()
 	log := zerolog.Nop()
 	resolveTTL := time.Duration(0)
 	namedTunnel := &connection.NamedTunnelConfig{
@@ -50,17 +48,11 @@ func TestWaitForBackoffFallback(t *testing.T) {
 		&log,
 	)
 	assert.NoError(t, err)
-	config := &TunnelConfig{
-		Log:              &log,
-		ProtocolSelector: protocolSelector,
-		Observer:         connection.NewObserver(nil, nil, false),
-	}
-	connIndex := uint8(1)
 
 	initProtocol := protocolSelector.Current()
 	assert.Equal(t, connection.HTTP2, initProtocol)
 
-	protocallFallback := &protocallFallback{
+	protocolFallback := &protocolFallback{
 		backoff,
 		initProtocol,
 		false,
@@ -68,29 +60,33 @@ func TestWaitForBackoffFallback(t *testing.T) {
 
 	// Retry #0 and #1. At retry #2, we switch protocol, so the fallback loop has one more retry than this
 	for i := 0; i < int(maxRetries-1); i++ {
-		err := waitForBackoff(ctx, &log, protocallFallback, config, connIndex, fmt.Errorf("some error"))
-		assert.NoError(t, err)
-		assert.Equal(t, initProtocol, protocallFallback.protocol)
+		protocolFallback.BackoffTimer() // simulate retry
+		ok := selectNextProtocol(&log, protocolFallback, protocolSelector)
+		assert.True(t, ok)
+		assert.Equal(t, initProtocol, protocolFallback.protocol)
 	}
 
 	// Retry fallback protocol
 	for i := 0; i < int(maxRetries); i++ {
-		err := waitForBackoff(ctx, &log, protocallFallback, config, connIndex, fmt.Errorf("some error"))
-		assert.NoError(t, err)
+		protocolFallback.BackoffTimer() // simulate retry
+		ok := selectNextProtocol(&log, protocolFallback, protocolSelector)
+		assert.True(t, ok)
 		fallback, ok := protocolSelector.Fallback()
 		assert.True(t, ok)
-		assert.Equal(t, fallback, protocallFallback.protocol)
+		assert.Equal(t, fallback, protocolFallback.protocol)
 	}
 
 	currentGlobalProtocol := protocolSelector.Current()
 	assert.Equal(t, initProtocol, currentGlobalProtocol)
 
 	// No protocol to fallback, return error
-	err = waitForBackoff(ctx, &log, protocallFallback, config, connIndex, fmt.Errorf("some error"))
-	assert.Error(t, err)
+	protocolFallback.BackoffTimer() // simulate retry
+	ok := selectNextProtocol(&log, protocolFallback, protocolSelector)
+	assert.False(t, ok)
 
-	protocallFallback.reset()
-	err = waitForBackoff(ctx, &log, protocallFallback, config, connIndex, fmt.Errorf("new error"))
-	assert.NoError(t, err)
-	assert.Equal(t, initProtocol, protocallFallback.protocol)
+	protocolFallback.reset()
+	protocolFallback.BackoffTimer() // simulate retry
+	ok = selectNextProtocol(&log, protocolFallback, protocolSelector)
+	assert.True(t, ok)
+	assert.Equal(t, initProtocol, protocolFallback.protocol)
 }

socks/connection_handler_test.go

@@ -42,7 +42,7 @@ func startTestServer(t *testing.T, httpHandler func(w http.ResponseWriter, r *ht
 	// create a socks server
 	requestHandler := NewRequestHandler(NewNetDialer())
 	socksServer := NewConnectionHandler(requestHandler)
-	listener, err := net.Listen("tcp", ":8086")
+	listener, err := net.Listen("tcp", "localhost:8086")
 	assert.NoError(t, err)
 
 	go func() {
@@ -58,7 +58,7 @@ func startTestServer(t *testing.T, httpHandler func(w http.ResponseWriter, r *ht
 	mux.HandleFunc("/", httpHandler)
 
 	// start the servers
-	go http.ListenAndServe(":8085", mux)
+	go http.ListenAndServe("localhost:8085", mux)
 }
 
 func respondWithJSON(w http.ResponseWriter, v interface{}, status int) {

tunneldns/tunnel.go

@@ -50,6 +50,7 @@ func Command(hidden bool) *cli.Command {
 				Value:   "localhost",
 				EnvVars: []string{"TUNNEL_DNS_ADDRESS"},
 			},
+			// Note TUN-3758 , we use Int because UInt is not supported with altsrc
 			&cli.IntFlag{
 				Name:    "port",
 				Usage:   "Listen on given port for the DNS over HTTPS proxy server.",
@@ -87,7 +88,8 @@ func Run(c *cli.Context) error {
 
 	listener, err := CreateListener(
 		c.String("address"),
-		uint16(c.Uint("port")),
+		// Note TUN-3758 , we use Int because UInt is not supported with altsrc
+		uint16(c.Int("port")),
 		c.StringSlice("upstream"),
 		c.StringSlice("bootstrap"),
 		log,

vendor/github.com/mattn/go-colorable/.travis.yml

@@ -0,0 +1,15 @@
+language: go
+sudo: false
+go:
+  - 1.13.x
+  - tip
+
+before_install:
+  - go get -t -v ./...
+
+script:
+  - ./go.test.sh
+
+after_success:
+  - bash <(curl -s https://codecov.io/bash)
+

vendor/github.com/mattn/go-colorable/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Yasuhiro Matsumoto
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

vendor/github.com/mattn/go-colorable/README.md

@@ -0,0 +1,48 @@
+# go-colorable
+
+[![Build Status](https://travis-ci.org/mattn/go-colorable.svg?branch=master)](https://travis-ci.org/mattn/go-colorable)
+[![Codecov](https://codecov.io/gh/mattn/go-colorable/branch/master/graph/badge.svg)](https://codecov.io/gh/mattn/go-colorable)
+[![GoDoc](https://godoc.org/github.com/mattn/go-colorable?status.svg)](http://godoc.org/github.com/mattn/go-colorable)
+[![Go Report Card](https://goreportcard.com/badge/mattn/go-colorable)](https://goreportcard.com/report/mattn/go-colorable)
+
+Colorable writer for windows.
+
+For example, most of logger packages doesn't show colors on windows. (I know we can do it with ansicon. But I don't want.)
+This package is possible to handle escape sequence for ansi color on windows.
+
+## Too Bad!
+
+![](https://raw.githubusercontent.com/mattn/go-colorable/gh-pages/bad.png)
+
+
+## So Good!
+
+![](https://raw.githubusercontent.com/mattn/go-colorable/gh-pages/good.png)
+
+## Usage
+
+```go
+logrus.SetFormatter(&logrus.TextFormatter{ForceColors: true})
+logrus.SetOutput(colorable.NewColorableStdout())
+
+logrus.Info("succeeded")
+logrus.Warn("not correct")
+logrus.Error("something error")
+logrus.Fatal("panic")
+```
+
+You can compile above code on non-windows OSs.
+
+## Installation
+
+```
+$ go get github.com/mattn/go-colorable
+```
+
+# License
+
+MIT
+
+# Author
+
+Yasuhiro Matsumoto (a.k.a mattn)

vendor/github.com/mattn/go-colorable/colorable_appengine.go

@@ -0,0 +1,37 @@
+// +build appengine
+
+package colorable
+
+import (
+	"io"
+	"os"
+
+	_ "github.com/mattn/go-isatty"
+)
+
+// NewColorable returns new instance of Writer which handles escape sequence.
+func NewColorable(file *os.File) io.Writer {
+	if file == nil {
+		panic("nil passed instead of *os.File to NewColorable()")
+	}
+
+	return file
+}
+
+// NewColorableStdout returns new instance of Writer which handles escape sequence for stdout.
+func NewColorableStdout() io.Writer {
+	return os.Stdout
+}
+
+// NewColorableStderr returns new instance of Writer which handles escape sequence for stderr.
+func NewColorableStderr() io.Writer {
+	return os.Stderr
+}
+
+// EnableColorsStdout enable colors if possible.
+func EnableColorsStdout(enabled *bool) func() {
+	if enabled != nil {
+		*enabled = true
+	}
+	return func() {}
+}

vendor/github.com/mattn/go-colorable/colorable_others.go

@@ -0,0 +1,38 @@
+// +build !windows
+// +build !appengine
+
+package colorable
+
+import (
+	"io"
+	"os"
+
+	_ "github.com/mattn/go-isatty"
+)
+
+// NewColorable returns new instance of Writer which handles escape sequence.
+func NewColorable(file *os.File) io.Writer {
+	if file == nil {
+		panic("nil passed instead of *os.File to NewColorable()")
+	}
+
+	return file
+}
+
+// NewColorableStdout returns new instance of Writer which handles escape sequence for stdout.
+func NewColorableStdout() io.Writer {
+	return os.Stdout
+}
+
+// NewColorableStderr returns new instance of Writer which handles escape sequence for stderr.
+func NewColorableStderr() io.Writer {
+	return os.Stderr
+}
+
+// EnableColorsStdout enable colors if possible.
+func EnableColorsStdout(enabled *bool) func() {
+	if enabled != nil {
+		*enabled = true
+	}
+	return func() {}
+}

vendor/github.com/mattn/go-colorable/go.mod

@@ -0,0 +1,8 @@
+module github.com/mattn/go-colorable
+
+require (
+	github.com/mattn/go-isatty v0.0.12
+	golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae // indirect
+)
+
+go 1.13

vendor/github.com/mattn/go-colorable/go.sum

@@ -0,0 +1,5 @@
+github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
+github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae h1:/WDfKMnPU+m5M4xB+6x4kaepxRw6jWvR5iDRdvjHgy8=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

vendor/github.com/mattn/go-colorable/go.test.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+set -e
+echo "" > coverage.txt
+
+for d in $(go list ./... | grep -v vendor); do
+    go test -race -coverprofile=profile.out -covermode=atomic "$d"
+    if [ -f profile.out ]; then
+        cat profile.out >> coverage.txt
+        rm profile.out
+    fi
+done

vendor/github.com/mattn/go-colorable/noncolorable.go

@@ -0,0 +1,55 @@
+package colorable
+
+import (
+	"bytes"
+	"io"
+)
+
+// NonColorable holds writer but removes escape sequence.
+type NonColorable struct {
+	out io.Writer
+}
+
+// NewNonColorable returns new instance of Writer which removes escape sequence from Writer.
+func NewNonColorable(w io.Writer) io.Writer {
+	return &NonColorable{out: w}
+}
+
+// Write writes data on console
+func (w *NonColorable) Write(data []byte) (n int, err error) {
+	er := bytes.NewReader(data)
+	var bw [1]byte
+loop:
+	for {
+		c1, err := er.ReadByte()
+		if err != nil {
+			break loop
+		}
+		if c1 != 0x1b {
+			bw[0] = c1
+			w.out.Write(bw[:])
+			continue
+		}
+		c2, err := er.ReadByte()
+		if err != nil {
+			break loop
+		}
+		if c2 != 0x5b {
+			continue
+		}
+
+		var buf bytes.Buffer
+		for {
+			c, err := er.ReadByte()
+			if err != nil {
+				break loop
+			}
+			if ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') || c == '@' {
+				break
+			}
+			buf.Write([]byte(string(c)))
+		}
+	}
+
+	return len(data), nil
+}

vendor/github.com/mattn/go-isatty/.travis.yml

@@ -0,0 +1,14 @@
+language: go
+sudo: false
+go:
+  - 1.13.x
+  - tip
+
+before_install:
+  - go get -t -v ./...
+
+script:
+  - ./go.test.sh
+
+after_success:
+  - bash <(curl -s https://codecov.io/bash)

vendor/github.com/mattn/go-isatty/LICENSE

@@ -0,0 +1,9 @@
+Copyright (c) Yasuhiro MATSUMOTO <mattn.jp@gmail.com>
+
+MIT License (Expat)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

vendor/github.com/mattn/go-isatty/README.md

@@ -0,0 +1,50 @@
+# go-isatty
+
+[![Godoc Reference](https://godoc.org/github.com/mattn/go-isatty?status.svg)](http://godoc.org/github.com/mattn/go-isatty)
+[![Codecov](https://codecov.io/gh/mattn/go-isatty/branch/master/graph/badge.svg)](https://codecov.io/gh/mattn/go-isatty)
+[![Coverage Status](https://coveralls.io/repos/github/mattn/go-isatty/badge.svg?branch=master)](https://coveralls.io/github/mattn/go-isatty?branch=master)
+[![Go Report Card](https://goreportcard.com/badge/mattn/go-isatty)](https://goreportcard.com/report/mattn/go-isatty)
+
+isatty for golang
+
+## Usage
+
+```go
+package main
+
+import (
+	"fmt"
+	"github.com/mattn/go-isatty"
+	"os"
+)
+
+func main() {
+	if isatty.IsTerminal(os.Stdout.Fd()) {
+		fmt.Println("Is Terminal")
+	} else if isatty.IsCygwinTerminal(os.Stdout.Fd()) {
+		fmt.Println("Is Cygwin/MSYS2 Terminal")
+	} else {
+		fmt.Println("Is Not Terminal")
+	}
+}
+```
+
+## Installation
+
+```
+$ go get github.com/mattn/go-isatty
+```
+
+## License
+
+MIT
+
+## Author
+
+Yasuhiro Matsumoto (a.k.a mattn)
+
+## Thanks
+
+* k-takata: base idea for IsCygwinTerminal
+
+    https://github.com/k-takata/go-iscygpty

vendor/github.com/mattn/go-isatty/doc.go

@@ -0,0 +1,2 @@
+// Package isatty implements interface to isatty
+package isatty

vendor/github.com/mattn/go-isatty/go.mod

@@ -0,0 +1,5 @@
+module github.com/mattn/go-isatty
+
+go 1.12
+
+require golang.org/x/sys v0.0.0-20200116001909-b77594299b42

vendor/github.com/mattn/go-isatty/go.sum

@@ -0,0 +1,2 @@
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42 h1:vEOn+mP2zCOVzKckCZy6YsCtDblrpj/w7B9nxGNELpg=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

vendor/github.com/mattn/go-isatty/go.test.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+set -e
+echo "" > coverage.txt
+
+for d in $(go list ./... | grep -v vendor); do
+    go test -race -coverprofile=profile.out -covermode=atomic "$d"
+    if [ -f profile.out ]; then
+        cat profile.out >> coverage.txt
+        rm profile.out
+    fi
+done

vendor/github.com/mattn/go-isatty/isatty_bsd.go

@@ -0,0 +1,18 @@
+// +build darwin freebsd openbsd netbsd dragonfly
+// +build !appengine
+
+package isatty
+
+import "golang.org/x/sys/unix"
+
+// IsTerminal return true if the file descriptor is terminal.
+func IsTerminal(fd uintptr) bool {
+	_, err := unix.IoctlGetTermios(int(fd), unix.TIOCGETA)
+	return err == nil
+}
+
+// IsCygwinTerminal return true if the file descriptor is a cygwin or msys2
+// terminal. This is also always false on this environment.
+func IsCygwinTerminal(fd uintptr) bool {
+	return false
+}

vendor/github.com/mattn/go-isatty/isatty_others.go

@@ -0,0 +1,15 @@
+// +build appengine js nacl
+
+package isatty
+
+// IsTerminal returns true if the file descriptor is terminal which
+// is always false on js and appengine classic which is a sandboxed PaaS.
+func IsTerminal(fd uintptr) bool {
+	return false
+}
+
+// IsCygwinTerminal() return true if the file descriptor is a cygwin or msys2
+// terminal. This is also always false on this environment.
+func IsCygwinTerminal(fd uintptr) bool {
+	return false
+}

vendor/github.com/mattn/go-isatty/isatty_plan9.go

@@ -0,0 +1,22 @@
+// +build plan9
+
+package isatty
+
+import (
+	"syscall"
+)
+
+// IsTerminal returns true if the given file descriptor is a terminal.
+func IsTerminal(fd uintptr) bool {
+	path, err := syscall.Fd2path(int(fd))
+	if err != nil {
+		return false
+	}
+	return path == "/dev/cons" || path == "/mnt/term/dev/cons"
+}
+
+// IsCygwinTerminal return true if the file descriptor is a cygwin or msys2
+// terminal. This is also always false on this environment.
+func IsCygwinTerminal(fd uintptr) bool {
+	return false
+}

vendor/github.com/mattn/go-isatty/isatty_solaris.go

@@ -0,0 +1,22 @@
+// +build solaris
+// +build !appengine
+
+package isatty
+
+import (
+	"golang.org/x/sys/unix"
+)
+
+// IsTerminal returns true if the given file descriptor is a terminal.
+// see: http://src.illumos.org/source/xref/illumos-gate/usr/src/lib/libbc/libc/gen/common/isatty.c
+func IsTerminal(fd uintptr) bool {
+	var termio unix.Termio
+	err := unix.IoctlSetTermio(int(fd), unix.TCGETA, &termio)
+	return err == nil
+}
+
+// IsCygwinTerminal return true if the file descriptor is a cygwin or msys2
+// terminal. This is also always false on this environment.
+func IsCygwinTerminal(fd uintptr) bool {
+	return false
+}

vendor/github.com/mattn/go-isatty/isatty_tcgets.go

@@ -0,0 +1,18 @@
+// +build linux aix
+// +build !appengine
+
+package isatty
+
+import "golang.org/x/sys/unix"
+
+// IsTerminal return true if the file descriptor is terminal.
+func IsTerminal(fd uintptr) bool {
+	_, err := unix.IoctlGetTermios(int(fd), unix.TCGETS)
+	return err == nil
+}
+
+// IsCygwinTerminal return true if the file descriptor is a cygwin or msys2
+// terminal. This is also always false on this environment.
+func IsCygwinTerminal(fd uintptr) bool {
+	return false
+}

vendor/github.com/mattn/go-isatty/isatty_windows.go

@@ -0,0 +1,125 @@
+// +build windows
+// +build !appengine
+
+package isatty
+
+import (
+	"errors"
+	"strings"
+	"syscall"
+	"unicode/utf16"
+	"unsafe"
+)
+
+const (
+	objectNameInfo uintptr = 1
+	fileNameInfo           = 2
+	fileTypePipe           = 3
+)
+
+var (
+	kernel32                         = syscall.NewLazyDLL("kernel32.dll")
+	ntdll                            = syscall.NewLazyDLL("ntdll.dll")
+	procGetConsoleMode               = kernel32.NewProc("GetConsoleMode")
+	procGetFileInformationByHandleEx = kernel32.NewProc("GetFileInformationByHandleEx")
+	procGetFileType                  = kernel32.NewProc("GetFileType")
+	procNtQueryObject                = ntdll.NewProc("NtQueryObject")
+)
+
+func init() {
+	// Check if GetFileInformationByHandleEx is available.
+	if procGetFileInformationByHandleEx.Find() != nil {
+		procGetFileInformationByHandleEx = nil
+	}
+}
+
+// IsTerminal return true if the file descriptor is terminal.
+func IsTerminal(fd uintptr) bool {
+	var st uint32
+	r, _, e := syscall.Syscall(procGetConsoleMode.Addr(), 2, fd, uintptr(unsafe.Pointer(&st)), 0)
+	return r != 0 && e == 0
+}
+
+// Check pipe name is used for cygwin/msys2 pty.
+// Cygwin/MSYS2 PTY has a name like:
+//   \{cygwin,msys}-XXXXXXXXXXXXXXXX-ptyN-{from,to}-master
+func isCygwinPipeName(name string) bool {
+	token := strings.Split(name, "-")
+	if len(token) < 5 {
+		return false
+	}
+
+	if token[0] != `\msys` &&
+		token[0] != `\cygwin` &&
+		token[0] != `\Device\NamedPipe\msys` &&
+		token[0] != `\Device\NamedPipe\cygwin` {
+		return false
+	}
+
+	if token[1] == "" {
+		return false
+	}
+
+	if !strings.HasPrefix(token[2], "pty") {
+		return false
+	}
+
+	if token[3] != `from` && token[3] != `to` {
+		return false
+	}
+
+	if token[4] != "master" {
+		return false
+	}
+
+	return true
+}
+
+// getFileNameByHandle use the undocomented ntdll NtQueryObject to get file full name from file handler
+// since GetFileInformationByHandleEx is not avilable under windows Vista and still some old fashion
+// guys are using Windows XP, this is a workaround for those guys, it will also work on system from
+// Windows vista to 10
+// see https://stackoverflow.com/a/18792477 for details
+func getFileNameByHandle(fd uintptr) (string, error) {
+	if procNtQueryObject == nil {
+		return "", errors.New("ntdll.dll: NtQueryObject not supported")
+	}
+
+	var buf [4 + syscall.MAX_PATH]uint16
+	var result int
+	r, _, e := syscall.Syscall6(procNtQueryObject.Addr(), 5,
+		fd, objectNameInfo, uintptr(unsafe.Pointer(&buf)), uintptr(2*len(buf)), uintptr(unsafe.Pointer(&result)), 0)
+	if r != 0 {
+		return "", e
+	}
+	return string(utf16.Decode(buf[4 : 4+buf[0]/2])), nil
+}
+
+// IsCygwinTerminal() return true if the file descriptor is a cygwin or msys2
+// terminal.
+func IsCygwinTerminal(fd uintptr) bool {
+	if procGetFileInformationByHandleEx == nil {
+		name, err := getFileNameByHandle(fd)
+		if err != nil {
+			return false
+		}
+		return isCygwinPipeName(name)
+	}
+
+	// Cygwin/msys's pty is a pipe.
+	ft, _, e := syscall.Syscall(procGetFileType.Addr(), 1, fd, 0, 0)
+	if ft != fileTypePipe || e != 0 {
+		return false
+	}
+
+	var buf [2 + syscall.MAX_PATH]uint16
+	r, _, e := syscall.Syscall6(procGetFileInformationByHandleEx.Addr(),
+		4, fd, fileNameInfo, uintptr(unsafe.Pointer(&buf)),
+		uintptr(len(buf)*2), 0, 0)
+	if r == 0 || e != 0 {
+		return false
+	}
+
+	l := *(*uint32)(unsafe.Pointer(&buf))
+	return isCygwinPipeName(string(utf16.Decode(buf[2 : 2+l/2])))
+}

vendor/github.com/mattn/go-isatty/renovate.json

@@ -0,0 +1,8 @@
+{
+  "extends": [
+    "config:base"
+  ],
+  "postUpdateOptions": [
+    "gomodTidy"
+  ]
+}

vendor/golang.org/x/sys/cpu/asm_aix_ppc64.s

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build !gccgo
+// +build gc
 
 #include "textflag.h"
 

vendor/golang.org/x/sys/cpu/cpu.go

@@ -29,26 +29,46 @@ type CacheLinePad struct{ _ [cacheLineSize]byte }
 // and HasAVX2 are only set if the OS supports XMM and YMM
 // registers in addition to the CPUID feature bit being set.
 var X86 struct {
-	_            CacheLinePad
-	HasAES       bool // AES hardware implementation (AES NI)
-	HasADX       bool // Multi-precision add-carry instruction extensions
-	HasAVX       bool // Advanced vector extension
-	HasAVX2      bool // Advanced vector extension 2
-	HasBMI1      bool // Bit manipulation instruction set 1
-	HasBMI2      bool // Bit manipulation instruction set 2
-	HasERMS      bool // Enhanced REP for MOVSB and STOSB
-	HasFMA       bool // Fused-multiply-add instructions
-	HasOSXSAVE   bool // OS supports XSAVE/XRESTOR for saving/restoring XMM registers.
-	HasPCLMULQDQ bool // PCLMULQDQ instruction - most often used for AES-GCM
-	HasPOPCNT    bool // Hamming weight instruction POPCNT.
-	HasRDRAND    bool // RDRAND instruction (on-chip random number generator)
-	HasRDSEED    bool // RDSEED instruction (on-chip random number generator)
-	HasSSE2      bool // Streaming SIMD extension 2 (always available on amd64)
-	HasSSE3      bool // Streaming SIMD extension 3
-	HasSSSE3     bool // Supplemental streaming SIMD extension 3
-	HasSSE41     bool // Streaming SIMD extension 4 and 4.1
-	HasSSE42     bool // Streaming SIMD extension 4 and 4.2
-	_            CacheLinePad
+	_                   CacheLinePad
+	HasAES              bool // AES hardware implementation (AES NI)
+	HasADX              bool // Multi-precision add-carry instruction extensions
+	HasAVX              bool // Advanced vector extension
+	HasAVX2             bool // Advanced vector extension 2
+	HasAVX512           bool // Advanced vector extension 512
+	HasAVX512F          bool // Advanced vector extension 512 Foundation Instructions
+	HasAVX512CD         bool // Advanced vector extension 512 Conflict Detection Instructions
+	HasAVX512ER         bool // Advanced vector extension 512 Exponential and Reciprocal Instructions
+	HasAVX512PF         bool // Advanced vector extension 512 Prefetch Instructions Instructions
+	HasAVX512VL         bool // Advanced vector extension 512 Vector Length Extensions
+	HasAVX512BW         bool // Advanced vector extension 512 Byte and Word Instructions
+	HasAVX512DQ         bool // Advanced vector extension 512 Doubleword and Quadword Instructions
+	HasAVX512IFMA       bool // Advanced vector extension 512 Integer Fused Multiply Add
+	HasAVX512VBMI       bool // Advanced vector extension 512 Vector Byte Manipulation Instructions
+	HasAVX5124VNNIW     bool // Advanced vector extension 512 Vector Neural Network Instructions Word variable precision
+	HasAVX5124FMAPS     bool // Advanced vector extension 512 Fused Multiply Accumulation Packed Single precision
+	HasAVX512VPOPCNTDQ  bool // Advanced vector extension 512 Double and quad word population count instructions
+	HasAVX512VPCLMULQDQ bool // Advanced vector extension 512 Vector carry-less multiply operations
+	HasAVX512VNNI       bool // Advanced vector extension 512 Vector Neural Network Instructions
+	HasAVX512GFNI       bool // Advanced vector extension 512 Galois field New Instructions
+	HasAVX512VAES       bool // Advanced vector extension 512 Vector AES instructions
+	HasAVX512VBMI2      bool // Advanced vector extension 512 Vector Byte Manipulation Instructions 2
+	HasAVX512BITALG     bool // Advanced vector extension 512 Bit Algorithms
+	HasAVX512BF16       bool // Advanced vector extension 512 BFloat16 Instructions
+	HasBMI1             bool // Bit manipulation instruction set 1
+	HasBMI2             bool // Bit manipulation instruction set 2
+	HasERMS             bool // Enhanced REP for MOVSB and STOSB
+	HasFMA              bool // Fused-multiply-add instructions
+	HasOSXSAVE          bool // OS supports XSAVE/XRESTOR for saving/restoring XMM registers.
+	HasPCLMULQDQ        bool // PCLMULQDQ instruction - most often used for AES-GCM
+	HasPOPCNT           bool // Hamming weight instruction POPCNT.
+	HasRDRAND           bool // RDRAND instruction (on-chip random number generator)
+	HasRDSEED           bool // RDSEED instruction (on-chip random number generator)
+	HasSSE2             bool // Streaming SIMD extension 2 (always available on amd64)
+	HasSSE3             bool // Streaming SIMD extension 3
+	HasSSSE3            bool // Supplemental streaming SIMD extension 3
+	HasSSE41            bool // Streaming SIMD extension 4 and 4.1
+	HasSSE42            bool // Streaming SIMD extension 4 and 4.2
+	_                   CacheLinePad
 }
 
 // ARM64 contains the supported CPU features of the

vendor/golang.org/x/sys/cpu/cpu_arm64.go

@@ -39,31 +39,34 @@ func initOptions() {
 
 func archInit() {
 	switch runtime.GOOS {
-	case "android", "darwin", "netbsd":
-		// Android and iOS don't seem to allow reading these registers.
-		//
-		// NetBSD:
-		// ID_AA64ISAR0_EL1 is a privileged register and cannot be read from EL0.
-		// It can be read via sysctl(3). Example for future implementers:
-		// https://nxr.netbsd.org/xref/src/usr.sbin/cpuctl/arch/aarch64.c
-		//
-		// Fake the minimal features expected by
-		// TestARM64minimalFeatures.
-		ARM64.HasASIMD = true
-		ARM64.HasFP = true
-	case "linux":
+	case "freebsd":
+		readARM64Registers()
+	case "linux", "netbsd":
 		doinit()
 	default:
-		readARM64Registers()
+		// Most platforms don't seem to allow reading these registers.
+		//
+		// OpenBSD:
+		// See https://golang.org/issue/31746
+		setMinimalFeatures()
 	}
 }
 
+// setMinimalFeatures fakes the minimal ARM64 features expected by
+// TestARM64minimalFeatures.
+func setMinimalFeatures() {
+	ARM64.HasASIMD = true
+	ARM64.HasFP = true
+}
+
 func readARM64Registers() {
 	Initialized = true
 
-	// ID_AA64ISAR0_EL1
-	isar0 := getisar0()
+	parseARM64SystemRegisters(getisar0(), getisar1(), getpfr0())
+}
 
+func parseARM64SystemRegisters(isar0, isar1, pfr0 uint64) {
+	// ID_AA64ISAR0_EL1
 	switch extractBits(isar0, 4, 7) {
 	case 1:
 		ARM64.HasAES = true
@@ -121,8 +124,6 @@ func readARM64Registers() {
 	}
 
 	// ID_AA64ISAR1_EL1
-	isar1 := getisar1()
-
 	switch extractBits(isar1, 0, 3) {
 	case 1:
 		ARM64.HasDCPOP = true
@@ -144,8 +145,6 @@ func readARM64Registers() {
 	}
 
 	// ID_AA64PFR0_EL1
-	pfr0 := getpfr0()
-
 	switch extractBits(pfr0, 16, 19) {
 	case 0:
 		ARM64.HasFP = true

vendor/golang.org/x/sys/cpu/cpu_arm64.s

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build !gccgo
+// +build gc
 
 #include "textflag.h"
 

vendor/golang.org/x/sys/cpu/cpu_gc_arm64.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build !gccgo
+// +build gc
 
 package cpu
 

vendor/golang.org/x/sys/cpu/cpu_gc_s390x.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build !gccgo
+// +build gc
 
 package cpu
 

vendor/golang.org/x/sys/cpu/cpu_gc_x86.go

@@ -3,7 +3,7 @@
 // license that can be found in the LICENSE file.
 
 // +build 386 amd64 amd64p32
-// +build !gccgo
+// +build gc
 
 package cpu
 

vendor/golang.org/x/sys/cpu/cpu_linux_s390x.go

@@ -17,86 +17,7 @@ const (
 	hwcap_VXE    = 8192
 )
 
-// bitIsSet reports whether the bit at index is set. The bit index
-// is in big endian order, so bit index 0 is the leftmost bit.
-func bitIsSet(bits []uint64, index uint) bool {
-	return bits[index/64]&((1<<63)>>(index%64)) != 0
-}
-
-// function is the code for the named cryptographic function.
-type function uint8
-
-const (
-	// KM{,A,C,CTR} function codes
-	aes128 function = 18 // AES-128
-	aes192 function = 19 // AES-192
-	aes256 function = 20 // AES-256
-
-	// K{I,L}MD function codes
-	sha1     function = 1  // SHA-1
-	sha256   function = 2  // SHA-256
-	sha512   function = 3  // SHA-512
-	sha3_224 function = 32 // SHA3-224
-	sha3_256 function = 33 // SHA3-256
-	sha3_384 function = 34 // SHA3-384
-	sha3_512 function = 35 // SHA3-512
-	shake128 function = 36 // SHAKE-128
-	shake256 function = 37 // SHAKE-256
-
-	// KLMD function codes
-	ghash function = 65 // GHASH
-)
-
-// queryResult contains the result of a Query function
-// call. Bits are numbered in big endian order so the
-// leftmost bit (the MSB) is at index 0.
-type queryResult struct {
-	bits [2]uint64
-}
-
-// Has reports whether the given functions are present.
-func (q *queryResult) Has(fns ...function) bool {
-	if len(fns) == 0 {
-		panic("no function codes provided")
-	}
-	for _, f := range fns {
-		if !bitIsSet(q.bits[:], uint(f)) {
-			return false
-		}
-	}
-	return true
-}
-
-// facility is a bit index for the named facility.
-type facility uint8
-
-const (
-	// cryptography facilities
-	msa4 facility = 77  // message-security-assist extension 4
-	msa8 facility = 146 // message-security-assist extension 8
-)
-
-// facilityList contains the result of an STFLE call.
-// Bits are numbered in big endian order so the
-// leftmost bit (the MSB) is at index 0.
-type facilityList struct {
-	bits [4]uint64
-}
-
-// Has reports whether the given facilities are present.
-func (s *facilityList) Has(fs ...facility) bool {
-	if len(fs) == 0 {
-		panic("no facility bits provided")
-	}
-	for _, f := range fs {
-		if !bitIsSet(s.bits[:], uint(f)) {
-			return false
-		}
-	}
-	return true
-}
-
-func doinit() {
+func initS390Xbase() {
 	// test HWCAP bit vector
 	has := func(featureMask uint) bool {
 		return hwCap&featureMask == featureMask
@@ -116,44 +37,4 @@ func doinit() {
 	if S390X.HasVX {
 		S390X.HasVXE = has(hwcap_VXE)
 	}
-
-	// We need implementations of stfle, km and so on
-	// to detect cryptographic features.
-	if !haveAsmFunctions() {
-		return
-	}
-
-	// optional cryptographic functions
-	if S390X.HasMSA {
-		aes := []function{aes128, aes192, aes256}
-
-		// cipher message
-		km, kmc := kmQuery(), kmcQuery()
-		S390X.HasAES = km.Has(aes...)
-		S390X.HasAESCBC = kmc.Has(aes...)
-		if S390X.HasSTFLE {
-			facilities := stfle()
-			if facilities.Has(msa4) {
-				kmctr := kmctrQuery()
-				S390X.HasAESCTR = kmctr.Has(aes...)
-			}
-			if facilities.Has(msa8) {
-				kma := kmaQuery()
-				S390X.HasAESGCM = kma.Has(aes...)
-			}
-		}
-
-		// compute message digest
-		kimd := kimdQuery() // intermediate (no padding)
-		klmd := klmdQuery() // last (padding)
-		S390X.HasSHA1 = kimd.Has(sha1) && klmd.Has(sha1)
-		S390X.HasSHA256 = kimd.Has(sha256) && klmd.Has(sha256)
-		S390X.HasSHA512 = kimd.Has(sha512) && klmd.Has(sha512)
-		S390X.HasGHASH = kimd.Has(ghash) // KLMD-GHASH does not exist
-		sha3 := []function{
-			sha3_224, sha3_256, sha3_384, sha3_512,
-			shake128, shake256,
-		}
-		S390X.HasSHA3 = kimd.Has(sha3...) && klmd.Has(sha3...)
-	}
 }

vendor/golang.org/x/sys/cpu/cpu_netbsd_arm64.go

@@ -0,0 +1,173 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package cpu
+
+import (
+	"syscall"
+	"unsafe"
+)
+
+// Minimal copy of functionality from x/sys/unix so the cpu package can call
+// sysctl without depending on x/sys/unix.
+
+const (
+	_CTL_QUERY = -2
+
+	_SYSCTL_VERS_1 = 0x1000000
+)
+
+var _zero uintptr
+
+func sysctl(mib []int32, old *byte, oldlen *uintptr, new *byte, newlen uintptr) (err error) {
+	var _p0 unsafe.Pointer
+	if len(mib) > 0 {
+		_p0 = unsafe.Pointer(&mib[0])
+	} else {
+		_p0 = unsafe.Pointer(&_zero)
+	}
+	_, _, errno := syscall.Syscall6(
+		syscall.SYS___SYSCTL,
+		uintptr(_p0),
+		uintptr(len(mib)),
+		uintptr(unsafe.Pointer(old)),
+		uintptr(unsafe.Pointer(oldlen)),
+		uintptr(unsafe.Pointer(new)),
+		uintptr(newlen))
+	if errno != 0 {
+		return errno
+	}
+	return nil
+}
+
+type sysctlNode struct {
+	Flags          uint32
+	Num            int32
+	Name           [32]int8
+	Ver            uint32
+	__rsvd         uint32
+	Un             [16]byte
+	_sysctl_size   [8]byte
+	_sysctl_func   [8]byte
+	_sysctl_parent [8]byte
+	_sysctl_desc   [8]byte
+}
+
+func sysctlNodes(mib []int32) ([]sysctlNode, error) {
+	var olen uintptr
+
+	// Get a list of all sysctl nodes below the given MIB by performing
+	// a sysctl for the given MIB with CTL_QUERY appended.
+	mib = append(mib, _CTL_QUERY)
+	qnode := sysctlNode{Flags: _SYSCTL_VERS_1}
+	qp := (*byte)(unsafe.Pointer(&qnode))
+	sz := unsafe.Sizeof(qnode)
+	if err := sysctl(mib, nil, &olen, qp, sz); err != nil {
+		return nil, err
+	}
+
+	// Now that we know the size, get the actual nodes.
+	nodes := make([]sysctlNode, olen/sz)
+	np := (*byte)(unsafe.Pointer(&nodes[0]))
+	if err := sysctl(mib, np, &olen, qp, sz); err != nil {
+		return nil, err
+	}
+
+	return nodes, nil
+}
+
+func nametomib(name string) ([]int32, error) {
+	// Split name into components.
+	var parts []string
+	last := 0
+	for i := 0; i < len(name); i++ {
+		if name[i] == '.' {
+			parts = append(parts, name[last:i])
+			last = i + 1
+		}
+	}
+	parts = append(parts, name[last:])
+
+	mib := []int32{}
+	// Discover the nodes and construct the MIB OID.
+	for partno, part := range parts {
+		nodes, err := sysctlNodes(mib)
+		if err != nil {
+			return nil, err
+		}
+		for _, node := range nodes {
+			n := make([]byte, 0)
+			for i := range node.Name {
+				if node.Name[i] != 0 {
+					n = append(n, byte(node.Name[i]))
+				}
+			}
+			if string(n) == part {
+				mib = append(mib, int32(node.Num))
+				break
+			}
+		}
+		if len(mib) != partno+1 {
+			return nil, err
+		}
+	}
+
+	return mib, nil
+}
+
+// aarch64SysctlCPUID is struct aarch64_sysctl_cpu_id from NetBSD's <aarch64/armreg.h>
+type aarch64SysctlCPUID struct {
+	midr      uint64 /* Main ID Register */
+	revidr    uint64 /* Revision ID Register */
+	mpidr     uint64 /* Multiprocessor Affinity Register */
+	aa64dfr0  uint64 /* A64 Debug Feature Register 0 */
+	aa64dfr1  uint64 /* A64 Debug Feature Register 1 */
+	aa64isar0 uint64 /* A64 Instruction Set Attribute Register 0 */
+	aa64isar1 uint64 /* A64 Instruction Set Attribute Register 1 */
+	aa64mmfr0 uint64 /* A64 Memory Model Feature Register 0 */
+	aa64mmfr1 uint64 /* A64 Memory Model Feature Register 1 */
+	aa64mmfr2 uint64 /* A64 Memory Model Feature Register 2 */
+	aa64pfr0  uint64 /* A64 Processor Feature Register 0 */
+	aa64pfr1  uint64 /* A64 Processor Feature Register 1 */
+	aa64zfr0  uint64 /* A64 SVE Feature ID Register 0 */
+	mvfr0     uint32 /* Media and VFP Feature Register 0 */
+	mvfr1     uint32 /* Media and VFP Feature Register 1 */
+	mvfr2     uint32 /* Media and VFP Feature Register 2 */
+	pad       uint32
+	clidr     uint64 /* Cache Level ID Register */
+	ctr       uint64 /* Cache Type Register */
+}
+
+func sysctlCPUID(name string) (*aarch64SysctlCPUID, error) {
+	mib, err := nametomib(name)
+	if err != nil {
+		return nil, err
+	}
+
+	out := aarch64SysctlCPUID{}
+	n := unsafe.Sizeof(out)
+	_, _, errno := syscall.Syscall6(
+		syscall.SYS___SYSCTL,
+		uintptr(unsafe.Pointer(&mib[0])),
+		uintptr(len(mib)),
+		uintptr(unsafe.Pointer(&out)),
+		uintptr(unsafe.Pointer(&n)),
+		uintptr(0),
+		uintptr(0))
+	if errno != 0 {
+		return nil, errno
+	}
+	return &out, nil
+}
+
+func doinit() {
+	cpuid, err := sysctlCPUID("machdep.cpu0.cpu_id")
+	if err != nil {
+		setMinimalFeatures()
+		return
+	}
+	parseARM64SystemRegisters(cpuid.aa64isar0, cpuid.aa64isar1, cpuid.aa64pfr0)
+
+	Initialized = true
+}

vendor/golang.org/x/sys/cpu/cpu_other_arm64.go

@@ -2,7 +2,8 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build !linux,arm64
+// +build !linux,!netbsd
+// +build arm64
 
 package cpu
 

vendor/golang.org/x/sys/cpu/cpu_other_mips64x.go

@@ -0,0 +1,12 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !linux
+// +build mips64 mips64le
+
+package cpu
+
+func archInit() {
+	Initialized = true
+}

vendor/golang.org/x/sys/cpu/cpu_s390x.go

@@ -8,10 +8,10 @@ const cacheLineSize = 256
 
 func initOptions() {
 	options = []option{
-		{Name: "zarch", Feature: &S390X.HasZARCH},
-		{Name: "stfle", Feature: &S390X.HasSTFLE},
-		{Name: "ldisp", Feature: &S390X.HasLDISP},
-		{Name: "eimm", Feature: &S390X.HasEIMM},
+		{Name: "zarch", Feature: &S390X.HasZARCH, Required: true},
+		{Name: "stfle", Feature: &S390X.HasSTFLE, Required: true},
+		{Name: "ldisp", Feature: &S390X.HasLDISP, Required: true},
+		{Name: "eimm", Feature: &S390X.HasEIMM, Required: true},
 		{Name: "dfp", Feature: &S390X.HasDFP},
 		{Name: "etf3eh", Feature: &S390X.HasETF3EH},
 		{Name: "msa", Feature: &S390X.HasMSA},
@@ -28,3 +28,145 @@ func initOptions() {
 		{Name: "vxe", Feature: &S390X.HasVXE},
 	}
 }
+
+// bitIsSet reports whether the bit at index is set. The bit index
+// is in big endian order, so bit index 0 is the leftmost bit.
+func bitIsSet(bits []uint64, index uint) bool {
+	return bits[index/64]&((1<<63)>>(index%64)) != 0
+}
+
+// facility is a bit index for the named facility.
+type facility uint8
+
+const (
+	// mandatory facilities
+	zarch  facility = 1  // z architecture mode is active
+	stflef facility = 7  // store-facility-list-extended
+	ldisp  facility = 18 // long-displacement
+	eimm   facility = 21 // extended-immediate
+
+	// miscellaneous facilities
+	dfp    facility = 42 // decimal-floating-point
+	etf3eh facility = 30 // extended-translation 3 enhancement
+
+	// cryptography facilities
+	msa  facility = 17  // message-security-assist
+	msa3 facility = 76  // message-security-assist extension 3
+	msa4 facility = 77  // message-security-assist extension 4
+	msa5 facility = 57  // message-security-assist extension 5
+	msa8 facility = 146 // message-security-assist extension 8
+	msa9 facility = 155 // message-security-assist extension 9
+
+	// vector facilities
+	vx   facility = 129 // vector facility
+	vxe  facility = 135 // vector-enhancements 1
+	vxe2 facility = 148 // vector-enhancements 2
+)
+
+// facilityList contains the result of an STFLE call.
+// Bits are numbered in big endian order so the
+// leftmost bit (the MSB) is at index 0.
+type facilityList struct {
+	bits [4]uint64
+}
+
+// Has reports whether the given facilities are present.
+func (s *facilityList) Has(fs ...facility) bool {
+	if len(fs) == 0 {
+		panic("no facility bits provided")
+	}
+	for _, f := range fs {
+		if !bitIsSet(s.bits[:], uint(f)) {
+			return false
+		}
+	}
+	return true
+}
+
+// function is the code for the named cryptographic function.
+type function uint8
+
+const (
+	// KM{,A,C,CTR} function codes
+	aes128 function = 18 // AES-128
+	aes192 function = 19 // AES-192
+	aes256 function = 20 // AES-256
+
+	// K{I,L}MD function codes
+	sha1     function = 1  // SHA-1
+	sha256   function = 2  // SHA-256
+	sha512   function = 3  // SHA-512
+	sha3_224 function = 32 // SHA3-224
+	sha3_256 function = 33 // SHA3-256
+	sha3_384 function = 34 // SHA3-384
+	sha3_512 function = 35 // SHA3-512
+	shake128 function = 36 // SHAKE-128
+	shake256 function = 37 // SHAKE-256
+
+	// KLMD function codes
+	ghash function = 65 // GHASH
+)
+
+// queryResult contains the result of a Query function
+// call. Bits are numbered in big endian order so the
+// leftmost bit (the MSB) is at index 0.
+type queryResult struct {
+	bits [2]uint64
+}
+
+// Has reports whether the given functions are present.
+func (q *queryResult) Has(fns ...function) bool {
+	if len(fns) == 0 {
+		panic("no function codes provided")
+	}
+	for _, f := range fns {
+		if !bitIsSet(q.bits[:], uint(f)) {
+			return false
+		}
+	}
+	return true
+}
+
+func doinit() {
+	initS390Xbase()
+
+	// We need implementations of stfle, km and so on
+	// to detect cryptographic features.
+	if !haveAsmFunctions() {
+		return
+	}
+
+	// optional cryptographic functions
+	if S390X.HasMSA {
+		aes := []function{aes128, aes192, aes256}
+
+		// cipher message
+		km, kmc := kmQuery(), kmcQuery()
+		S390X.HasAES = km.Has(aes...)
+		S390X.HasAESCBC = kmc.Has(aes...)
+		if S390X.HasSTFLE {
+			facilities := stfle()
+			if facilities.Has(msa4) {
+				kmctr := kmctrQuery()
+				S390X.HasAESCTR = kmctr.Has(aes...)
+			}
+			if facilities.Has(msa8) {
+				kma := kmaQuery()
+				S390X.HasAESGCM = kma.Has(aes...)
+			}
+		}
+
+		// compute message digest
+		kimd := kimdQuery() // intermediate (no padding)
+		klmd := klmdQuery() // last (padding)
+		S390X.HasSHA1 = kimd.Has(sha1) && klmd.Has(sha1)
+		S390X.HasSHA256 = kimd.Has(sha256) && klmd.Has(sha256)
+		S390X.HasSHA512 = kimd.Has(sha512) && klmd.Has(sha512)
+		S390X.HasGHASH = kimd.Has(ghash) // KLMD-GHASH does not exist
+		sha3 := []function{
+			sha3_224, sha3_256, sha3_384, sha3_512,
+			shake128, shake256,
+		}
+		S390X.HasSHA3 = kimd.Has(sha3...) && klmd.Has(sha3...)
+	}
+}

vendor/golang.org/x/sys/cpu/cpu_s390x.s

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build !gccgo
+// +build gc
 
 #include "textflag.h"
 

vendor/golang.org/x/sys/cpu/cpu_x86.go

@@ -16,6 +16,26 @@ func initOptions() {
 		{Name: "aes", Feature: &X86.HasAES},
 		{Name: "avx", Feature: &X86.HasAVX},
 		{Name: "avx2", Feature: &X86.HasAVX2},
+		{Name: "avx512", Feature: &X86.HasAVX512},
+		{Name: "avx512f", Feature: &X86.HasAVX512F},
+		{Name: "avx512cd", Feature: &X86.HasAVX512CD},
+		{Name: "avx512er", Feature: &X86.HasAVX512ER},
+		{Name: "avx512pf", Feature: &X86.HasAVX512PF},
+		{Name: "avx512vl", Feature: &X86.HasAVX512VL},
+		{Name: "avx512bw", Feature: &X86.HasAVX512BW},
+		{Name: "avx512dq", Feature: &X86.HasAVX512DQ},
+		{Name: "avx512ifma", Feature: &X86.HasAVX512IFMA},
+		{Name: "avx512vbmi", Feature: &X86.HasAVX512VBMI},
+		{Name: "avx512vnniw", Feature: &X86.HasAVX5124VNNIW},
+		{Name: "avx5124fmaps", Feature: &X86.HasAVX5124FMAPS},
+		{Name: "avx512vpopcntdq", Feature: &X86.HasAVX512VPOPCNTDQ},
+		{Name: "avx512vpclmulqdq", Feature: &X86.HasAVX512VPCLMULQDQ},
+		{Name: "avx512vnni", Feature: &X86.HasAVX512VNNI},
+		{Name: "avx512gfni", Feature: &X86.HasAVX512GFNI},
+		{Name: "avx512vaes", Feature: &X86.HasAVX512VAES},
+		{Name: "avx512vbmi2", Feature: &X86.HasAVX512VBMI2},
+		{Name: "avx512bitalg", Feature: &X86.HasAVX512BITALG},
+		{Name: "avx512bf16", Feature: &X86.HasAVX512BF16},
 		{Name: "bmi1", Feature: &X86.HasBMI1},
 		{Name: "bmi2", Feature: &X86.HasBMI2},
 		{Name: "erms", Feature: &X86.HasERMS},
@@ -59,12 +79,15 @@ func archInit() {
 	X86.HasOSXSAVE = isSet(27, ecx1)
 	X86.HasRDRAND = isSet(30, ecx1)
 
-	osSupportsAVX := false
+	var osSupportsAVX, osSupportsAVX512 bool
 	// For XGETBV, OSXSAVE bit is required and sufficient.
 	if X86.HasOSXSAVE {
 		eax, _ := xgetbv()
 		// Check if XMM and YMM registers have OS support.
 		osSupportsAVX = isSet(1, eax) && isSet(2, eax)
+
+		// Check if OPMASK and ZMM registers have OS support.
+		osSupportsAVX512 = osSupportsAVX && isSet(5, eax) && isSet(6, eax) && isSet(7, eax)
 	}
 
 	X86.HasAVX = isSet(28, ecx1) && osSupportsAVX
@@ -73,7 +96,7 @@ func archInit() {
 		return
 	}
 
-	_, ebx7, _, _ := cpuid(7, 0)
+	_, ebx7, ecx7, edx7 := cpuid(7, 0)
 	X86.HasBMI1 = isSet(3, ebx7)
 	X86.HasAVX2 = isSet(5, ebx7) && osSupportsAVX
 	X86.HasBMI2 = isSet(8, ebx7)
@@ -81,6 +104,30 @@ func archInit() {
 	X86.HasRDSEED = isSet(18, ebx7)
 	X86.HasADX = isSet(19, ebx7)
 
+	X86.HasAVX512 = isSet(16, ebx7) && osSupportsAVX512 // Because avx-512 foundation is the core required extension
+	if X86.HasAVX512 {
+		X86.HasAVX512F = true
+		X86.HasAVX512CD = isSet(28, ebx7)
+		X86.HasAVX512ER = isSet(27, ebx7)
+		X86.HasAVX512PF = isSet(26, ebx7)
+		X86.HasAVX512VL = isSet(31, ebx7)
+		X86.HasAVX512BW = isSet(30, ebx7)
+		X86.HasAVX512DQ = isSet(17, ebx7)
+		X86.HasAVX512IFMA = isSet(21, ebx7)
+		X86.HasAVX512VBMI = isSet(1, ecx7)
+		X86.HasAVX5124VNNIW = isSet(2, edx7)
+		X86.HasAVX5124FMAPS = isSet(3, edx7)
+		X86.HasAVX512VPOPCNTDQ = isSet(14, ecx7)
+		X86.HasAVX512VPCLMULQDQ = isSet(10, ecx7)
+		X86.HasAVX512VNNI = isSet(11, ecx7)
+		X86.HasAVX512GFNI = isSet(8, ecx7)
+		X86.HasAVX512VAES = isSet(9, ecx7)
+		X86.HasAVX512VBMI2 = isSet(6, ecx7)
+		X86.HasAVX512BITALG = isSet(12, ecx7)
+
+		eax71, _, _, _ := cpuid(7, 1)
+		X86.HasAVX512BF16 = isSet(5, eax71)
+	}
 }
 
 func isSet(bitpos uint, value uint32) bool {

vendor/golang.org/x/sys/cpu/cpu_x86.s

@@ -3,7 +3,7 @@
 // license that can be found in the LICENSE file.
 
 // +build 386 amd64 amd64p32
-// +build !gccgo
+// +build gc
 
 #include "textflag.h"
 

vendor/golang.org/x/sys/cpu/cpu_zos.go

@@ -0,0 +1,10 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package cpu
+
+func archInit() {
+	doinit()
+	Initialized = true
+}

vendor/golang.org/x/sys/cpu/cpu_zos_s390x.go

@@ -0,0 +1,25 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package cpu
+
+func initS390Xbase() {
+	// get the facilities list
+	facilities := stfle()
+
+	// mandatory
+	S390X.HasZARCH = facilities.Has(zarch)
+	S390X.HasSTFLE = facilities.Has(stflef)
+	S390X.HasLDISP = facilities.Has(ldisp)
+	S390X.HasEIMM = facilities.Has(eimm)
+
+	// optional
+	S390X.HasETF3EH = facilities.Has(etf3eh)
+	S390X.HasDFP = facilities.Has(dfp)
+	S390X.HasMSA = facilities.Has(msa)
+	S390X.HasVX = facilities.Has(vx)
+	if S390X.HasVX {
+		S390X.HasVXE = facilities.Has(vxe)
+	}
+}

vendor/golang.org/x/sys/cpu/syscall_aix_ppc64_gc.go

@@ -7,7 +7,7 @@
 // (See golang.org/issue/32102)
 
 // +build aix,ppc64
-// +build !gccgo
+// +build gc
 
 package cpu
 

vendor/golang.org/x/sys/plan9/asm.s

@@ -0,0 +1,8 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+#include "textflag.h"
+
+TEXT ·use(SB),NOSPLIT,$0
+	RET

vendor/golang.org/x/sys/plan9/asm_plan9_386.s

@@ -0,0 +1,30 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+#include "textflag.h"
+
+//
+// System call support for 386, Plan 9
+//
+
+// Just jump to package syscall's implementation for all these functions.
+// The runtime may know about them.
+
+TEXT	·Syscall(SB),NOSPLIT,$0-32
+	JMP	syscall·Syscall(SB)
+
+TEXT	·Syscall6(SB),NOSPLIT,$0-44
+	JMP	syscall·Syscall6(SB)
+
+TEXT ·RawSyscall(SB),NOSPLIT,$0-28
+	JMP	syscall·RawSyscall(SB)
+
+TEXT ·RawSyscall6(SB),NOSPLIT,$0-40
+	JMP	syscall·RawSyscall6(SB)
+
+TEXT ·seek(SB),NOSPLIT,$0-36
+	JMP	syscall·seek(SB)
+
+TEXT ·exit(SB),NOSPLIT,$4-4
+	JMP	syscall·exit(SB)

vendor/golang.org/x/sys/plan9/asm_plan9_amd64.s

@@ -0,0 +1,30 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+#include "textflag.h"
+
+//
+// System call support for amd64, Plan 9
+//
+
+// Just jump to package syscall's implementation for all these functions.
+// The runtime may know about them.
+
+TEXT	·Syscall(SB),NOSPLIT,$0-64
+	JMP	syscall·Syscall(SB)
+
+TEXT	·Syscall6(SB),NOSPLIT,$0-88
+	JMP	syscall·Syscall6(SB)
+
+TEXT ·RawSyscall(SB),NOSPLIT,$0-56
+	JMP	syscall·RawSyscall(SB)
+
+TEXT	·RawSyscall6(SB),NOSPLIT,$0-80
+	JMP	syscall·RawSyscall6(SB)
+
+TEXT ·seek(SB),NOSPLIT,$0-56
+	JMP	syscall·seek(SB)
+
+TEXT ·exit(SB),NOSPLIT,$8-8
+	JMP	syscall·exit(SB)

vendor/golang.org/x/sys/plan9/asm_plan9_arm.s

@@ -0,0 +1,25 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+#include "textflag.h"
+
+// System call support for plan9 on arm
+
+// Just jump to package syscall's implementation for all these functions.
+// The runtime may know about them.
+
+TEXT ·Syscall(SB),NOSPLIT,$0-32
+	JMP	syscall·Syscall(SB)
+
+TEXT ·Syscall6(SB),NOSPLIT,$0-44
+	JMP	syscall·Syscall6(SB)
+
+TEXT ·RawSyscall(SB),NOSPLIT,$0-28
+	JMP	syscall·RawSyscall(SB)
+
+TEXT ·RawSyscall6(SB),NOSPLIT,$0-40
+	JMP	syscall·RawSyscall6(SB)
+
+TEXT ·seek(SB),NOSPLIT,$0-36
+	JMP	syscall·exit(SB)

vendor/golang.org/x/sys/plan9/const_plan9.go

@@ -0,0 +1,70 @@
+package plan9
+
+// Plan 9 Constants
+
+// Open modes
+const (
+	O_RDONLY  = 0
+	O_WRONLY  = 1
+	O_RDWR    = 2
+	O_TRUNC   = 16
+	O_CLOEXEC = 32
+	O_EXCL    = 0x1000
+)
+
+// Rfork flags
+const (
+	RFNAMEG  = 1 << 0
+	RFENVG   = 1 << 1
+	RFFDG    = 1 << 2
+	RFNOTEG  = 1 << 3
+	RFPROC   = 1 << 4
+	RFMEM    = 1 << 5
+	RFNOWAIT = 1 << 6
+	RFCNAMEG = 1 << 10
+	RFCENVG  = 1 << 11
+	RFCFDG   = 1 << 12
+	RFREND   = 1 << 13
+	RFNOMNT  = 1 << 14
+)
+
+// Qid.Type bits
+const (
+	QTDIR    = 0x80
+	QTAPPEND = 0x40
+	QTEXCL   = 0x20
+	QTMOUNT  = 0x10
+	QTAUTH   = 0x08
+	QTTMP    = 0x04
+	QTFILE   = 0x00
+)
+
+// Dir.Mode bits
+const (
+	DMDIR    = 0x80000000
+	DMAPPEND = 0x40000000
+	DMEXCL   = 0x20000000
+	DMMOUNT  = 0x10000000
+	DMAUTH   = 0x08000000
+	DMTMP    = 0x04000000
+	DMREAD   = 0x4
+	DMWRITE  = 0x2
+	DMEXEC   = 0x1
+)
+
+const (
+	STATMAX    = 65535
+	ERRMAX     = 128
+	STATFIXLEN = 49
+)
+
+// Mount and bind flags
+const (
+	MREPL   = 0x0000
+	MBEFORE = 0x0001
+	MAFTER  = 0x0002
+	MORDER  = 0x0003
+	MCREATE = 0x0004
+	MCACHE  = 0x0010
+	MMASK   = 0x0017
+)

vendor/golang.org/x/sys/plan9/dir_plan9.go

@@ -0,0 +1,212 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Plan 9 directory marshalling. See intro(5).
+
+package plan9
+
+import "errors"
+
+var (
+	ErrShortStat = errors.New("stat buffer too short")
+	ErrBadStat   = errors.New("malformed stat buffer")
+	ErrBadName   = errors.New("bad character in file name")
+)
+
+// A Qid represents a 9P server's unique identification for a file.
+type Qid struct {
+	Path uint64 // the file server's unique identification for the file
+	Vers uint32 // version number for given Path
+	Type uint8  // the type of the file (plan9.QTDIR for example)
+}
+
+// A Dir contains the metadata for a file.
+type Dir struct {
+	// system-modified data
+	Type uint16 // server type
+	Dev  uint32 // server subtype
+
+	// file data
+	Qid    Qid    // unique id from server
+	Mode   uint32 // permissions
+	Atime  uint32 // last read time
+	Mtime  uint32 // last write time
+	Length int64  // file length
+	Name   string // last element of path
+	Uid    string // owner name
+	Gid    string // group name
+	Muid   string // last modifier name
+}
+
+var nullDir = Dir{
+	Type: ^uint16(0),
+	Dev:  ^uint32(0),
+	Qid: Qid{
+		Path: ^uint64(0),
+		Vers: ^uint32(0),
+		Type: ^uint8(0),
+	},
+	Mode:   ^uint32(0),
+	Atime:  ^uint32(0),
+	Mtime:  ^uint32(0),
+	Length: ^int64(0),
+}
+
+// Null assigns special "don't touch" values to members of d to
+// avoid modifying them during plan9.Wstat.
+func (d *Dir) Null() { *d = nullDir }
+
+// Marshal encodes a 9P stat message corresponding to d into b
+//
+// If there isn't enough space in b for a stat message, ErrShortStat is returned.
+func (d *Dir) Marshal(b []byte) (n int, err error) {
+	n = STATFIXLEN + len(d.Name) + len(d.Uid) + len(d.Gid) + len(d.Muid)
+	if n > len(b) {
+		return n, ErrShortStat
+	}
+
+	for _, c := range d.Name {
+		if c == '/' {
+			return n, ErrBadName
+		}
+	}
+
+	b = pbit16(b, uint16(n)-2)
+	b = pbit16(b, d.Type)
+	b = pbit32(b, d.Dev)
+	b = pbit8(b, d.Qid.Type)
+	b = pbit32(b, d.Qid.Vers)
+	b = pbit64(b, d.Qid.Path)
+	b = pbit32(b, d.Mode)
+	b = pbit32(b, d.Atime)
+	b = pbit32(b, d.Mtime)
+	b = pbit64(b, uint64(d.Length))
+	b = pstring(b, d.Name)
+	b = pstring(b, d.Uid)
+	b = pstring(b, d.Gid)
+	b = pstring(b, d.Muid)
+
+	return n, nil
+}
+
+// UnmarshalDir decodes a single 9P stat message from b and returns the resulting Dir.
+//
+// If b is too small to hold a valid stat message, ErrShortStat is returned.
+//
+// If the stat message itself is invalid, ErrBadStat is returned.
+func UnmarshalDir(b []byte) (*Dir, error) {
+	if len(b) < STATFIXLEN {
+		return nil, ErrShortStat
+	}
+	size, buf := gbit16(b)
+	if len(b) != int(size)+2 {
+		return nil, ErrBadStat
+	}
+	b = buf
+
+	var d Dir
+	d.Type, b = gbit16(b)
+	d.Dev, b = gbit32(b)
+	d.Qid.Type, b = gbit8(b)
+	d.Qid.Vers, b = gbit32(b)
+	d.Qid.Path, b = gbit64(b)
+	d.Mode, b = gbit32(b)
+	d.Atime, b = gbit32(b)
+	d.Mtime, b = gbit32(b)
+
+	n, b := gbit64(b)
+	d.Length = int64(n)
+
+	var ok bool
+	if d.Name, b, ok = gstring(b); !ok {
+		return nil, ErrBadStat
+	}
+	if d.Uid, b, ok = gstring(b); !ok {
+		return nil, ErrBadStat
+	}
+	if d.Gid, b, ok = gstring(b); !ok {
+		return nil, ErrBadStat
+	}
+	if d.Muid, b, ok = gstring(b); !ok {
+		return nil, ErrBadStat
+	}
+
+	return &d, nil
+}
+
+// pbit8 copies the 8-bit number v to b and returns the remaining slice of b.
+func pbit8(b []byte, v uint8) []byte {
+	b[0] = byte(v)
+	return b[1:]
+}
+
+// pbit16 copies the 16-bit number v to b in little-endian order and returns the remaining slice of b.
+func pbit16(b []byte, v uint16) []byte {
+	b[0] = byte(v)
+	b[1] = byte(v >> 8)
+	return b[2:]
+}
+
+// pbit32 copies the 32-bit number v to b in little-endian order and returns the remaining slice of b.
+func pbit32(b []byte, v uint32) []byte {
+	b[0] = byte(v)
+	b[1] = byte(v >> 8)
+	b[2] = byte(v >> 16)
+	b[3] = byte(v >> 24)
+	return b[4:]
+}
+
+// pbit64 copies the 64-bit number v to b in little-endian order and returns the remaining slice of b.
+func pbit64(b []byte, v uint64) []byte {
+	b[0] = byte(v)
+	b[1] = byte(v >> 8)
+	b[2] = byte(v >> 16)
+	b[3] = byte(v >> 24)
+	b[4] = byte(v >> 32)
+	b[5] = byte(v >> 40)
+	b[6] = byte(v >> 48)
+	b[7] = byte(v >> 56)
+	return b[8:]
+}
+
+// pstring copies the string s to b, prepending it with a 16-bit length in little-endian order, and
+// returning the remaining slice of b..
+func pstring(b []byte, s string) []byte {
+	b = pbit16(b, uint16(len(s)))
+	n := copy(b, s)
+	return b[n:]
+}
+
+// gbit8 reads an 8-bit number from b and returns it with the remaining slice of b.
+func gbit8(b []byte) (uint8, []byte) {
+	return uint8(b[0]), b[1:]
+}
+
+// gbit16 reads a 16-bit number in little-endian order from b and returns it with the remaining slice of b.
+func gbit16(b []byte) (uint16, []byte) {
+	return uint16(b[0]) | uint16(b[1])<<8, b[2:]
+}
+
+// gbit32 reads a 32-bit number in little-endian order from b and returns it with the remaining slice of b.
+func gbit32(b []byte) (uint32, []byte) {
+	return uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24, b[4:]
+}
+
+// gbit64 reads a 64-bit number in little-endian order from b and returns it with the remaining slice of b.
+func gbit64(b []byte) (uint64, []byte) {
+	lo := uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24
+	hi := uint32(b[4]) | uint32(b[5])<<8 | uint32(b[6])<<16 | uint32(b[7])<<24
+	return uint64(lo) | uint64(hi)<<32, b[8:]
+}
+
+// gstring reads a string from b, prefixed with a 16-bit length in little-endian order.
+// It returns the string with the remaining slice of b and a boolean. If the length is
+// greater than the number of bytes in b, the boolean will be false.
+func gstring(b []byte) (string, []byte, bool) {
+	n, b := gbit16(b)
+	if int(n) > len(b) {
+		return "", b, false
+	}
+	return string(b[:n]), b[n:], true
+}