Release 2021.2.2

Also changed the socks test code so that it binds to localhost, so that
we don't get popups saying "would you like to allow socks.test to use
the network"

.dockerignore

@@ -0,0 +1 @@
+.git

.gitignore

@@ -18,3 +18,4 @@ cloudflared-x86-64*
 .DS_Store
 *-session.log
 ssh_server_tests/.env
+.cover

Dockerfile

@@ -1,7 +1,7 @@
 # use a builder image for building cloudflare
 ARG TARGET_GOOS
 ARG TARGET_GOARCH
-FROM golang:1.15.6 as builder
+FROM golang:1.15.7 as builder
 ENV GO111MODULE=on \
     CGO_ENABLED=0 \
     TARGET_GOOS=${TARGET_GOOS} \

Makefile

@@ -1,10 +1,19 @@
 VERSION       := $(shell git describe --tags --always --dirty="-dev" --match "[0-9][0-9][0-9][0-9].*.*")
-DATE          := $(shell date -u '+%Y-%m-%d-%H%M UTC')
-VERSION_FLAGS := -ldflags='-X "main.Version=$(VERSION)" -X "main.BuildTime=$(DATE)"'
 MSI_VERSION   := $(shell git tag -l --sort=v:refname | grep "w" | tail -1 | cut -c2-)
 #MSI_VERSION expects the format of the tag to be: (wX.X.X). Starts with the w character to not break cfsetup.
 #e.g. w3.0.1 or w4.2.10. It trims off the w character when creating the MSI.
 
+ifeq ($(FIPS), true)
+	GO_BUILD_TAGS := $(GO_BUILD_TAGS) fips
+endif
+
+ifneq ($(GO_BUILD_TAGS),)
+	GO_BUILD_TAGS := -tags $(GO_BUILD_TAGS)
+endif
+
+DATE          := $(shell date -u '+%Y-%m-%d-%H%M UTC')
+VERSION_FLAGS := -ldflags='-X "main.Version=$(VERSION)" -X "main.BuildTime=$(DATE)"'
+
 IMPORT_PATH   := github.com/cloudflare/cloudflared
 PACKAGE_DIR   := $(CURDIR)/packaging
 INSTALL_BINDIR := /usr/bin/
@@ -71,7 +80,7 @@ clean:
 
 .PHONY: cloudflared
 cloudflared: tunnel-deps
-	GOOS=$(TARGET_OS) GOARCH=$(TARGET_ARCH) go build -v -mod=vendor $(VERSION_FLAGS) $(IMPORT_PATH)/cmd/cloudflared
+	GOOS=$(TARGET_OS) GOARCH=$(TARGET_ARCH) go build -v -mod=vendor $(GO_BUILD_TAGS) $(VERSION_FLAGS) $(IMPORT_PATH)/cmd/cloudflared
 
 .PHONY: container
 container:
@@ -79,7 +88,13 @@ container:
 
 .PHONY: test
 test: vet
+ifndef CI
 	go test -v -mod=vendor -race $(VERSION_FLAGS) ./...
+else
+	@mkdir -p .cover
+	go test -v -mod=vendor -race $(VERSION_FLAGS) -coverprofile=".cover/c.out" ./...
+	go tool cover -html ".cover/c.out" -o .cover/all.html
+endif
 
 .PHONY: test-ssh-server
 test-ssh-server:

RELEASE_NOTES

@@ -1,3 +1,33 @@
+2021.2.2
+- 2021-02-04 TUN-3864: Users can choose where credentials file is written after creating a tunnel
+- 2021-02-04 TUN-3869: Improve reliability of graceful shutdown.
+- 2021-02-07 TUN-3878: Do not supply -tags when none are specified
+- 2021-02-04 TUN-3635: Send event when unregistering tunnel for gracful shutdown so /ready endpoint reports down status befoe connections finish handling pending requests.
+- 2021-02-08 TUN-3890: Code coverage for cloudflared in CI
+- 2021-02-09 AUTH-3375 exchangeOrgToken deleted cookie fix
+- 2020-11-18 Update error message to use login command
+
+2021.2.1
+- 2021-02-04 TUN-3858: Do not suffix cloudflared version with -fips
+
+2021.2.0
+- 2021-02-01 TUN-3837: Remove automation_email from cloudflared status page test
+- 2021-02-03 TUN-3848: Use transport logger for h2mux
+- 2021-02-03 TUN-3854: cloudflared tunnel list flags to sort output
+- 2021-01-21 TUN-3195: Don't colorize console logs when stderr is not a terminal
+- 2021-01-20 Fixed connection error handling by removing duplicated errors, standardizing on non-pointer error types
+- 2021-01-20 TUN-3118: Changed graceful shutdown to immediately unregister tunnel from the edge, keep the connection open until the edge drops it or grace period expires
+- 2021-01-25 TUN-3165: Add reference to Argo Tunnel documentation in the help output
+- 2021-01-25 TUN-3806: Use a .dockerignore
+- 2021-01-21 TUN-3795: Use RFC-3339 style date format for logs, produce timestamp in UTC
+- 2021-01-26 TUN-3795: Removed errant test
+- 2021-01-25 TUN-3792: Handle graceful shutdown correctly when running as a windows service. Only expose one shutdown channel globally, which now triggers the graceful shutdown sequence across all modes. Removed separate handling of zero-duration grace period, instead it's checked only when we need to wait for exit.
+- 2021-01-27 TUN-3811: Better error reporting on http2 connection termination. Registration errors from control loop are now propagated out of the connection server code. Unified error handling between h2mux and http2 connections so we log and retry errors the same way, regardless of underlying transport.
+- 2021-01-28 TUN-3830: Use Go 1.15.7
+- 2021-01-28 TUN-3826: Use go-fips when building cloudflared for linux/amd64
+- 2021-01-19 TUN-3777: Fix /ready endpoint for classic tunnels
+- 2021-01-19 TUN-3773: Add back pprof endpoints
+
 2021.1.5
 - 2021-01-15 TUN-3594: Log ingress response at debug level
 - 2021-01-15 TUN-3765: Fix doubly nested log output by `logfile` option

cfsetup.yaml

@@ -1,26 +1,30 @@
-pinned_go: &pinned_go go=1.15.6-1
+pinned_go: &pinned_go go=1.15.7-1
+pinned_go_fips: &pinned_go_fips go-fips=1.15.5-3
+
 build_dir: &build_dir /cfsetup_build
 default-flavor: buster
 stretch: &stretch
   build:
     build_dir: *build_dir
     builddeps:
-      - *pinned_go
+      - *pinned_go_fips
       - build-essential
     post-cache:
       - export GOOS=linux
       - export GOARCH=amd64
+      - export FIPS=true
       - make cloudflared
   build-deb:
     build_dir: *build_dir
     builddeps:
-      - *pinned_go
+      - *pinned_go_fips
       - build-essential
       - fakeroot
       - rubygem-fpm
     post-cache:
       - export GOOS=linux
       - export GOARCH=amd64
+      - export FIPS=true
       - make cloudflared-deb
   build-deb-arm64:
     build_dir: *build_dir
@@ -36,7 +40,7 @@ stretch: &stretch
   publish-deb:
     build_dir: *build_dir
     builddeps:
-      - *pinned_go
+      - *pinned_go_fips
       - build-essential
       - fakeroot
       - rubygem-fpm
@@ -44,20 +48,22 @@ stretch: &stretch
     post-cache:
       - export GOOS=linux
       - export GOARCH=amd64
+      - export FIPS=true
       - make publish-deb
   release-linux-amd64:
     build_dir: *build_dir
     builddeps:
-      - *pinned_go
+      - *pinned_go_fips
       - build-essential
     post-cache:
       - export GOOS=linux
       - export GOARCH=amd64
+      - export FIPS=true
       - make release
   github-release-linux-amd64:
     build_dir: *build_dir
     builddeps:
-      - *pinned_go
+      - *pinned_go_fips
       - build-essential
       - python3-setuptools
       - python3-pip
@@ -66,6 +72,7 @@ stretch: &stretch
     post-cache:
       - export GOOS=linux
       - export GOARCH=amd64
+      - export FIPS=true
       - make github-release
   release-linux-armv6:
     build_dir: *build_dir
@@ -185,11 +192,12 @@ stretch: &stretch
   test:
     build_dir: *build_dir
     builddeps:
-      - *pinned_go
+      - *pinned_go_fips
       - build-essential
     post-cache:
       - export GOOS=linux
       - export GOARCH=amd64
+      - export FIPS=true
       # cd to a non-module directory: https://github.com/golang/go/issues/24250
       - (cd / && go get github.com/BurntSushi/go-sumtype)
       - export PATH="$HOME/go/bin:$PATH"
@@ -248,8 +256,8 @@ centos-7:
       - https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
     pre-cache:
       - yum install -y fakeroot
-      - wget https://golang.org/dl/go1.15.6.linux-amd64.tar.gz -P /tmp/
-      - tar -C /usr/local -xzf /tmp/go1.15.6.linux-amd64.tar.gz
+      - wget https://golang.org/dl/go1.15.7.linux-amd64.tar.gz -P /tmp/
+      - tar -C /usr/local -xzf /tmp/go1.15.7.linux-amd64.tar.gz
     post-cache:
       - export PATH=$PATH:/usr/local/go/bin
       - export GOOS=linux
@@ -260,8 +268,8 @@ centos-7:
     builddeps: *el7_builddeps
     pre-cache:
       - yum install -y fakeroot
-      - wget https://golang.org/dl/go1.15.6.linux-amd64.tar.gz -P /tmp/
-      - tar -C /usr/local -xzf /tmp/go1.15.6.linux-amd64.tar.gz
+      - wget https://golang.org/dl/go1.15.7.linux-amd64.tar.gz -P /tmp/
+      - tar -C /usr/local -xzf /tmp/go1.15.7.linux-amd64.tar.gz
     post-cache:
       - export PATH=$PATH:/usr/local/go/bin
       - export GOOS=linux

cmd/cloudflared/access/cmd.go

@@ -55,12 +55,11 @@ const sentryDSN = "https://56a9c9fa5c364ab28f34b14f35ea0f1b@sentry.io/189878"
 
 var (
 	shutdownC      chan struct{}
-	graceShutdownC chan struct{}
 )
 
 // Init will initialize and store vars from the main program
-func Init(s, g chan struct{}) {
-	shutdownC, graceShutdownC = s, g
+func Init(shutdown chan struct{}) {
+	shutdownC = shutdown
 }
 
 // Flags return the global flags for Access related commands (hopefully none)
@@ -292,7 +291,7 @@ func generateToken(c *cli.Context) error {
 	}
 	tok, err := token.GetAppTokenIfExists(appURL)
 	if err != nil || tok == "" {
-		fmt.Fprintln(os.Stderr, "Unable to find token for provided application. Please run token command to generate token.")
+		fmt.Fprintln(os.Stderr, "Unable to find token for provided application. Please run login command to generate token.")
 		return err
 	}
 

cmd/cloudflared/fips.go

@@ -0,0 +1,5 @@
+// +build fips
+
+package main
+
+import _ "crypto/tls/fipsonly"

cmd/cloudflared/linux_service.go

@@ -16,7 +16,7 @@ import (
 	"github.com/urfave/cli/v2"
 )
 
-func runApp(app *cli.App, shutdownC, graceShutdownC chan struct{}) {
+func runApp(app *cli.App, graceShutdownC chan struct{}) {
 	app.Commands = append(app.Commands, &cli.Command{
 		Name:  "service",
 		Usage: "Manages the Argo Tunnel system service",

cmd/cloudflared/macos_service.go

@@ -17,7 +17,7 @@ const (
 	launchdIdentifier = "com.cloudflare.cloudflared"
 )
 
-func runApp(app *cli.App, shutdownC, graceShutdownC chan struct{}) {
+func runApp(app *cli.App, graceShutdownC chan struct{}) {
 	app.Commands = append(app.Commands, &cli.Command{
 		Name:  "service",
 		Usage: "Manages the Argo Tunnel launch agent",

cmd/cloudflared/main.go

@@ -46,10 +46,7 @@ func main() {
 	metrics.RegisterBuildInfo(BuildTime, Version)
 	raven.SetRelease(Version)
 
-	// Force shutdown channel used by the app. When closed, app must terminate.
-	// Windows service manager closes this channel when it receives shutdown command.
-	shutdownC := make(chan struct{})
-	// Graceful shutdown channel used by the app. When closed, app must terminate.
+	// Graceful shutdown channel used by the app. When closed, app must terminate gracefully.
 	// Windows service manager closes this channel when it receives stop command.
 	graceShutdownC := make(chan struct{})
 
@@ -73,16 +70,18 @@ func main() {
 	app.Version = fmt.Sprintf("%s (built %s)", Version, BuildTime)
 	app.Description = `cloudflared connects your machine or user identity to Cloudflare's global network.
 	You can use it to authenticate a session to reach an API behind Access, route web traffic to this machine,
-	and configure access control.`
+	and configure access control.
+
+	See https://developers.cloudflare.com/argo-tunnel/ for more in-depth documentation.`
 	app.Flags = flags()
-	app.Action = action(Version, shutdownC, graceShutdownC)
+	app.Action = action(graceShutdownC)
 	app.Before = tunnel.SetFlagsFromConfigFile
 	app.Commands = commands(cli.ShowVersion)
 
-	tunnel.Init(Version, shutdownC, graceShutdownC) // we need this to support the tunnel sub command...
-	access.Init(shutdownC, graceShutdownC)
+	tunnel.Init(Version, graceShutdownC) // we need this to support the tunnel sub command...
+	access.Init(graceShutdownC)
 	updater.Init(Version)
-	runApp(app, shutdownC, graceShutdownC)
+	runApp(app, graceShutdownC)
 }
 
 func commands(version func(c *cli.Context)) []*cli.Command {
@@ -143,10 +142,10 @@ func isEmptyInvocation(c *cli.Context) bool {
 	return c.NArg() == 0 && c.NumFlags() == 0
 }
 
-func action(version string, shutdownC, graceShutdownC chan struct{}) cli.ActionFunc {
+func action(graceShutdownC chan struct{}) cli.ActionFunc {
 	return cliutil.ErrorHandler(func(c *cli.Context) (err error) {
 		if isEmptyInvocation(c) {
-			return handleServiceMode(c, shutdownC)
+			return handleServiceMode(c, graceShutdownC)
 		}
 		tags := make(map[string]string)
 		tags["hostname"] = c.String("hostname")

cmd/cloudflared/token/token.go

@@ -301,7 +301,9 @@ func exchangeOrgToken(appURL *url.URL, orgToken string) (string, error) {
 	resp.Body.Close()
 	var appToken string
 	for _, c := range resp.Cookies() {
-		if c.Name == tokenHeader {
+		//if Org token revoked on exchange, getTokensFromEdge instead
+		validAppToken := c.Name == tokenHeader && time.Now().Before(c.Expires)
+		if validAppToken {
 			appToken = c.Value
 			break
 		}

cmd/cloudflared/tunnel/cmd.go

@@ -32,7 +32,6 @@ import (
 	"github.com/coreos/go-systemd/daemon"
 	"github.com/facebookgo/grace/gracenet"
 	"github.com/getsentry/raven-go"
-	"github.com/google/uuid"
 	"github.com/mitchellh/go-homedir"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
@@ -87,7 +86,6 @@ const (
 )
 
 var (
-	shutdownC      chan struct{}
 	graceShutdownC chan struct{}
 	version        string
 )
@@ -156,7 +154,7 @@ func TunnelCommand(c *cli.Context) error {
 		return err
 	}
 	if name := c.String("name"); name != "" { // Start a named tunnel
-		return runAdhocNamedTunnel(sc, name)
+		return runAdhocNamedTunnel(sc, name, c.String(CredFileFlag))
 	}
 	if ref := config.GetConfiguration().TunnelID; ref != "" {
 		return fmt.Errorf("Use `cloudflared tunnel run` to start tunnel %s", ref)
@@ -166,15 +164,15 @@ func TunnelCommand(c *cli.Context) error {
 	return runClassicTunnel(sc)
 }
 
-func Init(v string, s, g chan struct{}) {
-	version, shutdownC, graceShutdownC = v, s, g
+func Init(ver string, gracefulShutdown chan struct{}) {
+	version, graceShutdownC = ver, gracefulShutdown
 }
 
 // runAdhocNamedTunnel create, route and run a named tunnel in one command
-func runAdhocNamedTunnel(sc *subcommandContext, name string) error {
+func runAdhocNamedTunnel(sc *subcommandContext, name, credentialsOutputPath string) error {
 	tunnel, ok, err := sc.tunnelActive(name)
 	if err != nil || !ok {
-		tunnel, err = sc.create(name)
+		tunnel, err = sc.create(name, credentialsOutputPath)
 		if err != nil {
 			return errors.Wrap(err, "failed to create tunnel")
 		}
@@ -199,7 +197,7 @@ func runAdhocNamedTunnel(sc *subcommandContext, name string) error {
 
 // runClassicTunnel creates a "classic" non-named tunnel
 func runClassicTunnel(sc *subcommandContext) error {
-	return StartServer(sc.c, version, shutdownC, graceShutdownC, nil, sc.log, sc.isUIEnabled)
+	return StartServer(sc.c, version, nil, sc.log, sc.isUIEnabled)
 }
 
 func routeFromFlag(c *cli.Context) (tunnelstore.Route, bool) {
@@ -215,8 +213,6 @@ func routeFromFlag(c *cli.Context) (tunnelstore.Route, bool) {
 func StartServer(
 	c *cli.Context,
 	version string,
-	shutdownC,
-	graceShutdownC chan struct{},
 	namedTunnel *connection.NamedTunnelConfig,
 	log *zerolog.Logger,
 	isUIEnabled bool,
@@ -225,8 +221,6 @@ func StartServer(
 	var wg sync.WaitGroup
 	listeners := gracenet.Net{}
 	errC := make(chan error)
-	connectedSignal := signal.New(make(chan struct{}))
-	dnsReadySignal := make(chan struct{})
 
 	if config.GetConfiguration().Source() == "" {
 		log.Info().Msg(config.ErrNoConfigFile.Error())
@@ -269,36 +263,29 @@ func StartServer(
 	buildInfo.Log(log)
 	logClientOptions(c, log)
 
+	// this context drives the server, when it's cancelled tunnel and all other components (origins, dns, etc...) should stop
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	go waitForSignal(graceShutdownC, log)
+
 	if c.IsSet("proxy-dns") {
+		dnsReadySignal := make(chan struct{})
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
-			errC <- runDNSProxyServer(c, dnsReadySignal, shutdownC, log)
+			errC <- runDNSProxyServer(c, dnsReadySignal, ctx.Done(), log)
 		}()
-	} else {
-		close(dnsReadySignal)
+		// Wait for proxy-dns to come up (if used)
+		<-dnsReadySignal
 	}
 
-	// Wait for proxy-dns to come up (if used)
-	<-dnsReadySignal
-
+	connectedSignal := signal.New(make(chan struct{}))
 	go notifySystemd(connectedSignal)
 	if c.IsSet("pidfile") {
 		go writePidFile(connectedSignal, c.String("pidfile"), log)
 	}
 
-	cloudflaredID, err := uuid.NewRandom()
-	if err != nil {
-		log.Err(err).Msg("Cannot generate cloudflared ID")
-		return err
-	}
-
-	ctx, cancel := context.WithCancel(context.Background())
-	go func() {
-		<-shutdownC
-		cancel()
-	}()
-
 	// update needs to be after DNS proxy is up to resolve equinox server address
 	if updater.IsAutoupdateEnabled(c, log) {
 		autoupdateFreq := c.Duration("autoupdate-freq")
@@ -315,7 +302,7 @@ func StartServer(
 	if dnsProxyStandAlone(c) {
 		connectedSignal.Notify()
 		// no grace period, handle SIGINT/SIGTERM immediately
-		return waitToShutdown(&wg, errC, shutdownC, graceShutdownC, 0, log)
+		return waitToShutdown(&wg, cancel, errC, graceShutdownC, 0, log)
 	}
 
 	url := c.String("url")
@@ -326,11 +313,11 @@ func StartServer(
 		return fmt.Errorf(errText)
 	}
 
-	transportLog := logger.CreateTransportLoggerFromContext(c, isUIEnabled)
+	logTransport := logger.CreateTransportLoggerFromContext(c, isUIEnabled)
 
-	observer := connection.NewObserver(log, isUIEnabled)
+	observer := connection.NewObserver(log, logTransport, isUIEnabled)
 
-	tunnelConfig, ingressRules, err := prepareTunnelConfig(c, buildInfo, version, log, observer, namedTunnel)
+	tunnelConfig, ingressRules, err := prepareTunnelConfig(c, buildInfo, version, log, logTransport, observer, namedTunnel)
 	if err != nil {
 		log.Err(err).Msg("Couldn't start tunnel")
 		return err
@@ -347,10 +334,10 @@ func StartServer(
 		defer wg.Done()
 		readinessServer := metrics.NewReadyServer(log)
 		observer.RegisterSink(readinessServer)
-		errC <- metrics.ServeMetrics(metricsListener, shutdownC, readinessServer, log)
+		errC <- metrics.ServeMetrics(metricsListener, ctx.Done(), readinessServer, log)
 	}()
 
-	if err := ingressRules.StartOrigins(&wg, log, shutdownC, errC); err != nil {
+	if err := ingressRules.StartOrigins(&wg, log, ctx.Done(), errC); err != nil {
 		return err
 	}
 
@@ -362,8 +349,11 @@ func StartServer(
 
 	wg.Add(1)
 	go func() {
-		defer wg.Done()
-		errC <- origin.StartTunnelDaemon(ctx, tunnelConfig, connectedSignal, cloudflaredID, reconnectCh)
+		defer func() {
+			wg.Done()
+			log.Info().Msg("Tunnel server stopped")
+		}()
+		errC <- origin.StartTunnelDaemon(ctx, tunnelConfig, connectedSignal, reconnectCh, graceShutdownC)
 	}()
 
 	if isUIEnabled {
@@ -374,11 +364,11 @@ func StartServer(
 			&ingressRules,
 			tunnelConfig.HAConnections,
 		)
-		app := tunnelUI.Launch(ctx, log, transportLog)
+		app := tunnelUI.Launch(ctx, log, logTransport)
 		observer.RegisterSink(app)
 	}
 
-	return waitToShutdown(&wg, errC, shutdownC, graceShutdownC, c.Duration("grace-period"), log)
+	return waitToShutdown(&wg, cancel, errC, graceShutdownC, c.Duration("grace-period"), log)
 }
 
 func SetFlagsFromConfigFile(c *cli.Context) error {
@@ -402,30 +392,44 @@ func SetFlagsFromConfigFile(c *cli.Context) error {
 }
 
 func waitToShutdown(wg *sync.WaitGroup,
-	errC chan error,
-	shutdownC, graceShutdownC chan struct{},
+	cancelServerContext func(),
+	errC <-chan error,
+	graceShutdownC <-chan struct{},
 	gracePeriod time.Duration,
 	log *zerolog.Logger,
 ) error {
 	var err error
-	if gracePeriod > 0 {
-		err = waitForSignalWithGraceShutdown(errC, shutdownC, graceShutdownC, gracePeriod, log)
-	} else {
-		err = waitForSignal(errC, shutdownC, log)
-		close(graceShutdownC)
+	select {
+	case err = <-errC:
+		log.Error().Err(err).Msg("Initiating shutdown")
+	case <-graceShutdownC:
+		log.Debug().Msg("Graceful shutdown signalled")
+		if gracePeriod > 0 {
+			// wait for either grace period or service termination
+			select {
+			case <-time.Tick(gracePeriod):
+			case <-errC:
+			}
+		}
 	}
 
-	if err != nil {
-		log.Err(err).Msg("Quitting due to error")
-	} else {
-		log.Info().Msg("Quitting...")
-	}
-	// Wait for clean exit, discarding all errors
+	// stop server context
+	cancelServerContext()
+
+	// Wait for clean exit, discarding all errors while we wait
+	stopDiscarding := make(chan struct{})
 	go func() {
-		for range errC {
+		for {
+			select {
+			case <-errC: // ignore
+			case <-stopDiscarding:
+				return
+			}
 		}
 	}()
 	wg.Wait()
+	close(stopDiscarding)
+
 	return err
 }
 
@@ -535,6 +539,7 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 	flags = append(flags, configureLoggingFlags(shouldHide)...)
 	flags = append(flags, configureProxyDNSFlags(shouldHide)...)
 	flags = append(flags, []cli.Flag{
+		credentialsFileFlag,
 		altsrc.NewBoolFlag(&cli.BoolFlag{
 			Name:   "is-autoupdated",
 			Usage:  "Signal the new process that Argo Tunnel client has been autoupdated",
@@ -1040,7 +1045,7 @@ func stdinControl(reconnectCh chan origin.ReconnectSignal, log *zerolog.Logger)
 						continue
 					}
 				}
-				log.Info().Msgf("Sending reconnect signal %+v", reconnect)
+				log.Info().Msgf("Sending %+v", reconnect)
 				reconnectCh <- reconnect
 			default:
 				log.Info().Str(LogFieldCommand, command).Msg("Unknown command")

cmd/cloudflared/tunnel/configuration.go

@@ -149,7 +149,7 @@ func prepareTunnelConfig(
 	c *cli.Context,
 	buildInfo *buildinfo.BuildInfo,
 	version string,
-	log *zerolog.Logger,
+	log, logTransport *zerolog.Logger,
 	observer *connection.Observer,
 	namedTunnel *connection.NamedTunnelConfig,
 ) (*origin.TunnelConfig, ingress.Ingress, error) {
@@ -252,9 +252,9 @@ func prepareTunnelConfig(
 		ReplaceExisting: c.Bool("force"),
 	}
 	muxerConfig := &connection.MuxerConfig{
-		HeartbeatInterval:  c.Duration("heartbeat-interval"),
+		HeartbeatInterval: c.Duration("heartbeat-interval"),
 		// Note TUN-3758 , we use Int because UInt is not supported with altsrc
-		MaxHeartbeats:      uint64(c.Int("heartbeat-count")),
+		MaxHeartbeats: uint64(c.Int("heartbeat-count")),
 		// Note TUN-3758 , we use Int because UInt is not supported with altsrc
 		CompressionSetting: h2mux.CompressionSetting(uint64(c.Int("compression-quality"))),
 		MetricsUpdateFreq:  c.Duration("metrics-update-freq"),
@@ -272,6 +272,7 @@ func prepareTunnelConfig(
 		LBPool:           c.String("lb-pool"),
 		Tags:             tags,
 		Log:              log,
+		LogTransport:     logTransport,
 		Observer:         observer,
 		ReportedVersion:  version,
 		// Note TUN-3758 , we use Int because UInt is not supported with altsrc

cmd/cloudflared/tunnel/server.go

@@ -8,7 +8,7 @@ import (
 	"github.com/urfave/cli/v2"
 )
 
-func runDNSProxyServer(c *cli.Context, dnsReadySignal, shutdownC chan struct{}, log *zerolog.Logger) error {
+func runDNSProxyServer(c *cli.Context, dnsReadySignal chan struct{}, shutdownC <-chan struct{}, log *zerolog.Logger) error {
 	port := c.Int("proxy-dns-port")
 	if port <= 0 || port > 65535 {
 		return errors.New("The 'proxy-dns-port' must be a valid port number in <1, 65535> range.")
@@ -26,5 +26,6 @@ func runDNSProxyServer(c *cli.Context, dnsReadySignal, shutdownC chan struct{},
 	}
 	<-shutdownC
 	_ = listener.Stop()
+	log.Info().Msg("DNS server stopped")
 	return nil
 }

cmd/cloudflared/tunnel/signal.go

@@ -4,84 +4,20 @@ import (
 	"os"
 	"os/signal"
 	"syscall"
-	"time"
 
 	"github.com/rs/zerolog"
 )
 
-const LogFieldSignal = "signal"
-
-// waitForSignal notifies all routines to shutdownC immediately by closing the
-// shutdownC when one of the routines in main exits, or when this process receives
-// SIGTERM/SIGINT
-func waitForSignal(errC chan error, shutdownC chan struct{}, log *zerolog.Logger) error {
+// waitForSignal closes graceShutdownC to indicate that we should start graceful shutdown sequence
+func waitForSignal(graceShutdownC chan struct{}, logger *zerolog.Logger) {
 	signals := make(chan os.Signal, 10)
 	signal.Notify(signals, syscall.SIGTERM, syscall.SIGINT)
 	defer signal.Stop(signals)
 
 	select {
-	case err := <-errC:
-		log.Err(err).Msg("terminating due to error")
-		close(shutdownC)
-		return err
-	case s := <-signals:
-		log.Info().Str(LogFieldSignal, s.String()).Msg("terminating due to signal")
-		close(shutdownC)
-	case <-shutdownC:
-	}
-	return nil
-}
-
-// waitForSignalWithGraceShutdown notifies all routines to shutdown immediately
-// by closing the shutdownC when one of the routines in main exits.
-// When this process recieves SIGTERM/SIGINT, it closes the graceShutdownC to
-// notify certain routines to start graceful shutdown. When grace period is over,
-// or when some routine exits, it notifies the rest of the routines to shutdown
-// immediately by closing shutdownC.
-// In the case of handling commands from Windows Service Manager, closing graceShutdownC
-// initiate graceful shutdown.
-func waitForSignalWithGraceShutdown(errC chan error,
-	shutdownC, graceShutdownC chan struct{},
-	gracePeriod time.Duration,
-	logger *zerolog.Logger,
-) error {
-	signals := make(chan os.Signal, 10)
-	signal.Notify(signals, syscall.SIGTERM, syscall.SIGINT)
-	defer signal.Stop(signals)
-
-	select {
-	case err := <-errC:
-		logger.Info().Msgf("Initiating graceful shutdown due to %v ...", err)
-		close(graceShutdownC)
-		close(shutdownC)
-		return err
 	case s := <-signals:
 		logger.Info().Msgf("Initiating graceful shutdown due to signal %s ...", s)
 		close(graceShutdownC)
-		waitForGracePeriod(signals, errC, shutdownC, gracePeriod)
 	case <-graceShutdownC:
-		waitForGracePeriod(signals, errC, shutdownC, gracePeriod)
-	case <-shutdownC:
-		close(graceShutdownC)
 	}
-
-	return nil
-}
-
-func waitForGracePeriod(signals chan os.Signal,
-	errC chan error,
-	shutdownC chan struct{},
-	gracePeriod time.Duration,
-) {
-	// Unregister signal handler early, so the client can send a second SIGTERM/SIGINT
-	// to force shutdown cloudflared
-	signal.Stop(signals)
-	graceTimerTick := time.Tick(gracePeriod)
-	// send close signal via shutdownC when grace period expires or when an
-	// error is encountered.
-	select {
-	case <-graceTimerTick:
-	case <-errC:
-	}
-	close(shutdownC)
-}
+}

cmd/cloudflared/tunnel/signal_test.go

@@ -2,11 +2,12 @@ package tunnel
 
 import (
 	"fmt"
-	"github.com/rs/zerolog"
+	"sync"
 	"syscall"
 	"testing"
 	"time"
 
+	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -18,40 +19,21 @@ var (
 	graceShutdownErr = fmt.Errorf("receive grace shutdown")
 )
 
-func testChannelClosed(t *testing.T, c chan struct{}) {
+func channelClosed(c chan struct{}) bool {
 	select {
 	case <-c:
-		return
+		return true
 	default:
-		t.Fatal("Channel should be closed")
+		return false
 	}
 }
 
-func TestWaitForSignal(t *testing.T) {
+func TestSignalShutdown(t *testing.T) {
 	log := zerolog.Nop()
 
-	// Test handling server error
-	errC := make(chan error)
-	shutdownC := make(chan struct{})
-
-	go func() {
-		errC <- serverErr
-	}()
-
-	// received error, shutdownC should be closed
-	err := waitForSignal(errC, shutdownC, &log)
-	assert.Equal(t, serverErr, err)
-	testChannelClosed(t, shutdownC)
-
 	// Test handling SIGTERM & SIGINT
 	for _, sig := range []syscall.Signal{syscall.SIGTERM, syscall.SIGINT} {
-		errC = make(chan error)
-		shutdownC = make(chan struct{})
-
-		go func(shutdownC chan struct{}) {
-			<-shutdownC
-			errC <- shutdownErr
-		}(shutdownC)
+		graceShutdownC := make(chan struct{})
 
 		go func(sig syscall.Signal) {
 			// sleep for a tick to prevent sending signal before calling waitForSignal
@@ -59,99 +41,64 @@ func TestWaitForSignal(t *testing.T) {
 			_ = syscall.Kill(syscall.Getpid(), sig)
 		}(sig)
 
-		err = waitForSignal(errC, shutdownC, &log)
-		assert.Equal(t, nil, err)
-		assert.Equal(t, shutdownErr, <-errC)
-		testChannelClosed(t, shutdownC)
+		time.AfterFunc(time.Second, func() {
+			select {
+			case <-graceShutdownC:
+			default:
+				close(graceShutdownC)
+				t.Fatal("waitForSignal timed out")
+			}
+		})
+
+		waitForSignal(graceShutdownC, &log)
+		assert.True(t, channelClosed(graceShutdownC))
 	}
 }
 
-func TestWaitForSignalWithGraceShutdown(t *testing.T) {
-	// Test server returning error
-	errC := make(chan error)
-	shutdownC := make(chan struct{})
-	graceshutdownC := make(chan struct{})
+func TestWaitForShutdown(t *testing.T) {
+	log := zerolog.Nop()
 
+	errC := make(chan error)
+	graceShutdownC := make(chan struct{})
+	const gracePeriod = 5 * time.Second
+
+	contextCancelled := false
+	cancel := func() {
+		contextCancelled = true
+	}
+	var wg sync.WaitGroup
+
+	// on, error stop immediately
+	contextCancelled = false
+	startTime := time.Now()
 	go func() {
 		errC <- serverErr
 	}()
-
-	log := zerolog.Nop()
-
-	// received error, both shutdownC and graceshutdownC should be closed
-	err := waitForSignalWithGraceShutdown(errC, shutdownC, graceshutdownC, tick, &log)
+	err := waitToShutdown(&wg, cancel, errC, graceShutdownC, gracePeriod, &log)
 	assert.Equal(t, serverErr, err)
-	testChannelClosed(t, shutdownC)
-	testChannelClosed(t, graceshutdownC)
+	assert.True(t, contextCancelled)
+	assert.False(t, channelClosed(graceShutdownC))
+	assert.True(t, time.Now().Sub(startTime) < time.Second) // check that wait ended early
 
-	// shutdownC closed, graceshutdownC should also be closed and no error
-	errC = make(chan error)
-	shutdownC = make(chan struct{})
-	graceshutdownC = make(chan struct{})
-	close(shutdownC)
-	err = waitForSignalWithGraceShutdown(errC, shutdownC, graceshutdownC, tick, &log)
-	assert.NoError(t, err)
-	testChannelClosed(t, shutdownC)
-	testChannelClosed(t, graceshutdownC)
+	// on graceful shutdown, ignore error but stop as soon as an error arrives
+	contextCancelled = false
+	startTime = time.Now()
+	go func() {
+		close(graceShutdownC)
+		time.Sleep(tick)
+		errC <- serverErr
+	}()
+	err = waitToShutdown(&wg, cancel, errC, graceShutdownC, gracePeriod, &log)
+	assert.Nil(t, err)
+	assert.True(t, contextCancelled)
+	assert.True(t, time.Now().Sub(startTime) < time.Second) // check that wait ended early
 
-	// graceshutdownC closed, shutdownC should also be closed and no error
-	errC = make(chan error)
-	shutdownC = make(chan struct{})
-	graceshutdownC = make(chan struct{})
-	close(graceshutdownC)
-	err = waitForSignalWithGraceShutdown(errC, shutdownC, graceshutdownC, tick, &log)
-	assert.NoError(t, err)
-	testChannelClosed(t, shutdownC)
-	testChannelClosed(t, graceshutdownC)
-
-	// Test handling SIGTERM & SIGINT
-	for _, sig := range []syscall.Signal{syscall.SIGTERM, syscall.SIGINT} {
-		errC := make(chan error)
-		shutdownC = make(chan struct{})
-		graceshutdownC = make(chan struct{})
-
-		go func(shutdownC, graceshutdownC chan struct{}) {
-			<-graceshutdownC
-			<-shutdownC
-			errC <- graceShutdownErr
-		}(shutdownC, graceshutdownC)
-
-		go func(sig syscall.Signal) {
-			// sleep for a tick to prevent sending signal before calling waitForSignalWithGraceShutdown
-			time.Sleep(tick)
-			_ = syscall.Kill(syscall.Getpid(), sig)
-		}(sig)
-
-		err = waitForSignalWithGraceShutdown(errC, shutdownC, graceshutdownC, tick, &log)
-		assert.Equal(t, nil, err)
-		assert.Equal(t, graceShutdownErr, <-errC)
-		testChannelClosed(t, shutdownC)
-		testChannelClosed(t, graceshutdownC)
-	}
-
-	// Test handling SIGTERM & SIGINT, server send error before end of grace period
-	for _, sig := range []syscall.Signal{syscall.SIGTERM, syscall.SIGINT} {
-		errC := make(chan error)
-		shutdownC = make(chan struct{})
-		graceshutdownC = make(chan struct{})
-
-		go func(shutdownC, graceshutdownC chan struct{}) {
-			<-graceshutdownC
-			errC <- graceShutdownErr
-			<-shutdownC
-			errC <- shutdownErr
-		}(shutdownC, graceshutdownC)
-
-		go func(sig syscall.Signal) {
-			// sleep for a tick to prevent sending signal before calling waitForSignalWithGraceShutdown
-			time.Sleep(tick)
-			_ = syscall.Kill(syscall.Getpid(), sig)
-		}(sig)
-
-		err = waitForSignalWithGraceShutdown(errC, shutdownC, graceshutdownC, tick, &log)
-		assert.Equal(t, nil, err)
-		assert.Equal(t, shutdownErr, <-errC)
-		testChannelClosed(t, shutdownC)
-		testChannelClosed(t, graceshutdownC)
-	}
+	// with graceShutdownC closed stop right away without grace period
+	contextCancelled = false
+	startTime = time.Now()
+	err = waitToShutdown(&wg, cancel, errC, graceShutdownC, 0, &log)
+	assert.Nil(t, err)
+	assert.True(t, contextCancelled)
+	assert.True(t, time.Now().Sub(startTime) < time.Second) // check that wait ended early
 }
+

cmd/cloudflared/tunnel/subcommand_context.go

@@ -147,7 +147,7 @@ func (sc *subcommandContext) readTunnelCredentials(credFinder CredFinder) (conne
 	return credentials, nil
 }
 
-func (sc *subcommandContext) create(name string) (*tunnelstore.Tunnel, error) {
+func (sc *subcommandContext) create(name string, credentialsOutputPath string) (*tunnelstore.Tunnel, error) {
 	client, err := sc.client()
 	if err != nil {
 		return nil, errors.Wrap(err, "couldn't create client to talk to Argo Tunnel backend")
@@ -173,7 +173,7 @@ func (sc *subcommandContext) create(name string) (*tunnelstore.Tunnel, error) {
 		TunnelID:     tunnel.ID,
 		TunnelName:   name,
 	}
-	filePath, writeFileErr := writeTunnelCredentials(credential.certPath, &tunnelCredentials)
+	filePath, writeFileErr := writeTunnelCredentials(credential.certPath, credentialsOutputPath, &tunnelCredentials)
 	if writeFileErr != nil {
 		var errorLines []string
 		errorLines = append(errorLines, fmt.Sprintf("Your tunnel '%v' was created with ID %v. However, cloudflared couldn't write to the tunnel credentials file at %v.json.", tunnel.Name, tunnel.ID, tunnel.ID))
@@ -274,8 +274,6 @@ func (sc *subcommandContext) run(tunnelID uuid.UUID) error {
 	return StartServer(
 		sc.c,
 		version,
-		shutdownC,
-		graceShutdownC,
 		&connection.NamedTunnelConfig{Credentials: credentials},
 		sc.log,
 		sc.isUIEnabled,

cmd/cloudflared/tunnel/subcommands.go

@@ -28,6 +28,7 @@ import (
 )
 
 const (
+	allSortByOptions  = "name, id, createdAt, deletedAt, numConnections"
 	CredFileFlagAlias = "cred-file"
 	CredFileFlag      = "credentials-file"
 
@@ -67,6 +68,17 @@ var (
 		Aliases: []string{"o"},
 		Usage:   "Render output using given `FORMAT`. Valid options are 'json' or 'yaml'",
 	})
+	sortByFlag = &cli.StringFlag{
+		Name:    "sort-by",
+		Value:   "name",
+		Usage:   fmt.Sprintf("Sorts the list of tunnels by the given field. Valid options are {%s}", allSortByOptions),
+		EnvVars: []string{"TUNNEL_LIST_SORT_BY"},
+	}
+	invertSortFlag = &cli.BoolFlag{
+		Name:    "invert-sort",
+		Usage:   "Inverts the sort order of the tunnel list.",
+		EnvVars: []string{"TUNNEL_LIST_INVERT_SORT"},
+	}
 	forceFlag = altsrc.NewBoolFlag(&cli.BoolFlag{
 		Name:    "force",
 		Aliases: []string{"f"},
@@ -78,7 +90,7 @@ var (
 	credentialsFileFlag = altsrc.NewStringFlag(&cli.StringFlag{
 		Name:    CredFileFlag,
 		Aliases: []string{CredFileFlagAlias},
-		Usage:   "File path of tunnel credentials",
+		Usage:   "Filepath at which to read/write the tunnel credentials",
 		EnvVars: []string{"TUNNEL_CRED_FILE"},
 	})
 	forceDeleteFlag = &cli.BoolFlag{
@@ -109,7 +121,7 @@ func buildCreateCommand() *cli.Command {
   For example, to create a tunnel named 'my-tunnel' run:
 
   $ cloudflared tunnel create my-tunnel`,
-		Flags:              []cli.Flag{outputFormatFlag},
+		Flags:              []cli.Flag{outputFormatFlag, credentialsFileFlag},
 		CustomHelpTemplate: commandHelpTemplate(),
 	}
 }
@@ -132,7 +144,7 @@ func createCommand(c *cli.Context) error {
 	}
 	name := c.Args().First()
 
-	_, err = sc.create(name)
+	_, err = sc.create(name, c.String(CredFileFlag))
 	return errors.Wrap(err, "failed to create tunnel")
 }
 
@@ -142,12 +154,18 @@ func tunnelFilePath(tunnelID uuid.UUID, directory string) (string, error) {
 	return homedir.Expand(filePath)
 }
 
+// If an `outputFile` is given, write the credentials there.
+// Otherwise, write it to the same directory as the originCert,
+// with the filename `<tunnel id>.json`.
 func writeTunnelCredentials(
-	originCertPath string,
+	originCertPath, outputFile string,
 	credentials *connection.Credentials,
 ) (filePath string, err error) {
-	originCertDir := filepath.Dir(originCertPath)
-	filePath, err = tunnelFilePath(credentials.TunnelID, originCertDir)
+	filePath = outputFile
+	if outputFile == "" {
+		originCertDir := filepath.Dir(originCertPath)
+		filePath, err = tunnelFilePath(credentials.TunnelID, originCertDir)
+	}
 	if err != nil {
 		return "", err
 	}
@@ -161,12 +179,21 @@ func writeTunnelCredentials(
 
 func buildListCommand() *cli.Command {
 	return &cli.Command{
-		Name:               "list",
-		Action:             cliutil.ErrorHandler(listCommand),
-		Usage:              "List existing tunnels",
-		UsageText:          "cloudflared tunnel [tunnel command options] list [subcommand options]",
-		Description:        "cloudflared tunnel list will display all active tunnels, their created time and associated connections. Use -d flag to include deleted tunnels. See the list of options to filter the list",
-		Flags:              []cli.Flag{outputFormatFlag, showDeletedFlag, listNameFlag, listExistedAtFlag, listIDFlag, showRecentlyDisconnected},
+		Name:        "list",
+		Action:      cliutil.ErrorHandler(listCommand),
+		Usage:       "List existing tunnels",
+		UsageText:   "cloudflared tunnel [tunnel command options] list [subcommand options]",
+		Description: "cloudflared tunnel list will display all active tunnels, their created time and associated connections. Use -d flag to include deleted tunnels. See the list of options to filter the list",
+		Flags: []cli.Flag{
+			outputFormatFlag,
+			showDeletedFlag,
+			listNameFlag,
+			listExistedAtFlag,
+			listIDFlag,
+			showRecentlyDisconnected,
+			sortByFlag,
+			invertSortFlag,
+		},
 		CustomHelpTemplate: commandHelpTemplate(),
 	}
 }
@@ -200,6 +227,36 @@ func listCommand(c *cli.Context) error {
 		return err
 	}
 
+	// Sort the tunnels
+	sortBy := c.String("sort-by")
+	invalidSortField := false
+	sort.Slice(tunnels, func(i, j int) bool {
+		cmp := func() bool {
+			switch sortBy {
+			case "name":
+				return tunnels[i].Name < tunnels[j].Name
+			case "id":
+				return tunnels[i].ID.String() < tunnels[j].ID.String()
+			case "createdAt":
+				return tunnels[i].CreatedAt.Unix() < tunnels[j].CreatedAt.Unix()
+			case "deletedAt":
+				return tunnels[i].DeletedAt.Unix() < tunnels[j].DeletedAt.Unix()
+			case "numConnections":
+				return len(tunnels[i].Connections) < len(tunnels[j].Connections)
+			default:
+				invalidSortField = true
+				return tunnels[i].Name < tunnels[j].Name
+			}
+		}()
+		if c.Bool("invert-sort") {
+			return !cmp
+		}
+		return cmp
+	})
+	if invalidSortField {
+		sc.log.Error().Msgf("%s is not a valid sort field. Valid sort fields are %s. Defaulting to 'name'.", sortBy, allSortByOptions)
+	}
+
 	if outputFormat := c.String(outputFormatFlag.Name); outputFormat != "" {
 		return renderOutput(outputFormat, tunnels)
 	}

cmd/cloudflared/ui/launch_ui.go

@@ -15,7 +15,6 @@ import (
 
 type connState struct {
 	location string
-	state    connection.Status
 }
 
 type uiModel struct {
@@ -32,6 +31,7 @@ type palette struct {
 	defaultText  string
 	disconnected string
 	reconnecting string
+	unregistered string
 }
 
 func NewUIModel(version, hostname, metricsURL string, ing *ingress.Ingress, haConnections int) *uiModel {
@@ -67,6 +67,7 @@ func (data *uiModel) Launch(
 		defaultText:  "white",
 		disconnected: "red",
 		reconnecting: "orange",
+		unregistered: "orange",
 	}
 
 	app := tview.NewApplication()
@@ -128,14 +129,14 @@ func (data *uiModel) Launch(
 		switch event.EventType {
 		case connection.Connected:
 			data.setConnTableCell(event, connTable, palette)
-		case connection.Disconnected, connection.Reconnecting:
+		case connection.Disconnected, connection.Reconnecting, connection.Unregistering:
 			data.changeConnStatus(event, connTable, log, palette)
 		case connection.SetURL:
 			tunnelHostText.SetText(event.URL)
 			data.edgeURL = event.URL
 		case connection.RegisteringTunnel:
 			if data.edgeURL == "" {
-				tunnelHostText.SetText("Registering tunnel...")
+				tunnelHostText.SetText(fmt.Sprintf("Registering tunnel connection %d...", event.Index))
 			}
 		}
 		app.Draw()
@@ -167,10 +168,7 @@ func (data *uiModel) changeConnStatus(event connection.Event, table *tview.Table
 
 	locationState := event.Location
 
-	if event.EventType == connection.Disconnected {
-		connState.state = connection.Disconnected
-	} else if event.EventType == connection.Reconnecting {
-		connState.state = connection.Reconnecting
+	if event.EventType == connection.Reconnecting {
 		locationState = "Reconnecting..."
 	}
 
@@ -196,7 +194,6 @@ func (data *uiModel) setConnTableCell(event connection.Event, table *tview.Table
 	connectionNum := index + 1
 
 	// Update slice to keep track of connection location and state in UI table
-	data.connections[index].state = connection.Connected
 	data.connections[index].location = event.Location
 
 	// Update text in table cell to show disconnected state
@@ -218,6 +215,8 @@ func newCellText(palette palette, connectionNum int, location string, connectedS
 		dotColor = palette.disconnected
 	case connection.Reconnecting:
 		dotColor = palette.reconnecting
+	case connection.Unregistering:
+		dotColor = palette.unregistered
 	}
 
 	return fmt.Sprintf(connFmtString, dotColor, palette.defaultText, connectionNum, location)

cmd/cloudflared/windows_service.go

@@ -40,21 +40,21 @@ const (
 	LogFieldWindowsServiceName = "windowsServiceName"
 )
 
-func runApp(app *cli.App, shutdownC, graceShutdownC chan struct{}) {
+func runApp(app *cli.App, graceShutdownC chan struct{}) {
 	app.Commands = append(app.Commands, &cli.Command{
 		Name:  "service",
 		Usage: "Manages the Argo Tunnel Windows service",
 		Subcommands: []*cli.Command{
-			&cli.Command{
+			{
 				Name:   "install",
 				Usage:  "Install Argo Tunnel as a Windows service",
 				Action: installWindowsService,
-			},
-			&cli.Command{
+						},
+			{
 				Name:   "uninstall",
 				Usage:  "Uninstall the Argo Tunnel service",
 				Action: uninstallWindowsService,
-			},
+						},
 		},
 	})
 
@@ -82,7 +82,7 @@ func runApp(app *cli.App, shutdownC, graceShutdownC chan struct{}) {
 	// Run executes service name by calling windowsService which is a Handler
 	// interface that implements Execute method.
 	// It will set service status to stop after Execute returns
-	err = svc.Run(windowsServiceName, &windowsService{app: app, shutdownC: shutdownC, graceShutdownC: graceShutdownC})
+	err = svc.Run(windowsServiceName, &windowsService{app: app, graceShutdownC: graceShutdownC})
 	if err != nil {
 		if errno, ok := err.(syscall.Errno); ok && int(errno) == serviceControllerConnectionFailure {
 			// Hack: assume this is a false negative from the IsAnInteractiveSession() check above.
@@ -96,11 +96,11 @@ func runApp(app *cli.App, shutdownC, graceShutdownC chan struct{}) {
 
 type windowsService struct {
 	app            *cli.App
-	shutdownC      chan struct{}
 	graceShutdownC chan struct{}
 }
 
-// called by the package code at the start of the service
+// Execute is called by the service manager when service starts, the state
+// of the service will be set to Stopped when this function returns.
 func (s *windowsService) Execute(serviceArgs []string, r <-chan svc.ChangeRequest, statusChan chan<- svc.Status) (ssec bool, errno uint32) {
 	log := logger.Create(nil)
 	elog, err := eventlog.Open(windowsServiceName)
@@ -128,13 +128,12 @@ func (s *windowsService) Execute(serviceArgs []string, r <-chan svc.ChangeReques
 	}
 	elog.Info(1, fmt.Sprintf("%s service arguments: %v", windowsServiceName, args))
 
-	const cmdsAccepted = svc.AcceptStop | svc.AcceptShutdown
 	statusChan <- svc.Status{State: svc.StartPending}
 	errC := make(chan error)
 	go func() {
 		errC <- s.app.Run(args)
 	}()
-	statusChan <- svc.Status{State: svc.Running, Accepts: cmdsAccepted}
+	statusChan <- svc.Status{State: svc.Running, Accepts: svc.AcceptStop | svc.AcceptShutdown}
 
 	for {
 		select {
@@ -143,17 +142,25 @@ func (s *windowsService) Execute(serviceArgs []string, r <-chan svc.ChangeReques
 			case svc.Interrogate:
 				statusChan <- c.CurrentStatus
 			case svc.Stop, svc.Shutdown:
-				close(s.graceShutdownC)
-				statusChan <- svc.Status{State: svc.Stopped, Accepts: cmdsAccepted}
+				if s.graceShutdownC != nil {
+					// start graceful shutdown
+					elog.Info(1, "cloudflared starting graceful shutdown")
+					close(s.graceShutdownC)
+					s.graceShutdownC = nil
+					statusChan <- svc.Status{State: svc.StopPending}
+					continue
+				}
+				// repeated attempts at graceful shutdown forces immediate stop
+				elog.Info(1, "cloudflared terminating immediately")
 				statusChan <- svc.Status{State: svc.StopPending}
-				return
+				return false, 0
 			default:
 				elog.Error(1, fmt.Sprintf("unexpected control request #%d", c))
 			}
 		case err := <-errC:
-			ssec = true
 			if err != nil {
 				elog.Error(1, fmt.Sprintf("cloudflared terminated with error %v", err))
+				ssec = true
 				errno = 1
 			} else {
 				elog.Info(1, "cloudflared terminated without error")

connection/connection_test.go

@@ -27,7 +27,6 @@ var (
 		Scheme: "https",
 		Host:   "connectiontest.argotunnel.com",
 	}
-	testObserver        = NewObserver(&log, false)
 	testLargeResp = make([]byte, largeFileSize)
 )
 

connection/errors.go

@@ -4,55 +4,40 @@ import (
 	"github.com/cloudflare/cloudflared/edgediscovery"
 	"github.com/cloudflare/cloudflared/h2mux"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
-	"github.com/prometheus/client_golang/prometheus"
 )
 
 const (
 	DuplicateConnectionError = "EDUPCONN"
 )
 
-// RegisterTunnel error from client
-type clientRegisterTunnelError struct {
-	cause error
-}
-
-func newRPCError(cause error, counter *prometheus.CounterVec, name rpcName) clientRegisterTunnelError {
-	counter.WithLabelValues(cause.Error(), string(name)).Inc()
-	return clientRegisterTunnelError{cause: cause}
-}
-
-func (e clientRegisterTunnelError) Error() string {
-	return e.cause.Error()
-}
-
 type DupConnRegisterTunnelError struct{}
 
-var errDuplicationConnection = &DupConnRegisterTunnelError{}
+var errDuplicationConnection = DupConnRegisterTunnelError{}
 
 func (e DupConnRegisterTunnelError) Error() string {
 	return "already connected to this server, trying another address"
 }
 
 // RegisterTunnel error from server
-type serverRegisterTunnelError struct {
-	cause     error
-	permanent bool
+type ServerRegisterTunnelError struct {
+	Cause     error
+	Permanent bool
 }
 
-func (e serverRegisterTunnelError) Error() string {
-	return e.cause.Error()
+func (e ServerRegisterTunnelError) Error() string {
+	return e.Cause.Error()
 }
 
-func serverRegistrationErrorFromRPC(err error) *serverRegisterTunnelError {
+func serverRegistrationErrorFromRPC(err error) ServerRegisterTunnelError {
 	if retryable, ok := err.(*tunnelpogs.RetryableError); ok {
-		return &serverRegisterTunnelError{
-			cause:     retryable.Unwrap(),
-			permanent: false,
+		return ServerRegisterTunnelError{
+			Cause:     retryable.Unwrap(),
+			Permanent: false,
 		}
 	}
-	return &serverRegisterTunnelError{
-		cause:     err,
-		permanent: true,
+	return ServerRegisterTunnelError{
+		Cause:     err,
+		Permanent: true,
 	}
 }
 
@@ -62,6 +47,8 @@ func (e muxerShutdownError) Error() string {
 	return "muxer shutdown"
 }
 
+var errMuxerStopped = muxerShutdownError{}
+
 func isHandshakeErrRecoverable(err error, connIndex uint8, observer *Observer) bool {
 	log := observer.log.With().
 		Uint8(LogFieldConnIndex, connIndex).

connection/event.go

@@ -22,4 +22,6 @@ const (
 	SetURL
 	// RegisteringTunnel means the non-named tunnel is registering its connection.
 	RegisteringTunnel
+	// We're unregistering tunnel from the edge in preparation for a disconnect
+	Unregistering
 )

connection/h2mux.go

@@ -2,6 +2,7 @@ package connection
 
 import (
 	"context"
+	"io"
 	"net"
 	"net/http"
 	"time"
@@ -28,7 +29,12 @@ type h2muxConnection struct {
 	connIndexStr string
 	connIndex    uint8
 
-	observer *Observer
+	observer          *Observer
+	gracefulShutdownC <-chan struct{}
+	stoppedGracefully bool
+
+	// newRPCClientFunc allows us to mock RPCs during testing
+	newRPCClientFunc func(context.Context, io.ReadWriteCloser, *zerolog.Logger) NamedTunnelRPCClient
 }
 
 type MuxerConfig struct {
@@ -57,18 +63,21 @@ func NewH2muxConnection(
 	edgeConn net.Conn,
 	connIndex uint8,
 	observer *Observer,
+	gracefulShutdownC <-chan struct{},
 ) (*h2muxConnection, error, bool) {
 	h := &h2muxConnection{
-		config:       config,
-		muxerConfig:  muxerConfig,
-		connIndexStr: uint8ToString(connIndex),
-		connIndex:    connIndex,
-		observer:     observer,
+		config:            config,
+		muxerConfig:       muxerConfig,
+		connIndexStr:      uint8ToString(connIndex),
+		connIndex:         connIndex,
+		observer:          observer,
+		gracefulShutdownC: gracefulShutdownC,
+		newRPCClientFunc:  newRegistrationRPCClient,
 	}
 
 	// Establish a muxed connection with the edge
 	// Client mux handshake with agent server
-	muxer, err := h2mux.Handshake(edgeConn, edgeConn, *muxerConfig.H2MuxerConfig(h, observer.log), h2mux.ActiveStreams)
+	muxer, err := h2mux.Handshake(edgeConn, edgeConn, *muxerConfig.H2MuxerConfig(h, observer.logTransport), h2mux.ActiveStreams)
 	if err != nil {
 		recoverable := isHandshakeErrRecoverable(err, connIndex, observer)
 		return nil, err, recoverable
@@ -77,21 +86,14 @@ func NewH2muxConnection(
 	return h, nil, false
 }
 
-func (h *h2muxConnection) ServeNamedTunnel(ctx context.Context, namedTunnel *NamedTunnelConfig, credentialManager CredentialManager, connOptions *tunnelpogs.ConnectionOptions, connectedFuse ConnectedFuse) error {
+func (h *h2muxConnection) ServeNamedTunnel(ctx context.Context, namedTunnel *NamedTunnelConfig, connOptions *tunnelpogs.ConnectionOptions, connectedFuse ConnectedFuse) error {
 	errGroup, serveCtx := errgroup.WithContext(ctx)
 	errGroup.Go(func() error {
 		return h.serveMuxer(serveCtx)
 	})
 
 	errGroup.Go(func() error {
-		stream, err := h.newRPCStream(serveCtx, register)
-		if err != nil {
-			return err
-		}
-		rpcClient := newRegistrationRPCClient(ctx, stream, h.observer.log)
-		defer rpcClient.Close()
-
-		if err = rpcClient.RegisterConnection(serveCtx, namedTunnel, connOptions, h.connIndex, h.observer); err != nil {
+		if err := h.registerNamedTunnel(serveCtx, namedTunnel, connOptions); err != nil {
 			return err
 		}
 		connectedFuse.Connected()
@@ -102,7 +104,15 @@ func (h *h2muxConnection) ServeNamedTunnel(ctx context.Context, namedTunnel *Nam
 		h.controlLoop(serveCtx, connectedFuse, true)
 		return nil
 	})
-	return errGroup.Wait()
+
+	err := errGroup.Wait()
+	if err == errMuxerStopped {
+		if h.stoppedGracefully {
+			return nil
+		}
+		h.observer.log.Info().Uint8(LogFieldConnIndex, h.connIndex).Msg("Unexpected muxer shutdown")
+	}
+	return err
 }
 
 func (h *h2muxConnection) ServeClassicTunnel(ctx context.Context, classicTunnel *ClassicTunnelConfig, credentialManager CredentialManager, registrationOptions *tunnelpogs.RegistrationOptions, connectedFuse ConnectedFuse) error {
@@ -134,7 +144,15 @@ func (h *h2muxConnection) ServeClassicTunnel(ctx context.Context, classicTunnel
 		h.controlLoop(serveCtx, connectedFuse, false)
 		return nil
 	})
-	return errGroup.Wait()
+
+	err := errGroup.Wait()
+	if err == errMuxerStopped {
+		if h.stoppedGracefully {
+			return nil
+		}
+		h.observer.log.Info().Uint8(LogFieldConnIndex, h.connIndex).Msg("Unexpected muxer shutdown")
+	}
+	return err
 }
 
 func (h *h2muxConnection) serveMuxer(ctx context.Context) error {
@@ -143,22 +161,36 @@ func (h *h2muxConnection) serveMuxer(ctx context.Context) error {
 	// here to notify other routines to stop
 	err := h.muxer.Serve(ctx)
 	if err == nil {
-		return muxerShutdownError{}
+		return errMuxerStopped
 	}
 	return err
 }
 
 func (h *h2muxConnection) controlLoop(ctx context.Context, connectedFuse ConnectedFuse, isNamedTunnel bool) {
 	updateMetricsTickC := time.Tick(h.muxerConfig.MetricsUpdateFreq)
+	var shutdownCompleted <-chan struct{}
 	for {
 		select {
-		case <-ctx.Done():
-			// UnregisterTunnel blocks until the RPC call returns
+		case <-h.gracefulShutdownC:
 			if connectedFuse.IsConnected() {
 				h.unregister(isNamedTunnel)
 			}
-			h.muxer.Shutdown()
+			h.stoppedGracefully = true
+			h.gracefulShutdownC = nil
+			shutdownCompleted = h.muxer.Shutdown()
+
+		case <-shutdownCompleted:
 			return
+
+		case <-ctx.Done():
+			// UnregisterTunnel blocks until the RPC call returns
+			if !h.stoppedGracefully && connectedFuse.IsConnected() {
+				h.unregister(isNamedTunnel)
+			}
+			h.muxer.Shutdown()
+			// don't wait for shutdown to finish when context is closed, this is the hard termination path
+			return
+
 		case <-updateMetricsTickC:
 			h.observer.metrics.updateMuxerMetrics(h.connIndexStr, h.muxer.Metrics())
 		}

connection/h2mux_test.go

@@ -11,10 +11,12 @@ import (
 	"testing"
 	"time"
 
-	"github.com/cloudflare/cloudflared/h2mux"
 	"github.com/gobwas/ws/wsutil"
+	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+
+	"github.com/cloudflare/cloudflared/h2mux"
 )
 
 var (
@@ -31,14 +33,22 @@ func newH2MuxConnection(t require.TestingT) (*h2muxConnection, *h2mux.Muxer) {
 	edgeMuxChan := make(chan *h2mux.Muxer)
 	go func() {
 		edgeMuxConfig := h2mux.MuxerConfig{
-			Log: testObserver.log,
+			Log: &log,
+			Handler: h2mux.MuxedStreamFunc(func(stream *h2mux.MuxedStream) error {
+				// we only expect RPC traffic in client->edge direction, provide minimal support for mocking
+				require.True(t, stream.IsRPCStream())
+				return stream.WriteHeaders([]h2mux.Header{
+					{Name:  ":status", Value: "200"},
+				})
+			}),
 		}
 		edgeMux, err := h2mux.Handshake(edgeConn, edgeConn, edgeMuxConfig, h2mux.ActiveStreams)
 		require.NoError(t, err)
 		edgeMuxChan <- edgeMux
 	}()
 	var connIndex = uint8(0)
-	h2muxConn, err, _ := NewH2muxConnection(testConfig, testMuxerConfig, originConn, connIndex, testObserver)
+	testObserver := NewObserver(&log, &log, false)
+	h2muxConn, err, _ := NewH2muxConnection(testConfig, testMuxerConfig, originConn, connIndex, testObserver, nil)
 	require.NoError(t, err)
 	return h2muxConn, <-edgeMuxChan
 }
@@ -168,6 +178,55 @@ func TestServeStreamWS(t *testing.T) {
 	wg.Wait()
 }
 
+func TestGracefulShutdownH2Mux(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	h2muxConn, edgeMux := newH2MuxConnection(t)
+
+	shutdownC := make(chan struct{})
+	unregisteredC := make(chan struct{})
+	h2muxConn.gracefulShutdownC = shutdownC
+	h2muxConn.newRPCClientFunc = func(_ context.Context, _ io.ReadWriteCloser, _ *zerolog.Logger) NamedTunnelRPCClient {
+		return &mockNamedTunnelRPCClient{
+			registered:   nil,
+			unregistered: unregisteredC,
+		}
+	}
+
+	var wg sync.WaitGroup
+	wg.Add(3)
+	go func() {
+		defer wg.Done()
+		_ = edgeMux.Serve(ctx)
+	}()
+	go func() {
+		defer wg.Done()
+		_ = h2muxConn.serveMuxer(ctx)
+	}()
+
+	go func() {
+		defer wg.Done()
+		h2muxConn.controlLoop(ctx, &mockConnectedFuse{}, true)
+	}()
+
+	time.Sleep(100 * time.Millisecond)
+	close(shutdownC)
+
+	select {
+	case <-unregisteredC:
+		break // ok
+	case <-time.Tick(time.Second):
+		assert.Fail(t, "timed out waiting for control loop to unregister")
+	}
+
+	cancel()
+	wg.Wait()
+
+	assert.True(t, h2muxConn.stoppedGracefully)
+	assert.Nil(t, h2muxConn.gracefulShutdownC)
+}
+
 func hasHeader(stream *h2mux.MuxedStream, name, val string) bool {
 	for _, header := range stream.Headers {
 		if header.Name == name && header.Value == val {

connection/http2.go

@@ -2,6 +2,7 @@ package connection
 
 import (
 	"context"
+	"fmt"
 	"io"
 	"math"
 	"net"
@@ -22,6 +23,8 @@ const (
 	controlStreamUpgrade  = "control-stream"
 )
 
+var errEdgeConnectionClosed = fmt.Errorf("connection with edge closed")
+
 type http2Connection struct {
 	conn         net.Conn
 	server       *http2.Server
@@ -31,10 +34,14 @@ type http2Connection struct {
 	observer     *Observer
 	connIndexStr string
 	connIndex    uint8
-	wg           *sync.WaitGroup
 	// newRPCClientFunc allows us to mock RPCs during testing
 	newRPCClientFunc func(context.Context, io.ReadWriteCloser, *zerolog.Logger) NamedTunnelRPCClient
-	connectedFuse    ConnectedFuse
+
+	activeRequestsWG  sync.WaitGroup
+	connectedFuse     ConnectedFuse
+	gracefulShutdownC <-chan struct{}
+	stoppedGracefully bool
+	controlStreamErr  error // result of running control stream handler
 }
 
 func NewHTTP2Connection(
@@ -45,25 +52,26 @@ func NewHTTP2Connection(
 	observer *Observer,
 	connIndex uint8,
 	connectedFuse ConnectedFuse,
+	gracefulShutdownC <-chan struct{},
 ) *http2Connection {
 	return &http2Connection{
 		conn: conn,
 		server: &http2.Server{
 			MaxConcurrentStreams: math.MaxUint32,
 		},
-		config:           config,
-		namedTunnel:      namedTunnelConfig,
-		connOptions:      connOptions,
-		observer:         observer,
-		connIndexStr:     uint8ToString(connIndex),
-		connIndex:        connIndex,
-		wg:               &sync.WaitGroup{},
-		newRPCClientFunc: newRegistrationRPCClient,
-		connectedFuse:    connectedFuse,
+		config:            config,
+		namedTunnel:       namedTunnelConfig,
+		connOptions:       connOptions,
+		observer:          observer,
+		connIndexStr:      uint8ToString(connIndex),
+		connIndex:         connIndex,
+		newRPCClientFunc:  newRegistrationRPCClient,
+		connectedFuse:     connectedFuse,
+		gracefulShutdownC: gracefulShutdownC,
 	}
 }
 
-func (c *http2Connection) Serve(ctx context.Context) {
+func (c *http2Connection) Serve(ctx context.Context) error {
 	go func() {
 		<-ctx.Done()
 		c.close()
@@ -72,11 +80,21 @@ func (c *http2Connection) Serve(ctx context.Context) {
 		Context: ctx,
 		Handler: c,
 	})
+
+	switch {
+	case c.stoppedGracefully:
+		return nil
+	case c.controlStreamErr != nil:
+		return c.controlStreamErr
+	default:
+		c.observer.log.Info().Uint8(LogFieldConnIndex, c.connIndex).Msg("Lost connection with the edge")
+		return errEdgeConnectionClosed
+	}
 }
 
 func (c *http2Connection) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	c.wg.Add(1)
-	defer c.wg.Done()
+	c.activeRequestsWG.Add(1)
+	defer c.activeRequestsWG.Done()
 
 	respWriter := &http2RespWriter{
 		r: r.Body,
@@ -93,6 +111,7 @@ func (c *http2Connection) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	if isControlStreamUpgrade(r) {
 		respWriter.shouldFlush = true
 		err = c.serveControlStream(r.Context(), respWriter)
+		c.controlStreamErr = err
 	} else if isWebsocketUpgrade(r) {
 		respWriter.shouldFlush = true
 		stripWebsocketUpgradeHeader(r)
@@ -115,14 +134,23 @@ func (c *http2Connection) serveControlStream(ctx context.Context, respWriter *ht
 	}
 	c.connectedFuse.Connected()
 
-	<-ctx.Done()
+	// wait for connection termination or start of graceful shutdown
+	select {
+	case <-ctx.Done():
+		break
+	case <-c.gracefulShutdownC:
+		c.stoppedGracefully = true
+	}
+
+	c.observer.sendUnregisteringEvent(c.connIndex)
 	rpcClient.GracefulShutdown(ctx, c.config.GracePeriod)
+	c.observer.log.Info().Uint8(LogFieldConnIndex, c.connIndex).Msg("Unregistered tunnel connection")
 	return nil
 }
 
 func (c *http2Connection) close() {
 	// Wait for all serve HTTP handlers to return
-	c.wg.Wait()
+	c.activeRequestsWG.Wait()
 	c.conn.Close()
 }
 

connection/http2_test.go

@@ -12,6 +12,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/assert"
+
 	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 
@@ -33,9 +35,10 @@ func newTestHTTP2Connection() (*http2Connection, net.Conn) {
 		testConfig,
 		&NamedTunnelConfig{},
 		&pogs.ConnectionOptions{},
-		testObserver,
+		NewObserver(&log, &log, false),
 		connIndex,
 		mockConnectedFuse{},
+		nil,
 	), edgeConn
 }
 
@@ -241,10 +244,72 @@ func TestServeControlStream(t *testing.T) {
 	<-rpcClientFactory.registered
 	cancel()
 	<-rpcClientFactory.unregistered
+	assert.False(t, http2Conn.stoppedGracefully)
 
 	wg.Wait()
 }
 
+func TestGracefulShutdownHTTP2(t *testing.T) {
+	http2Conn, edgeConn := newTestHTTP2Connection()
+
+	rpcClientFactory := mockRPCClientFactory{
+		registered:   make(chan struct{}),
+		unregistered: make(chan struct{}),
+	}
+	events := &eventCollectorSink{}
+	http2Conn.newRPCClientFunc = rpcClientFactory.newMockRPCClient
+	http2Conn.observer.RegisterSink(events)
+	shutdownC := make(chan struct{})
+	http2Conn.gracefulShutdownC = shutdownC
+
+	ctx, cancel := context.WithCancel(context.Background())
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		http2Conn.Serve(ctx)
+	}()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://localhost:8080/", nil)
+	require.NoError(t, err)
+	req.Header.Set(internalUpgradeHeader, controlStreamUpgrade)
+
+	edgeHTTP2Conn, err := testTransport.NewClientConn(edgeConn)
+	require.NoError(t, err)
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		_, _ = edgeHTTP2Conn.RoundTrip(req)
+	}()
+
+	select {
+	case <-rpcClientFactory.registered:
+		break //ok
+	case <-time.Tick(time.Second):
+		t.Fatal("timeout out waiting for registration")
+	}
+
+	// signal graceful shutdown
+	close(shutdownC)
+
+	select {
+	case <-rpcClientFactory.unregistered:
+		break //ok
+	case <-time.Tick(time.Second):
+		t.Fatal("timeout out waiting for unregistered signal")
+	}
+	assert.True(t, http2Conn.stoppedGracefully)
+
+	cancel()
+	wg.Wait()
+
+	events.assertSawEvent(t, Event{
+		Index:     http2Conn.connIndex,
+		EventType: Unregistering,
+	})
+}
+
 func benchmarkServeHTTP(b *testing.B, test testRequest) {
 	http2Conn, edgeConn := newTestHTTP2Connection()
 
@@ -281,6 +346,7 @@ func benchmarkServeHTTP(b *testing.B, test testRequest) {
 	cancel()
 	wg.Wait()
 }
+
 func BenchmarkServeHTTPSimple(b *testing.B) {
 	test := testRequest{
 		name:           "ok",

connection/observer.go

@@ -17,6 +17,7 @@ const (
 
 type Observer struct {
 	log             *zerolog.Logger
+	logTransport    *zerolog.Logger
 	metrics         *tunnelMetrics
 	tunnelEventChan chan Event
 	uiEnabled       bool
@@ -27,9 +28,10 @@ type EventSink interface {
 	OnTunnelEvent(event Event)
 }
 
-func NewObserver(log *zerolog.Logger, uiEnabled bool) *Observer {
+func NewObserver(log, logTransport *zerolog.Logger, uiEnabled bool) *Observer {
 	o := &Observer{
 		log:             log,
+		logTransport:    logTransport,
 		metrics:         newTunnelMetrics(),
 		uiEnabled:       uiEnabled,
 		tunnelEventChan: make(chan Event, observerChannelBufferSize),
@@ -99,8 +101,8 @@ func trialZoneMsg(url string) []string {
 	}
 }
 
-func (o *Observer) sendRegisteringEvent() {
-	o.sendEvent(Event{EventType: RegisteringTunnel})
+func (o *Observer) sendRegisteringEvent(connIndex uint8) {
+	o.sendEvent(Event{Index: connIndex, EventType: RegisteringTunnel})
 }
 
 func (o *Observer) sendConnectedEvent(connIndex uint8, location string) {
@@ -115,6 +117,10 @@ func (o *Observer) SendReconnect(connIndex uint8) {
 	o.sendEvent(Event{Index: connIndex, EventType: Reconnecting})
 }
 
+func (o *Observer) sendUnregisteringEvent(connIndex uint8) {
+	o.sendEvent(Event{Index: connIndex, EventType: Unregistering})
+}
+
 func (o *Observer) SendDisconnect(connIndex uint8) {
 	o.sendEvent(Event{Index: connIndex, EventType: Disconnected})
 }

connection/observer_test.go

@@ -44,7 +44,7 @@ func TestRegisterServerLocation(t *testing.T) {
 }
 
 func TestObserverEventsDontBlock(t *testing.T) {
-	observer := NewObserver(&log, false)
+	observer := NewObserver(&log, &log, false)
 	var mu sync.Mutex
 	observer.RegisterSink(EventSinkFunc(func(_ Event) {
 		// callback will block if lock is already held
@@ -53,16 +53,34 @@ func TestObserverEventsDontBlock(t *testing.T) {
 	}))
 
 	timeout := time.AfterFunc(5*time.Second, func() {
-		mu.Unlock()  // release the callback on timer expiration
+		mu.Unlock() // release the callback on timer expiration
 		t.Fatal("observer is blocked")
 	})
 
-	mu.Lock()  // block the callback
-	for i := 0; i < 2 * observerChannelBufferSize; i++ {
-		observer.sendRegisteringEvent()
+	mu.Lock() // block the callback
+	for i := 0; i < 2*observerChannelBufferSize; i++ {
+		observer.sendRegisteringEvent(0)
 	}
 	if pending := timeout.Stop(); pending {
 		// release the callback if timer hasn't expired yet
 		mu.Unlock()
 	}
 }
+
+
+type eventCollectorSink struct {
+	observedEvents []Event
+	mu sync.Mutex
+}
+
+func (s *eventCollectorSink) OnTunnelEvent(event Event) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.observedEvents = append(s.observedEvents, event)
+}
+
+func (s *eventCollectorSink) assertSawEvent(t *testing.T, event Event) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	assert.Contains(t, s.observedEvents, event)
+}

connection/rpc.go

@@ -138,7 +138,7 @@ const (
 )
 
 func (h *h2muxConnection) registerTunnel(ctx context.Context, credentialSetter CredentialManager, classicTunnel *ClassicTunnelConfig, registrationOptions *tunnelpogs.RegistrationOptions) error {
-	h.observer.sendRegisteringEvent()
+	h.observer.sendRegisteringEvent(registrationOptions.ConnectionID)
 
 	stream, err := h.newRPCStream(ctx, register)
 	if err != nil {
@@ -209,9 +209,9 @@ func (h *h2muxConnection) processRegisterTunnelError(err tunnelpogs.TunnelRegist
 		return errDuplicationConnection
 	}
 	h.observer.metrics.regFail.WithLabelValues("server_error", string(name)).Inc()
-	return serverRegisterTunnelError{
-		cause:     err,
-		permanent: err.IsPermanent(),
+	return ServerRegisterTunnelError{
+		Cause:     err,
+		Permanent: err.IsPermanent(),
 	}
 }
 
@@ -272,17 +272,38 @@ func (h *h2muxConnection) logServerInfo(ctx context.Context, rpcClient *tunnelSe
 	return nil
 }
 
+func (h *h2muxConnection) registerNamedTunnel(
+	ctx context.Context,
+	namedTunnel *NamedTunnelConfig,
+	connOptions *tunnelpogs.ConnectionOptions,
+) error {
+	stream, err := h.newRPCStream(ctx, register)
+	if err != nil {
+		return err
+	}
+	rpcClient := h.newRPCClientFunc(ctx, stream, h.observer.log)
+	defer rpcClient.Close()
+
+	if err = rpcClient.RegisterConnection(ctx, namedTunnel, connOptions, h.connIndex, h.observer); err != nil {
+		return err
+	}
+	return nil
+}
+
 func (h *h2muxConnection) unregister(isNamedTunnel bool) {
+	h.observer.sendUnregisteringEvent(h.connIndex)
+
 	unregisterCtx, cancel := context.WithTimeout(context.Background(), h.config.GracePeriod)
 	defer cancel()
 
-	stream, err := h.newRPCStream(unregisterCtx, register)
+	stream, err := h.newRPCStream(unregisterCtx, unregister)
 	if err != nil {
 		return
 	}
+	defer stream.Close()
 
 	if isNamedTunnel {
-		rpcClient := newRegistrationRPCClient(unregisterCtx, stream, h.observer.log)
+		rpcClient := h.newRPCClientFunc(unregisterCtx, stream, h.observer.log)
 		defer rpcClient.Close()
 
 		rpcClient.GracefulShutdown(unregisterCtx, h.config.GracePeriod)
@@ -293,4 +314,6 @@ func (h *h2muxConnection) unregister(isNamedTunnel bool) {
 		// gracePeriod is encoded in int64 using capnproto
 		_ = rpcClient.client.UnregisterTunnel(unregisterCtx, h.config.GracePeriod.Nanoseconds())
 	}
+
+	h.observer.log.Info().Uint8(LogFieldConnIndex, h.connIndex).Msg("Unregistered tunnel connection")
 }

dev.Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.15.6 as builder
+FROM golang:1.15.7 as builder
 ENV GO111MODULE=on \
     CGO_ENABLED=0
 WORKDIR /go/src/github.com/cloudflare/cloudflared/

go.mod

@@ -60,7 +60,8 @@ require (
 	golang.org/x/net v0.0.0-20200904194848-62affa334b73
 	golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43 // indirect
 	golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208
-	golang.org/x/sys v0.0.0-20200909081042-eff7692f9009
+	golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4
+	golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf
 	google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d // indirect
 	google.golang.org/grpc v1.32.0 // indirect
 	gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect

go.sum

@@ -817,6 +817,12 @@ golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200909081042-eff7692f9009 h1:W0lCpv29Hv0UaM1LXb9QlBHLNP8UFfcKjblhVCWftOM=
 golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 h1:myAQVi0cGEoqQVR5POX+8RR2mrocKqNN1hmeMqhX27k=
+golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf h1:MZ2shdL+ZM/XzY3ZGOnh4Nlpnxz5GSOhOmtHo3iPU6M=
+golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

h2mux/muxmetrics.go

@@ -142,7 +142,7 @@ func (updater *muxMetricsUpdaterImpl) run(log *zerolog.Logger) error {
 	for {
 		select {
 		case <-updater.abortChan:
-			log.Info().Msgf("mux - metrics: Stopping mux metrics updater")
+			log.Debug().Msgf("mux - metrics: Stopping mux metrics updater")
 			return nil
 		case roundTripMeasurement := <-updater.updateRTTChan:
 			go updater.rttData.update(roundTripMeasurement)

hello/hello_test.go

@@ -27,7 +27,7 @@ func TestCreateTLSListenerOnlyHostSuccess(t *testing.T) {
 }
 
 func TestCreateTLSListenerOnlyPortSuccess(t *testing.T) {
-	listener, err := CreateTLSListener(":8888")
+	listener, err := CreateTLSListener("localhost:8888")
 	if err != nil {
 		t.Fatal(err)
 	}

logger/create.go

@@ -7,11 +7,13 @@ import (
 	"path"
 	"path/filepath"
 	"sync"
+	"time"
 
 	"github.com/mattn/go-colorable"
 	"github.com/rs/zerolog"
 	fallbacklog "github.com/rs/zerolog/log"
 	"github.com/urfave/cli/v2"
+	"golang.org/x/term"
 	"gopkg.in/natefinch/lumberjack.v2"
 )
 
@@ -29,8 +31,18 @@ const (
 
 	dirPermMode  = 0744 // rwxr--r--
 	filePermMode = 0644 // rw-r--r--
+
+	consoleTimeFormat = time.RFC3339
 )
 
+func init() {
+	zerolog.TimestampFunc = utcNow
+}
+
+func utcNow() time.Time {
+	return time.Now().UTC()
+}
+
 func fallbackLogger(err error) *zerolog.Logger {
 	failLog := fallbacklog.With().Logger()
 	fallbacklog.Error().Msgf("Falling back to a default logger due to logger setup failure: %s", err)
@@ -52,7 +64,6 @@ func (t resilientMultiWriter) Write(p []byte) (n int, err error) {
 	return len(p), nil
 }
 
-
 func newZerolog(loggerConfig *Config) *zerolog.Logger {
 	var writers []io.Writer
 
@@ -138,9 +149,11 @@ func Create(loggerConfig *Config) *zerolog.Logger {
 }
 
 func createConsoleLogger(config ConsoleConfig) io.Writer {
+	consoleOut := os.Stderr
 	return zerolog.ConsoleWriter{
-		Out:     colorable.NewColorableStderr(),
-		NoColor: config.noColor,
+		Out:     colorable.NewColorable(consoleOut),
+		NoColor: config.noColor || !term.IsTerminal(int(consoleOut.Fd())),
+		TimeFormat: consoleTimeFormat,
 	}
 }
 

metrics/metrics.go

@@ -10,6 +10,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/gorilla/mux"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/rs/zerolog"
@@ -21,16 +22,19 @@ const (
 	startupTime     = time.Millisecond * 500
 )
 
-func newMetricsHandler(readyServer *ReadyServer) *http.ServeMux {
-	mux := http.NewServeMux()
-	mux.Handle("/metrics", promhttp.Handler())
-	mux.HandleFunc("/healthcheck", func(w http.ResponseWriter, r *http.Request) {
+func newMetricsHandler(readyServer *ReadyServer) *mux.Router {
+	router := mux.NewRouter()
+	router.PathPrefix("/debug/").Handler(http.DefaultServeMux)
+
+	router.Handle("/metrics", promhttp.Handler())
+	router.HandleFunc("/healthcheck", func(w http.ResponseWriter, r *http.Request) {
 		_, _ = fmt.Fprintf(w, "OK\n")
 	})
 	if readyServer != nil {
-		mux.Handle("/ready", readyServer)
+		router.Handle("/ready", readyServer)
 	}
-	return mux
+
+	return router
 }
 
 func ServeMetrics(

metrics/readiness.go

@@ -32,7 +32,7 @@ func (rs *ReadyServer) OnTunnelEvent(c conn.Event) {
 		rs.Lock()
 		rs.isConnected[int(c.Index)] = true
 		rs.Unlock()
-	case conn.Disconnected, conn.Reconnecting, conn.RegisteringTunnel:
+	case conn.Disconnected, conn.Reconnecting, conn.RegisteringTunnel, conn.Unregistering:
 		rs.Lock()
 		rs.isConnected[int(c.Index)] = false
 		rs.Unlock()

metrics/readiness_test.go

@@ -87,7 +87,7 @@ func TestReadinessEventHandling(t *testing.T) {
 	})
 	code, ready = rs.makeResponse()
 	assert.EqualValues(t, http.StatusOK, code)
-	assert.EqualValues(t, 2,  ready)
+	assert.EqualValues(t, 2, ready)
 
 	// one reconnecting => still ok
 	rs.OnTunnelEvent(connection.Event{
@@ -98,6 +98,31 @@ func TestReadinessEventHandling(t *testing.T) {
 	assert.EqualValues(t, http.StatusOK, code)
 	assert.EqualValues(t, 1, ready)
 
+	// Regression test for TUN-3777
+	rs.OnTunnelEvent(connection.Event{
+		Index:     1,
+		EventType: connection.RegisteringTunnel,
+	})
+	code, ready = rs.makeResponse()
+	assert.NotEqualValues(t, http.StatusOK, code)
+	assert.Zero(t, ready)
+
+	// other connected then unregistered  => not ok
+	rs.OnTunnelEvent(connection.Event{
+		Index:     1,
+		EventType: connection.Connected,
+	})
+	code, ready = rs.makeResponse()
+	assert.EqualValues(t, http.StatusOK, code)
+	assert.EqualValues(t, 1, ready)
+	rs.OnTunnelEvent(connection.Event{
+		Index:     1,
+		EventType: connection.Unregistering,
+	})
+	code, ready = rs.makeResponse()
+	assert.NotEqualValues(t, http.StatusOK, code)
+	assert.Zero(t, ready)
+
 	// other disconnected => not ok
 	rs.OnTunnelEvent(connection.Event{
 		Index:     1,

origin/cloudflare_status_page_test.go

@@ -137,8 +137,7 @@ func TestParseStatusPage(t *testing.T) {
 								"page_id": "yh6f0r4529hb",
 								"group_id": "1km35smx8p41",
 								"group": false,
-								"only_show_if_degraded": false,
-								"automation_email": "component+g4tb35rs9yw7@notifications.statuspage.io"
+								"only_show_if_degraded": false
 							}
 						],
 						"impact": "minor"
@@ -208,8 +207,7 @@ func TestParseStatusPage(t *testing.T) {
 								"page_id": "yh6f0r4529hb",
 								"group_id": "1km35smx8p41",
 								"group": false,
-								"only_show_if_degraded": false,
-								"automation_email": "component+4c231tkdlpcl@notifications.statuspage.io"
+								"only_show_if_degraded": false
 							}
 						],
 						"impact": "minor"

origin/external_control.go

@@ -10,11 +10,11 @@ type ReconnectSignal struct {
 }
 
 // Error allows us to use ReconnectSignal as a special error to force connection abort
-func (r *ReconnectSignal) Error() string {
+func (r ReconnectSignal) Error() string {
 	return "reconnect signal"
 }
 
-func (r *ReconnectSignal) DelayBeforeReconnect() {
+func (r ReconnectSignal) DelayBeforeReconnect() {
 	if r.Delay > 0 {
 		time.Sleep(r.Delay)
 	}

origin/supervisor.go

@@ -3,6 +3,7 @@ package origin
 import (
 	"context"
 	"errors"
+	"fmt"
 	"net"
 	"time"
 
@@ -50,23 +51,31 @@ type Supervisor struct {
 	nextConnectedIndex  int
 	nextConnectedSignal chan struct{}
 
-	log *zerolog.Logger
+	log          *zerolog.Logger
+	logTransport *zerolog.Logger
 
 	reconnectCredentialManager *reconnectCredentialManager
 	useReconnectToken          bool
+
+	reconnectCh       chan ReconnectSignal
+	gracefulShutdownC <-chan struct{}
 }
 
+var errEarlyShutdown = errors.New("shutdown started")
+
 type tunnelError struct {
 	index int
 	addr  *net.TCPAddr
 	err   error
 }
 
-func NewSupervisor(config *TunnelConfig, cloudflaredUUID uuid.UUID) (*Supervisor, error) {
-	var (
-		edgeIPs *edgediscovery.Edge
-		err     error
-	)
+func NewSupervisor(config *TunnelConfig, reconnectCh chan ReconnectSignal, gracefulShutdownC <-chan struct{}) (*Supervisor, error) {
+	cloudflaredUUID, err := uuid.NewRandom()
+	if err != nil {
+		return nil, fmt.Errorf("failed to generate cloudflared instance ID: %w", err)
+	}
+
+	var edgeIPs *edgediscovery.Edge
 	if len(config.EdgeAddrs) > 0 {
 		edgeIPs, err = edgediscovery.StaticEdge(config.Log, config.EdgeAddrs)
 	} else {
@@ -88,13 +97,22 @@ func NewSupervisor(config *TunnelConfig, cloudflaredUUID uuid.UUID) (*Supervisor
 		tunnelErrors:               make(chan tunnelError),
 		tunnelsConnecting:          map[int]chan struct{}{},
 		log:                        config.Log,
+		logTransport:               config.LogTransport,
 		reconnectCredentialManager: newReconnectCredentialManager(connection.MetricsNamespace, connection.TunnelSubsystem, config.HAConnections),
 		useReconnectToken:          useReconnectToken,
+		reconnectCh:                reconnectCh,
+		gracefulShutdownC:          gracefulShutdownC,
 	}, nil
 }
 
-func (s *Supervisor) Run(ctx context.Context, connectedSignal *signal.Signal, reconnectCh chan ReconnectSignal) error {
-	if err := s.initialize(ctx, connectedSignal, reconnectCh); err != nil {
+func (s *Supervisor) Run(
+	ctx context.Context,
+	connectedSignal *signal.Signal,
+) error {
+	if err := s.initialize(ctx, connectedSignal); err != nil {
+		if err == errEarlyShutdown {
+			return nil
+		}
 		return err
 	}
 	var tunnelsWaiting []int
@@ -117,6 +135,7 @@ func (s *Supervisor) Run(ctx context.Context, connectedSignal *signal.Signal, re
 		}
 	}
 
+	shuttingDown := false
 	for {
 		select {
 		// Context cancelled
@@ -130,8 +149,8 @@ func (s *Supervisor) Run(ctx context.Context, connectedSignal *signal.Signal, re
 		// (note that this may also be caused by context cancellation)
 		case tunnelError := <-s.tunnelErrors:
 			tunnelsActive--
-			if tunnelError.err != nil {
-				s.log.Err(tunnelError.err).Msg("supervisor: Tunnel disconnected")
+			if tunnelError.err != nil && !shuttingDown {
+				s.log.Err(tunnelError.err).Int(connection.LogFieldConnIndex, tunnelError.index).Msg("Connection terminated")
 				tunnelsWaiting = append(tunnelsWaiting, tunnelError.index)
 				s.waitForNextTunnel(tunnelError.index)
 
@@ -139,14 +158,16 @@ func (s *Supervisor) Run(ctx context.Context, connectedSignal *signal.Signal, re
 					backoffTimer = backoff.BackoffTimer()
 				}
 
-				// Previously we'd mark the edge address as bad here, but now we'll just silently use
-				// another.
+				// Previously we'd mark the edge address as bad here, but now we'll just silently use another.
+			} else if tunnelsActive == 0 {
+				// all connected tunnels exited gracefully, no more work to do
+				return nil
 			}
 		// Backoff was set and its timer expired
 		case <-backoffTimer:
 			backoffTimer = nil
 			for _, index := range tunnelsWaiting {
-				go s.startTunnel(ctx, index, s.newConnectedTunnelSignal(index), reconnectCh)
+				go s.startTunnel(ctx, index, s.newConnectedTunnelSignal(index))
 			}
 			tunnelsActive += len(tunnelsWaiting)
 			tunnelsWaiting = nil
@@ -166,31 +187,38 @@ func (s *Supervisor) Run(ctx context.Context, connectedSignal *signal.Signal, re
 				// No more tunnels outstanding, clear backoff timer
 				backoff.SetGracePeriod()
 			}
+		case <-s.gracefulShutdownC:
+			shuttingDown = true
 		}
 	}
 }
 
 // Returns nil if initialization succeeded, else the initialization error.
-func (s *Supervisor) initialize(ctx context.Context, connectedSignal *signal.Signal, reconnectCh chan ReconnectSignal) error {
-	availableAddrs := int(s.edgeIPs.AvailableAddrs())
+func (s *Supervisor) initialize(
+	ctx context.Context,
+	connectedSignal *signal.Signal,
+) error {
+	availableAddrs := s.edgeIPs.AvailableAddrs()
 	if s.config.HAConnections > availableAddrs {
 		s.log.Info().Msgf("You requested %d HA connections but I can give you at most %d.", s.config.HAConnections, availableAddrs)
 		s.config.HAConnections = availableAddrs
 	}
 
-	go s.startFirstTunnel(ctx, connectedSignal, reconnectCh)
+	go s.startFirstTunnel(ctx, connectedSignal)
 	select {
 	case <-ctx.Done():
 		<-s.tunnelErrors
 		return ctx.Err()
 	case tunnelError := <-s.tunnelErrors:
 		return tunnelError.err
+	case <-s.gracefulShutdownC:
+		return errEarlyShutdown
 	case <-connectedSignal.Wait():
 	}
 	// At least one successful connection, so start the rest
 	for i := 1; i < s.config.HAConnections; i++ {
 		ch := signal.New(make(chan struct{}))
-		go s.startTunnel(ctx, i, ch, reconnectCh)
+		go s.startTunnel(ctx, i, ch)
 		time.Sleep(registrationInterval)
 	}
 	return nil
@@ -198,7 +226,10 @@ func (s *Supervisor) initialize(ctx context.Context, connectedSignal *signal.Sig
 
 // startTunnel starts the first tunnel connection. The resulting error will be sent on
 // s.tunnelErrors. It will send a signal via connectedSignal if registration succeed
-func (s *Supervisor) startFirstTunnel(ctx context.Context, connectedSignal *signal.Signal, reconnectCh chan ReconnectSignal) {
+func (s *Supervisor) startFirstTunnel(
+	ctx context.Context,
+	connectedSignal *signal.Signal,
+) {
 	var (
 		addr *net.TCPAddr
 		err  error
@@ -221,7 +252,8 @@ func (s *Supervisor) startFirstTunnel(ctx context.Context, connectedSignal *sign
 		firstConnIndex,
 		connectedSignal,
 		s.cloudflaredUUID,
-		reconnectCh,
+		s.reconnectCh,
+		s.gracefulShutdownC,
 	)
 	// If the first tunnel disconnects, keep restarting it.
 	edgeErrors := 0
@@ -253,14 +285,19 @@ func (s *Supervisor) startFirstTunnel(ctx context.Context, connectedSignal *sign
 			firstConnIndex,
 			connectedSignal,
 			s.cloudflaredUUID,
-			reconnectCh,
+			s.reconnectCh,
+			s.gracefulShutdownC,
 		)
 	}
 }
 
 // startTunnel starts a new tunnel connection. The resulting error will be sent on
 // s.tunnelErrors.
-func (s *Supervisor) startTunnel(ctx context.Context, index int, connectedSignal *signal.Signal, reconnectCh chan ReconnectSignal) {
+func (s *Supervisor) startTunnel(
+	ctx context.Context,
+	index int,
+	connectedSignal *signal.Signal,
+) {
 	var (
 		addr *net.TCPAddr
 		err  error
@@ -281,7 +318,8 @@ func (s *Supervisor) startTunnel(ctx context.Context, index int, connectedSignal
 		uint8(index),
 		connectedSignal,
 		s.cloudflaredUUID,
-		reconnectCh,
+		s.reconnectCh,
+		s.gracefulShutdownC,
 	)
 }
 
@@ -324,7 +362,7 @@ func (s *Supervisor) authenticate(ctx context.Context, numPreviousAttempts int)
 		// This callback is invoked by h2mux when the edge initiates a stream.
 		return nil // noop
 	})
-	muxerConfig := s.config.MuxerConfig.H2MuxerConfig(handler, s.log)
+	muxerConfig := s.config.MuxerConfig.H2MuxerConfig(handler, s.logTransport)
 	muxer, err := h2mux.Handshake(edgeConn, edgeConn, *muxerConfig, h2mux.ActiveStreams)
 	if err != nil {
 		return nil, err

origin/tunnel.go

@@ -51,6 +51,7 @@ type TunnelConfig struct {
 	LBPool           string
 	Tags             []tunnelpogs.Tag
 	Log              *zerolog.Logger
+	LogTransport     *zerolog.Logger
 	Observer         *connection.Observer
 	ReportedVersion  string
 	Retries          uint
@@ -63,31 +64,6 @@ type TunnelConfig struct {
 	EdgeTLSConfigs   map[connection.Protocol]*tls.Config
 }
 
-type muxerShutdownError struct{}
-
-func (e muxerShutdownError) Error() string {
-	return "muxer shutdown"
-}
-
-// RegisterTunnel error from server
-type serverRegisterTunnelError struct {
-	cause     error
-	permanent bool
-}
-
-func (e serverRegisterTunnelError) Error() string {
-	return e.cause.Error()
-}
-
-// RegisterTunnel error from client
-type clientRegisterTunnelError struct {
-	cause error
-}
-
-func (e clientRegisterTunnelError) Error() string {
-	return e.cause.Error()
-}
-
 func (c *TunnelConfig) RegistrationOptions(connectionID uint8, OriginLocalIP string, uuid uuid.UUID) *tunnelpogs.RegistrationOptions {
 	policy := tunnelrpc.ExistingTunnelPolicy_balance
 	if c.HAConnections <= 1 && c.LBPool == "" {
@@ -132,12 +108,18 @@ func (c *TunnelConfig) SupportedFeatures() []string {
 	return features
 }
 
-func StartTunnelDaemon(ctx context.Context, config *TunnelConfig, connectedSignal *signal.Signal, cloudflaredID uuid.UUID, reconnectCh chan ReconnectSignal) error {
-	s, err := NewSupervisor(config, cloudflaredID)
+func StartTunnelDaemon(
+	ctx context.Context,
+	config *TunnelConfig,
+	connectedSignal *signal.Signal,
+	reconnectCh chan ReconnectSignal,
+	graceShutdownC <-chan struct{},
+) error {
+	s, err := NewSupervisor(config, reconnectCh, graceShutdownC)
 	if err != nil {
 		return err
 	}
-	return s.Run(ctx, connectedSignal, reconnectCh)
+	return s.Run(ctx, connectedSignal)
 }
 
 func ServeTunnelLoop(
@@ -149,13 +131,14 @@ func ServeTunnelLoop(
 	connectedSignal *signal.Signal,
 	cloudflaredUUID uuid.UUID,
 	reconnectCh chan ReconnectSignal,
+	gracefulShutdownC <-chan struct{},
 ) error {
 	haConnections.Inc()
 	defer haConnections.Dec()
 
 	connLog := config.Log.With().Uint8(connection.LogFieldConnIndex, connIndex).Logger()
 
-	protocallFallback := &protocallFallback{
+	protocolFallback := &protocolFallback{
 		BackoffHandler{MaxRetries: config.Retries},
 		config.ProtocolSelector.Current(),
 		false,
@@ -179,94 +162,99 @@ func ServeTunnelLoop(
 			addr,
 			connIndex,
 			connectedFuse,
-			protocallFallback,
+			protocolFallback,
 			cloudflaredUUID,
 			reconnectCh,
-			protocallFallback.protocol,
+			protocolFallback.protocol,
+			gracefulShutdownC,
 		)
 		if !recoverable {
 			return err
 		}
 
-		err = waitForBackoff(ctx, &connLog, protocallFallback, config, connIndex, err)
-		if err != nil {
+		config.Observer.SendReconnect(connIndex)
+
+		duration, ok := protocolFallback.GetBackoffDuration(ctx)
+		if !ok {
 			return err
 		}
+		connLog.Info().Msgf("Retrying connection in %s seconds", duration)
+
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-gracefulShutdownC:
+			return nil
+		case <-protocolFallback.BackoffTimer():
+			if !selectNextProtocol(&connLog, protocolFallback, config.ProtocolSelector) {
+				return err
+			}
+		}
 	}
 }
 
-// protocallFallback is a wrapper around backoffHandler that will try fallback option when backoff reaches
+// protocolFallback is a wrapper around backoffHandler that will try fallback option when backoff reaches
 // max retries
-type protocallFallback struct {
+type protocolFallback struct {
 	BackoffHandler
 	protocol   connection.Protocol
 	inFallback bool
 }
 
-func (pf *protocallFallback) reset() {
+func (pf *protocolFallback) reset() {
 	pf.resetNow()
 	pf.inFallback = false
 }
 
-func (pf *protocallFallback) fallback(fallback connection.Protocol) {
+func (pf *protocolFallback) fallback(fallback connection.Protocol) {
 	pf.resetNow()
 	pf.protocol = fallback
 	pf.inFallback = true
 }
 
-// Expect err to always be non nil
-func waitForBackoff(
-	ctx context.Context,
-	log *zerolog.Logger,
-	protobackoff *protocallFallback,
-	config *TunnelConfig,
-	connIndex uint8,
-	err error,
-) error {
-	duration, ok := protobackoff.GetBackoffDuration(ctx)
-	if !ok {
-		return err
-	}
-
-	config.Observer.SendReconnect(connIndex)
-	log.Info().
-		Err(err).
-		Msgf("Retrying connection in %s seconds", duration)
-	protobackoff.Backoff(ctx)
-
-	if protobackoff.ReachedMaxRetries() {
-		fallback, hasFallback := config.ProtocolSelector.Fallback()
+// selectNextProtocol picks connection protocol for the next retry iteration,
+// returns true if it was able to pick the protocol, false if we are out of options and should stop retrying
+func selectNextProtocol(
+	connLog *zerolog.Logger,
+	protocolBackoff *protocolFallback,
+	selector connection.ProtocolSelector,
+) bool {
+	if protocolBackoff.ReachedMaxRetries() {
+		fallback, hasFallback := selector.Fallback()
 		if !hasFallback {
-			return err
+			return false
 		}
 		// Already using fallback protocol, no point to retry
-		if protobackoff.protocol == fallback {
-			return err
+		if protocolBackoff.protocol == fallback {
+			return false
 		}
-		log.Info().Msgf("Fallback to use %s", fallback)
-		protobackoff.fallback(fallback)
-	} else if !protobackoff.inFallback {
-		current := config.ProtocolSelector.Current()
-		if protobackoff.protocol != current {
-			protobackoff.protocol = current
-			config.Log.Info().Msgf("Change protocol to %s", current)
+		connLog.Info().Msgf("Switching to fallback protocol %s", fallback)
+		protocolBackoff.fallback(fallback)
+	} else if !protocolBackoff.inFallback {
+		current := selector.Current()
+		if protocolBackoff.protocol != current {
+			protocolBackoff.protocol = current
+			connLog.Info().Msgf("Changing protocol to %s", current)
 		}
 	}
-	return nil
+	return true
 }
 
+// ServeTunnel runs a single tunnel connection, returns nil on graceful shutdown,
+// on error returns a flag indicating if error can be retried
 func ServeTunnel(
 	ctx context.Context,
-	log *zerolog.Logger,
+	connLong *zerolog.Logger,
 	credentialManager *reconnectCredentialManager,
 	config *TunnelConfig,
 	addr *net.TCPAddr,
 	connIndex uint8,
 	fuse *h2mux.BooleanFuse,
-	backoff *protocallFallback,
+	backoff *protocolFallback,
 	cloudflaredUUID uuid.UUID,
 	reconnectCh chan ReconnectSignal,
 	protocol connection.Protocol,
+	gracefulShutdownC <-chan struct{},
 ) (err error, recoverable bool) {
 	// Treat panics as recoverable errors
 	defer func() {
@@ -291,26 +279,78 @@ func ServeTunnel(
 		fuse:    fuse,
 		backoff: backoff,
 	}
+
 	if protocol == connection.HTTP2 {
 		connOptions := config.ConnectionOptions(edgeConn.LocalAddr().String(), uint8(backoff.retries))
-		return ServeHTTP2(ctx, log, config, edgeConn, connOptions, connIndex, connectedFuse, reconnectCh)
+		err = ServeHTTP2(
+			ctx,
+			connLong,
+			config,
+			edgeConn,
+			connOptions,
+			connIndex,
+			connectedFuse,
+			reconnectCh,
+			gracefulShutdownC,
+		)
+	} else {
+		err = ServeH2mux(
+			ctx,
+			connLong,
+			credentialManager,
+			config,
+			edgeConn,
+			connIndex,
+			connectedFuse,
+			cloudflaredUUID,
+			reconnectCh,
+			gracefulShutdownC,
+		)
 	}
-	return ServeH2mux(
-		ctx,
-		log,
-		credentialManager,
-		config,
-		edgeConn,
-		connIndex,
-		connectedFuse,
-		cloudflaredUUID,
-		reconnectCh,
-	)
+
+	if err != nil {
+		switch err := err.(type) {
+		case connection.DupConnRegisterTunnelError:
+			// don't retry this connection anymore, let supervisor pick a new address
+			return err, false
+		case connection.ServerRegisterTunnelError:
+			connLong.Err(err).Msg("Register tunnel error from server side")
+			// Don't send registration error return from server to Sentry. They are
+			// logged on server side
+			if incidents := config.IncidentLookup.ActiveIncidents(); len(incidents) > 0 {
+				connLong.Error().Msg(activeIncidentsMsg(incidents))
+			}
+			return err.Cause, !err.Permanent
+		case ReconnectSignal:
+			connLong.Info().
+				Uint8(connection.LogFieldConnIndex, connIndex).
+				Msgf("Restarting connection due to reconnect signal in %s", err.Delay)
+			err.DelayBeforeReconnect()
+			return err, true
+		default:
+			if err == context.Canceled {
+				connLong.Debug().Err(err).Msgf("Serve tunnel error")
+				return err, false
+			}
+			connLong.Err(err).Msgf("Serve tunnel error")
+			_, permanent := err.(unrecoverableError)
+			return err, !permanent
+		}
+	}
+	return nil, false
+}
+
+type unrecoverableError struct {
+	err error
+}
+
+func (r unrecoverableError) Error() string {
+	return r.err.Error()
 }
 
 func ServeH2mux(
 	ctx context.Context,
-	log *zerolog.Logger,
+	connLog *zerolog.Logger,
 	credentialManager *reconnectCredentialManager,
 	config *TunnelConfig,
 	edgeConn net.Conn,
@@ -318,8 +358,9 @@ func ServeH2mux(
 	connectedFuse *connectedFuse,
 	cloudflaredUUID uuid.UUID,
 	reconnectCh chan ReconnectSignal,
-) (err error, recoverable bool) {
-	config.Log.Debug().Msgf("Connecting via h2mux")
+	gracefulShutdownC <-chan struct{},
+) error {
+	connLog.Debug().Msgf("Connecting via h2mux")
 	// Returns error from parsing the origin URL or handshake errors
 	handler, err, recoverable := connection.NewH2muxConnection(
 		config.ConnectionConfig,
@@ -327,74 +368,46 @@ func ServeH2mux(
 		edgeConn,
 		connIndex,
 		config.Observer,
+		gracefulShutdownC,
 	)
 	if err != nil {
-		return err, recoverable
+		if !recoverable {
+			return unrecoverableError{err}
+		}
+		return err
 	}
 
 	errGroup, serveCtx := errgroup.WithContext(ctx)
 
-	errGroup.Go(func() (err error) {
+	errGroup.Go(func() error {
 		if config.NamedTunnel != nil {
 			connOptions := config.ConnectionOptions(edgeConn.LocalAddr().String(), uint8(connectedFuse.backoff.retries))
-			return handler.ServeNamedTunnel(serveCtx, config.NamedTunnel, credentialManager, connOptions, connectedFuse)
+			return handler.ServeNamedTunnel(serveCtx, config.NamedTunnel, connOptions, connectedFuse)
 		}
 		registrationOptions := config.RegistrationOptions(connIndex, edgeConn.LocalAddr().String(), cloudflaredUUID)
 		return handler.ServeClassicTunnel(serveCtx, config.ClassicTunnel, credentialManager, registrationOptions, connectedFuse)
 	})
 
-	errGroup.Go(listenReconnect(serveCtx, reconnectCh))
+	errGroup.Go(func() error {
+		return listenReconnect(serveCtx, reconnectCh, gracefulShutdownC)
+	})
 
-	err = errGroup.Wait()
-	if err != nil {
-		switch err := err.(type) {
-		case *connection.DupConnRegisterTunnelError:
-			// don't retry this connection anymore, let supervisor pick new a address
-			return err, false
-		case *serverRegisterTunnelError:
-			log.Err(err).Msg("Register tunnel error from server side")
-			// Don't send registration error return from server to Sentry. They are
-			// logged on server side
-			if incidents := config.IncidentLookup.ActiveIncidents(); len(incidents) > 0 {
-				log.Error().Msg(activeIncidentsMsg(incidents))
-			}
-			return err.cause, !err.permanent
-		case *clientRegisterTunnelError:
-			log.Err(err).Msg("Register tunnel error on client side")
-			return err, true
-		case *muxerShutdownError:
-			log.Info().Msg("Muxer shutdown")
-			return err, true
-		case *ReconnectSignal:
-			log.Info().
-				Uint8(connection.LogFieldConnIndex, connIndex).
-				Msgf("Restarting connection due to reconnect signal in %s", err.Delay)
-			err.DelayBeforeReconnect()
-			return err, true
-		default:
-			if err == context.Canceled {
-				log.Debug().Err(err).Msgf("Serve tunnel error")
-				return err, false
-			}
-			log.Err(err).Msgf("Serve tunnel error")
-			return err, true
-		}
-	}
-	return nil, true
+	return errGroup.Wait()
 }
 
 func ServeHTTP2(
 	ctx context.Context,
-	log *zerolog.Logger,
+	connLog *zerolog.Logger,
 	config *TunnelConfig,
 	tlsServerConn net.Conn,
 	connOptions *tunnelpogs.ConnectionOptions,
 	connIndex uint8,
 	connectedFuse connection.ConnectedFuse,
 	reconnectCh chan ReconnectSignal,
-) (err error, recoverable bool) {
-	log.Debug().Msgf("Connecting via http2")
-	server := connection.NewHTTP2Connection(
+	gracefulShutdownC <-chan struct{},
+) error {
+	connLog.Debug().Msgf("Connecting via http2")
+	h2conn := connection.NewHTTP2Connection(
 		tlsServerConn,
 		config.ConnectionConfig,
 		config.NamedTunnel,
@@ -402,37 +415,40 @@ func ServeHTTP2(
 		config.Observer,
 		connIndex,
 		connectedFuse,
+		gracefulShutdownC,
 	)
 
 	errGroup, serveCtx := errgroup.WithContext(ctx)
 	errGroup.Go(func() error {
-		server.Serve(serveCtx)
-		return fmt.Errorf("connection with edge closed")
+		return h2conn.Serve(serveCtx)
 	})
 
-	errGroup.Go(listenReconnect(serveCtx, reconnectCh))
+	errGroup.Go(func() error {
+		err := listenReconnect(serveCtx, reconnectCh, gracefulShutdownC)
+		if err != nil {
+			// forcefully break the connection (this is only used for testing)
+			_ = tlsServerConn.Close()
+		}
+		return err
+	})
 
-	err = errGroup.Wait()
-	if err != nil {
-		return err, true
-	}
-	return nil, false
+	return errGroup.Wait()
 }
 
-func listenReconnect(ctx context.Context, reconnectCh <-chan ReconnectSignal) func() error {
-	return func() error {
-		select {
-		case reconnect := <-reconnectCh:
-			return &reconnect
-		case <-ctx.Done():
-			return nil
-		}
+func listenReconnect(ctx context.Context, reconnectCh <-chan ReconnectSignal, gracefulShutdownCh <-chan struct{}) error {
+	select {
+	case reconnect := <-reconnectCh:
+		return reconnect
+	case <-gracefulShutdownCh:
+		return nil
+	case <-ctx.Done():
+		return nil
 	}
 }
 
 type connectedFuse struct {
 	fuse    *h2mux.BooleanFuse
-	backoff *protocallFallback
+	backoff *protocolFallback
 }
 
 func (cf *connectedFuse) Connected() {

origin/tunnel_test.go

@@ -1,8 +1,6 @@
 package origin
 
 import (
-	"context"
-	"fmt"
 	"testing"
 	"time"
 
@@ -25,13 +23,13 @@ func (dmf *dynamicMockFetcher) fetch() connection.PercentageFetcher {
 		return dmf.percentage, nil
 	}
 }
+
 func TestWaitForBackoffFallback(t *testing.T) {
 	maxRetries := uint(3)
 	backoff := BackoffHandler{
 		MaxRetries: maxRetries,
 		BaseTime:   time.Millisecond * 10,
 	}
-	ctx := context.Background()
 	log := zerolog.Nop()
 	resolveTTL := time.Duration(0)
 	namedTunnel := &connection.NamedTunnelConfig{
@@ -50,17 +48,11 @@ func TestWaitForBackoffFallback(t *testing.T) {
 		&log,
 	)
 	assert.NoError(t, err)
-	config := &TunnelConfig{
-		Log:              &log,
-		ProtocolSelector: protocolSelector,
-		Observer:         connection.NewObserver(nil, false),
-	}
-	connIndex := uint8(1)
 
 	initProtocol := protocolSelector.Current()
 	assert.Equal(t, connection.HTTP2, initProtocol)
 
-	protocallFallback := &protocallFallback{
+	protocolFallback := &protocolFallback{
 		backoff,
 		initProtocol,
 		false,
@@ -68,29 +60,33 @@ func TestWaitForBackoffFallback(t *testing.T) {
 
 	// Retry #0 and #1. At retry #2, we switch protocol, so the fallback loop has one more retry than this
 	for i := 0; i < int(maxRetries-1); i++ {
-		err := waitForBackoff(ctx, &log, protocallFallback, config, connIndex, fmt.Errorf("some error"))
-		assert.NoError(t, err)
-		assert.Equal(t, initProtocol, protocallFallback.protocol)
+		protocolFallback.BackoffTimer() // simulate retry
+		ok := selectNextProtocol(&log, protocolFallback, protocolSelector)
+		assert.True(t, ok)
+		assert.Equal(t, initProtocol, protocolFallback.protocol)
 	}
 
 	// Retry fallback protocol
 	for i := 0; i < int(maxRetries); i++ {
-		err := waitForBackoff(ctx, &log, protocallFallback, config, connIndex, fmt.Errorf("some error"))
-		assert.NoError(t, err)
+		protocolFallback.BackoffTimer() // simulate retry
+		ok := selectNextProtocol(&log, protocolFallback, protocolSelector)
+		assert.True(t, ok)
 		fallback, ok := protocolSelector.Fallback()
 		assert.True(t, ok)
-		assert.Equal(t, fallback, protocallFallback.protocol)
+		assert.Equal(t, fallback, protocolFallback.protocol)
 	}
 
 	currentGlobalProtocol := protocolSelector.Current()
 	assert.Equal(t, initProtocol, currentGlobalProtocol)
 
 	// No protocol to fallback, return error
-	err = waitForBackoff(ctx, &log, protocallFallback, config, connIndex, fmt.Errorf("some error"))
-	assert.Error(t, err)
+	protocolFallback.BackoffTimer() // simulate retry
+	ok := selectNextProtocol(&log, protocolFallback, protocolSelector)
+	assert.False(t, ok)
 
-	protocallFallback.reset()
-	err = waitForBackoff(ctx, &log, protocallFallback, config, connIndex, fmt.Errorf("new error"))
-	assert.NoError(t, err)
-	assert.Equal(t, initProtocol, protocallFallback.protocol)
+	protocolFallback.reset()
+	protocolFallback.BackoffTimer() // simulate retry
+	ok = selectNextProtocol(&log, protocolFallback, protocolSelector)
+	assert.True(t, ok)
+	assert.Equal(t, initProtocol, protocolFallback.protocol)
 }

socks/connection_handler_test.go

@@ -42,7 +42,7 @@ func startTestServer(t *testing.T, httpHandler func(w http.ResponseWriter, r *ht
 	// create a socks server
 	requestHandler := NewRequestHandler(NewNetDialer())
 	socksServer := NewConnectionHandler(requestHandler)
-	listener, err := net.Listen("tcp", ":8086")
+	listener, err := net.Listen("tcp", "localhost:8086")
 	assert.NoError(t, err)
 
 	go func() {
@@ -58,7 +58,7 @@ func startTestServer(t *testing.T, httpHandler func(w http.ResponseWriter, r *ht
 	mux.HandleFunc("/", httpHandler)
 
 	// start the servers
-	go http.ListenAndServe(":8085", mux)
+	go http.ListenAndServe("localhost:8085", mux)
 }
 
 func respondWithJSON(w http.ResponseWriter, v interface{}, status int) {

vendor/golang.org/x/sys/cpu/asm_aix_ppc64.s

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build !gccgo
+// +build gc
 
 #include "textflag.h"
 

vendor/golang.org/x/sys/cpu/cpu.go

@@ -29,26 +29,46 @@ type CacheLinePad struct{ _ [cacheLineSize]byte }
 // and HasAVX2 are only set if the OS supports XMM and YMM
 // registers in addition to the CPUID feature bit being set.
 var X86 struct {
-	_            CacheLinePad
-	HasAES       bool // AES hardware implementation (AES NI)
-	HasADX       bool // Multi-precision add-carry instruction extensions
-	HasAVX       bool // Advanced vector extension
-	HasAVX2      bool // Advanced vector extension 2
-	HasBMI1      bool // Bit manipulation instruction set 1
-	HasBMI2      bool // Bit manipulation instruction set 2
-	HasERMS      bool // Enhanced REP for MOVSB and STOSB
-	HasFMA       bool // Fused-multiply-add instructions
-	HasOSXSAVE   bool // OS supports XSAVE/XRESTOR for saving/restoring XMM registers.
-	HasPCLMULQDQ bool // PCLMULQDQ instruction - most often used for AES-GCM
-	HasPOPCNT    bool // Hamming weight instruction POPCNT.
-	HasRDRAND    bool // RDRAND instruction (on-chip random number generator)
-	HasRDSEED    bool // RDSEED instruction (on-chip random number generator)
-	HasSSE2      bool // Streaming SIMD extension 2 (always available on amd64)
-	HasSSE3      bool // Streaming SIMD extension 3
-	HasSSSE3     bool // Supplemental streaming SIMD extension 3
-	HasSSE41     bool // Streaming SIMD extension 4 and 4.1
-	HasSSE42     bool // Streaming SIMD extension 4 and 4.2
-	_            CacheLinePad
+	_                   CacheLinePad
+	HasAES              bool // AES hardware implementation (AES NI)
+	HasADX              bool // Multi-precision add-carry instruction extensions
+	HasAVX              bool // Advanced vector extension
+	HasAVX2             bool // Advanced vector extension 2
+	HasAVX512           bool // Advanced vector extension 512
+	HasAVX512F          bool // Advanced vector extension 512 Foundation Instructions
+	HasAVX512CD         bool // Advanced vector extension 512 Conflict Detection Instructions
+	HasAVX512ER         bool // Advanced vector extension 512 Exponential and Reciprocal Instructions
+	HasAVX512PF         bool // Advanced vector extension 512 Prefetch Instructions Instructions
+	HasAVX512VL         bool // Advanced vector extension 512 Vector Length Extensions
+	HasAVX512BW         bool // Advanced vector extension 512 Byte and Word Instructions
+	HasAVX512DQ         bool // Advanced vector extension 512 Doubleword and Quadword Instructions
+	HasAVX512IFMA       bool // Advanced vector extension 512 Integer Fused Multiply Add
+	HasAVX512VBMI       bool // Advanced vector extension 512 Vector Byte Manipulation Instructions
+	HasAVX5124VNNIW     bool // Advanced vector extension 512 Vector Neural Network Instructions Word variable precision
+	HasAVX5124FMAPS     bool // Advanced vector extension 512 Fused Multiply Accumulation Packed Single precision
+	HasAVX512VPOPCNTDQ  bool // Advanced vector extension 512 Double and quad word population count instructions
+	HasAVX512VPCLMULQDQ bool // Advanced vector extension 512 Vector carry-less multiply operations
+	HasAVX512VNNI       bool // Advanced vector extension 512 Vector Neural Network Instructions
+	HasAVX512GFNI       bool // Advanced vector extension 512 Galois field New Instructions
+	HasAVX512VAES       bool // Advanced vector extension 512 Vector AES instructions
+	HasAVX512VBMI2      bool // Advanced vector extension 512 Vector Byte Manipulation Instructions 2
+	HasAVX512BITALG     bool // Advanced vector extension 512 Bit Algorithms
+	HasAVX512BF16       bool // Advanced vector extension 512 BFloat16 Instructions
+	HasBMI1             bool // Bit manipulation instruction set 1
+	HasBMI2             bool // Bit manipulation instruction set 2
+	HasERMS             bool // Enhanced REP for MOVSB and STOSB
+	HasFMA              bool // Fused-multiply-add instructions
+	HasOSXSAVE          bool // OS supports XSAVE/XRESTOR for saving/restoring XMM registers.
+	HasPCLMULQDQ        bool // PCLMULQDQ instruction - most often used for AES-GCM
+	HasPOPCNT           bool // Hamming weight instruction POPCNT.
+	HasRDRAND           bool // RDRAND instruction (on-chip random number generator)
+	HasRDSEED           bool // RDSEED instruction (on-chip random number generator)
+	HasSSE2             bool // Streaming SIMD extension 2 (always available on amd64)
+	HasSSE3             bool // Streaming SIMD extension 3
+	HasSSSE3            bool // Supplemental streaming SIMD extension 3
+	HasSSE41            bool // Streaming SIMD extension 4 and 4.1
+	HasSSE42            bool // Streaming SIMD extension 4 and 4.2
+	_                   CacheLinePad
 }
 
 // ARM64 contains the supported CPU features of the

vendor/golang.org/x/sys/cpu/cpu_arm64.go

@@ -39,31 +39,34 @@ func initOptions() {
 
 func archInit() {
 	switch runtime.GOOS {
-	case "android", "darwin", "netbsd":
-		// Android and iOS don't seem to allow reading these registers.
-		//
-		// NetBSD:
-		// ID_AA64ISAR0_EL1 is a privileged register and cannot be read from EL0.
-		// It can be read via sysctl(3). Example for future implementers:
-		// https://nxr.netbsd.org/xref/src/usr.sbin/cpuctl/arch/aarch64.c
-		//
-		// Fake the minimal features expected by
-		// TestARM64minimalFeatures.
-		ARM64.HasASIMD = true
-		ARM64.HasFP = true
-	case "linux":
+	case "freebsd":
+		readARM64Registers()
+	case "linux", "netbsd":
 		doinit()
 	default:
-		readARM64Registers()
+		// Most platforms don't seem to allow reading these registers.
+		//
+		// OpenBSD:
+		// See https://golang.org/issue/31746
+		setMinimalFeatures()
 	}
 }
 
+// setMinimalFeatures fakes the minimal ARM64 features expected by
+// TestARM64minimalFeatures.
+func setMinimalFeatures() {
+	ARM64.HasASIMD = true
+	ARM64.HasFP = true
+}
+
 func readARM64Registers() {
 	Initialized = true
 
-	// ID_AA64ISAR0_EL1
-	isar0 := getisar0()
+	parseARM64SystemRegisters(getisar0(), getisar1(), getpfr0())
+}
 
+func parseARM64SystemRegisters(isar0, isar1, pfr0 uint64) {
+	// ID_AA64ISAR0_EL1
 	switch extractBits(isar0, 4, 7) {
 	case 1:
 		ARM64.HasAES = true
@@ -121,8 +124,6 @@ func readARM64Registers() {
 	}
 
 	// ID_AA64ISAR1_EL1
-	isar1 := getisar1()
-
 	switch extractBits(isar1, 0, 3) {
 	case 1:
 		ARM64.HasDCPOP = true
@@ -144,8 +145,6 @@ func readARM64Registers() {
 	}
 
 	// ID_AA64PFR0_EL1
-	pfr0 := getpfr0()
-
 	switch extractBits(pfr0, 16, 19) {
 	case 0:
 		ARM64.HasFP = true

vendor/golang.org/x/sys/cpu/cpu_arm64.s

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build !gccgo
+// +build gc
 
 #include "textflag.h"
 

vendor/golang.org/x/sys/cpu/cpu_gc_arm64.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build !gccgo
+// +build gc
 
 package cpu
 

vendor/golang.org/x/sys/cpu/cpu_gc_s390x.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build !gccgo
+// +build gc
 
 package cpu
 

vendor/golang.org/x/sys/cpu/cpu_gc_x86.go

@@ -3,7 +3,7 @@
 // license that can be found in the LICENSE file.
 
 // +build 386 amd64 amd64p32
-// +build !gccgo
+// +build gc
 
 package cpu
 

vendor/golang.org/x/sys/cpu/cpu_linux_s390x.go

@@ -17,86 +17,7 @@ const (
 	hwcap_VXE    = 8192
 )
 
-// bitIsSet reports whether the bit at index is set. The bit index
-// is in big endian order, so bit index 0 is the leftmost bit.
-func bitIsSet(bits []uint64, index uint) bool {
-	return bits[index/64]&((1<<63)>>(index%64)) != 0
-}
-
-// function is the code for the named cryptographic function.
-type function uint8
-
-const (
-	// KM{,A,C,CTR} function codes
-	aes128 function = 18 // AES-128
-	aes192 function = 19 // AES-192
-	aes256 function = 20 // AES-256
-
-	// K{I,L}MD function codes
-	sha1     function = 1  // SHA-1
-	sha256   function = 2  // SHA-256
-	sha512   function = 3  // SHA-512
-	sha3_224 function = 32 // SHA3-224
-	sha3_256 function = 33 // SHA3-256
-	sha3_384 function = 34 // SHA3-384
-	sha3_512 function = 35 // SHA3-512
-	shake128 function = 36 // SHAKE-128
-	shake256 function = 37 // SHAKE-256
-
-	// KLMD function codes
-	ghash function = 65 // GHASH
-)
-
-// queryResult contains the result of a Query function
-// call. Bits are numbered in big endian order so the
-// leftmost bit (the MSB) is at index 0.
-type queryResult struct {
-	bits [2]uint64
-}
-
-// Has reports whether the given functions are present.
-func (q *queryResult) Has(fns ...function) bool {
-	if len(fns) == 0 {
-		panic("no function codes provided")
-	}
-	for _, f := range fns {
-		if !bitIsSet(q.bits[:], uint(f)) {
-			return false
-		}
-	}
-	return true
-}
-
-// facility is a bit index for the named facility.
-type facility uint8
-
-const (
-	// cryptography facilities
-	msa4 facility = 77  // message-security-assist extension 4
-	msa8 facility = 146 // message-security-assist extension 8
-)
-
-// facilityList contains the result of an STFLE call.
-// Bits are numbered in big endian order so the
-// leftmost bit (the MSB) is at index 0.
-type facilityList struct {
-	bits [4]uint64
-}
-
-// Has reports whether the given facilities are present.
-func (s *facilityList) Has(fs ...facility) bool {
-	if len(fs) == 0 {
-		panic("no facility bits provided")
-	}
-	for _, f := range fs {
-		if !bitIsSet(s.bits[:], uint(f)) {
-			return false
-		}
-	}
-	return true
-}
-
-func doinit() {
+func initS390Xbase() {
 	// test HWCAP bit vector
 	has := func(featureMask uint) bool {
 		return hwCap&featureMask == featureMask
@@ -116,44 +37,4 @@ func doinit() {
 	if S390X.HasVX {
 		S390X.HasVXE = has(hwcap_VXE)
 	}
-
-	// We need implementations of stfle, km and so on
-	// to detect cryptographic features.
-	if !haveAsmFunctions() {
-		return
-	}
-
-	// optional cryptographic functions
-	if S390X.HasMSA {
-		aes := []function{aes128, aes192, aes256}
-
-		// cipher message
-		km, kmc := kmQuery(), kmcQuery()
-		S390X.HasAES = km.Has(aes...)
-		S390X.HasAESCBC = kmc.Has(aes...)
-		if S390X.HasSTFLE {
-			facilities := stfle()
-			if facilities.Has(msa4) {
-				kmctr := kmctrQuery()
-				S390X.HasAESCTR = kmctr.Has(aes...)
-			}
-			if facilities.Has(msa8) {
-				kma := kmaQuery()
-				S390X.HasAESGCM = kma.Has(aes...)
-			}
-		}
-
-		// compute message digest
-		kimd := kimdQuery() // intermediate (no padding)
-		klmd := klmdQuery() // last (padding)
-		S390X.HasSHA1 = kimd.Has(sha1) && klmd.Has(sha1)
-		S390X.HasSHA256 = kimd.Has(sha256) && klmd.Has(sha256)
-		S390X.HasSHA512 = kimd.Has(sha512) && klmd.Has(sha512)
-		S390X.HasGHASH = kimd.Has(ghash) // KLMD-GHASH does not exist
-		sha3 := []function{
-			sha3_224, sha3_256, sha3_384, sha3_512,
-			shake128, shake256,
-		}
-		S390X.HasSHA3 = kimd.Has(sha3...) && klmd.Has(sha3...)
-	}
 }

vendor/golang.org/x/sys/cpu/cpu_netbsd_arm64.go

@@ -0,0 +1,173 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package cpu
+
+import (
+	"syscall"
+	"unsafe"
+)
+
+// Minimal copy of functionality from x/sys/unix so the cpu package can call
+// sysctl without depending on x/sys/unix.
+
+const (
+	_CTL_QUERY = -2
+
+	_SYSCTL_VERS_1 = 0x1000000
+)
+
+var _zero uintptr
+
+func sysctl(mib []int32, old *byte, oldlen *uintptr, new *byte, newlen uintptr) (err error) {
+	var _p0 unsafe.Pointer
+	if len(mib) > 0 {
+		_p0 = unsafe.Pointer(&mib[0])
+	} else {
+		_p0 = unsafe.Pointer(&_zero)
+	}
+	_, _, errno := syscall.Syscall6(
+		syscall.SYS___SYSCTL,
+		uintptr(_p0),
+		uintptr(len(mib)),
+		uintptr(unsafe.Pointer(old)),
+		uintptr(unsafe.Pointer(oldlen)),
+		uintptr(unsafe.Pointer(new)),
+		uintptr(newlen))
+	if errno != 0 {
+		return errno
+	}
+	return nil
+}
+
+type sysctlNode struct {
+	Flags          uint32
+	Num            int32
+	Name           [32]int8
+	Ver            uint32
+	__rsvd         uint32
+	Un             [16]byte
+	_sysctl_size   [8]byte
+	_sysctl_func   [8]byte
+	_sysctl_parent [8]byte
+	_sysctl_desc   [8]byte
+}
+
+func sysctlNodes(mib []int32) ([]sysctlNode, error) {
+	var olen uintptr
+
+	// Get a list of all sysctl nodes below the given MIB by performing
+	// a sysctl for the given MIB with CTL_QUERY appended.
+	mib = append(mib, _CTL_QUERY)
+	qnode := sysctlNode{Flags: _SYSCTL_VERS_1}
+	qp := (*byte)(unsafe.Pointer(&qnode))
+	sz := unsafe.Sizeof(qnode)
+	if err := sysctl(mib, nil, &olen, qp, sz); err != nil {
+		return nil, err
+	}
+
+	// Now that we know the size, get the actual nodes.
+	nodes := make([]sysctlNode, olen/sz)
+	np := (*byte)(unsafe.Pointer(&nodes[0]))
+	if err := sysctl(mib, np, &olen, qp, sz); err != nil {
+		return nil, err
+	}
+
+	return nodes, nil
+}
+
+func nametomib(name string) ([]int32, error) {
+	// Split name into components.
+	var parts []string
+	last := 0
+	for i := 0; i < len(name); i++ {
+		if name[i] == '.' {
+			parts = append(parts, name[last:i])
+			last = i + 1
+		}
+	}
+	parts = append(parts, name[last:])
+
+	mib := []int32{}
+	// Discover the nodes and construct the MIB OID.
+	for partno, part := range parts {
+		nodes, err := sysctlNodes(mib)
+		if err != nil {
+			return nil, err
+		}
+		for _, node := range nodes {
+			n := make([]byte, 0)
+			for i := range node.Name {
+				if node.Name[i] != 0 {
+					n = append(n, byte(node.Name[i]))
+				}
+			}
+			if string(n) == part {
+				mib = append(mib, int32(node.Num))
+				break
+			}
+		}
+		if len(mib) != partno+1 {
+			return nil, err
+		}
+	}
+
+	return mib, nil
+}
+
+// aarch64SysctlCPUID is struct aarch64_sysctl_cpu_id from NetBSD's <aarch64/armreg.h>
+type aarch64SysctlCPUID struct {
+	midr      uint64 /* Main ID Register */
+	revidr    uint64 /* Revision ID Register */
+	mpidr     uint64 /* Multiprocessor Affinity Register */
+	aa64dfr0  uint64 /* A64 Debug Feature Register 0 */
+	aa64dfr1  uint64 /* A64 Debug Feature Register 1 */
+	aa64isar0 uint64 /* A64 Instruction Set Attribute Register 0 */
+	aa64isar1 uint64 /* A64 Instruction Set Attribute Register 1 */
+	aa64mmfr0 uint64 /* A64 Memory Model Feature Register 0 */
+	aa64mmfr1 uint64 /* A64 Memory Model Feature Register 1 */
+	aa64mmfr2 uint64 /* A64 Memory Model Feature Register 2 */
+	aa64pfr0  uint64 /* A64 Processor Feature Register 0 */
+	aa64pfr1  uint64 /* A64 Processor Feature Register 1 */
+	aa64zfr0  uint64 /* A64 SVE Feature ID Register 0 */
+	mvfr0     uint32 /* Media and VFP Feature Register 0 */
+	mvfr1     uint32 /* Media and VFP Feature Register 1 */
+	mvfr2     uint32 /* Media and VFP Feature Register 2 */
+	pad       uint32
+	clidr     uint64 /* Cache Level ID Register */
+	ctr       uint64 /* Cache Type Register */
+}
+
+func sysctlCPUID(name string) (*aarch64SysctlCPUID, error) {
+	mib, err := nametomib(name)
+	if err != nil {
+		return nil, err
+	}
+
+	out := aarch64SysctlCPUID{}
+	n := unsafe.Sizeof(out)
+	_, _, errno := syscall.Syscall6(
+		syscall.SYS___SYSCTL,
+		uintptr(unsafe.Pointer(&mib[0])),
+		uintptr(len(mib)),
+		uintptr(unsafe.Pointer(&out)),
+		uintptr(unsafe.Pointer(&n)),
+		uintptr(0),
+		uintptr(0))
+	if errno != 0 {
+		return nil, errno
+	}
+	return &out, nil
+}
+
+func doinit() {
+	cpuid, err := sysctlCPUID("machdep.cpu0.cpu_id")
+	if err != nil {
+		setMinimalFeatures()
+		return
+	}
+	parseARM64SystemRegisters(cpuid.aa64isar0, cpuid.aa64isar1, cpuid.aa64pfr0)
+
+	Initialized = true
+}

vendor/golang.org/x/sys/cpu/cpu_other_arm64.go

@@ -2,7 +2,8 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build !linux,arm64
+// +build !linux,!netbsd
+// +build arm64
 
 package cpu
 

vendor/golang.org/x/sys/cpu/cpu_other_mips64x.go

@@ -0,0 +1,12 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !linux
+// +build mips64 mips64le
+
+package cpu
+
+func archInit() {
+	Initialized = true
+}

vendor/golang.org/x/sys/cpu/cpu_s390x.go

@@ -8,10 +8,10 @@ const cacheLineSize = 256
 
 func initOptions() {
 	options = []option{
-		{Name: "zarch", Feature: &S390X.HasZARCH},
-		{Name: "stfle", Feature: &S390X.HasSTFLE},
-		{Name: "ldisp", Feature: &S390X.HasLDISP},
-		{Name: "eimm", Feature: &S390X.HasEIMM},
+		{Name: "zarch", Feature: &S390X.HasZARCH, Required: true},
+		{Name: "stfle", Feature: &S390X.HasSTFLE, Required: true},
+		{Name: "ldisp", Feature: &S390X.HasLDISP, Required: true},
+		{Name: "eimm", Feature: &S390X.HasEIMM, Required: true},
 		{Name: "dfp", Feature: &S390X.HasDFP},
 		{Name: "etf3eh", Feature: &S390X.HasETF3EH},
 		{Name: "msa", Feature: &S390X.HasMSA},
@@ -28,3 +28,145 @@ func initOptions() {
 		{Name: "vxe", Feature: &S390X.HasVXE},
 	}
 }
+
+// bitIsSet reports whether the bit at index is set. The bit index
+// is in big endian order, so bit index 0 is the leftmost bit.
+func bitIsSet(bits []uint64, index uint) bool {
+	return bits[index/64]&((1<<63)>>(index%64)) != 0
+}
+
+// facility is a bit index for the named facility.
+type facility uint8
+
+const (
+	// mandatory facilities
+	zarch  facility = 1  // z architecture mode is active
+	stflef facility = 7  // store-facility-list-extended
+	ldisp  facility = 18 // long-displacement
+	eimm   facility = 21 // extended-immediate
+
+	// miscellaneous facilities
+	dfp    facility = 42 // decimal-floating-point
+	etf3eh facility = 30 // extended-translation 3 enhancement
+
+	// cryptography facilities
+	msa  facility = 17  // message-security-assist
+	msa3 facility = 76  // message-security-assist extension 3
+	msa4 facility = 77  // message-security-assist extension 4
+	msa5 facility = 57  // message-security-assist extension 5
+	msa8 facility = 146 // message-security-assist extension 8
+	msa9 facility = 155 // message-security-assist extension 9
+
+	// vector facilities
+	vx   facility = 129 // vector facility
+	vxe  facility = 135 // vector-enhancements 1
+	vxe2 facility = 148 // vector-enhancements 2
+)
+
+// facilityList contains the result of an STFLE call.
+// Bits are numbered in big endian order so the
+// leftmost bit (the MSB) is at index 0.
+type facilityList struct {
+	bits [4]uint64
+}
+
+// Has reports whether the given facilities are present.
+func (s *facilityList) Has(fs ...facility) bool {
+	if len(fs) == 0 {
+		panic("no facility bits provided")
+	}
+	for _, f := range fs {
+		if !bitIsSet(s.bits[:], uint(f)) {
+			return false
+		}
+	}
+	return true
+}
+
+// function is the code for the named cryptographic function.
+type function uint8
+
+const (
+	// KM{,A,C,CTR} function codes
+	aes128 function = 18 // AES-128
+	aes192 function = 19 // AES-192
+	aes256 function = 20 // AES-256
+
+	// K{I,L}MD function codes
+	sha1     function = 1  // SHA-1
+	sha256   function = 2  // SHA-256
+	sha512   function = 3  // SHA-512
+	sha3_224 function = 32 // SHA3-224
+	sha3_256 function = 33 // SHA3-256
+	sha3_384 function = 34 // SHA3-384
+	sha3_512 function = 35 // SHA3-512
+	shake128 function = 36 // SHAKE-128
+	shake256 function = 37 // SHAKE-256
+
+	// KLMD function codes
+	ghash function = 65 // GHASH
+)
+
+// queryResult contains the result of a Query function
+// call. Bits are numbered in big endian order so the
+// leftmost bit (the MSB) is at index 0.
+type queryResult struct {
+	bits [2]uint64
+}
+
+// Has reports whether the given functions are present.
+func (q *queryResult) Has(fns ...function) bool {
+	if len(fns) == 0 {
+		panic("no function codes provided")
+	}
+	for _, f := range fns {
+		if !bitIsSet(q.bits[:], uint(f)) {
+			return false
+		}
+	}
+	return true
+}
+
+func doinit() {
+	initS390Xbase()
+
+	// We need implementations of stfle, km and so on
+	// to detect cryptographic features.
+	if !haveAsmFunctions() {
+		return
+	}
+
+	// optional cryptographic functions
+	if S390X.HasMSA {
+		aes := []function{aes128, aes192, aes256}
+
+		// cipher message
+		km, kmc := kmQuery(), kmcQuery()
+		S390X.HasAES = km.Has(aes...)
+		S390X.HasAESCBC = kmc.Has(aes...)
+		if S390X.HasSTFLE {
+			facilities := stfle()
+			if facilities.Has(msa4) {
+				kmctr := kmctrQuery()
+				S390X.HasAESCTR = kmctr.Has(aes...)
+			}
+			if facilities.Has(msa8) {
+				kma := kmaQuery()
+				S390X.HasAESGCM = kma.Has(aes...)
+			}
+		}
+
+		// compute message digest
+		kimd := kimdQuery() // intermediate (no padding)
+		klmd := klmdQuery() // last (padding)
+		S390X.HasSHA1 = kimd.Has(sha1) && klmd.Has(sha1)
+		S390X.HasSHA256 = kimd.Has(sha256) && klmd.Has(sha256)
+		S390X.HasSHA512 = kimd.Has(sha512) && klmd.Has(sha512)
+		S390X.HasGHASH = kimd.Has(ghash) // KLMD-GHASH does not exist
+		sha3 := []function{
+			sha3_224, sha3_256, sha3_384, sha3_512,
+			shake128, shake256,
+		}
+		S390X.HasSHA3 = kimd.Has(sha3...) && klmd.Has(sha3...)
+	}
+}

vendor/golang.org/x/sys/cpu/cpu_s390x.s

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build !gccgo
+// +build gc
 
 #include "textflag.h"
 

vendor/golang.org/x/sys/cpu/cpu_x86.go

@@ -16,6 +16,26 @@ func initOptions() {
 		{Name: "aes", Feature: &X86.HasAES},
 		{Name: "avx", Feature: &X86.HasAVX},
 		{Name: "avx2", Feature: &X86.HasAVX2},
+		{Name: "avx512", Feature: &X86.HasAVX512},
+		{Name: "avx512f", Feature: &X86.HasAVX512F},
+		{Name: "avx512cd", Feature: &X86.HasAVX512CD},
+		{Name: "avx512er", Feature: &X86.HasAVX512ER},
+		{Name: "avx512pf", Feature: &X86.HasAVX512PF},
+		{Name: "avx512vl", Feature: &X86.HasAVX512VL},
+		{Name: "avx512bw", Feature: &X86.HasAVX512BW},
+		{Name: "avx512dq", Feature: &X86.HasAVX512DQ},
+		{Name: "avx512ifma", Feature: &X86.HasAVX512IFMA},
+		{Name: "avx512vbmi", Feature: &X86.HasAVX512VBMI},
+		{Name: "avx512vnniw", Feature: &X86.HasAVX5124VNNIW},
+		{Name: "avx5124fmaps", Feature: &X86.HasAVX5124FMAPS},
+		{Name: "avx512vpopcntdq", Feature: &X86.HasAVX512VPOPCNTDQ},
+		{Name: "avx512vpclmulqdq", Feature: &X86.HasAVX512VPCLMULQDQ},
+		{Name: "avx512vnni", Feature: &X86.HasAVX512VNNI},
+		{Name: "avx512gfni", Feature: &X86.HasAVX512GFNI},
+		{Name: "avx512vaes", Feature: &X86.HasAVX512VAES},
+		{Name: "avx512vbmi2", Feature: &X86.HasAVX512VBMI2},
+		{Name: "avx512bitalg", Feature: &X86.HasAVX512BITALG},
+		{Name: "avx512bf16", Feature: &X86.HasAVX512BF16},
 		{Name: "bmi1", Feature: &X86.HasBMI1},
 		{Name: "bmi2", Feature: &X86.HasBMI2},
 		{Name: "erms", Feature: &X86.HasERMS},
@@ -59,12 +79,15 @@ func archInit() {
 	X86.HasOSXSAVE = isSet(27, ecx1)
 	X86.HasRDRAND = isSet(30, ecx1)
 
-	osSupportsAVX := false
+	var osSupportsAVX, osSupportsAVX512 bool
 	// For XGETBV, OSXSAVE bit is required and sufficient.
 	if X86.HasOSXSAVE {
 		eax, _ := xgetbv()
 		// Check if XMM and YMM registers have OS support.
 		osSupportsAVX = isSet(1, eax) && isSet(2, eax)
+
+		// Check if OPMASK and ZMM registers have OS support.
+		osSupportsAVX512 = osSupportsAVX && isSet(5, eax) && isSet(6, eax) && isSet(7, eax)
 	}
 
 	X86.HasAVX = isSet(28, ecx1) && osSupportsAVX
@@ -73,7 +96,7 @@ func archInit() {
 		return
 	}
 
-	_, ebx7, _, _ := cpuid(7, 0)
+	_, ebx7, ecx7, edx7 := cpuid(7, 0)
 	X86.HasBMI1 = isSet(3, ebx7)
 	X86.HasAVX2 = isSet(5, ebx7) && osSupportsAVX
 	X86.HasBMI2 = isSet(8, ebx7)
@@ -81,6 +104,30 @@ func archInit() {
 	X86.HasRDSEED = isSet(18, ebx7)
 	X86.HasADX = isSet(19, ebx7)
 
+	X86.HasAVX512 = isSet(16, ebx7) && osSupportsAVX512 // Because avx-512 foundation is the core required extension
+	if X86.HasAVX512 {
+		X86.HasAVX512F = true
+		X86.HasAVX512CD = isSet(28, ebx7)
+		X86.HasAVX512ER = isSet(27, ebx7)
+		X86.HasAVX512PF = isSet(26, ebx7)
+		X86.HasAVX512VL = isSet(31, ebx7)
+		X86.HasAVX512BW = isSet(30, ebx7)
+		X86.HasAVX512DQ = isSet(17, ebx7)
+		X86.HasAVX512IFMA = isSet(21, ebx7)
+		X86.HasAVX512VBMI = isSet(1, ecx7)
+		X86.HasAVX5124VNNIW = isSet(2, edx7)
+		X86.HasAVX5124FMAPS = isSet(3, edx7)
+		X86.HasAVX512VPOPCNTDQ = isSet(14, ecx7)
+		X86.HasAVX512VPCLMULQDQ = isSet(10, ecx7)
+		X86.HasAVX512VNNI = isSet(11, ecx7)
+		X86.HasAVX512GFNI = isSet(8, ecx7)
+		X86.HasAVX512VAES = isSet(9, ecx7)
+		X86.HasAVX512VBMI2 = isSet(6, ecx7)
+		X86.HasAVX512BITALG = isSet(12, ecx7)
+
+		eax71, _, _, _ := cpuid(7, 1)
+		X86.HasAVX512BF16 = isSet(5, eax71)
+	}
 }
 
 func isSet(bitpos uint, value uint32) bool {

vendor/golang.org/x/sys/cpu/cpu_x86.s

@@ -3,7 +3,7 @@
 // license that can be found in the LICENSE file.
 
 // +build 386 amd64 amd64p32
-// +build !gccgo
+// +build gc
 
 #include "textflag.h"
 

vendor/golang.org/x/sys/cpu/cpu_zos.go

@@ -0,0 +1,10 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package cpu
+
+func archInit() {
+	doinit()
+	Initialized = true
+}

vendor/golang.org/x/sys/cpu/cpu_zos_s390x.go

@@ -0,0 +1,25 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package cpu
+
+func initS390Xbase() {
+	// get the facilities list
+	facilities := stfle()
+
+	// mandatory
+	S390X.HasZARCH = facilities.Has(zarch)
+	S390X.HasSTFLE = facilities.Has(stflef)
+	S390X.HasLDISP = facilities.Has(ldisp)
+	S390X.HasEIMM = facilities.Has(eimm)
+
+	// optional
+	S390X.HasETF3EH = facilities.Has(etf3eh)
+	S390X.HasDFP = facilities.Has(dfp)
+	S390X.HasMSA = facilities.Has(msa)
+	S390X.HasVX = facilities.Has(vx)
+	if S390X.HasVX {
+		S390X.HasVXE = facilities.Has(vxe)
+	}
+}

vendor/golang.org/x/sys/cpu/syscall_aix_ppc64_gc.go

@@ -7,7 +7,7 @@
 // (See golang.org/issue/32102)
 
 // +build aix,ppc64
-// +build !gccgo
+// +build gc
 
 package cpu
 

vendor/golang.org/x/sys/plan9/asm.s

@@ -0,0 +1,8 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+#include "textflag.h"
+
+TEXT ·use(SB),NOSPLIT,$0
+	RET

vendor/golang.org/x/sys/plan9/asm_plan9_386.s

@@ -0,0 +1,30 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+#include "textflag.h"
+
+//
+// System call support for 386, Plan 9
+//
+
+// Just jump to package syscall's implementation for all these functions.
+// The runtime may know about them.
+
+TEXT	·Syscall(SB),NOSPLIT,$0-32
+	JMP	syscall·Syscall(SB)
+
+TEXT	·Syscall6(SB),NOSPLIT,$0-44
+	JMP	syscall·Syscall6(SB)
+
+TEXT ·RawSyscall(SB),NOSPLIT,$0-28
+	JMP	syscall·RawSyscall(SB)
+
+TEXT ·RawSyscall6(SB),NOSPLIT,$0-40
+	JMP	syscall·RawSyscall6(SB)
+
+TEXT ·seek(SB),NOSPLIT,$0-36
+	JMP	syscall·seek(SB)
+
+TEXT ·exit(SB),NOSPLIT,$4-4
+	JMP	syscall·exit(SB)

vendor/golang.org/x/sys/plan9/asm_plan9_amd64.s

@@ -0,0 +1,30 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+#include "textflag.h"
+
+//
+// System call support for amd64, Plan 9
+//
+
+// Just jump to package syscall's implementation for all these functions.
+// The runtime may know about them.
+
+TEXT	·Syscall(SB),NOSPLIT,$0-64
+	JMP	syscall·Syscall(SB)
+
+TEXT	·Syscall6(SB),NOSPLIT,$0-88
+	JMP	syscall·Syscall6(SB)
+
+TEXT ·RawSyscall(SB),NOSPLIT,$0-56
+	JMP	syscall·RawSyscall(SB)
+
+TEXT	·RawSyscall6(SB),NOSPLIT,$0-80
+	JMP	syscall·RawSyscall6(SB)
+
+TEXT ·seek(SB),NOSPLIT,$0-56
+	JMP	syscall·seek(SB)
+
+TEXT ·exit(SB),NOSPLIT,$8-8
+	JMP	syscall·exit(SB)

vendor/golang.org/x/sys/plan9/asm_plan9_arm.s

@@ -0,0 +1,25 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+#include "textflag.h"
+
+// System call support for plan9 on arm
+
+// Just jump to package syscall's implementation for all these functions.
+// The runtime may know about them.
+
+TEXT ·Syscall(SB),NOSPLIT,$0-32
+	JMP	syscall·Syscall(SB)
+
+TEXT ·Syscall6(SB),NOSPLIT,$0-44
+	JMP	syscall·Syscall6(SB)
+
+TEXT ·RawSyscall(SB),NOSPLIT,$0-28
+	JMP	syscall·RawSyscall(SB)
+
+TEXT ·RawSyscall6(SB),NOSPLIT,$0-40
+	JMP	syscall·RawSyscall6(SB)
+
+TEXT ·seek(SB),NOSPLIT,$0-36
+	JMP	syscall·exit(SB)

vendor/golang.org/x/sys/plan9/const_plan9.go

@@ -0,0 +1,70 @@
+package plan9
+
+// Plan 9 Constants
+
+// Open modes
+const (
+	O_RDONLY  = 0
+	O_WRONLY  = 1
+	O_RDWR    = 2
+	O_TRUNC   = 16
+	O_CLOEXEC = 32
+	O_EXCL    = 0x1000
+)
+
+// Rfork flags
+const (
+	RFNAMEG  = 1 << 0
+	RFENVG   = 1 << 1
+	RFFDG    = 1 << 2
+	RFNOTEG  = 1 << 3
+	RFPROC   = 1 << 4
+	RFMEM    = 1 << 5
+	RFNOWAIT = 1 << 6
+	RFCNAMEG = 1 << 10
+	RFCENVG  = 1 << 11
+	RFCFDG   = 1 << 12
+	RFREND   = 1 << 13
+	RFNOMNT  = 1 << 14
+)
+
+// Qid.Type bits
+const (
+	QTDIR    = 0x80
+	QTAPPEND = 0x40
+	QTEXCL   = 0x20
+	QTMOUNT  = 0x10
+	QTAUTH   = 0x08
+	QTTMP    = 0x04
+	QTFILE   = 0x00
+)
+
+// Dir.Mode bits
+const (
+	DMDIR    = 0x80000000
+	DMAPPEND = 0x40000000
+	DMEXCL   = 0x20000000
+	DMMOUNT  = 0x10000000
+	DMAUTH   = 0x08000000
+	DMTMP    = 0x04000000
+	DMREAD   = 0x4
+	DMWRITE  = 0x2
+	DMEXEC   = 0x1
+)
+
+const (
+	STATMAX    = 65535
+	ERRMAX     = 128
+	STATFIXLEN = 49
+)
+
+// Mount and bind flags
+const (
+	MREPL   = 0x0000
+	MBEFORE = 0x0001
+	MAFTER  = 0x0002
+	MORDER  = 0x0003
+	MCREATE = 0x0004
+	MCACHE  = 0x0010
+	MMASK   = 0x0017
+)

vendor/golang.org/x/sys/plan9/dir_plan9.go

@@ -0,0 +1,212 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Plan 9 directory marshalling. See intro(5).
+
+package plan9
+
+import "errors"
+
+var (
+	ErrShortStat = errors.New("stat buffer too short")
+	ErrBadStat   = errors.New("malformed stat buffer")
+	ErrBadName   = errors.New("bad character in file name")
+)
+
+// A Qid represents a 9P server's unique identification for a file.
+type Qid struct {
+	Path uint64 // the file server's unique identification for the file
+	Vers uint32 // version number for given Path
+	Type uint8  // the type of the file (plan9.QTDIR for example)
+}
+
+// A Dir contains the metadata for a file.
+type Dir struct {
+	// system-modified data
+	Type uint16 // server type
+	Dev  uint32 // server subtype
+
+	// file data
+	Qid    Qid    // unique id from server
+	Mode   uint32 // permissions
+	Atime  uint32 // last read time
+	Mtime  uint32 // last write time
+	Length int64  // file length
+	Name   string // last element of path
+	Uid    string // owner name
+	Gid    string // group name
+	Muid   string // last modifier name
+}
+
+var nullDir = Dir{
+	Type: ^uint16(0),
+	Dev:  ^uint32(0),
+	Qid: Qid{
+		Path: ^uint64(0),
+		Vers: ^uint32(0),
+		Type: ^uint8(0),
+	},
+	Mode:   ^uint32(0),
+	Atime:  ^uint32(0),
+	Mtime:  ^uint32(0),
+	Length: ^int64(0),
+}
+
+// Null assigns special "don't touch" values to members of d to
+// avoid modifying them during plan9.Wstat.
+func (d *Dir) Null() { *d = nullDir }
+
+// Marshal encodes a 9P stat message corresponding to d into b
+//
+// If there isn't enough space in b for a stat message, ErrShortStat is returned.
+func (d *Dir) Marshal(b []byte) (n int, err error) {
+	n = STATFIXLEN + len(d.Name) + len(d.Uid) + len(d.Gid) + len(d.Muid)
+	if n > len(b) {
+		return n, ErrShortStat
+	}
+
+	for _, c := range d.Name {
+		if c == '/' {
+			return n, ErrBadName
+		}
+	}
+
+	b = pbit16(b, uint16(n)-2)
+	b = pbit16(b, d.Type)
+	b = pbit32(b, d.Dev)
+	b = pbit8(b, d.Qid.Type)
+	b = pbit32(b, d.Qid.Vers)
+	b = pbit64(b, d.Qid.Path)
+	b = pbit32(b, d.Mode)
+	b = pbit32(b, d.Atime)
+	b = pbit32(b, d.Mtime)
+	b = pbit64(b, uint64(d.Length))
+	b = pstring(b, d.Name)
+	b = pstring(b, d.Uid)
+	b = pstring(b, d.Gid)
+	b = pstring(b, d.Muid)
+
+	return n, nil
+}
+
+// UnmarshalDir decodes a single 9P stat message from b and returns the resulting Dir.
+//
+// If b is too small to hold a valid stat message, ErrShortStat is returned.
+//
+// If the stat message itself is invalid, ErrBadStat is returned.
+func UnmarshalDir(b []byte) (*Dir, error) {
+	if len(b) < STATFIXLEN {
+		return nil, ErrShortStat
+	}
+	size, buf := gbit16(b)
+	if len(b) != int(size)+2 {
+		return nil, ErrBadStat
+	}
+	b = buf
+
+	var d Dir
+	d.Type, b = gbit16(b)
+	d.Dev, b = gbit32(b)
+	d.Qid.Type, b = gbit8(b)
+	d.Qid.Vers, b = gbit32(b)
+	d.Qid.Path, b = gbit64(b)
+	d.Mode, b = gbit32(b)
+	d.Atime, b = gbit32(b)
+	d.Mtime, b = gbit32(b)
+
+	n, b := gbit64(b)
+	d.Length = int64(n)
+
+	var ok bool
+	if d.Name, b, ok = gstring(b); !ok {
+		return nil, ErrBadStat
+	}
+	if d.Uid, b, ok = gstring(b); !ok {
+		return nil, ErrBadStat
+	}
+	if d.Gid, b, ok = gstring(b); !ok {
+		return nil, ErrBadStat
+	}
+	if d.Muid, b, ok = gstring(b); !ok {
+		return nil, ErrBadStat
+	}
+
+	return &d, nil
+}
+
+// pbit8 copies the 8-bit number v to b and returns the remaining slice of b.
+func pbit8(b []byte, v uint8) []byte {
+	b[0] = byte(v)
+	return b[1:]
+}
+
+// pbit16 copies the 16-bit number v to b in little-endian order and returns the remaining slice of b.
+func pbit16(b []byte, v uint16) []byte {
+	b[0] = byte(v)
+	b[1] = byte(v >> 8)
+	return b[2:]
+}
+
+// pbit32 copies the 32-bit number v to b in little-endian order and returns the remaining slice of b.
+func pbit32(b []byte, v uint32) []byte {
+	b[0] = byte(v)
+	b[1] = byte(v >> 8)
+	b[2] = byte(v >> 16)
+	b[3] = byte(v >> 24)
+	return b[4:]
+}
+
+// pbit64 copies the 64-bit number v to b in little-endian order and returns the remaining slice of b.
+func pbit64(b []byte, v uint64) []byte {
+	b[0] = byte(v)
+	b[1] = byte(v >> 8)
+	b[2] = byte(v >> 16)
+	b[3] = byte(v >> 24)
+	b[4] = byte(v >> 32)
+	b[5] = byte(v >> 40)
+	b[6] = byte(v >> 48)
+	b[7] = byte(v >> 56)
+	return b[8:]
+}
+
+// pstring copies the string s to b, prepending it with a 16-bit length in little-endian order, and
+// returning the remaining slice of b..
+func pstring(b []byte, s string) []byte {
+	b = pbit16(b, uint16(len(s)))
+	n := copy(b, s)
+	return b[n:]
+}
+
+// gbit8 reads an 8-bit number from b and returns it with the remaining slice of b.
+func gbit8(b []byte) (uint8, []byte) {
+	return uint8(b[0]), b[1:]
+}
+
+// gbit16 reads a 16-bit number in little-endian order from b and returns it with the remaining slice of b.
+func gbit16(b []byte) (uint16, []byte) {
+	return uint16(b[0]) | uint16(b[1])<<8, b[2:]
+}
+
+// gbit32 reads a 32-bit number in little-endian order from b and returns it with the remaining slice of b.
+func gbit32(b []byte) (uint32, []byte) {
+	return uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24, b[4:]
+}
+
+// gbit64 reads a 64-bit number in little-endian order from b and returns it with the remaining slice of b.
+func gbit64(b []byte) (uint64, []byte) {
+	lo := uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24
+	hi := uint32(b[4]) | uint32(b[5])<<8 | uint32(b[6])<<16 | uint32(b[7])<<24
+	return uint64(lo) | uint64(hi)<<32, b[8:]
+}
+
+// gstring reads a string from b, prefixed with a 16-bit length in little-endian order.
+// It returns the string with the remaining slice of b and a boolean. If the length is
+// greater than the number of bytes in b, the boolean will be false.
+func gstring(b []byte) (string, []byte, bool) {
+	n, b := gbit16(b)
+	if int(n) > len(b) {
+		return "", b, false
+	}
+	return string(b[:n]), b[n:], true
+}

vendor/golang.org/x/sys/plan9/env_plan9.go

@@ -0,0 +1,31 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Plan 9 environment variables.
+
+package plan9
+
+import (
+	"syscall"
+)
+
+func Getenv(key string) (value string, found bool) {
+	return syscall.Getenv(key)
+}
+
+func Setenv(key, value string) error {
+	return syscall.Setenv(key, value)
+}
+
+func Clearenv() {
+	syscall.Clearenv()
+}
+
+func Environ() []string {
+	return syscall.Environ()
+}
+
+func Unsetenv(key string) error {
+	return syscall.Unsetenv(key)
+}

vendor/golang.org/x/sys/plan9/errors_plan9.go

@@ -0,0 +1,50 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package plan9
+
+import "syscall"
+
+// Constants
+const (
+	// Invented values to support what package os expects.
+	O_CREAT    = 0x02000
+	O_APPEND   = 0x00400
+	O_NOCTTY   = 0x00000
+	O_NONBLOCK = 0x00000
+	O_SYNC     = 0x00000
+	O_ASYNC    = 0x00000
+
+	S_IFMT   = 0x1f000
+	S_IFIFO  = 0x1000
+	S_IFCHR  = 0x2000
+	S_IFDIR  = 0x4000
+	S_IFBLK  = 0x6000
+	S_IFREG  = 0x8000
+	S_IFLNK  = 0xa000
+	S_IFSOCK = 0xc000
+)
+
+// Errors
+var (
+	EINVAL       = syscall.NewError("bad arg in system call")
+	ENOTDIR      = syscall.NewError("not a directory")
+	EISDIR       = syscall.NewError("file is a directory")
+	ENOENT       = syscall.NewError("file does not exist")
+	EEXIST       = syscall.NewError("file already exists")
+	EMFILE       = syscall.NewError("no free file descriptors")
+	EIO          = syscall.NewError("i/o error")
+	ENAMETOOLONG = syscall.NewError("file name too long")
+	EINTR        = syscall.NewError("interrupted")
+	EPERM        = syscall.NewError("permission denied")
+	EBUSY        = syscall.NewError("no free devices")
+	ETIMEDOUT    = syscall.NewError("connection timed out")
+	EPLAN9       = syscall.NewError("not supported by plan 9")
+
+	// The following errors do not correspond to any
+	// Plan 9 system messages. Invented to support
+	// what package os and others expect.
+	EACCES       = syscall.NewError("access permission denied")
+	EAFNOSUPPORT = syscall.NewError("address family not supported by protocol")
+)

vendor/golang.org/x/sys/plan9/mkall.sh

@@ -0,0 +1,150 @@
+#!/usr/bin/env bash
+# Copyright 2009 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+# The plan9 package provides access to the raw system call
+# interface of the underlying operating system.  Porting Go to
+# a new architecture/operating system combination requires
+# some manual effort, though there are tools that automate
+# much of the process.  The auto-generated files have names
+# beginning with z.
+#
+# This script runs or (given -n) prints suggested commands to generate z files
+# for the current system.  Running those commands is not automatic.
+# This script is documentation more than anything else.
+#
+# * asm_${GOOS}_${GOARCH}.s
+#
+# This hand-written assembly file implements system call dispatch.
+# There are three entry points:
+#
+# 	func Syscall(trap, a1, a2, a3 uintptr) (r1, r2, err uintptr);
+# 	func Syscall6(trap, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, err uintptr);
+# 	func RawSyscall(trap, a1, a2, a3 uintptr) (r1, r2, err uintptr);
+#
+# The first and second are the standard ones; they differ only in
+# how many arguments can be passed to the kernel.
+# The third is for low-level use by the ForkExec wrapper;
+# unlike the first two, it does not call into the scheduler to
+# let it know that a system call is running.
+#
+# * syscall_${GOOS}.go
+#
+# This hand-written Go file implements system calls that need
+# special handling and lists "//sys" comments giving prototypes
+# for ones that can be auto-generated.  Mksyscall reads those
+# comments to generate the stubs.
+#
+# * syscall_${GOOS}_${GOARCH}.go
+#
+# Same as syscall_${GOOS}.go except that it contains code specific
+# to ${GOOS} on one particular architecture.
+#
+# * types_${GOOS}.c
+#
+# This hand-written C file includes standard C headers and then
+# creates typedef or enum names beginning with a dollar sign
+# (use of $ in variable names is a gcc extension).  The hardest
+# part about preparing this file is figuring out which headers to
+# include and which symbols need to be #defined to get the
+# actual data structures that pass through to the kernel system calls.
+# Some C libraries present alternate versions for binary compatibility
+# and translate them on the way in and out of system calls, but
+# there is almost always a #define that can get the real ones.
+# See types_darwin.c and types_linux.c for examples.
+#
+# * zerror_${GOOS}_${GOARCH}.go
+#
+# This machine-generated file defines the system's error numbers,
+# error strings, and signal numbers.  The generator is "mkerrors.sh".
+# Usually no arguments are needed, but mkerrors.sh will pass its
+# arguments on to godefs.
+#
+# * zsyscall_${GOOS}_${GOARCH}.go
+#
+# Generated by mksyscall.pl; see syscall_${GOOS}.go above.
+#
+# * zsysnum_${GOOS}_${GOARCH}.go
+#
+# Generated by mksysnum_${GOOS}.
+#
+# * ztypes_${GOOS}_${GOARCH}.go
+#
+# Generated by godefs; see types_${GOOS}.c above.
+
+GOOSARCH="${GOOS}_${GOARCH}"
+
+# defaults
+mksyscall="go run mksyscall.go"
+mkerrors="./mkerrors.sh"
+zerrors="zerrors_$GOOSARCH.go"
+mksysctl=""
+zsysctl="zsysctl_$GOOSARCH.go"
+mksysnum=
+mktypes=
+run="sh"
+
+case "$1" in
+-syscalls)
+	for i in zsyscall*go
+	do
+		sed 1q $i | sed 's;^// ;;' | sh > _$i && gofmt < _$i > $i
+		rm _$i
+	done
+	exit 0
+	;;
+-n)
+	run="cat"
+	shift
+esac
+
+case "$#" in
+0)
+	;;
+*)
+	echo 'usage: mkall.sh [-n]' 1>&2
+	exit 2
+esac
+
+case "$GOOSARCH" in
+_* | *_ | _)
+	echo 'undefined $GOOS_$GOARCH:' "$GOOSARCH" 1>&2
+	exit 1
+	;;
+plan9_386)
+	mkerrors=
+	mksyscall="go run mksyscall.go -l32 -plan9 -tags plan9,386"
+	mksysnum="./mksysnum_plan9.sh /n/sources/plan9/sys/src/libc/9syscall/sys.h"
+	mktypes="XXX"
+	;;
+plan9_amd64)
+	mkerrors=
+	mksyscall="go run mksyscall.go -l32 -plan9 -tags plan9,amd64"
+	mksysnum="./mksysnum_plan9.sh /n/sources/plan9/sys/src/libc/9syscall/sys.h"
+	mktypes="XXX"
+	;;
+plan9_arm)
+	mkerrors=
+	mksyscall="go run mksyscall.go -l32 -plan9 -tags plan9,arm"
+	mksysnum="./mksysnum_plan9.sh /n/sources/plan9/sys/src/libc/9syscall/sys.h"
+	mktypes="XXX"
+	;;
+*)
+	echo 'unrecognized $GOOS_$GOARCH: ' "$GOOSARCH" 1>&2
+	exit 1
+	;;
+esac
+
+(
+	if [ -n "$mkerrors" ]; then echo "$mkerrors |gofmt >$zerrors"; fi
+	case "$GOOS" in
+	plan9)
+		syscall_goos="syscall_$GOOS.go"
+		if [ -n "$mksyscall" ]; then echo "$mksyscall $syscall_goos |gofmt >zsyscall_$GOOSARCH.go"; fi
+		;;
+	esac
+	if [ -n "$mksysctl" ]; then echo "$mksysctl |gofmt >$zsysctl"; fi
+	if [ -n "$mksysnum" ]; then echo "$mksysnum |gofmt >zsysnum_$GOOSARCH.go"; fi
+	if [ -n "$mktypes" ]; then echo "$mktypes types_$GOOS.go |gofmt >ztypes_$GOOSARCH.go"; fi
+) | $run

vendor/golang.org/x/sys/plan9/mkerrors.sh

@@ -0,0 +1,246 @@
+#!/usr/bin/env bash
+# Copyright 2009 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+# Generate Go code listing errors and other #defined constant
+# values (ENAMETOOLONG etc.), by asking the preprocessor
+# about the definitions.
+
+unset LANG
+export LC_ALL=C
+export LC_CTYPE=C
+
+CC=${CC:-gcc}
+
+uname=$(uname)
+
+includes='
+#include <sys/types.h>
+#include <sys/file.h>
+#include <fcntl.h>
+#include <dirent.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <netinet/ip.h>
+#include <netinet/ip6.h>
+#include <netinet/tcp.h>
+#include <errno.h>
+#include <sys/signal.h>
+#include <signal.h>
+#include <sys/resource.h>
+'
+
+ccflags="$@"
+
+# Write go tool cgo -godefs input.
+(
+	echo package plan9
+	echo
+	echo '/*'
+	indirect="includes_$(uname)"
+	echo "${!indirect} $includes"
+	echo '*/'
+	echo 'import "C"'
+	echo
+	echo 'const ('
+
+	# The gcc command line prints all the #defines
+	# it encounters while processing the input
+	echo "${!indirect} $includes" | $CC -x c - -E -dM $ccflags |
+	awk '
+		$1 != "#define" || $2 ~ /\(/ || $3 == "" {next}
+
+		$2 ~ /^E([ABCD]X|[BIS]P|[SD]I|S|FL)$/ {next}  # 386 registers
+		$2 ~ /^(SIGEV_|SIGSTKSZ|SIGRT(MIN|MAX))/ {next}
+		$2 ~ /^(SCM_SRCRT)$/ {next}
+		$2 ~ /^(MAP_FAILED)$/ {next}
+
+		$2 !~ /^ETH_/ &&
+		$2 !~ /^EPROC_/ &&
+		$2 !~ /^EQUIV_/ &&
+		$2 !~ /^EXPR_/ &&
+		$2 ~ /^E[A-Z0-9_]+$/ ||
+		$2 ~ /^B[0-9_]+$/ ||
+		$2 ~ /^V[A-Z0-9]+$/ ||
+		$2 ~ /^CS[A-Z0-9]/ ||
+		$2 ~ /^I(SIG|CANON|CRNL|EXTEN|MAXBEL|STRIP|UTF8)$/ ||
+		$2 ~ /^IGN/ ||
+		$2 ~ /^IX(ON|ANY|OFF)$/ ||
+		$2 ~ /^IN(LCR|PCK)$/ ||
+		$2 ~ /(^FLU?SH)|(FLU?SH$)/ ||
+		$2 ~ /^C(LOCAL|READ)$/ ||
+		$2 == "BRKINT" ||
+		$2 == "HUPCL" ||
+		$2 == "PENDIN" ||
+		$2 == "TOSTOP" ||
+		$2 ~ /^PAR/ ||
+		$2 ~ /^SIG[^_]/ ||
+		$2 ~ /^O[CNPFP][A-Z]+[^_][A-Z]+$/ ||
+		$2 ~ /^IN_/ ||
+		$2 ~ /^LOCK_(SH|EX|NB|UN)$/ ||
+		$2 ~ /^(AF|SOCK|SO|SOL|IPPROTO|IP|IPV6|ICMP6|TCP|EVFILT|NOTE|EV|SHUT|PROT|MAP|PACKET|MSG|SCM|MCL|DT|MADV|PR)_/ ||
+		$2 == "ICMPV6_FILTER" ||
+		$2 == "SOMAXCONN" ||
+		$2 == "NAME_MAX" ||
+		$2 == "IFNAMSIZ" ||
+		$2 ~ /^CTL_(MAXNAME|NET|QUERY)$/ ||
+		$2 ~ /^SYSCTL_VERS/ ||
+		$2 ~ /^(MS|MNT)_/ ||
+		$2 ~ /^TUN(SET|GET|ATTACH|DETACH)/ ||
+		$2 ~ /^(O|F|FD|NAME|S|PTRACE|PT)_/ ||
+		$2 ~ /^LINUX_REBOOT_CMD_/ ||
+		$2 ~ /^LINUX_REBOOT_MAGIC[12]$/ ||
+		$2 !~ "NLA_TYPE_MASK" &&
+		$2 ~ /^(NETLINK|NLM|NLMSG|NLA|IFA|IFAN|RT|RTCF|RTN|RTPROT|RTNH|ARPHRD|ETH_P)_/ ||
+		$2 ~ /^SIOC/ ||
+		$2 ~ /^TIOC/ ||
+		$2 !~ "RTF_BITS" &&
+		$2 ~ /^(IFF|IFT|NET_RT|RTM|RTF|RTV|RTA|RTAX)_/ ||
+		$2 ~ /^BIOC/ ||
+		$2 ~ /^RUSAGE_(SELF|CHILDREN|THREAD)/ ||
+		$2 ~ /^RLIMIT_(AS|CORE|CPU|DATA|FSIZE|NOFILE|STACK)|RLIM_INFINITY/ ||
+		$2 ~ /^PRIO_(PROCESS|PGRP|USER)/ ||
+		$2 ~ /^CLONE_[A-Z_]+/ ||
+		$2 !~ /^(BPF_TIMEVAL)$/ &&
+		$2 ~ /^(BPF|DLT)_/ ||
+		$2 !~ "WMESGLEN" &&
+		$2 ~ /^W[A-Z0-9]+$/ {printf("\t%s = C.%s\n", $2, $2)}
+		$2 ~ /^__WCOREFLAG$/ {next}
+		$2 ~ /^__W[A-Z0-9]+$/ {printf("\t%s = C.%s\n", substr($2,3), $2)}
+
+		{next}
+	' | sort
+
+	echo ')'
+) >_const.go
+
+# Pull out the error names for later.
+errors=$(
+	echo '#include <errno.h>' | $CC -x c - -E -dM $ccflags |
+	awk '$1=="#define" && $2 ~ /^E[A-Z0-9_]+$/ { print $2 }' |
+	sort
+)
+
+# Pull out the signal names for later.
+signals=$(
+	echo '#include <signal.h>' | $CC -x c - -E -dM $ccflags |
+	awk '$1=="#define" && $2 ~ /^SIG[A-Z0-9]+$/ { print $2 }' |
+	egrep -v '(SIGSTKSIZE|SIGSTKSZ|SIGRT)' |
+	sort
+)
+
+# Again, writing regexps to a file.
+echo '#include <errno.h>' | $CC -x c - -E -dM $ccflags |
+	awk '$1=="#define" && $2 ~ /^E[A-Z0-9_]+$/ { print "^\t" $2 "[ \t]*=" }' |
+	sort >_error.grep
+echo '#include <signal.h>' | $CC -x c - -E -dM $ccflags |
+	awk '$1=="#define" && $2 ~ /^SIG[A-Z0-9]+$/ { print "^\t" $2 "[ \t]*=" }' |
+	egrep -v '(SIGSTKSIZE|SIGSTKSZ|SIGRT)' |
+	sort >_signal.grep
+
+echo '// mkerrors.sh' "$@"
+echo '// Code generated by the command above; DO NOT EDIT.'
+echo
+go tool cgo -godefs -- "$@" _const.go >_error.out
+cat _error.out | grep -vf _error.grep | grep -vf _signal.grep
+echo
+echo '// Errors'
+echo 'const ('
+cat _error.out | grep -f _error.grep | sed 's/=\(.*\)/= Errno(\1)/'
+echo ')'
+
+echo
+echo '// Signals'
+echo 'const ('
+cat _error.out | grep -f _signal.grep | sed 's/=\(.*\)/= Signal(\1)/'
+echo ')'
+
+# Run C program to print error and syscall strings.
+(
+	echo -E "
+#include <stdio.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <ctype.h>
+#include <string.h>
+#include <signal.h>
+
+#define nelem(x) (sizeof(x)/sizeof((x)[0]))
+
+enum { A = 'A', Z = 'Z', a = 'a', z = 'z' }; // avoid need for single quotes below
+
+int errors[] = {
+"
+	for i in $errors
+	do
+		echo -E '	'$i,
+	done
+
+	echo -E "
+};
+
+int signals[] = {
+"
+	for i in $signals
+	do
+		echo -E '	'$i,
+	done
+
+	# Use -E because on some systems bash builtin interprets \n itself.
+	echo -E '
+};
+
+static int
+intcmp(const void *a, const void *b)
+{
+	return *(int*)a - *(int*)b;
+}
+
+int
+main(void)
+{
+	int i, j, e;
+	char buf[1024], *p;
+
+	printf("\n\n// Error table\n");
+	printf("var errors = [...]string {\n");
+	qsort(errors, nelem(errors), sizeof errors[0], intcmp);
+	for(i=0; i<nelem(errors); i++) {
+		e = errors[i];
+		if(i > 0 && errors[i-1] == e)
+			continue;
+		strcpy(buf, strerror(e));
+		// lowercase first letter: Bad -> bad, but STREAM -> STREAM.
+		if(A <= buf[0] && buf[0] <= Z && a <= buf[1] && buf[1] <= z)
+			buf[0] += a - A;
+		printf("\t%d: \"%s\",\n", e, buf);
+	}
+	printf("}\n\n");
+	
+	printf("\n\n// Signal table\n");
+	printf("var signals = [...]string {\n");
+	qsort(signals, nelem(signals), sizeof signals[0], intcmp);
+	for(i=0; i<nelem(signals); i++) {
+		e = signals[i];
+		if(i > 0 && signals[i-1] == e)
+			continue;
+		strcpy(buf, strsignal(e));
+		// lowercase first letter: Bad -> bad, but STREAM -> STREAM.
+		if(A <= buf[0] && buf[0] <= Z && a <= buf[1] && buf[1] <= z)
+			buf[0] += a - A;
+		// cut trailing : number.
+		p = strrchr(buf, ":"[0]);
+		if(p)
+			*p = '\0';
+		printf("\t%d: \"%s\",\n", e, buf);
+	}
+	printf("}\n\n");
+
+	return 0;
+}
+
+'
+) >_errors.c
+
+$CC $ccflags -o _errors _errors.c && $GORUN ./_errors && rm -f _errors.c _errors _const.go _error.grep _signal.grep _error.out

vendor/golang.org/x/sys/plan9/mksysnum_plan9.sh

@@ -0,0 +1,23 @@
+#!/bin/sh
+# Copyright 2009 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+COMMAND="mksysnum_plan9.sh $@"
+
+cat <<EOF
+// $COMMAND
+// MACHINE GENERATED BY THE ABOVE COMMAND; DO NOT EDIT
+
+package plan9
+
+const(
+EOF
+
+SP='[ 	]' # space or tab
+sed "s/^#define${SP}\\([A-Z0-9_][A-Z0-9_]*\\)${SP}${SP}*\\([0-9][0-9]*\\)/SYS_\\1=\\2/g" \
+	< $1 | grep -v SYS__
+
+cat <<EOF
+)
+EOF

vendor/golang.org/x/sys/plan9/pwd_go15_plan9.go

@@ -0,0 +1,21 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build go1.5
+
+package plan9
+
+import "syscall"
+
+func fixwd() {
+	syscall.Fixwd()
+}
+
+func Getwd() (wd string, err error) {
+	return syscall.Getwd()
+}
+
+func Chdir(path string) error {
+	return syscall.Chdir(path)
+}

vendor/golang.org/x/sys/plan9/pwd_plan9.go

@@ -0,0 +1,23 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !go1.5
+
+package plan9
+
+func fixwd() {
+}
+
+func Getwd() (wd string, err error) {
+	fd, err := open(".", O_RDONLY)
+	if err != nil {
+		return "", err
+	}
+	defer Close(fd)
+	return Fd2path(fd)
+}
+
+func Chdir(path string) error {
+	return chdir(path)
+}

vendor/golang.org/x/sys/plan9/race.go

@@ -0,0 +1,30 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build plan9,race
+
+package plan9
+
+import (
+	"runtime"
+	"unsafe"
+)
+
+const raceenabled = true
+
+func raceAcquire(addr unsafe.Pointer) {
+	runtime.RaceAcquire(addr)
+}
+
+func raceReleaseMerge(addr unsafe.Pointer) {
+	runtime.RaceReleaseMerge(addr)
+}
+
+func raceReadRange(addr unsafe.Pointer, len int) {
+	runtime.RaceReadRange(addr, len)
+}
+
+func raceWriteRange(addr unsafe.Pointer, len int) {
+	runtime.RaceWriteRange(addr, len)
+}

vendor/golang.org/x/sys/plan9/race0.go

@@ -0,0 +1,25 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build plan9,!race
+
+package plan9
+
+import (
+	"unsafe"
+)
+
+const raceenabled = false
+
+func raceAcquire(addr unsafe.Pointer) {
+}
+
+func raceReleaseMerge(addr unsafe.Pointer) {
+}
+
+func raceReadRange(addr unsafe.Pointer, len int) {
+}
+
+func raceWriteRange(addr unsafe.Pointer, len int) {
+}

vendor/golang.org/x/sys/plan9/str.go

@@ -0,0 +1,22 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build plan9
+
+package plan9
+
+func itoa(val int) string { // do it here rather than with fmt to avoid dependency
+	if val < 0 {
+		return "-" + itoa(-val)
+	}
+	var buf [32]byte // big enough for int64
+	i := len(buf) - 1
+	for val >= 10 {
+		buf[i] = byte(val%10 + '0')
+		i--
+		val /= 10
+	}
+	buf[i] = byte(val + '0')
+	return string(buf[i:])
+}

vendor/golang.org/x/sys/plan9/syscall.go

@@ -0,0 +1,116 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build plan9
+
+// Package plan9 contains an interface to the low-level operating system
+// primitives. OS details vary depending on the underlying system, and
+// by default, godoc will display the OS-specific documentation for the current
+// system. If you want godoc to display documentation for another
+// system, set $GOOS and $GOARCH to the desired system. For example, if
+// you want to view documentation for freebsd/arm on linux/amd64, set $GOOS
+// to freebsd and $GOARCH to arm.
+//
+// The primary use of this package is inside other packages that provide a more
+// portable interface to the system, such as "os", "time" and "net".  Use
+// those packages rather than this one if you can.
+//
+// For details of the functions and data types in this package consult
+// the manuals for the appropriate operating system.
+//
+// These calls return err == nil to indicate success; otherwise
+// err represents an operating system error describing the failure and
+// holds a value of type syscall.ErrorString.
+package plan9 // import "golang.org/x/sys/plan9"
+
+import (
+	"bytes"
+	"strings"
+	"unsafe"
+
+	"golang.org/x/sys/internal/unsafeheader"
+)
+
+// ByteSliceFromString returns a NUL-terminated slice of bytes
+// containing the text of s. If s contains a NUL byte at any
+// location, it returns (nil, EINVAL).
+func ByteSliceFromString(s string) ([]byte, error) {
+	if strings.IndexByte(s, 0) != -1 {
+		return nil, EINVAL
+	}
+	a := make([]byte, len(s)+1)
+	copy(a, s)
+	return a, nil
+}
+
+// BytePtrFromString returns a pointer to a NUL-terminated array of
+// bytes containing the text of s. If s contains a NUL byte at any
+// location, it returns (nil, EINVAL).
+func BytePtrFromString(s string) (*byte, error) {
+	a, err := ByteSliceFromString(s)
+	if err != nil {
+		return nil, err
+	}
+	return &a[0], nil
+}
+
+// ByteSliceToString returns a string form of the text represented by the slice s, with a terminating NUL and any
+// bytes after the NUL removed.
+func ByteSliceToString(s []byte) string {
+	if i := bytes.IndexByte(s, 0); i != -1 {
+		s = s[:i]
+	}
+	return string(s)
+}
+
+// BytePtrToString takes a pointer to a sequence of text and returns the corresponding string.
+// If the pointer is nil, it returns the empty string. It assumes that the text sequence is terminated
+// at a zero byte; if the zero byte is not present, the program may crash.
+func BytePtrToString(p *byte) string {
+	if p == nil {
+		return ""
+	}
+	if *p == 0 {
+		return ""
+	}
+
+	// Find NUL terminator.
+	n := 0
+	for ptr := unsafe.Pointer(p); *(*byte)(ptr) != 0; n++ {
+		ptr = unsafe.Pointer(uintptr(ptr) + 1)
+	}
+
+	var s []byte
+	h := (*unsafeheader.Slice)(unsafe.Pointer(&s))
+	h.Data = unsafe.Pointer(p)
+	h.Len = n
+	h.Cap = n
+
+	return string(s)
+}
+
+// Single-word zero for use when we need a valid pointer to 0 bytes.
+// See mksyscall.pl.
+var _zero uintptr
+
+func (ts *Timespec) Unix() (sec int64, nsec int64) {
+	return int64(ts.Sec), int64(ts.Nsec)
+}
+
+func (tv *Timeval) Unix() (sec int64, nsec int64) {
+	return int64(tv.Sec), int64(tv.Usec) * 1000
+}
+
+func (ts *Timespec) Nano() int64 {
+	return int64(ts.Sec)*1e9 + int64(ts.Nsec)
+}
+
+func (tv *Timeval) Nano() int64 {
+	return int64(tv.Sec)*1e9 + int64(tv.Usec)*1000
+}
+
+// use is a no-op, but the compiler cannot see that it is.
+// Calling use(p) ensures that p is kept live until that point.
+//go:noescape
+func use(p unsafe.Pointer)

vendor/golang.org/x/sys/plan9/syscall_plan9.go

@@ -0,0 +1,349 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Plan 9 system calls.
+// This file is compiled as ordinary Go code,
+// but it is also input to mksyscall,
+// which parses the //sys lines and generates system call stubs.
+// Note that sometimes we use a lowercase //sys name and
+// wrap it in our own nicer implementation.
+
+package plan9
+
+import (
+	"bytes"
+	"syscall"
+	"unsafe"
+)
+
+// A Note is a string describing a process note.
+// It implements the os.Signal interface.
+type Note string
+
+func (n Note) Signal() {}
+
+func (n Note) String() string {
+	return string(n)
+}
+
+var (
+	Stdin  = 0
+	Stdout = 1
+	Stderr = 2
+)
+
+// For testing: clients can set this flag to force
+// creation of IPv6 sockets to return EAFNOSUPPORT.
+var SocketDisableIPv6 bool
+
+func Syscall(trap, a1, a2, a3 uintptr) (r1, r2 uintptr, err syscall.ErrorString)
+func Syscall6(trap, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err syscall.ErrorString)
+func RawSyscall(trap, a1, a2, a3 uintptr) (r1, r2, err uintptr)
+func RawSyscall6(trap, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, err uintptr)
+
+func atoi(b []byte) (n uint) {
+	n = 0
+	for i := 0; i < len(b); i++ {
+		n = n*10 + uint(b[i]-'0')
+	}
+	return
+}
+
+func cstring(s []byte) string {
+	i := bytes.IndexByte(s, 0)
+	if i == -1 {
+		i = len(s)
+	}
+	return string(s[:i])
+}
+
+func errstr() string {
+	var buf [ERRMAX]byte
+
+	RawSyscall(SYS_ERRSTR, uintptr(unsafe.Pointer(&buf[0])), uintptr(len(buf)), 0)
+
+	buf[len(buf)-1] = 0
+	return cstring(buf[:])
+}
+
+// Implemented in assembly to import from runtime.
+func exit(code int)
+
+func Exit(code int) { exit(code) }
+
+func readnum(path string) (uint, error) {
+	var b [12]byte
+
+	fd, e := Open(path, O_RDONLY)
+	if e != nil {
+		return 0, e
+	}
+	defer Close(fd)
+
+	n, e := Pread(fd, b[:], 0)
+
+	if e != nil {
+		return 0, e
+	}
+
+	m := 0
+	for ; m < n && b[m] == ' '; m++ {
+	}
+
+	return atoi(b[m : n-1]), nil
+}
+
+func Getpid() (pid int) {
+	n, _ := readnum("#c/pid")
+	return int(n)
+}
+
+func Getppid() (ppid int) {
+	n, _ := readnum("#c/ppid")
+	return int(n)
+}
+
+func Read(fd int, p []byte) (n int, err error) {
+	return Pread(fd, p, -1)
+}
+
+func Write(fd int, p []byte) (n int, err error) {
+	return Pwrite(fd, p, -1)
+}
+
+var ioSync int64
+
+//sys	fd2path(fd int, buf []byte) (err error)
+func Fd2path(fd int) (path string, err error) {
+	var buf [512]byte
+
+	e := fd2path(fd, buf[:])
+	if e != nil {
+		return "", e
+	}
+	return cstring(buf[:]), nil
+}
+
+//sys	pipe(p *[2]int32) (err error)
+func Pipe(p []int) (err error) {
+	if len(p) != 2 {
+		return syscall.ErrorString("bad arg in system call")
+	}
+	var pp [2]int32
+	err = pipe(&pp)
+	p[0] = int(pp[0])
+	p[1] = int(pp[1])
+	return
+}
+
+// Underlying system call writes to newoffset via pointer.
+// Implemented in assembly to avoid allocation.
+func seek(placeholder uintptr, fd int, offset int64, whence int) (newoffset int64, err string)
+
+func Seek(fd int, offset int64, whence int) (newoffset int64, err error) {
+	newoffset, e := seek(0, fd, offset, whence)
+
+	if newoffset == -1 {
+		err = syscall.ErrorString(e)
+	}
+	return
+}
+
+func Mkdir(path string, mode uint32) (err error) {
+	fd, err := Create(path, O_RDONLY, DMDIR|mode)
+
+	if fd != -1 {
+		Close(fd)
+	}
+
+	return
+}
+
+type Waitmsg struct {
+	Pid  int
+	Time [3]uint32
+	Msg  string
+}
+
+func (w Waitmsg) Exited() bool   { return true }
+func (w Waitmsg) Signaled() bool { return false }
+
+func (w Waitmsg) ExitStatus() int {
+	if len(w.Msg) == 0 {
+		// a normal exit returns no message
+		return 0
+	}
+	return 1
+}
+
+//sys	await(s []byte) (n int, err error)
+func Await(w *Waitmsg) (err error) {
+	var buf [512]byte
+	var f [5][]byte
+
+	n, err := await(buf[:])
+
+	if err != nil || w == nil {
+		return
+	}
+
+	nf := 0
+	p := 0
+	for i := 0; i < n && nf < len(f)-1; i++ {
+		if buf[i] == ' ' {
+			f[nf] = buf[p:i]
+			p = i + 1
+			nf++
+		}
+	}
+	f[nf] = buf[p:]
+	nf++
+
+	if nf != len(f) {
+		return syscall.ErrorString("invalid wait message")
+	}
+	w.Pid = int(atoi(f[0]))
+	w.Time[0] = uint32(atoi(f[1]))
+	w.Time[1] = uint32(atoi(f[2]))
+	w.Time[2] = uint32(atoi(f[3]))
+	w.Msg = cstring(f[4])
+	if w.Msg == "''" {
+		// await() returns '' for no error
+		w.Msg = ""
+	}
+	return
+}
+
+func Unmount(name, old string) (err error) {
+	fixwd()
+	oldp, err := BytePtrFromString(old)
+	if err != nil {
+		return err
+	}
+	oldptr := uintptr(unsafe.Pointer(oldp))
+
+	var r0 uintptr
+	var e syscall.ErrorString
+
+	// bind(2) man page: If name is zero, everything bound or mounted upon old is unbound or unmounted.
+	if name == "" {
+		r0, _, e = Syscall(SYS_UNMOUNT, _zero, oldptr, 0)
+	} else {
+		namep, err := BytePtrFromString(name)
+		if err != nil {
+			return err
+		}
+		r0, _, e = Syscall(SYS_UNMOUNT, uintptr(unsafe.Pointer(namep)), oldptr, 0)
+	}
+
+	if int32(r0) == -1 {
+		err = e
+	}
+	return
+}
+
+func Fchdir(fd int) (err error) {
+	path, err := Fd2path(fd)
+
+	if err != nil {
+		return
+	}
+
+	return Chdir(path)
+}
+
+type Timespec struct {
+	Sec  int32
+	Nsec int32
+}
+
+type Timeval struct {
+	Sec  int32
+	Usec int32
+}
+
+func NsecToTimeval(nsec int64) (tv Timeval) {
+	nsec += 999 // round up to microsecond
+	tv.Usec = int32(nsec % 1e9 / 1e3)
+	tv.Sec = int32(nsec / 1e9)
+	return
+}
+
+func nsec() int64 {
+	var scratch int64
+
+	r0, _, _ := Syscall(SYS_NSEC, uintptr(unsafe.Pointer(&scratch)), 0, 0)
+	// TODO(aram): remove hack after I fix _nsec in the pc64 kernel.
+	if r0 == 0 {
+		return scratch
+	}
+	return int64(r0)
+}
+
+func Gettimeofday(tv *Timeval) error {
+	nsec := nsec()
+	*tv = NsecToTimeval(nsec)
+	return nil
+}
+
+func Getpagesize() int { return 0x1000 }
+
+func Getegid() (egid int) { return -1 }
+func Geteuid() (euid int) { return -1 }
+func Getgid() (gid int)   { return -1 }
+func Getuid() (uid int)   { return -1 }
+
+func Getgroups() (gids []int, err error) {
+	return make([]int, 0), nil
+}
+
+//sys	open(path string, mode int) (fd int, err error)
+func Open(path string, mode int) (fd int, err error) {
+	fixwd()
+	return open(path, mode)
+}
+
+//sys	create(path string, mode int, perm uint32) (fd int, err error)
+func Create(path string, mode int, perm uint32) (fd int, err error) {
+	fixwd()
+	return create(path, mode, perm)
+}
+
+//sys	remove(path string) (err error)
+func Remove(path string) error {
+	fixwd()
+	return remove(path)
+}
+
+//sys	stat(path string, edir []byte) (n int, err error)
+func Stat(path string, edir []byte) (n int, err error) {
+	fixwd()
+	return stat(path, edir)
+}
+
+//sys	bind(name string, old string, flag int) (err error)
+func Bind(name string, old string, flag int) (err error) {
+	fixwd()
+	return bind(name, old, flag)
+}
+
+//sys	mount(fd int, afd int, old string, flag int, aname string) (err error)
+func Mount(fd int, afd int, old string, flag int, aname string) (err error) {
+	fixwd()
+	return mount(fd, afd, old, flag, aname)
+}
+
+//sys	wstat(path string, edir []byte) (err error)
+func Wstat(path string, edir []byte) (err error) {
+	fixwd()
+	return wstat(path, edir)
+}
+
+//sys	chdir(path string) (err error)
+//sys	Dup(oldfd int, newfd int) (fd int, err error)
+//sys	Pread(fd int, p []byte, offset int64) (n int, err error)
+//sys	Pwrite(fd int, p []byte, offset int64) (n int, err error)
+//sys	Close(fd int) (err error)
+//sys	Fstat(fd int, edir []byte) (n int, err error)
+//sys	Fwstat(fd int, edir []byte) (err error)

vendor/golang.org/x/sys/plan9/zsyscall_plan9_386.go

@@ -0,0 +1,284 @@
+// go run mksyscall.go -l32 -plan9 -tags plan9,386 syscall_plan9.go
+// Code generated by the command above; see README.md. DO NOT EDIT.
+
+// +build plan9,386
+
+package plan9
+
+import "unsafe"
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
+func fd2path(fd int, buf []byte) (err error) {
+	var _p0 unsafe.Pointer
+	if len(buf) > 0 {
+		_p0 = unsafe.Pointer(&buf[0])
+	} else {
+		_p0 = unsafe.Pointer(&_zero)
+	}
+	r0, _, e1 := Syscall(SYS_FD2PATH, uintptr(fd), uintptr(_p0), uintptr(len(buf)))
+	if int32(r0) == -1 {
+		err = e1
+	}
+	return
+}
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
+func pipe(p *[2]int32) (err error) {
+	r0, _, e1 := Syscall(SYS_PIPE, uintptr(unsafe.Pointer(p)), 0, 0)
+	if int32(r0) == -1 {
+		err = e1
+	}
+	return
+}
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
+func await(s []byte) (n int, err error) {
+	var _p0 unsafe.Pointer
+	if len(s) > 0 {
+		_p0 = unsafe.Pointer(&s[0])
+	} else {
+		_p0 = unsafe.Pointer(&_zero)
+	}
+	r0, _, e1 := Syscall(SYS_AWAIT, uintptr(_p0), uintptr(len(s)), 0)
+	n = int(r0)
+	if int32(r0) == -1 {
+		err = e1
+	}
+	return
+}
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
+func open(path string, mode int) (fd int, err error) {
+	var _p0 *byte
+	_p0, err = BytePtrFromString(path)
+	if err != nil {
+		return
+	}
+	r0, _, e1 := Syscall(SYS_OPEN, uintptr(unsafe.Pointer(_p0)), uintptr(mode), 0)
+	fd = int(r0)
+	if int32(r0) == -1 {
+		err = e1
+	}
+	return
+}
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
+func create(path string, mode int, perm uint32) (fd int, err error) {
+	var _p0 *byte
+	_p0, err = BytePtrFromString(path)
+	if err != nil {
+		return
+	}
+	r0, _, e1 := Syscall(SYS_CREATE, uintptr(unsafe.Pointer(_p0)), uintptr(mode), uintptr(perm))
+	fd = int(r0)
+	if int32(r0) == -1 {
+		err = e1
+	}
+	return
+}
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
+func remove(path string) (err error) {
+	var _p0 *byte
+	_p0, err = BytePtrFromString(path)
+	if err != nil {
+		return
+	}
+	r0, _, e1 := Syscall(SYS_REMOVE, uintptr(unsafe.Pointer(_p0)), 0, 0)
+	if int32(r0) == -1 {
+		err = e1
+	}
+	return
+}
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
+func stat(path string, edir []byte) (n int, err error) {
+	var _p0 *byte
+	_p0, err = BytePtrFromString(path)
+	if err != nil {
+		return
+	}
+	var _p1 unsafe.Pointer
+	if len(edir) > 0 {
+		_p1 = unsafe.Pointer(&edir[0])
+	} else {
+		_p1 = unsafe.Pointer(&_zero)
+	}
+	r0, _, e1 := Syscall(SYS_STAT, uintptr(unsafe.Pointer(_p0)), uintptr(_p1), uintptr(len(edir)))
+	n = int(r0)
+	if int32(r0) == -1 {
+		err = e1
+	}
+	return
+}
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
+func bind(name string, old string, flag int) (err error) {
+	var _p0 *byte
+	_p0, err = BytePtrFromString(name)
+	if err != nil {
+		return
+	}
+	var _p1 *byte
+	_p1, err = BytePtrFromString(old)
+	if err != nil {
+		return
+	}
+	r0, _, e1 := Syscall(SYS_BIND, uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(_p1)), uintptr(flag))
+	if int32(r0) == -1 {
+		err = e1
+	}
+	return
+}
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
+func mount(fd int, afd int, old string, flag int, aname string) (err error) {
+	var _p0 *byte
+	_p0, err = BytePtrFromString(old)
+	if err != nil {
+		return
+	}
+	var _p1 *byte
+	_p1, err = BytePtrFromString(aname)
+	if err != nil {
+		return
+	}
+	r0, _, e1 := Syscall6(SYS_MOUNT, uintptr(fd), uintptr(afd), uintptr(unsafe.Pointer(_p0)), uintptr(flag), uintptr(unsafe.Pointer(_p1)), 0)
+	if int32(r0) == -1 {
+		err = e1
+	}
+	return
+}
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
+func wstat(path string, edir []byte) (err error) {
+	var _p0 *byte
+	_p0, err = BytePtrFromString(path)
+	if err != nil {
+		return
+	}
+	var _p1 unsafe.Pointer
+	if len(edir) > 0 {
+		_p1 = unsafe.Pointer(&edir[0])
+	} else {
+		_p1 = unsafe.Pointer(&_zero)
+	}
+	r0, _, e1 := Syscall(SYS_WSTAT, uintptr(unsafe.Pointer(_p0)), uintptr(_p1), uintptr(len(edir)))
+	if int32(r0) == -1 {
+		err = e1
+	}
+	return
+}
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
+func chdir(path string) (err error) {
+	var _p0 *byte
+	_p0, err = BytePtrFromString(path)
+	if err != nil {
+		return
+	}
+	r0, _, e1 := Syscall(SYS_CHDIR, uintptr(unsafe.Pointer(_p0)), 0, 0)
+	if int32(r0) == -1 {
+		err = e1
+	}
+	return
+}
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
+func Dup(oldfd int, newfd int) (fd int, err error) {
+	r0, _, e1 := Syscall(SYS_DUP, uintptr(oldfd), uintptr(newfd), 0)
+	fd = int(r0)
+	if int32(r0) == -1 {
+		err = e1
+	}
+	return
+}
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
+func Pread(fd int, p []byte, offset int64) (n int, err error) {
+	var _p0 unsafe.Pointer
+	if len(p) > 0 {
+		_p0 = unsafe.Pointer(&p[0])
+	} else {
+		_p0 = unsafe.Pointer(&_zero)
+	}
+	r0, _, e1 := Syscall6(SYS_PREAD, uintptr(fd), uintptr(_p0), uintptr(len(p)), uintptr(offset), uintptr(offset>>32), 0)
+	n = int(r0)
+	if int32(r0) == -1 {
+		err = e1
+	}
+	return
+}
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
+func Pwrite(fd int, p []byte, offset int64) (n int, err error) {
+	var _p0 unsafe.Pointer
+	if len(p) > 0 {
+		_p0 = unsafe.Pointer(&p[0])
+	} else {
+		_p0 = unsafe.Pointer(&_zero)
+	}
+	r0, _, e1 := Syscall6(SYS_PWRITE, uintptr(fd), uintptr(_p0), uintptr(len(p)), uintptr(offset), uintptr(offset>>32), 0)
+	n = int(r0)
+	if int32(r0) == -1 {
+		err = e1
+	}
+	return
+}
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
+func Close(fd int) (err error) {
+	r0, _, e1 := Syscall(SYS_CLOSE, uintptr(fd), 0, 0)
+	if int32(r0) == -1 {
+		err = e1
+	}
+	return
+}
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
+func Fstat(fd int, edir []byte) (n int, err error) {
+	var _p0 unsafe.Pointer
+	if len(edir) > 0 {
+		_p0 = unsafe.Pointer(&edir[0])
+	} else {
+		_p0 = unsafe.Pointer(&_zero)
+	}
+	r0, _, e1 := Syscall(SYS_FSTAT, uintptr(fd), uintptr(_p0), uintptr(len(edir)))
+	n = int(r0)
+	if int32(r0) == -1 {
+		err = e1
+	}
+	return
+}
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
+func Fwstat(fd int, edir []byte) (err error) {
+	var _p0 unsafe.Pointer
+	if len(edir) > 0 {
+		_p0 = unsafe.Pointer(&edir[0])
+	} else {
+		_p0 = unsafe.Pointer(&_zero)
+	}
+	r0, _, e1 := Syscall(SYS_FWSTAT, uintptr(fd), uintptr(_p0), uintptr(len(edir)))
+	if int32(r0) == -1 {
+		err = e1
+	}
+	return
+}

vendor/golang.org/x/sys/plan9/zsyscall_plan9_amd64.go

@@ -0,0 +1,284 @@
+// go run mksyscall.go -l32 -plan9 -tags plan9,amd64 syscall_plan9.go
+// Code generated by the command above; see README.md. DO NOT EDIT.
+
+// +build plan9,amd64
+
+package plan9
+
+import "unsafe"
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
+func fd2path(fd int, buf []byte) (err error) {
+	var _p0 unsafe.Pointer
+	if len(buf) > 0 {
+		_p0 = unsafe.Pointer(&buf[0])
+	} else {
+		_p0 = unsafe.Pointer(&_zero)
+	}
+	r0, _, e1 := Syscall(SYS_FD2PATH, uintptr(fd), uintptr(_p0), uintptr(len(buf)))
+	if int32(r0) == -1 {
+		err = e1
+	}
+	return
+}
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
+func pipe(p *[2]int32) (err error) {
+	r0, _, e1 := Syscall(SYS_PIPE, uintptr(unsafe.Pointer(p)), 0, 0)
+	if int32(r0) == -1 {
+		err = e1
+	}
+	return
+}
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
+func await(s []byte) (n int, err error) {
+	var _p0 unsafe.Pointer
+	if len(s) > 0 {
+		_p0 = unsafe.Pointer(&s[0])
+	} else {
+		_p0 = unsafe.Pointer(&_zero)
+	}
+	r0, _, e1 := Syscall(SYS_AWAIT, uintptr(_p0), uintptr(len(s)), 0)
+	n = int(r0)
+	if int32(r0) == -1 {
+		err = e1
+	}
+	return
+}
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
+func open(path string, mode int) (fd int, err error) {
+	var _p0 *byte
+	_p0, err = BytePtrFromString(path)
+	if err != nil {
+		return
+	}
+	r0, _, e1 := Syscall(SYS_OPEN, uintptr(unsafe.Pointer(_p0)), uintptr(mode), 0)
+	fd = int(r0)
+	if int32(r0) == -1 {
+		err = e1
+	}
+	return
+}
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
+func create(path string, mode int, perm uint32) (fd int, err error) {
+	var _p0 *byte
+	_p0, err = BytePtrFromString(path)
+	if err != nil {
+		return
+	}
+	r0, _, e1 := Syscall(SYS_CREATE, uintptr(unsafe.Pointer(_p0)), uintptr(mode), uintptr(perm))
+	fd = int(r0)
+	if int32(r0) == -1 {
+		err = e1
+	}
+	return
+}
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
+func remove(path string) (err error) {
+	var _p0 *byte
+	_p0, err = BytePtrFromString(path)
+	if err != nil {
+		return
+	}
+	r0, _, e1 := Syscall(SYS_REMOVE, uintptr(unsafe.Pointer(_p0)), 0, 0)
+	if int32(r0) == -1 {
+		err = e1
+	}
+	return
+}
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
+func stat(path string, edir []byte) (n int, err error) {
+	var _p0 *byte
+	_p0, err = BytePtrFromString(path)
+	if err != nil {
+		return
+	}
+	var _p1 unsafe.Pointer
+	if len(edir) > 0 {
+		_p1 = unsafe.Pointer(&edir[0])
+	} else {
+		_p1 = unsafe.Pointer(&_zero)
+	}
+	r0, _, e1 := Syscall(SYS_STAT, uintptr(unsafe.Pointer(_p0)), uintptr(_p1), uintptr(len(edir)))
+	n = int(r0)
+	if int32(r0) == -1 {
+		err = e1
+	}
+	return
+}
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
+func bind(name string, old string, flag int) (err error) {
+	var _p0 *byte
+	_p0, err = BytePtrFromString(name)
+	if err != nil {
+		return
+	}
+	var _p1 *byte
+	_p1, err = BytePtrFromString(old)
+	if err != nil {
+		return
+	}
+	r0, _, e1 := Syscall(SYS_BIND, uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(_p1)), uintptr(flag))
+	if int32(r0) == -1 {
+		err = e1
+	}
+	return
+}
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
+func mount(fd int, afd int, old string, flag int, aname string) (err error) {
+	var _p0 *byte
+	_p0, err = BytePtrFromString(old)
+	if err != nil {
+		return
+	}
+	var _p1 *byte
+	_p1, err = BytePtrFromString(aname)
+	if err != nil {
+		return
+	}
+	r0, _, e1 := Syscall6(SYS_MOUNT, uintptr(fd), uintptr(afd), uintptr(unsafe.Pointer(_p0)), uintptr(flag), uintptr(unsafe.Pointer(_p1)), 0)
+	if int32(r0) == -1 {
+		err = e1
+	}
+	return
+}
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
+func wstat(path string, edir []byte) (err error) {
+	var _p0 *byte
+	_p0, err = BytePtrFromString(path)
+	if err != nil {
+		return
+	}
+	var _p1 unsafe.Pointer
+	if len(edir) > 0 {
+		_p1 = unsafe.Pointer(&edir[0])
+	} else {
+		_p1 = unsafe.Pointer(&_zero)
+	}
+	r0, _, e1 := Syscall(SYS_WSTAT, uintptr(unsafe.Pointer(_p0)), uintptr(_p1), uintptr(len(edir)))
+	if int32(r0) == -1 {
+		err = e1
+	}
+	return
+}
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
+func chdir(path string) (err error) {
+	var _p0 *byte
+	_p0, err = BytePtrFromString(path)
+	if err != nil {
+		return
+	}
+	r0, _, e1 := Syscall(SYS_CHDIR, uintptr(unsafe.Pointer(_p0)), 0, 0)
+	if int32(r0) == -1 {
+		err = e1
+	}
+	return
+}
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
+func Dup(oldfd int, newfd int) (fd int, err error) {
+	r0, _, e1 := Syscall(SYS_DUP, uintptr(oldfd), uintptr(newfd), 0)
+	fd = int(r0)
+	if int32(r0) == -1 {
+		err = e1
+	}
+	return
+}
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
+func Pread(fd int, p []byte, offset int64) (n int, err error) {
+	var _p0 unsafe.Pointer
+	if len(p) > 0 {
+		_p0 = unsafe.Pointer(&p[0])
+	} else {
+		_p0 = unsafe.Pointer(&_zero)
+	}
+	r0, _, e1 := Syscall6(SYS_PREAD, uintptr(fd), uintptr(_p0), uintptr(len(p)), uintptr(offset), uintptr(offset>>32), 0)
+	n = int(r0)
+	if int32(r0) == -1 {
+		err = e1
+	}
+	return
+}
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
+func Pwrite(fd int, p []byte, offset int64) (n int, err error) {
+	var _p0 unsafe.Pointer
+	if len(p) > 0 {
+		_p0 = unsafe.Pointer(&p[0])
+	} else {
+		_p0 = unsafe.Pointer(&_zero)
+	}
+	r0, _, e1 := Syscall6(SYS_PWRITE, uintptr(fd), uintptr(_p0), uintptr(len(p)), uintptr(offset), uintptr(offset>>32), 0)
+	n = int(r0)
+	if int32(r0) == -1 {
+		err = e1
+	}
+	return
+}
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
+func Close(fd int) (err error) {
+	r0, _, e1 := Syscall(SYS_CLOSE, uintptr(fd), 0, 0)
+	if int32(r0) == -1 {
+		err = e1
+	}
+	return
+}
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
+func Fstat(fd int, edir []byte) (n int, err error) {
+	var _p0 unsafe.Pointer
+	if len(edir) > 0 {
+		_p0 = unsafe.Pointer(&edir[0])
+	} else {
+		_p0 = unsafe.Pointer(&_zero)
+	}
+	r0, _, e1 := Syscall(SYS_FSTAT, uintptr(fd), uintptr(_p0), uintptr(len(edir)))
+	n = int(r0)
+	if int32(r0) == -1 {
+		err = e1
+	}
+	return
+}
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
+func Fwstat(fd int, edir []byte) (err error) {
+	var _p0 unsafe.Pointer
+	if len(edir) > 0 {
+		_p0 = unsafe.Pointer(&edir[0])
+	} else {
+		_p0 = unsafe.Pointer(&_zero)
+	}
+	r0, _, e1 := Syscall(SYS_FWSTAT, uintptr(fd), uintptr(_p0), uintptr(len(edir)))
+	if int32(r0) == -1 {
+		err = e1
+	}
+	return
+}

vendor/golang.org/x/sys/plan9/zsyscall_plan9_arm.go

@@ -0,0 +1,284 @@
+// go run mksyscall.go -l32 -plan9 -tags plan9,arm syscall_plan9.go
+// Code generated by the command above; see README.md. DO NOT EDIT.
+
+// +build plan9,arm
+
+package plan9
+
+import "unsafe"
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
+func fd2path(fd int, buf []byte) (err error) {
+	var _p0 unsafe.Pointer
+	if len(buf) > 0 {
+		_p0 = unsafe.Pointer(&buf[0])
+	} else {
+		_p0 = unsafe.Pointer(&_zero)
+	}
+	r0, _, e1 := Syscall(SYS_FD2PATH, uintptr(fd), uintptr(_p0), uintptr(len(buf)))
+	if int32(r0) == -1 {
+		err = e1
+	}
+	return
+}
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
+func pipe(p *[2]int32) (err error) {
+	r0, _, e1 := Syscall(SYS_PIPE, uintptr(unsafe.Pointer(p)), 0, 0)
+	if int32(r0) == -1 {
+		err = e1
+	}
+	return
+}
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
+func await(s []byte) (n int, err error) {
+	var _p0 unsafe.Pointer
+	if len(s) > 0 {
+		_p0 = unsafe.Pointer(&s[0])
+	} else {
+		_p0 = unsafe.Pointer(&_zero)
+	}
+	r0, _, e1 := Syscall(SYS_AWAIT, uintptr(_p0), uintptr(len(s)), 0)
+	n = int(r0)
+	if int32(r0) == -1 {
+		err = e1
+	}
+	return
+}
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
+func open(path string, mode int) (fd int, err error) {
+	var _p0 *byte
+	_p0, err = BytePtrFromString(path)
+	if err != nil {
+		return
+	}
+	r0, _, e1 := Syscall(SYS_OPEN, uintptr(unsafe.Pointer(_p0)), uintptr(mode), 0)
+	fd = int(r0)
+	if int32(r0) == -1 {
+		err = e1
+	}
+	return
+}
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
+func create(path string, mode int, perm uint32) (fd int, err error) {
+	var _p0 *byte
+	_p0, err = BytePtrFromString(path)
+	if err != nil {
+		return
+	}
+	r0, _, e1 := Syscall(SYS_CREATE, uintptr(unsafe.Pointer(_p0)), uintptr(mode), uintptr(perm))
+	fd = int(r0)
+	if int32(r0) == -1 {
+		err = e1
+	}
+	return
+}
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
+func remove(path string) (err error) {
+	var _p0 *byte
+	_p0, err = BytePtrFromString(path)
+	if err != nil {
+		return
+	}
+	r0, _, e1 := Syscall(SYS_REMOVE, uintptr(unsafe.Pointer(_p0)), 0, 0)
+	if int32(r0) == -1 {
+		err = e1
+	}
+	return
+}
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
+func stat(path string, edir []byte) (n int, err error) {
+	var _p0 *byte
+	_p0, err = BytePtrFromString(path)
+	if err != nil {
+		return
+	}
+	var _p1 unsafe.Pointer
+	if len(edir) > 0 {
+		_p1 = unsafe.Pointer(&edir[0])
+	} else {
+		_p1 = unsafe.Pointer(&_zero)
+	}
+	r0, _, e1 := Syscall(SYS_STAT, uintptr(unsafe.Pointer(_p0)), uintptr(_p1), uintptr(len(edir)))
+	n = int(r0)
+	if int32(r0) == -1 {
+		err = e1
+	}
+	return
+}
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
+func bind(name string, old string, flag int) (err error) {
+	var _p0 *byte
+	_p0, err = BytePtrFromString(name)
+	if err != nil {
+		return
+	}
+	var _p1 *byte
+	_p1, err = BytePtrFromString(old)
+	if err != nil {
+		return
+	}
+	r0, _, e1 := Syscall(SYS_BIND, uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(_p1)), uintptr(flag))
+	if int32(r0) == -1 {
+		err = e1
+	}
+	return
+}
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
+func mount(fd int, afd int, old string, flag int, aname string) (err error) {
+	var _p0 *byte
+	_p0, err = BytePtrFromString(old)
+	if err != nil {
+		return
+	}
+	var _p1 *byte
+	_p1, err = BytePtrFromString(aname)
+	if err != nil {
+		return
+	}
+	r0, _, e1 := Syscall6(SYS_MOUNT, uintptr(fd), uintptr(afd), uintptr(unsafe.Pointer(_p0)), uintptr(flag), uintptr(unsafe.Pointer(_p1)), 0)
+	if int32(r0) == -1 {
+		err = e1
+	}
+	return
+}
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
+func wstat(path string, edir []byte) (err error) {
+	var _p0 *byte
+	_p0, err = BytePtrFromString(path)
+	if err != nil {
+		return
+	}
+	var _p1 unsafe.Pointer
+	if len(edir) > 0 {
+		_p1 = unsafe.Pointer(&edir[0])
+	} else {
+		_p1 = unsafe.Pointer(&_zero)
+	}
+	r0, _, e1 := Syscall(SYS_WSTAT, uintptr(unsafe.Pointer(_p0)), uintptr(_p1), uintptr(len(edir)))
+	if int32(r0) == -1 {
+		err = e1
+	}
+	return
+}
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
+func chdir(path string) (err error) {
+	var _p0 *byte
+	_p0, err = BytePtrFromString(path)
+	if err != nil {
+		return
+	}
+	r0, _, e1 := Syscall(SYS_CHDIR, uintptr(unsafe.Pointer(_p0)), 0, 0)
+	if int32(r0) == -1 {
+		err = e1
+	}
+	return
+}
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
+func Dup(oldfd int, newfd int) (fd int, err error) {
+	r0, _, e1 := Syscall(SYS_DUP, uintptr(oldfd), uintptr(newfd), 0)
+	fd = int(r0)
+	if int32(r0) == -1 {
+		err = e1
+	}
+	return
+}
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
+func Pread(fd int, p []byte, offset int64) (n int, err error) {
+	var _p0 unsafe.Pointer
+	if len(p) > 0 {
+		_p0 = unsafe.Pointer(&p[0])
+	} else {
+		_p0 = unsafe.Pointer(&_zero)
+	}
+	r0, _, e1 := Syscall6(SYS_PREAD, uintptr(fd), uintptr(_p0), uintptr(len(p)), uintptr(offset), uintptr(offset>>32), 0)
+	n = int(r0)
+	if int32(r0) == -1 {
+		err = e1
+	}
+	return
+}
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
+func Pwrite(fd int, p []byte, offset int64) (n int, err error) {
+	var _p0 unsafe.Pointer
+	if len(p) > 0 {
+		_p0 = unsafe.Pointer(&p[0])
+	} else {
+		_p0 = unsafe.Pointer(&_zero)
+	}
+	r0, _, e1 := Syscall6(SYS_PWRITE, uintptr(fd), uintptr(_p0), uintptr(len(p)), uintptr(offset), uintptr(offset>>32), 0)
+	n = int(r0)
+	if int32(r0) == -1 {
+		err = e1
+	}
+	return
+}
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
+func Close(fd int) (err error) {
+	r0, _, e1 := Syscall(SYS_CLOSE, uintptr(fd), 0, 0)
+	if int32(r0) == -1 {
+		err = e1
+	}
+	return
+}
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
+func Fstat(fd int, edir []byte) (n int, err error) {
+	var _p0 unsafe.Pointer
+	if len(edir) > 0 {
+		_p0 = unsafe.Pointer(&edir[0])
+	} else {
+		_p0 = unsafe.Pointer(&_zero)
+	}
+	r0, _, e1 := Syscall(SYS_FSTAT, uintptr(fd), uintptr(_p0), uintptr(len(edir)))
+	n = int(r0)
+	if int32(r0) == -1 {
+		err = e1
+	}
+	return
+}
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
+func Fwstat(fd int, edir []byte) (err error) {
+	var _p0 unsafe.Pointer
+	if len(edir) > 0 {
+		_p0 = unsafe.Pointer(&edir[0])
+	} else {
+		_p0 = unsafe.Pointer(&_zero)
+	}
+	r0, _, e1 := Syscall(SYS_FWSTAT, uintptr(fd), uintptr(_p0), uintptr(len(edir)))
+	if int32(r0) == -1 {
+		err = e1
+	}
+	return
+}

vendor/golang.org/x/sys/plan9/zsysnum_plan9.go

@@ -0,0 +1,49 @@
+// mksysnum_plan9.sh /opt/plan9/sys/src/libc/9syscall/sys.h
+// MACHINE GENERATED BY THE ABOVE COMMAND; DO NOT EDIT
+
+package plan9
+
+const (
+	SYS_SYSR1       = 0
+	SYS_BIND        = 2
+	SYS_CHDIR       = 3
+	SYS_CLOSE       = 4
+	SYS_DUP         = 5
+	SYS_ALARM       = 6
+	SYS_EXEC        = 7
+	SYS_EXITS       = 8
+	SYS_FAUTH       = 10
+	SYS_SEGBRK      = 12
+	SYS_OPEN        = 14
+	SYS_OSEEK       = 16
+	SYS_SLEEP       = 17
+	SYS_RFORK       = 19
+	SYS_PIPE        = 21
+	SYS_CREATE      = 22
+	SYS_FD2PATH     = 23
+	SYS_BRK_        = 24
+	SYS_REMOVE      = 25
+	SYS_NOTIFY      = 28
+	SYS_NOTED       = 29
+	SYS_SEGATTACH   = 30
+	SYS_SEGDETACH   = 31
+	SYS_SEGFREE     = 32
+	SYS_SEGFLUSH    = 33
+	SYS_RENDEZVOUS  = 34
+	SYS_UNMOUNT     = 35
+	SYS_SEMACQUIRE  = 37
+	SYS_SEMRELEASE  = 38
+	SYS_SEEK        = 39
+	SYS_FVERSION    = 40
+	SYS_ERRSTR      = 41
+	SYS_STAT        = 42
+	SYS_FSTAT       = 43
+	SYS_WSTAT       = 44
+	SYS_FWSTAT      = 45
+	SYS_MOUNT       = 46
+	SYS_AWAIT       = 47
+	SYS_PREAD       = 50
+	SYS_PWRITE      = 51
+	SYS_TSEMACQUIRE = 52
+	SYS_NSEC        = 53
+)

vendor/golang.org/x/sys/unix/asm_aix_ppc64.s

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build !gccgo
+// +build gc
 
 #include "textflag.h"
 

vendor/golang.org/x/sys/unix/asm_darwin_386.s

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build !gccgo
+// +build gc
 
 #include "textflag.h"
 

vendor/golang.org/x/sys/unix/asm_darwin_amd64.s

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build !gccgo
+// +build gc
 
 #include "textflag.h"
 

vendor/golang.org/x/sys/unix/asm_darwin_arm.s

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build !gccgo
+// +build gc
 // +build arm,darwin
 
 #include "textflag.h"

vendor/golang.org/x/sys/unix/asm_darwin_arm64.s

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build !gccgo
+// +build gc
 // +build arm64,darwin
 
 #include "textflag.h"

vendor/golang.org/x/sys/unix/asm_dragonfly_amd64.s

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build !gccgo
+// +build gc
 
 #include "textflag.h"
 

vendor/golang.org/x/sys/unix/asm_freebsd_386.s

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build !gccgo
+// +build gc
 
 #include "textflag.h"
 

vendor/golang.org/x/sys/unix/asm_freebsd_amd64.s

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build !gccgo
+// +build gc
 
 #include "textflag.h"
 

vendor/golang.org/x/sys/unix/asm_freebsd_arm.s

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build !gccgo
+// +build gc
 
 #include "textflag.h"
 

vendor/golang.org/x/sys/unix/asm_freebsd_arm64.s

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build !gccgo
+// +build gc
 
 #include "textflag.h"
 

vendor/golang.org/x/sys/unix/asm_linux_386.s

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build !gccgo
+// +build gc
 
 #include "textflag.h"
 

vendor/golang.org/x/sys/unix/asm_linux_amd64.s

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build !gccgo
+// +build gc
 
 #include "textflag.h"
 

vendor/golang.org/x/sys/unix/asm_linux_arm.s

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build !gccgo
+// +build gc
 
 #include "textflag.h"
 

vendor/golang.org/x/sys/unix/asm_linux_arm64.s

@@ -4,7 +4,7 @@
 
 // +build linux
 // +build arm64
-// +build !gccgo
+// +build gc
 
 #include "textflag.h"