Release 2020.2.1

* perf(cloudflared): reuse memory from buffer pool to get better throughput

https://github.com/cloudflare/cloudflared/issues/160

Dockerfile

@@ -1,12 +1,28 @@
-FROM golang:1.12 as builder
-WORKDIR /go/src/github.com/cloudflare/cloudflared/
-RUN apt-get update && apt-get install -y --no-install-recommends upx
-# Run after `apt-get update` to improve rebuild scenarios
-COPY . .
-RUN make cloudflared
-RUN upx --no-progress cloudflared
+# use a builder image for building cloudflare
+FROM golang:1.13.3 as builder
+ENV GO111MODULE=on
+ENV CGO_ENABLED=0
+ENV GOOS=linux
 
-FROM gcr.io/distroless/base
-COPY --from=builder /go/src/github.com/cloudflare/cloudflared/cloudflared /usr/local/bin/
+WORKDIR /go/src/github.com/cloudflare/cloudflared/
+
+# copy our sources into the builder image
+COPY . .
+
+# compile cloudflared
+RUN make cloudflared
+
+# ---
+
+# use a distroless base image with glibc
+FROM gcr.io/distroless/base-debian10:nonroot
+
+# copy our compiled binary
+COPY --from=builder --chown=nonroot /go/src/github.com/cloudflare/cloudflared/cloudflared /usr/local/bin/
+
+# run as non-privileged user
+USER nonroot
+
+# command / entrypoint of container
 ENTRYPOINT ["cloudflared", "--no-autoupdate"]
 CMD ["version"]

RELEASE_NOTES

@@ -1,3 +1,47 @@
+2020.2.1
+- 2020-02-20 TUN-2745: Rename existing header management functions
+- 2020-02-21 TUN-2746: Add the new header management functions
+- 2020-02-25 perf(cloudflared): reuse memory from buffer pool to get better throughput (#161)
+- 2020-02-25 Tweak HTTP host header. Fixes #107 (#168)
+- 2020-02-25 TUN-2765: Add list of features to tunnelrpc
+- 2020-02-19 TUN-2725: Specify in code that --edge is for internal testing only
+- 2020-02-19 TUN-2703: Muxer.Serve terminates when its context is Done
+- 2020-02-09 TUN-2717: Function to serialize/deserialize HTTP headers
+- 2020-02-05 TUN-2714: New edge discovery. Connections try to reconnect to the same edge IP.
+
+2020.2.0
+- 2020-01-30 TUN-2651: Fix panic in h2mux reader when a stream error is encountered
+- 2020-01-27 TUN-2645: Revert "TUN-2645: Turn on reconnect tokens"
+- 2020-01-28 TUN-2693: Metrics for ReconnectTunnel
+- 2020-01-28 TUN-2696: Add unknown registerRPCName
+- 2020-01-28 TUN-2699: Metrics for Authenticate RPCs
+- 2020-01-28 TUN-2690: cloudflared reconnect uses wrong context
+- 2020-01-29 TUN-2707: Inconsistent cardinality in tunnel error metrics
+- 2020-01-13 TUN-2645: Turn on reconnect tokens
+- 2019-12-23 TUN-2646: Make --edge flag work again for local development
+
+2019.12.0
+- 2019-12-11 TUN-2631: only notify that activeStreamMap is closed if ignoreNewStreams=true
+- 2019-12-17 bug(cloudflared): Set the MaxIdleConnsPerHost of http.Transport to proxy-keepalive-connections (#155)
+- 2019-12-17 refactor(docker): optimize Dockerfile (#126)
+- 2019-12-19 Fix timer scheduling for systemd update service (#159)
+- 2019-12-13 TUN-2637: Manage edge IPs in a region-aware manner
+- 2019-12-03 bug(cloudflared): nil pointer deference on h2DictWriter Close() (#154)
+- 2019-12-03 TUN-2608: h2mux.Muxer.Shutdown always returns a non-nil channel
+- 2019-12-04 TUN-2555: origin/supervisor.go calls Authenticate
+- 2019-12-06 TUN-2554: cloudflared calls ReconnectTunnel
+- 2019-11-20 TUN-2575: Constructors + simpler conversions for AuthOutcome
+- 2019-11-22 Fix Docker build failure (#149)
+- 2019-11-21 TUN-2573: Refactor TunnelRegistration into PermanentRegistrationError, RetryableRegistrationError and SuccessfulTunnelRegistration
+- 2019-11-22 TUN-2582: EventDigest field in tunnelrpc
+- 2019-11-22 Fix "happy eyeballs" not being disabled since Golang 1.12 upgrade * The Dialer.DualStack setting is now ignored and deprecated; RFC 6555 Fast Fallback ("Happy Eyeballs") is now enabled by default. To disable, set Dialer.FallbackDelay to a negative value.
+- 2019-11-25 TUN-2591: ReconnectTunnel now sends EventDigest
+- 2019-11-21 TUN-2606: add DialEdge helpers
+- 2019-11-21 TUN-2607: add RPC stream helpers
+
+2019.11.3
+- 2019-11-20 TUN-2562: Update Cloudflare Origin CA RSA root
+
 2019.11.2
 - 2019-11-18 TUN-2567: AuthOutcome can be turned back into AuthResponse
 - 2019-11-18 TUN-2563: Exposes config_version metrics

buffer/pool.go

@@ -0,0 +1,29 @@
+package buffer
+
+import (
+	"sync"
+)
+
+type Pool struct {
+	// A Pool must not be copied after first use.
+	// https://golang.org/pkg/sync/#Pool
+	buffers sync.Pool
+}
+
+func NewPool(bufferSize int) *Pool {
+	return &Pool{
+		buffers: sync.Pool{
+			New: func() interface{} {
+				return make([]byte, bufferSize)
+			},
+		},
+	}
+}
+
+func (p *Pool) Get() []byte {
+	return p.buffers.Get().([]byte)
+}
+
+func (p *Pool) Put(buf []byte) {
+	p.buffers.Put(buf)
+}

cmd/cloudflared/linux_service.go

@@ -73,7 +73,7 @@ ExecStart=/bin/bash -c '{{ .Path }} update; code=$?; if [ $code -eq 64 ]; then s
 Description=Update Argo Tunnel
 
 [Timer]
-OnUnitActiveSec=1d
+OnCalendar=daily
 
 [Install]
 WantedBy=timers.target

cmd/cloudflared/tunnel/cmd.go

@@ -690,7 +690,7 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 		}),
 		altsrc.NewStringSliceFlag(&cli.StringSliceFlag{
 			Name:    "edge",
-			Usage:   "Address of the Cloudflare tunnel server.",
+			Usage:   "Address of the Cloudflare tunnel server. Only works in Cloudflare's internal testing environment.",
 			EnvVars: []string{"TUNNEL_EDGE"},
 			Hidden:  true,
 		}),
@@ -977,6 +977,12 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			EnvVars: []string{"TUNNEL_INTENT"},
 			Hidden:  true,
 		}),
+		altsrc.NewBoolFlag(&cli.BoolFlag{
+			Name:    "use-reconnect-token",
+			Usage:   "Test reestablishing connections with the new 'reconnect token' flow.",
+			EnvVars: []string{"TUNNEL_USE_RECONNECT_TOKEN"},
+			Hidden:  true,
+		}),
 		altsrc.NewDurationFlag(&cli.DurationFlag{
 			Name:    "dial-edge-timeout",
 			Usage:   "Maximum wait time to set up a connection with the edge",
@@ -1044,7 +1050,6 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			Usage:   "Absolute path of directory to save SSH host keys in",
 			EnvVars: []string{"HOST_KEY_PATH"},
 			Hidden:  true,
-
 		}),
 	}
-}
+}

cmd/cloudflared/tunnel/configuration.go

@@ -14,7 +14,7 @@ import (
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
-	"github.com/cloudflare/cloudflared/connection"
+	"github.com/cloudflare/cloudflared/edgediscovery"
 	"github.com/cloudflare/cloudflared/origin"
 	"github.com/cloudflare/cloudflared/tlsconfig"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
@@ -197,17 +197,21 @@ func prepareTunnelConfig(
 	httpTransport := &http.Transport{
 		Proxy:                 http.ProxyFromEnvironment,
 		MaxIdleConns:          c.Int("proxy-keepalive-connections"),
+		MaxIdleConnsPerHost:   c.Int("proxy-keepalive-connections"),
 		IdleConnTimeout:       c.Duration("proxy-keepalive-timeout"),
 		TLSHandshakeTimeout:   c.Duration("proxy-tls-timeout"),
 		ExpectContinueTimeout: 1 * time.Second,
 		TLSClientConfig:       &tls.Config{RootCAs: originCertPool, InsecureSkipVerify: c.IsSet("no-tls-verify")},
 	}
 
-	dialContext := (&net.Dialer{
+	dialer := &net.Dialer{
 		Timeout:   c.Duration("proxy-connect-timeout"),
 		KeepAlive: c.Duration("proxy-tcp-keepalive"),
-		DualStack: !c.Bool("proxy-no-happy-eyeballs"),
-	}).DialContext
+	}
+	if c.Bool("proxy-no-happy-eyeballs") {
+		dialer.FallbackDelay = -1 // As of Golang 1.12, a negative delay disables "happy eyeballs"
+	}
+	dialContext := dialer.DialContext
 
 	if c.IsSet("unix-socket") {
 		unixSocket, err := config.ValidateUnixSocket(c)
@@ -272,16 +276,17 @@ func prepareTunnelConfig(
 		TlsConfig:            toEdgeTLSConfig,
 		TransportLogger:      transportLogger,
 		UseDeclarativeTunnel: c.Bool("use-declarative-tunnels"),
+		UseReconnectToken:    c.Bool("use-reconnect-token"),
 	}, nil
 }
 
-func serviceDiscoverer(c *cli.Context, logger *logrus.Logger) (connection.EdgeServiceDiscoverer, error) {
+func serviceDiscoverer(c *cli.Context, logger *logrus.Logger) (*edgediscovery.Edge, error) {
 	// If --edge is specfied, resolve edge server addresses
 	if len(c.StringSlice("edge")) > 0 {
-		return connection.NewEdgeHostnameResolver(c.StringSlice("edge"))
+		return edgediscovery.StaticEdge(logger, c.StringSlice("edge"))
 	}
 	// Otherwise lookup edge server addresses through service discovery
-	return connection.NewEdgeAddrResolver(logger)
+	return edgediscovery.ResolveEdge(logger)
 }
 
 func isRunningFromTerminal() bool {

connection/connection.go

@@ -5,35 +5,27 @@ import (
 	"net"
 	"time"
 
-	"github.com/cloudflare/cloudflared/h2mux"
-	"github.com/cloudflare/cloudflared/tunnelrpc"
-	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 	"github.com/google/uuid"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 
-	rpc "zombiezen.com/go/capnproto2/rpc"
+	"github.com/cloudflare/cloudflared/h2mux"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
 const (
 	openStreamTimeout = 30 * time.Second
 )
 
-type dialError struct {
-	cause error
-}
-
-func (e dialError) Error() string {
-	return e.cause.Error()
-}
-
 type Connection struct {
-	id    uuid.UUID
-	muxer *h2mux.Muxer
+	id          uuid.UUID
+	muxer       *h2mux.Muxer
+	addr        *net.TCPAddr
+	isLongLived bool
+	longLivedID int
 }
 
-func newConnection(muxer *h2mux.Muxer, edgeIP *net.TCPAddr) (*Connection, error) {
+func newConnection(muxer *h2mux.Muxer, addr *net.TCPAddr) (*Connection, error) {
 	id, err := uuid.NewRandom()
 	if err != nil {
 		return nil, err
@@ -41,6 +33,7 @@ func newConnection(muxer *h2mux.Muxer, edgeIP *net.TCPAddr) (*Connection, error)
 	return &Connection{
 		id:    id,
 		muxer: muxer,
+		addr:  addr,
 	}, nil
 }
 
@@ -50,32 +43,15 @@ func (c *Connection) Serve(ctx context.Context) error {
 }
 
 // Connect is used to establish connections with cloudflare's edge network
-func (c *Connection) Connect(ctx context.Context, parameters *tunnelpogs.ConnectParameters, logger *logrus.Entry) (pogs.ConnectResult, error) {
-	openStreamCtx, cancel := context.WithTimeout(ctx, openStreamTimeout)
-	defer cancel()
-
-	rpcConn, err := c.newRPConn(openStreamCtx, logger)
+func (c *Connection) Connect(ctx context.Context, parameters *tunnelpogs.ConnectParameters, logger *logrus.Entry) (tunnelpogs.ConnectResult, error) {
+	tsClient, err := NewRPCClient(ctx, c.muxer, logger.WithField("rpc", "connect"), openStreamTimeout)
 	if err != nil {
 		return nil, errors.Wrap(err, "cannot create new RPC connection")
 	}
-	defer rpcConn.Close()
-
-	tsClient := tunnelpogs.TunnelServer_PogsClient{Client: rpcConn.Bootstrap(ctx)}
-
+	defer tsClient.Close()
 	return tsClient.Connect(ctx, parameters)
 }
 
 func (c *Connection) Shutdown() {
 	c.muxer.Shutdown()
 }
-
-func (c *Connection) newRPConn(ctx context.Context, logger *logrus.Entry) (*rpc.Conn, error) {
-	stream, err := c.muxer.OpenRPCStream(ctx)
-	if err != nil {
-		return nil, err
-	}
-	return rpc.NewConn(
-		tunnelrpc.NewTransportLogger(logger.WithField("rpc", "connect"), rpc.StreamTransport(stream)),
-		tunnelrpc.ConnLog(logger.WithField("rpc", "connect")),
-	), nil
-}

connection/dial.go

@@ -0,0 +1,54 @@
+package connection
+
+import (
+	"context"
+	"crypto/tls"
+	"net"
+	"time"
+
+	"github.com/pkg/errors"
+)
+
+// DialEdge makes a TLS connection to a Cloudflare edge node
+func DialEdge(
+	ctx context.Context,
+	timeout time.Duration,
+	tlsConfig *tls.Config,
+	edgeTCPAddr *net.TCPAddr,
+) (net.Conn, error) {
+	// Inherit from parent context so we can cancel (Ctrl-C) while dialing
+	dialCtx, dialCancel := context.WithTimeout(ctx, timeout)
+	defer dialCancel()
+
+	dialer := net.Dialer{}
+	edgeConn, err := dialer.DialContext(dialCtx, "tcp", edgeTCPAddr.String())
+	if err != nil {
+		return nil, newDialError(err, "DialContext error")
+	}
+	tlsEdgeConn := tls.Client(edgeConn, tlsConfig)
+	tlsEdgeConn.SetDeadline(time.Now().Add(timeout))
+
+	if err = tlsEdgeConn.Handshake(); err != nil {
+		return nil, newDialError(err, "TLS handshake with edge error")
+	}
+	// clear the deadline on the conn; h2mux has its own timeouts
+	tlsEdgeConn.SetDeadline(time.Time{})
+	return tlsEdgeConn, nil
+}
+
+// DialError is an error returned from DialEdge
+type DialError struct {
+	cause error
+}
+
+func newDialError(err error, message string) error {
+	return DialError{cause: errors.Wrap(err, message)}
+}
+
+func (e DialError) Error() string {
+	return e.cause.Error()
+}
+
+func (e DialError) Cause() error {
+	return e.cause
+}

connection/discovery.go

@@ -1,254 +0,0 @@
-package connection
-
-import (
-	"context"
-	"crypto/tls"
-	"fmt"
-	"net"
-	"sync"
-	"time"
-
-	"github.com/pkg/errors"
-	"github.com/sirupsen/logrus"
-)
-
-const (
-	// Used to discover HA origintunneld servers
-	srvService = "origintunneld"
-	srvProto   = "tcp"
-	srvName    = "argotunnel.com"
-
-	// Used to fallback to DoT when we can't use the default resolver to
-	// discover HA origintunneld servers (GitHub issue #75).
-	dotServerName = "cloudflare-dns.com"
-	dotServerAddr = "1.1.1.1:853"
-	dotTimeout    = time.Duration(15 * time.Second)
-
-	// SRV record resolution TTL
-	resolveEdgeAddrTTL = 1 * time.Hour
-)
-
-var friendlyDNSErrorLines = []string{
-	`Please try the following things to diagnose this issue:`,
-	`  1. ensure that argotunnel.com is returning "origintunneld" service records.`,
-	`     Run your system's equivalent of: dig srv _origintunneld._tcp.argotunnel.com`,
-	`  2. ensure that your DNS resolver is not returning compressed SRV records.`,
-	`     See GitHub issue https://github.com/golang/go/issues/27546`,
-	`     For example, you could use Cloudflare's 1.1.1.1 as your resolver:`,
-	`     https://developers.cloudflare.com/1.1.1.1/setting-up-1.1.1.1/`,
-}
-
-// EdgeServiceDiscoverer is an interface for looking up Cloudflare's edge network addresses
-type EdgeServiceDiscoverer interface {
-	// Addr returns an address to connect to cloudflare's edge network
-	Addr() *net.TCPAddr
-	// AvailableAddrs returns the number of unique addresses
-	AvailableAddrs() uint8
-	// Refresh rediscover Cloudflare's edge network addresses
-	Refresh() error
-}
-
-// EdgeAddrResolver discovers the addresses of Cloudflare's edge network through SRV record.
-// It implements EdgeServiceDiscoverer interface
-type EdgeAddrResolver struct {
-	sync.Mutex
-	// Addrs to connect to cloudflare's edge network
-	addrs []*net.TCPAddr
-	// index of the next element to use in addrs
-	nextAddrIndex int
-	logger        *logrus.Entry
-}
-
-func NewEdgeAddrResolver(logger *logrus.Logger) (EdgeServiceDiscoverer, error) {
-	r := &EdgeAddrResolver{
-		logger: logger.WithField("subsystem", " edgeAddrResolver"),
-	}
-	if err := r.Refresh(); err != nil {
-		return nil, err
-	}
-	return r, nil
-}
-
-func (r *EdgeAddrResolver) Addr() *net.TCPAddr {
-	r.Lock()
-	defer r.Unlock()
-	addr := r.addrs[r.nextAddrIndex]
-	r.nextAddrIndex = (r.nextAddrIndex + 1) % len(r.addrs)
-	return addr
-}
-
-func (r *EdgeAddrResolver) AvailableAddrs() uint8 {
-	r.Lock()
-	defer r.Unlock()
-	return uint8(len(r.addrs))
-}
-
-func (r *EdgeAddrResolver) Refresh() error {
-	newAddrs, err := EdgeDiscovery(r.logger)
-	if err != nil {
-		return err
-	}
-	r.Lock()
-	defer r.Unlock()
-	r.addrs = newAddrs
-	r.nextAddrIndex = 0
-	return nil
-}
-
-// HA service discovery lookup
-func EdgeDiscovery(logger *logrus.Entry) ([]*net.TCPAddr, error) {
-	_, addrs, err := net.LookupSRV(srvService, srvProto, srvName)
-	if err != nil {
-		// Try to fall back to DoT from Cloudflare directly.
-		//
-		// Note: Instead of DoT, we could also have used DoH. Either of these:
-		//     - directly via the JSON API (https://1.1.1.1/dns-query?ct=application/dns-json&name=_origintunneld._tcp.argotunnel.com&type=srv)
-		//     - indirectly via `tunneldns.NewUpstreamHTTPS()`
-		// But both of these cases miss out on a key feature from the stdlib:
-		//     "The returned records are sorted by priority and randomized by weight within a priority."
-		//     (https://golang.org/pkg/net/#Resolver.LookupSRV)
-		// Does this matter? I don't know. It may someday. Let's use DoT so we don't need to worry about it.
-		// See also: Go feature request for stdlib-supported DoH: https://github.com/golang/go/issues/27552
-		r := fallbackResolver(dotServerName, dotServerAddr)
-		ctx, cancel := context.WithTimeout(context.Background(), dotTimeout)
-		defer cancel()
-		_, fallbackAddrs, fallbackErr := r.LookupSRV(ctx, srvService, srvProto, srvName)
-		if fallbackErr != nil || len(fallbackAddrs) == 0 {
-			// use the original DNS error `err` in messages, not `fallbackErr`
-			logger.Errorln("Error looking up Cloudflare edge IPs: the DNS query failed:", err)
-			for _, s := range friendlyDNSErrorLines {
-				logger.Errorln(s)
-			}
-			return nil, errors.Wrap(err, "Could not lookup srv records on _origintunneld._tcp.argotunnel.com")
-		}
-		// Accept the fallback results and keep going
-		addrs = fallbackAddrs
-	}
-	var resolvedIPsPerCNAME [][]*net.TCPAddr
-	var lookupErr error
-	for _, addr := range addrs {
-		ips, err := resolveSRVToTCP(addr)
-		if err != nil || len(ips) == 0 {
-			// don't return early, we might be able to resolve other addresses
-			lookupErr = err
-			continue
-		}
-		resolvedIPsPerCNAME = append(resolvedIPsPerCNAME, ips)
-	}
-	ips := flattenServiceIPs(resolvedIPsPerCNAME)
-	if lookupErr == nil && len(ips) == 0 {
-		return nil, fmt.Errorf("Unknown service discovery error")
-	}
-	return ips, lookupErr
-}
-
-func resolveSRVToTCP(srv *net.SRV) ([]*net.TCPAddr, error) {
-	ips, err := net.LookupIP(srv.Target)
-	if err != nil {
-		return nil, err
-	}
-	addrs := make([]*net.TCPAddr, len(ips))
-	for i, ip := range ips {
-		addrs[i] = &net.TCPAddr{IP: ip, Port: int(srv.Port)}
-	}
-	return addrs, nil
-}
-
-// FlattenServiceIPs transposes and flattens the input slices such that the
-// first element of the n inner slices are the first n elements of the result.
-func flattenServiceIPs(ipsByService [][]*net.TCPAddr) []*net.TCPAddr {
-	var result []*net.TCPAddr
-	for len(ipsByService) > 0 {
-		filtered := ipsByService[:0]
-		for _, ips := range ipsByService {
-			if len(ips) == 0 {
-				// sanity check
-				continue
-			}
-			result = append(result, ips[0])
-			if len(ips) > 1 {
-				filtered = append(filtered, ips[1:])
-			}
-		}
-		ipsByService = filtered
-	}
-	return result
-}
-
-// Inspiration: https://github.com/artyom/dot/blob/master/dot.go
-func fallbackResolver(serverName, serverAddress string) *net.Resolver {
-	return &net.Resolver{
-		PreferGo: true,
-		Dial: func(ctx context.Context, _ string, _ string) (net.Conn, error) {
-			var dialer net.Dialer
-			conn, err := dialer.DialContext(ctx, "tcp", serverAddress)
-			if err != nil {
-				return nil, err
-			}
-			tlsConfig := &tls.Config{ServerName: serverName}
-			return tls.Client(conn, tlsConfig), nil
-		},
-	}
-}
-
-// EdgeHostnameResolver discovers the addresses of Cloudflare's edge network via a list of server hostnames.
-// It implements EdgeServiceDiscoverer interface, and is used mainly for testing connectivity.
-type EdgeHostnameResolver struct {
-	sync.Mutex
-	// hostnames of edge servers
-	hostnames []string
-	// Addrs to connect to cloudflare's edge network
-	addrs []*net.TCPAddr
-	// index of the next element to use in addrs
-	nextAddrIndex int
-}
-
-func NewEdgeHostnameResolver(edgeHostnames []string) (EdgeServiceDiscoverer, error) {
-	r := &EdgeHostnameResolver{
-		hostnames: edgeHostnames,
-	}
-	if err := r.Refresh(); err != nil {
-		return nil, err
-	}
-	return r, nil
-}
-
-func (r *EdgeHostnameResolver) Addr() *net.TCPAddr {
-	r.Lock()
-	defer r.Unlock()
-	addr := r.addrs[r.nextAddrIndex]
-	r.nextAddrIndex = (r.nextAddrIndex + 1) % len(r.addrs)
-	return addr
-}
-
-func (r *EdgeHostnameResolver) AvailableAddrs() uint8 {
-	r.Lock()
-	defer r.Unlock()
-	return uint8(len(r.addrs))
-}
-
-func (r *EdgeHostnameResolver) Refresh() error {
-	newAddrs, err := ResolveAddrs(r.hostnames)
-	if err != nil {
-		return err
-	}
-	r.Lock()
-	defer r.Unlock()
-	r.addrs = newAddrs
-	r.nextAddrIndex = 0
-	return nil
-}
-
-// Resolve TCP address given a list of addresses. Address can be a hostname, however, it will return at most one
-// of the hostname's IP addresses
-func ResolveAddrs(addrs []string) ([]*net.TCPAddr, error) {
-	var tcpAddrs []*net.TCPAddr
-	for _, addr := range addrs {
-		tcpAddr, err := net.ResolveTCPAddr("tcp", addr)
-		if err != nil {
-			return nil, err
-		}
-		tcpAddrs = append(tcpAddrs, tcpAddr)
-	}
-	return tcpAddrs, nil
-}

connection/discovery_test.go

@@ -1,63 +0,0 @@
-package connection
-
-import (
-	"net"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-type mockEdgeServiceDiscoverer struct {
-}
-
-func (mr *mockEdgeServiceDiscoverer) Addr() *net.TCPAddr {
-	return &net.TCPAddr{
-		IP:   net.ParseIP("127.0.0.1"),
-		Port: 63102,
-	}
-}
-
-func (mr *mockEdgeServiceDiscoverer) AvailableAddrs() uint8 {
-	return 1
-}
-
-func (mr *mockEdgeServiceDiscoverer) Refresh() error {
-	return nil
-}
-
-func TestFlattenServiceIPs(t *testing.T) {
-	result := flattenServiceIPs([][]*net.TCPAddr{
-		[]*net.TCPAddr{
-			&net.TCPAddr{Port: 1},
-			&net.TCPAddr{Port: 2},
-			&net.TCPAddr{Port: 3},
-			&net.TCPAddr{Port: 4},
-		},
-		[]*net.TCPAddr{
-			&net.TCPAddr{Port: 10},
-			&net.TCPAddr{Port: 12},
-			&net.TCPAddr{Port: 13},
-		},
-		[]*net.TCPAddr{
-			&net.TCPAddr{Port: 21},
-			&net.TCPAddr{Port: 22},
-			&net.TCPAddr{Port: 23},
-			&net.TCPAddr{Port: 24},
-			&net.TCPAddr{Port: 25},
-		},
-	})
-	assert.EqualValues(t, []*net.TCPAddr{
-		&net.TCPAddr{Port: 1},
-		&net.TCPAddr{Port: 10},
-		&net.TCPAddr{Port: 21},
-		&net.TCPAddr{Port: 2},
-		&net.TCPAddr{Port: 12},
-		&net.TCPAddr{Port: 22},
-		&net.TCPAddr{Port: 3},
-		&net.TCPAddr{Port: 13},
-		&net.TCPAddr{Port: 23},
-		&net.TCPAddr{Port: 4},
-		&net.TCPAddr{Port: 24},
-		&net.TCPAddr{Port: 25},
-	}, result)
-}

connection/manager.go

@@ -4,19 +4,19 @@ import (
 	"context"
 	"crypto/tls"
 	"fmt"
-	"net"
 	"sync"
 	"time"
 
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
-	"github.com/cloudflare/cloudflared/h2mux"
-	"github.com/cloudflare/cloudflared/streamhandler"
-	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
-	"github.com/prometheus/client_golang/prometheus"
-
 	"github.com/google/uuid"
 	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/sirupsen/logrus"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
+	"github.com/cloudflare/cloudflared/edgediscovery"
+	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/streamhandler"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
 const (
@@ -36,7 +36,7 @@ type EdgeManager struct {
 	// cloudflaredConfig is the cloudflared configuration that is determined when the process first starts
 	cloudflaredConfig *CloudflaredConfig
 	// serviceDiscoverer returns the next edge addr to connect to
-	serviceDiscoverer EdgeServiceDiscoverer
+	serviceDiscoverer *edgediscovery.Edge
 	// state is attributes of ConnectionManager that can change during runtime.
 	state *edgeManagerState
 
@@ -59,12 +59,12 @@ func newMetrics(namespace, subsystem string) *metrics {
 // EdgeManagerConfigurable is the configurable attributes of a EdgeConnectionManager
 type EdgeManagerConfigurable struct {
 	TunnelHostnames []h2mux.TunnelHostname
-	*pogs.EdgeConnectionConfig
+	*tunnelpogs.EdgeConnectionConfig
 }
 
 type CloudflaredConfig struct {
 	CloudflaredID uuid.UUID
-	Tags          []pogs.Tag
+	Tags          []tunnelpogs.Tag
 	BuildInfo     *buildinfo.BuildInfo
 	IntentLabel   string
 }
@@ -74,7 +74,7 @@ func NewEdgeManager(
 	edgeConnMgrConfigurable *EdgeManagerConfigurable,
 	userCredential []byte,
 	tlsConfig *tls.Config,
-	serviceDiscoverer EdgeServiceDiscoverer,
+	serviceDiscoverer *edgediscovery.Edge,
 	cloudflaredConfig *CloudflaredConfig,
 	logger *logrus.Logger,
 ) *EdgeManager {
@@ -92,27 +92,29 @@ func NewEdgeManager(
 func (em *EdgeManager) Run(ctx context.Context) error {
 	defer em.shutdown()
 
-	resolveEdgeIPTicker := time.Tick(resolveEdgeAddrTTL)
+	// Currently, declarative tunnels don't have any concept of a stable connection
+	// Each edge connection is transient and when it dies, it is replaced by a different one,
+	// not restarted.
+	// So in the future we should really change this so that n connections are stored individually
+	connIndex := 0
 	for {
 		select {
 		case <-ctx.Done():
 			return errors.Wrap(ctx.Err(), "EdgeConnectionManager terminated")
-		case <-resolveEdgeIPTicker:
-			if err := em.serviceDiscoverer.Refresh(); err != nil {
-				em.logger.WithError(err).Warn("Cannot refresh Cloudflare edge addresses")
-			}
 		default:
 			time.Sleep(1 * time.Second)
 		}
 		// Create/delete connection one at a time, so we don't need to adjust for connections that are being created/deleted
 		// in shouldCreateConnection or shouldReduceConnection calculation
 		if em.state.shouldCreateConnection(em.serviceDiscoverer.AvailableAddrs()) {
-			if connErr := em.newConnection(ctx); connErr != nil {
+			if connErr := em.newConnection(ctx, connIndex); connErr != nil {
 				if !connErr.ShouldRetry {
 					em.logger.WithError(connErr).Error(em.noRetryMessage())
 					return connErr
 				}
 				em.logger.WithError(connErr).Error("cannot create new connection")
+			} else {
+				connIndex++
 			}
 		} else if em.state.shouldReduceConnection() {
 			if err := em.closeConnection(ctx); err != nil {
@@ -127,13 +129,16 @@ func (em *EdgeManager) UpdateConfigurable(newConfigurable *EdgeManagerConfigurab
 	em.state.updateConfigurable(newConfigurable)
 }
 
-func (em *EdgeManager) newConnection(ctx context.Context) *pogs.ConnectError {
-	edgeIP := em.serviceDiscoverer.Addr()
-	edgeConn, err := em.dialEdge(ctx, edgeIP)
+func (em *EdgeManager) newConnection(ctx context.Context, index int) *tunnelpogs.ConnectError {
+	edgeTCPAddr, err := em.serviceDiscoverer.GetAddr(index)
+	if err != nil {
+		return retryConnection(fmt.Sprintf("edge address discovery error: %v", err))
+	}
+	configurable := em.state.getConfigurable()
+	edgeConn, err := DialEdge(ctx, configurable.Timeout, em.tlsConfig, edgeTCPAddr)
 	if err != nil {
 		return retryConnection(fmt.Sprintf("dial edge error: %v", err))
 	}
-	configurable := em.state.getConfigurable()
 	// Establish a muxed connection with the edge
 	// Client mux handshake with agent server
 	muxer, err := h2mux.Handshake(edgeConn, edgeConn, h2mux.MuxerConfig{
@@ -148,14 +153,14 @@ func (em *EdgeManager) newConnection(ctx context.Context) *pogs.ConnectError {
 		retryConnection(fmt.Sprintf("couldn't perform handshake with edge: %v", err))
 	}
 
-	h2muxConn, err := newConnection(muxer, edgeIP)
+	h2muxConn, err := newConnection(muxer, edgeTCPAddr)
 	if err != nil {
 		return retryConnection(fmt.Sprintf("couldn't create h2mux connection: %v", err))
 	}
 
 	go em.serveConn(ctx, h2muxConn)
 
-	connResult, err := h2muxConn.Connect(ctx, &pogs.ConnectParameters{
+	connResult, err := h2muxConn.Connect(ctx, &tunnelpogs.ConnectParameters{
 		CloudflaredID:       em.cloudflaredConfig.CloudflaredID,
 		CloudflaredVersion:  em.cloudflaredConfig.BuildInfo.CloudflaredVersion,
 		NumPreviousAttempts: 0,
@@ -187,6 +192,7 @@ func (em *EdgeManager) closeConnection(ctx context.Context) error {
 		return fmt.Errorf("no connection to close")
 	}
 	conn.Shutdown()
+	// teardown will be handled by EdgeManager.serveConn in another goroutine
 	return nil
 }
 
@@ -194,28 +200,7 @@ func (em *EdgeManager) serveConn(ctx context.Context, conn *Connection) {
 	err := conn.Serve(ctx)
 	em.logger.WithError(err).Warn("Connection closed")
 	em.state.closeConnection(conn)
-}
-
-func (em *EdgeManager) dialEdge(ctx context.Context, edgeIP *net.TCPAddr) (*tls.Conn, error) {
-	timeout := em.state.getConfigurable().Timeout
-	// Inherit from parent context so we can cancel (Ctrl-C) while dialing
-	dialCtx, dialCancel := context.WithTimeout(ctx, timeout)
-	defer dialCancel()
-
-	dialer := net.Dialer{DualStack: true}
-	edgeConn, err := dialer.DialContext(dialCtx, "tcp", edgeIP.String())
-	if err != nil {
-		return nil, dialError{cause: errors.Wrap(err, "DialContext error")}
-	}
-	tlsEdgeConn := tls.Client(edgeConn, em.tlsConfig)
-	tlsEdgeConn.SetDeadline(time.Now().Add(timeout))
-
-	if err = tlsEdgeConn.Handshake(); err != nil {
-		return nil, dialError{cause: errors.Wrap(err, "Handshake with edge error")}
-	}
-	// clear the deadline on the conn; h2mux has its own timeouts
-	tlsEdgeConn.SetDeadline(time.Time{})
-	return tlsEdgeConn, nil
+	em.serviceDiscoverer.GiveBack(conn.addr)
 }
 
 func (em *EdgeManager) noRetryMessage() string {
@@ -244,14 +229,14 @@ func newEdgeConnectionManagerState(configurable *EdgeManagerConfigurable, userCr
 	}
 }
 
-func (ems *edgeManagerState) shouldCreateConnection(availableEdgeAddrs uint8) bool {
+func (ems *edgeManagerState) shouldCreateConnection(availableEdgeAddrs int) bool {
 	ems.RLock()
 	defer ems.RUnlock()
-	expectedHAConns := ems.configurable.NumHAConnections
+	expectedHAConns := int(ems.configurable.NumHAConnections)
 	if availableEdgeAddrs < expectedHAConns {
 		expectedHAConns = availableEdgeAddrs
 	}
-	return uint8(len(ems.conns)) < expectedHAConns
+	return len(ems.conns) < expectedHAConns
 }
 
 func (ems *edgeManagerState) shouldReduceConnection() bool {
@@ -308,8 +293,8 @@ func (ems *edgeManagerState) getUserCredential() []byte {
 	return ems.userCredential
 }
 
-func retryConnection(cause string) *pogs.ConnectError {
-	return &pogs.ConnectError{
+func retryConnection(cause string) *tunnelpogs.ConnectError {
+	return &tunnelpogs.ConnectError{
 		Cause:       cause,
 		RetryAfter:  defaultRetryAfter,
 		ShouldRetry: true,

connection/manager_test.go

@@ -1,18 +1,19 @@
 package connection
 
 import (
+	"net"
 	"testing"
 	"time"
 
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
+	"github.com/google/uuid"
+	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
+	"github.com/cloudflare/cloudflared/edgediscovery"
 	"github.com/cloudflare/cloudflared/h2mux"
 	"github.com/cloudflare/cloudflared/streamhandler"
 	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
-
-	"github.com/google/uuid"
-	"github.com/sirupsen/logrus"
 )
 
 var (
@@ -48,14 +49,15 @@ func mockEdgeManager() *EdgeManager {
 	newConfigChan := make(chan<- *pogs.ClientConfig)
 	useConfigResultChan := make(<-chan *pogs.UseConfigurationResult)
 	logger := logrus.New()
+	edge := edgediscovery.MockEdge(logger, []*net.TCPAddr{})
 	return NewEdgeManager(
 		streamhandler.NewStreamHandler(newConfigChan, useConfigResultChan, logger),
 		configurable,
 		[]byte{},
 		nil,
-		&mockEdgeServiceDiscoverer{},
+		edge,
 		cloudflaredConfig,
-		logrus.New(),
+		logger,
 	)
 }
 

connection/mocks_for_test.go

@@ -0,0 +1,118 @@
+package connection
+
+import (
+	"fmt"
+	"math"
+	"math/rand"
+	"net"
+	"reflect"
+	"testing/quick"
+)
+
+type mockAddrs struct {
+	// a set of synthetic SRV records
+	addrMap map[net.SRV][]*net.TCPAddr
+	// the total number of addresses, aggregated across addrMap.
+	// For the convenience of test code that would otherwise have to compute
+	// this by hand every time.
+	numAddrs int
+}
+
+func newMockAddrs(port uint16, numRegions uint8, numAddrsPerRegion uint8) mockAddrs {
+	addrMap := make(map[net.SRV][]*net.TCPAddr)
+	numAddrs := 0
+
+	for r := uint8(0); r < numRegions; r++ {
+		var (
+			srv   = net.SRV{Target: fmt.Sprintf("test-region-%v.example.com", r), Port: port}
+			addrs []*net.TCPAddr
+		)
+		for a := uint8(0); a < numAddrsPerRegion; a++ {
+			addrs = append(addrs, &net.TCPAddr{
+				IP:   net.ParseIP(fmt.Sprintf("10.0.%v.%v", r, a)),
+				Port: int(port),
+			})
+		}
+		addrMap[srv] = addrs
+		numAddrs += len(addrs)
+	}
+	return mockAddrs{addrMap: addrMap, numAddrs: numAddrs}
+}
+
+var _ quick.Generator = mockAddrs{}
+
+func (mockAddrs) Generate(rand *rand.Rand, size int) reflect.Value {
+	port := uint16(rand.Intn(math.MaxUint16))
+	numRegions := uint8(1 + rand.Intn(10))
+	numAddrsPerRegion := uint8(1 + rand.Intn(32))
+	result := newMockAddrs(port, numRegions, numAddrsPerRegion)
+	return reflect.ValueOf(result)
+}
+
+// Returns a function compatible with net.LookupSRV that will return the SRV
+// records from mockAddrs.
+func mockNetLookupSRV(
+	m mockAddrs,
+) func(service, proto, name string) (cname string, addrs []*net.SRV, err error) {
+	var addrs []*net.SRV
+	for k := range m.addrMap {
+		addr := k
+		addrs = append(addrs, &addr)
+		// We can't just do
+		//   addrs = append(addrs, &k)
+		// `k` will be reused by subsequent loop iterations,
+		// so all the copies of `&k` would point to the same location.
+	}
+	return func(_, _, _ string) (string, []*net.SRV, error) {
+		return "", addrs, nil
+	}
+}
+
+// Returns a function compatible with net.LookupIP that translates the SRV records
+// from mockAddrs into IP addresses, based on the TCP addresses in mockAddrs.
+func mockNetLookupIP(
+	m mockAddrs,
+) func(host string) ([]net.IP, error) {
+	return func(host string) ([]net.IP, error) {
+		for srv, tcpAddrs := range m.addrMap {
+			if srv.Target != host {
+				continue
+			}
+			result := make([]net.IP, len(tcpAddrs))
+			for i, tcpAddr := range tcpAddrs {
+				result[i] = tcpAddr.IP
+			}
+			return result, nil
+		}
+		return nil, fmt.Errorf("No IPs for %v", host)
+	}
+}
+
+type mockEdgeServiceDiscoverer struct {
+}
+
+func (mr *mockEdgeServiceDiscoverer) Addr() (*net.TCPAddr, error) {
+	return &net.TCPAddr{
+		IP:   net.ParseIP("127.0.0.1"),
+		Port: 63102,
+	}, nil
+}
+
+func (mr *mockEdgeServiceDiscoverer) AnyAddr() (*net.TCPAddr, error) {
+	return &net.TCPAddr{
+		IP:   net.ParseIP("127.0.0.1"),
+		Port: 63102,
+	}, nil
+}
+
+func (mr *mockEdgeServiceDiscoverer) ReplaceAddr(addr *net.TCPAddr) {}
+
+func (mr *mockEdgeServiceDiscoverer) MarkAddrBad(addr *net.TCPAddr) {}
+
+func (mr *mockEdgeServiceDiscoverer) AvailableAddrs() int {
+	return 1
+}
+
+func (mr *mockEdgeServiceDiscoverer) Refresh() error {
+	return nil
+}

connection/rpc.go

@@ -0,0 +1,49 @@
+package connection
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/sirupsen/logrus"
+	rpc "zombiezen.com/go/capnproto2/rpc"
+
+	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/tunnelrpc"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+)
+
+// NewRPCClient creates and returns a new RPC client, which will communicate
+// using a stream on the given muxer
+func NewRPCClient(
+	ctx context.Context,
+	muxer *h2mux.Muxer,
+	logger *logrus.Entry,
+	openStreamTimeout time.Duration,
+) (client tunnelpogs.TunnelServer_PogsClient, err error) {
+	openStreamCtx, openStreamCancel := context.WithTimeout(ctx, openStreamTimeout)
+	defer openStreamCancel()
+	stream, err := muxer.OpenRPCStream(openStreamCtx)
+	if err != nil {
+		return
+	}
+
+	if !isRPCStreamResponse(stream.Headers) {
+		stream.Close()
+		err = fmt.Errorf("rpc: bad response headers: %v", stream.Headers)
+		return
+	}
+
+	conn := rpc.NewConn(
+		tunnelrpc.NewTransportLogger(logger, rpc.StreamTransport(stream)),
+		tunnelrpc.ConnLog(logger),
+	)
+	client = tunnelpogs.TunnelServer_PogsClient{Client: conn.Bootstrap(ctx), Conn: conn}
+	return client, nil
+}
+
+func isRPCStreamResponse(headers []h2mux.Header) bool {
+	return len(headers) == 1 &&
+		headers[0].Name == ":status" &&
+		headers[0].Value == "200"
+}

edgediscovery/allregions/discovery.go

@@ -0,0 +1,135 @@
+package allregions
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"net"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	// Used to discover HA origintunneld servers
+	srvService = "origintunneld"
+	srvProto   = "tcp"
+	srvName    = "argotunnel.com"
+
+	// Used to fallback to DoT when we can't use the default resolver to
+	// discover HA origintunneld servers (GitHub issue #75).
+	dotServerName = "cloudflare-dns.com"
+	dotServerAddr = "1.1.1.1:853"
+	dotTimeout    = time.Duration(15 * time.Second)
+
+	// SRV record resolution TTL
+	resolveEdgeAddrTTL = 1 * time.Hour
+
+	subsystemEdgeAddrResolver = "edgeAddrResolver"
+)
+
+// Redeclare network functions so they can be overridden in tests.
+var (
+	netLookupSRV = net.LookupSRV
+	netLookupIP  = net.LookupIP
+)
+
+// If the call to net.LookupSRV fails, try to fall back to DoT from Cloudflare directly.
+//
+// Note: Instead of DoT, we could also have used DoH. Either of these:
+//     - directly via the JSON API (https://1.1.1.1/dns-query?ct=application/dns-json&name=_origintunneld._tcp.argotunnel.com&type=srv)
+//     - indirectly via `tunneldns.NewUpstreamHTTPS()`
+// But both of these cases miss out on a key feature from the stdlib:
+//     "The returned records are sorted by priority and randomized by weight within a priority."
+//     (https://golang.org/pkg/net/#Resolver.LookupSRV)
+// Does this matter? I don't know. It may someday. Let's use DoT so we don't need to worry about it.
+// See also: Go feature request for stdlib-supported DoH: https://github.com/golang/go/issues/27552
+var fallbackLookupSRV = lookupSRVWithDOT
+
+var friendlyDNSErrorLines = []string{
+	`Please try the following things to diagnose this issue:`,
+	`  1. ensure that argotunnel.com is returning "origintunneld" service records.`,
+	`     Run your system's equivalent of: dig srv _origintunneld._tcp.argotunnel.com`,
+	`  2. ensure that your DNS resolver is not returning compressed SRV records.`,
+	`     See GitHub issue https://github.com/golang/go/issues/27546`,
+	`     For example, you could use Cloudflare's 1.1.1.1 as your resolver:`,
+	`     https://developers.cloudflare.com/1.1.1.1/setting-up-1.1.1.1/`,
+}
+
+// EdgeDiscovery implements HA service discovery lookup.
+func edgeDiscovery(logger *logrus.Entry) ([][]*net.TCPAddr, error) {
+	_, addrs, err := netLookupSRV(srvService, srvProto, srvName)
+	if err != nil {
+		_, fallbackAddrs, fallbackErr := fallbackLookupSRV(srvService, srvProto, srvName)
+		if fallbackErr != nil || len(fallbackAddrs) == 0 {
+			// use the original DNS error `err` in messages, not `fallbackErr`
+			logger.Errorln("Error looking up Cloudflare edge IPs: the DNS query failed:", err)
+			for _, s := range friendlyDNSErrorLines {
+				logger.Errorln(s)
+			}
+			return nil, errors.Wrapf(err, "Could not lookup srv records on _%v._%v.%v", srvService, srvProto, srvName)
+		}
+		// Accept the fallback results and keep going
+		addrs = fallbackAddrs
+	}
+
+	var resolvedIPsPerCNAME [][]*net.TCPAddr
+	for _, addr := range addrs {
+		ips, err := resolveSRVToTCP(addr)
+		if err != nil {
+			return nil, err
+		}
+		resolvedIPsPerCNAME = append(resolvedIPsPerCNAME, ips)
+	}
+
+	return resolvedIPsPerCNAME, nil
+}
+
+func lookupSRVWithDOT(service, proto, name string) (cname string, addrs []*net.SRV, err error) {
+	// Inspiration: https://github.com/artyom/dot/blob/master/dot.go
+	r := &net.Resolver{
+		PreferGo: true,
+		Dial: func(ctx context.Context, _ string, _ string) (net.Conn, error) {
+			var dialer net.Dialer
+			conn, err := dialer.DialContext(ctx, "tcp", dotServerAddr)
+			if err != nil {
+				return nil, err
+			}
+			tlsConfig := &tls.Config{ServerName: dotServerName}
+			return tls.Client(conn, tlsConfig), nil
+		},
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), dotTimeout)
+	defer cancel()
+	return r.LookupSRV(ctx, srvService, srvProto, srvName)
+}
+
+func resolveSRVToTCP(srv *net.SRV) ([]*net.TCPAddr, error) {
+	ips, err := netLookupIP(srv.Target)
+	if err != nil {
+		return nil, errors.Wrapf(err, "Couldn't resolve SRV record %v", srv)
+	}
+	if len(ips) == 0 {
+		return nil, fmt.Errorf("SRV record %v had no IPs", srv)
+	}
+	addrs := make([]*net.TCPAddr, len(ips))
+	for i, ip := range ips {
+		addrs[i] = &net.TCPAddr{IP: ip, Port: int(srv.Port)}
+	}
+	return addrs, nil
+}
+
+// ResolveAddrs resolves TCP address given a list of addresses. Address can be a hostname, however, it will return at most one
+// of the hostname's IP addresses
+func ResolveAddrs(addrs []string) ([]*net.TCPAddr, error) {
+	var tcpAddrs []*net.TCPAddr
+	for _, addr := range addrs {
+		tcpAddr, err := net.ResolveTCPAddr("tcp", addr)
+		if err != nil {
+			return nil, err
+		}
+		tcpAddrs = append(tcpAddrs, tcpAddr)
+	}
+	return tcpAddrs, nil
+}

edgediscovery/allregions/discovery_test.go

@@ -0,0 +1,32 @@
+package allregions
+
+import (
+	"testing"
+
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestEdgeDiscovery(t *testing.T) {
+	mockAddrs := newMockAddrs(19, 2, 5)
+	netLookupSRV = mockNetLookupSRV(mockAddrs)
+	netLookupIP = mockNetLookupIP(mockAddrs)
+
+	expectedAddrSet := map[string]bool{}
+	for _, addrs := range mockAddrs.addrMap {
+		for _, addr := range addrs {
+			expectedAddrSet[addr.String()] = true
+		}
+	}
+
+	addrLists, err := edgeDiscovery(logrus.New().WithFields(logrus.Fields{}))
+	assert.NoError(t, err)
+	actualAddrSet := map[string]bool{}
+	for _, addrs := range addrLists {
+		for _, addr := range addrs {
+			actualAddrSet[addr.String()] = true
+		}
+	}
+
+	assert.Equal(t, expectedAddrSet, actualAddrSet)
+}

edgediscovery/allregions/mocks_for_test.go

@@ -0,0 +1,89 @@
+package allregions
+
+import (
+	"fmt"
+	"math"
+	"math/rand"
+	"net"
+	"reflect"
+	"testing/quick"
+)
+
+type mockAddrs struct {
+	// a set of synthetic SRV records
+	addrMap map[net.SRV][]*net.TCPAddr
+	// the total number of addresses, aggregated across addrMap.
+	// For the convenience of test code that would otherwise have to compute
+	// this by hand every time.
+	numAddrs int
+}
+
+func newMockAddrs(port uint16, numRegions uint8, numAddrsPerRegion uint8) mockAddrs {
+	addrMap := make(map[net.SRV][]*net.TCPAddr)
+	numAddrs := 0
+
+	for r := uint8(0); r < numRegions; r++ {
+		var (
+			srv   = net.SRV{Target: fmt.Sprintf("test-region-%v.example.com", r), Port: port}
+			addrs []*net.TCPAddr
+		)
+		for a := uint8(0); a < numAddrsPerRegion; a++ {
+			addrs = append(addrs, &net.TCPAddr{
+				IP:   net.ParseIP(fmt.Sprintf("10.0.%v.%v", r, a)),
+				Port: int(port),
+			})
+		}
+		addrMap[srv] = addrs
+		numAddrs += len(addrs)
+	}
+	return mockAddrs{addrMap: addrMap, numAddrs: numAddrs}
+}
+
+var _ quick.Generator = mockAddrs{}
+
+func (mockAddrs) Generate(rand *rand.Rand, size int) reflect.Value {
+	port := uint16(rand.Intn(math.MaxUint16))
+	numRegions := uint8(1 + rand.Intn(10))
+	numAddrsPerRegion := uint8(1 + rand.Intn(32))
+	result := newMockAddrs(port, numRegions, numAddrsPerRegion)
+	return reflect.ValueOf(result)
+}
+
+// Returns a function compatible with net.LookupSRV that will return the SRV
+// records from mockAddrs.
+func mockNetLookupSRV(
+	m mockAddrs,
+) func(service, proto, name string) (cname string, addrs []*net.SRV, err error) {
+	var addrs []*net.SRV
+	for k := range m.addrMap {
+		addr := k
+		addrs = append(addrs, &addr)
+		// We can't just do
+		//   addrs = append(addrs, &k)
+		// `k` will be reused by subsequent loop iterations,
+		// so all the copies of `&k` would point to the same location.
+	}
+	return func(_, _, _ string) (string, []*net.SRV, error) {
+		return "", addrs, nil
+	}
+}
+
+// Returns a function compatible with net.LookupIP that translates the SRV records
+// from mockAddrs into IP addresses, based on the TCP addresses in mockAddrs.
+func mockNetLookupIP(
+	m mockAddrs,
+) func(host string) ([]net.IP, error) {
+	return func(host string) ([]net.IP, error) {
+		for srv, tcpAddrs := range m.addrMap {
+			if srv.Target != host {
+				continue
+			}
+			result := make([]net.IP, len(tcpAddrs))
+			for i, tcpAddr := range tcpAddrs {
+				result[i] = tcpAddr.IP
+			}
+			return result, nil
+		}
+		return nil, fmt.Errorf("No IPs for %v", host)
+	}
+}

edgediscovery/allregions/region.go

@@ -0,0 +1,78 @@
+package allregions
+
+import (
+	"net"
+)
+
+// Region contains cloudflared edge addresses. The edge is partitioned into several regions for
+// redundancy purposes.
+type Region struct {
+	connFor map[*net.TCPAddr]UsedBy
+}
+
+// NewRegion creates a region with the given addresses, which are all unused.
+func NewRegion(addrs []*net.TCPAddr) Region {
+	// The zero value of UsedBy is Unused(), so we can just initialize the map's values with their
+	// zero values.
+	m := make(map[*net.TCPAddr]UsedBy)
+	for _, addr := range addrs {
+		m[addr] = Unused()
+	}
+	return Region{connFor: m}
+}
+
+// AddrUsedBy finds the address used by the given connection in this region.
+// Returns nil if the connection isn't using any IP.
+func (r *Region) AddrUsedBy(connID int) *net.TCPAddr {
+	for addr, used := range r.connFor {
+		if used.Used && used.ConnID == connID {
+			return addr
+		}
+	}
+	return nil
+}
+
+// AvailableAddrs counts how many unused addresses this region contains.
+func (r Region) AvailableAddrs() int {
+	n := 0
+	for _, usedby := range r.connFor {
+		if !usedby.Used {
+			n++
+		}
+	}
+	return n
+}
+
+// GetUnusedIP returns a random unused address in this region.
+// Returns nil if all addresses are in use.
+func (r Region) GetUnusedIP(excluding *net.TCPAddr) *net.TCPAddr {
+	for addr, usedby := range r.connFor {
+		if !usedby.Used && addr != excluding {
+			return addr
+		}
+	}
+	return nil
+}
+
+// Use the address, assigning it to a proxy connection.
+func (r Region) Use(addr *net.TCPAddr, connID int) {
+	r.connFor[addr] = InUse(connID)
+}
+
+// GetAnyAddress returns an arbitrary address from the region.
+func (r Region) GetAnyAddress() *net.TCPAddr {
+	for addr := range r.connFor {
+		return addr
+	}
+	return nil
+}
+
+// GiveBack the address, ensuring it is no longer assigned to an IP.
+// Returns true if the address is in this region.
+func (r Region) GiveBack(addr *net.TCPAddr) (ok bool) {
+	if _, ok := r.connFor[addr]; !ok {
+		return false
+	}
+	r.connFor[addr] = Unused()
+	return true
+}

edgediscovery/allregions/region_test.go

@@ -0,0 +1,287 @@
+package allregions
+
+import (
+	"fmt"
+	"net"
+	"reflect"
+	"testing"
+)
+
+func TestRegion_New(t *testing.T) {
+	r := NewRegion([]*net.TCPAddr{&addr0, &addr1, &addr2})
+	fmt.Println(r.connFor)
+	if r.AvailableAddrs() != 3 {
+		t.Errorf("r.AvailableAddrs() == %v but want 3", r.AvailableAddrs())
+	}
+}
+
+func TestRegion_AddrUsedBy(t *testing.T) {
+	type fields struct {
+		connFor map[*net.TCPAddr]UsedBy
+	}
+	type args struct {
+		connID int
+	}
+	tests := []struct {
+		name   string
+		fields fields
+		args   args
+		want   *net.TCPAddr
+	}{
+		{
+			name: "happy trivial test",
+			fields: fields{connFor: map[*net.TCPAddr]UsedBy{
+				&addr0: InUse(0),
+			}},
+			args: args{connID: 0},
+			want: &addr0,
+		},
+		{
+			name: "sad trivial test",
+			fields: fields{connFor: map[*net.TCPAddr]UsedBy{
+				&addr0: InUse(0),
+			}},
+			args: args{connID: 1},
+			want: nil,
+		},
+		{
+			name: "sad test",
+			fields: fields{connFor: map[*net.TCPAddr]UsedBy{
+				&addr0: InUse(0),
+				&addr1: InUse(1),
+				&addr2: InUse(2),
+			}},
+			args: args{connID: 3},
+			want: nil,
+		},
+		{
+			name: "happy test",
+			fields: fields{connFor: map[*net.TCPAddr]UsedBy{
+				&addr0: InUse(0),
+				&addr1: InUse(1),
+				&addr2: InUse(2),
+			}},
+			args: args{connID: 1},
+			want: &addr1,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			r := &Region{
+				connFor: tt.fields.connFor,
+			}
+			if got := r.AddrUsedBy(tt.args.connID); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("Region.AddrUsedBy() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestRegion_AvailableAddrs(t *testing.T) {
+	type fields struct {
+		connFor map[*net.TCPAddr]UsedBy
+	}
+	tests := []struct {
+		name   string
+		fields fields
+		want   int
+	}{
+		{
+			name: "contains addresses",
+			fields: fields{connFor: map[*net.TCPAddr]UsedBy{
+				&addr0: InUse(0),
+				&addr1: Unused(),
+				&addr2: InUse(2),
+			}},
+			want: 1,
+		},
+		{
+			name: "all free",
+			fields: fields{connFor: map[*net.TCPAddr]UsedBy{
+				&addr0: Unused(),
+				&addr1: Unused(),
+				&addr2: Unused(),
+			}},
+			want: 3,
+		},
+		{
+			name: "all used",
+			fields: fields{connFor: map[*net.TCPAddr]UsedBy{
+				&addr0: InUse(0),
+				&addr1: InUse(1),
+				&addr2: InUse(2),
+			}},
+			want: 0,
+		},
+		{
+			name:   "empty",
+			fields: fields{connFor: map[*net.TCPAddr]UsedBy{}},
+			want:   0,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			r := Region{
+				connFor: tt.fields.connFor,
+			}
+			if got := r.AvailableAddrs(); got != tt.want {
+				t.Errorf("Region.AvailableAddrs() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestRegion_GetUnusedIP(t *testing.T) {
+	type fields struct {
+		connFor map[*net.TCPAddr]UsedBy
+	}
+	type args struct {
+		excluding *net.TCPAddr
+	}
+	tests := []struct {
+		name   string
+		fields fields
+		args   args
+		want   *net.TCPAddr
+	}{
+		{
+			name: "happy test with excluding set",
+			fields: fields{connFor: map[*net.TCPAddr]UsedBy{
+				&addr0: Unused(),
+				&addr1: Unused(),
+				&addr2: InUse(2),
+			}},
+			args: args{excluding: &addr0},
+			want: &addr1,
+		},
+		{
+			name: "happy test with no excluding",
+			fields: fields{connFor: map[*net.TCPAddr]UsedBy{
+				&addr0: InUse(0),
+				&addr1: Unused(),
+				&addr2: InUse(2),
+			}},
+			args: args{excluding: nil},
+			want: &addr1,
+		},
+		{
+			name: "sad test with no excluding",
+			fields: fields{connFor: map[*net.TCPAddr]UsedBy{
+				&addr0: InUse(0),
+				&addr1: InUse(1),
+				&addr2: InUse(2),
+			}},
+			args: args{excluding: nil},
+			want: nil,
+		},
+		{
+			name: "sad test with excluding",
+			fields: fields{connFor: map[*net.TCPAddr]UsedBy{
+				&addr0: Unused(),
+				&addr1: InUse(1),
+				&addr2: InUse(2),
+			}},
+			args: args{excluding: &addr0},
+			want: nil,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			r := Region{
+				connFor: tt.fields.connFor,
+			}
+			if got := r.GetUnusedIP(tt.args.excluding); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("Region.GetUnusedIP() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestRegion_GiveBack(t *testing.T) {
+	type fields struct {
+		connFor map[*net.TCPAddr]UsedBy
+	}
+	type args struct {
+		addr *net.TCPAddr
+	}
+	tests := []struct {
+		name           string
+		fields         fields
+		args           args
+		wantOk         bool
+		availableAfter int
+	}{
+		{
+			name: "sad test with excluding",
+			fields: fields{connFor: map[*net.TCPAddr]UsedBy{
+				&addr1: InUse(1),
+			}},
+			args:           args{addr: &addr1},
+			wantOk:         true,
+			availableAfter: 1,
+		},
+		{
+			name: "sad test with excluding",
+			fields: fields{connFor: map[*net.TCPAddr]UsedBy{
+				&addr1: InUse(1),
+			}},
+			args:           args{addr: &addr2},
+			wantOk:         false,
+			availableAfter: 0,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			r := Region{
+				connFor: tt.fields.connFor,
+			}
+			if gotOk := r.GiveBack(tt.args.addr); gotOk != tt.wantOk {
+				t.Errorf("Region.GiveBack() = %v, want %v", gotOk, tt.wantOk)
+			}
+			if tt.availableAfter != r.AvailableAddrs() {
+				t.Errorf("Region.AvailableAddrs() = %v, want %v", r.AvailableAddrs(), tt.availableAfter)
+			}
+		})
+	}
+}
+
+func TestRegion_GetAnyAddress(t *testing.T) {
+	type fields struct {
+		connFor map[*net.TCPAddr]UsedBy
+	}
+	tests := []struct {
+		name    string
+		fields  fields
+		wantNil bool
+	}{
+		{
+			name:    "Sad test -- GetAnyAddress should only fail if the region is empty",
+			fields:  fields{connFor: map[*net.TCPAddr]UsedBy{}},
+			wantNil: true,
+		},
+		{
+			name: "Happy test (all addresses unused)",
+			fields: fields{connFor: map[*net.TCPAddr]UsedBy{
+				&addr0: Unused(),
+			}},
+			wantNil: false,
+		},
+		{
+			name: "Happy test (GetAnyAddress can still return addresses used by proxy conns)",
+			fields: fields{connFor: map[*net.TCPAddr]UsedBy{
+				&addr0: InUse(2),
+			}},
+			wantNil: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			r := Region{
+				connFor: tt.fields.connFor,
+			}
+			if got := r.GetAnyAddress(); tt.wantNil != (got == nil) {
+				t.Errorf("Region.GetAnyAddress() = %v, but should it return nil? %v", got, tt.wantNil)
+			}
+		})
+	}
+}

edgediscovery/allregions/regions.go

@@ -0,0 +1,118 @@
+package allregions
+
+import (
+	"fmt"
+	"net"
+
+	"github.com/sirupsen/logrus"
+)
+
+// Regions stores Cloudflare edge network IPs, partitioned into two regions.
+// This is NOT thread-safe. Users of this package should use it with a lock.
+type Regions struct {
+	region1 Region
+	region2 Region
+}
+
+// ------------------------------------
+// Constructors
+// ------------------------------------
+
+// ResolveEdge resolves the Cloudflare edge, returning all regions discovered.
+func ResolveEdge(logger *logrus.Entry) (*Regions, error) {
+	addrLists, err := edgeDiscovery(logger)
+	if err != nil {
+		return nil, err
+	}
+	if len(addrLists) < 2 {
+		return nil, fmt.Errorf("expected at least 2 Cloudflare Regions regions, but SRV only returned %v", len(addrLists))
+	}
+	return &Regions{
+		region1: NewRegion(addrLists[0]),
+		region2: NewRegion(addrLists[1]),
+	}, nil
+}
+
+// StaticEdge creates a list of edge addresses from the list of hostnames.
+// Mainly used for testing connectivity.
+func StaticEdge(hostnames []string) (*Regions, error) {
+	addrs, err := ResolveAddrs(hostnames)
+	if err != nil {
+		return nil, err
+	}
+	return NewNoResolve(addrs), nil
+}
+
+// NewNoResolve doesn't resolve the edge. Instead it just uses the given addresses.
+// You probably only need this for testing.
+func NewNoResolve(addrs []*net.TCPAddr) *Regions {
+	region1 := make([]*net.TCPAddr, 0)
+	region2 := make([]*net.TCPAddr, 0)
+	for i, v := range addrs {
+		if i%2 == 0 {
+			region1 = append(region1, v)
+		} else {
+			region2 = append(region2, v)
+		}
+	}
+
+	return &Regions{
+		region1: NewRegion(region1),
+		region2: NewRegion(region2),
+	}
+}
+
+// ------------------------------------
+// Methods
+// ------------------------------------
+
+// GetAnyAddress returns an arbitrary address from the larger region.
+func (rs *Regions) GetAnyAddress() *net.TCPAddr {
+	if rs.region1.AvailableAddrs() > rs.region2.AvailableAddrs() {
+		return rs.region1.GetAnyAddress()
+	}
+	return rs.region2.GetAnyAddress()
+}
+
+// AddrUsedBy finds the address used by the given connection.
+// Returns nil if the connection isn't using an address.
+func (rs *Regions) AddrUsedBy(connID int) *net.TCPAddr {
+	if addr := rs.region1.AddrUsedBy(connID); addr != nil {
+		return addr
+	}
+	return rs.region2.AddrUsedBy(connID)
+}
+
+// GetUnusedAddr gets an unused addr from the edge, excluding the given addr. Prefer to use addresses
+// evenly across both regions.
+func (rs *Regions) GetUnusedAddr(excluding *net.TCPAddr, connID int) *net.TCPAddr {
+	var addr *net.TCPAddr
+	if rs.region1.AvailableAddrs() > rs.region2.AvailableAddrs() {
+		addr = rs.region1.GetUnusedIP(excluding)
+		rs.region1.Use(addr, connID)
+	} else {
+		addr = rs.region2.GetUnusedIP(excluding)
+		rs.region2.Use(addr, connID)
+	}
+
+	if addr == nil {
+		return nil
+	}
+
+	// Mark the address as used and return it
+	return addr
+}
+
+// AvailableAddrs returns how many edge addresses aren't used.
+func (rs *Regions) AvailableAddrs() int {
+	return rs.region1.AvailableAddrs() + rs.region2.AvailableAddrs()
+}
+
+// GiveBack the address so that other connections can use it.
+// Returns true if the address is in this edge.
+func (rs *Regions) GiveBack(addr *net.TCPAddr) bool {
+	if found := rs.region1.GiveBack(addr); found {
+		return found
+	}
+	return rs.region2.GiveBack(addr)
+}

edgediscovery/allregions/regions_test.go

@@ -0,0 +1,140 @@
+package allregions
+
+import (
+	"net"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+var (
+	addr0 = net.TCPAddr{
+		IP:   net.ParseIP("123.4.5.0"),
+		Port: 8000,
+		Zone: "",
+	}
+	addr1 = net.TCPAddr{
+		IP:   net.ParseIP("123.4.5.1"),
+		Port: 8000,
+		Zone: "",
+	}
+	addr2 = net.TCPAddr{
+		IP:   net.ParseIP("123.4.5.2"),
+		Port: 8000,
+		Zone: "",
+	}
+	addr3 = net.TCPAddr{
+		IP:   net.ParseIP("123.4.5.3"),
+		Port: 8000,
+		Zone: "",
+	}
+)
+
+func makeRegions() Regions {
+	r1 := NewRegion([]*net.TCPAddr{&addr0, &addr1})
+	r2 := NewRegion([]*net.TCPAddr{&addr2, &addr3})
+	return Regions{region1: r1, region2: r2}
+}
+
+func TestRegions_AddrUsedBy(t *testing.T) {
+	rs := makeRegions()
+	addr1 := rs.GetUnusedAddr(nil, 1)
+	assert.Equal(t, addr1, rs.AddrUsedBy(1))
+	addr2 := rs.GetUnusedAddr(nil, 2)
+	assert.Equal(t, addr2, rs.AddrUsedBy(2))
+	addr3 := rs.GetUnusedAddr(nil, 3)
+	assert.Equal(t, addr3, rs.AddrUsedBy(3))
+}
+
+func TestRegions_Giveback_Region1(t *testing.T) {
+	rs := makeRegions()
+	rs.region1.Use(&addr0, 0)
+	rs.region1.Use(&addr1, 1)
+	rs.region2.Use(&addr2, 2)
+	rs.region2.Use(&addr3, 3)
+
+	assert.Equal(t, 0, rs.AvailableAddrs())
+
+	rs.GiveBack(&addr0)
+	assert.Equal(t, &addr0, rs.GetUnusedAddr(nil, 3))
+}
+func TestRegions_Giveback_Region2(t *testing.T) {
+	rs := makeRegions()
+	rs.region1.Use(&addr0, 0)
+	rs.region1.Use(&addr1, 1)
+	rs.region2.Use(&addr2, 2)
+	rs.region2.Use(&addr3, 3)
+
+	assert.Equal(t, 0, rs.AvailableAddrs())
+
+	rs.GiveBack(&addr2)
+	assert.Equal(t, &addr2, rs.GetUnusedAddr(nil, 2))
+}
+
+func TestRegions_GetUnusedAddr_OneAddrLeft(t *testing.T) {
+	rs := makeRegions()
+
+	rs.region1.Use(&addr0, 0)
+	rs.region1.Use(&addr1, 1)
+	rs.region2.Use(&addr2, 2)
+
+	assert.Equal(t, 1, rs.AvailableAddrs())
+	assert.Equal(t, &addr3, rs.GetUnusedAddr(nil, 3))
+}
+
+func TestRegions_GetUnusedAddr_Excluding_Region1(t *testing.T) {
+	rs := makeRegions()
+
+	rs.region1.Use(&addr0, 0)
+	rs.region1.Use(&addr1, 1)
+
+	assert.Equal(t, 2, rs.AvailableAddrs())
+	assert.Equal(t, &addr3, rs.GetUnusedAddr(&addr2, 3))
+}
+
+func TestRegions_GetUnusedAddr_Excluding_Region2(t *testing.T) {
+	rs := makeRegions()
+
+	rs.region2.Use(&addr2, 0)
+	rs.region2.Use(&addr3, 1)
+
+	assert.Equal(t, 2, rs.AvailableAddrs())
+	assert.Equal(t, &addr1, rs.GetUnusedAddr(&addr0, 1))
+}
+
+func TestNewNoResolveBalancesRegions(t *testing.T) {
+	type args struct {
+		addrs []*net.TCPAddr
+	}
+	tests := []struct {
+		name string
+		args args
+	}{
+		{
+			name: "one address",
+			args: args{addrs: []*net.TCPAddr{&addr0}},
+		},
+		{
+			name: "two addresses",
+			args: args{addrs: []*net.TCPAddr{&addr0, &addr1}},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			regions := NewNoResolve(tt.args.addrs)
+			RegionsIsBalanced(t, regions)
+		})
+	}
+}
+
+func RegionsIsBalanced(t *testing.T, rs *Regions) {
+	delta := rs.region1.AvailableAddrs() - rs.region2.AvailableAddrs()
+	assert.True(t, abs(delta) <= 1)
+}
+
+func abs(x int) int {
+	if x >= 0 {
+		return x
+	}
+	return -x
+}

edgediscovery/allregions/usedby.go

@@ -0,0 +1,14 @@
+package allregions
+
+type UsedBy struct {
+	ConnID int
+	Used   bool
+}
+
+func InUse(connID int) UsedBy {
+	return UsedBy{ConnID: connID, Used: true}
+}
+
+func Unused() UsedBy {
+	return UsedBy{}
+}

edgediscovery/edgediscovery.go

@@ -0,0 +1,143 @@
+package edgediscovery
+
+import (
+	"fmt"
+	"net"
+	"sync"
+
+	"github.com/cloudflare/cloudflared/edgediscovery/allregions"
+
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	subsystem = "edgediscovery"
+)
+
+var errNoAddressesLeft = fmt.Errorf("There are no free edge addresses left")
+
+// Edge finds addresses on the Cloudflare edge and hands them out to connections.
+type Edge struct {
+	regions *allregions.Regions
+	sync.Mutex
+	logger *logrus.Entry
+}
+
+// ------------------------------------
+// Constructors
+// ------------------------------------
+
+// ResolveEdge runs the initial discovery of the Cloudflare edge, finding Addrs that can be allocated
+// to connections.
+func ResolveEdge(l *logrus.Logger) (*Edge, error) {
+	logger := l.WithField("subsystem", subsystem)
+	regions, err := allregions.ResolveEdge(logger)
+	if err != nil {
+		return new(Edge), err
+	}
+	return &Edge{
+		logger:  logger,
+		regions: regions,
+	}, nil
+}
+
+// StaticEdge creates a list of edge addresses from the list of hostnames. Mainly used for testing connectivity.
+func StaticEdge(l *logrus.Logger, hostnames []string) (*Edge, error) {
+	logger := l.WithField("subsystem", subsystem)
+	regions, err := allregions.StaticEdge(hostnames)
+	if err != nil {
+		return new(Edge), err
+	}
+	return &Edge{
+		logger:  logger,
+		regions: regions,
+	}, nil
+}
+
+// MockEdge creates a Cloudflare Edge from arbitrary TCP addresses. Used for testing.
+func MockEdge(l *logrus.Logger, addrs []*net.TCPAddr) *Edge {
+	logger := l.WithField("subsystem", subsystem)
+	regions := allregions.NewNoResolve(addrs)
+	return &Edge{
+		logger:  logger,
+		regions: regions,
+	}
+}
+
+// ------------------------------------
+// Methods
+// ------------------------------------
+
+// GetAddrForRPC gives this connection an edge Addr.
+func (ed *Edge) GetAddrForRPC() (*net.TCPAddr, error) {
+	ed.Lock()
+	defer ed.Unlock()
+	addr := ed.regions.GetAnyAddress()
+	if addr == nil {
+		return nil, errNoAddressesLeft
+	}
+	return addr, nil
+}
+
+// GetAddr gives this proxy connection an edge Addr. Prefer Addrs this connection has already used.
+func (ed *Edge) GetAddr(connID int) (*net.TCPAddr, error) {
+	ed.Lock()
+	defer ed.Unlock()
+	logger := ed.logger.WithFields(logrus.Fields{
+		"connID":   connID,
+		"function": "GetAddr",
+	})
+
+	// If this connection has already used an edge addr, return it.
+	if addr := ed.regions.AddrUsedBy(connID); addr != nil {
+		logger.Debug("Returning same address back to proxy connection")
+		return addr, nil
+	}
+
+	// Otherwise, give it an unused one
+	addr := ed.regions.GetUnusedAddr(nil, connID)
+	if addr == nil {
+		logger.Debug("No addresses left to give proxy connection")
+		return nil, errNoAddressesLeft
+	}
+	logger.Debug("Giving connection its new address")
+	return addr, nil
+}
+
+// GetDifferentAddr gives back the proxy connection's edge Addr and uses a new one.
+func (ed *Edge) GetDifferentAddr(connID int) (*net.TCPAddr, error) {
+	ed.Lock()
+	defer ed.Unlock()
+	logger := ed.logger.WithFields(logrus.Fields{
+		"connID":   connID,
+		"function": "GetDifferentAddr",
+	})
+
+	oldAddr := ed.regions.AddrUsedBy(connID)
+	if oldAddr != nil {
+		ed.regions.GiveBack(oldAddr)
+	}
+	addr := ed.regions.GetUnusedAddr(oldAddr, connID)
+	if addr == nil {
+		logger.Debug("No addresses left to give proxy connection")
+		return nil, errNoAddressesLeft
+	}
+	logger.Debug("Giving connection its new address")
+	return addr, nil
+}
+
+// AvailableAddrs returns how many unused addresses there are left.
+func (ed *Edge) AvailableAddrs() int {
+	ed.Lock()
+	defer ed.Unlock()
+	return ed.regions.AvailableAddrs()
+}
+
+// GiveBack the address so that other connections can use it.
+// Returns true if the address is in this edge.
+func (ed *Edge) GiveBack(addr *net.TCPAddr) bool {
+	ed.Lock()
+	defer ed.Unlock()
+	ed.logger.WithField("function", "GiveBack").Debug("Address now unused")
+	return ed.regions.GiveBack(addr)
+}

edgediscovery/edgediscovery_test.go

@@ -0,0 +1,130 @@
+package edgediscovery
+
+import (
+	"net"
+	"testing"
+
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+)
+
+var (
+	addr0 = net.TCPAddr{
+		IP:   net.ParseIP("123.0.0.0"),
+		Port: 8000,
+		Zone: "",
+	}
+	addr1 = net.TCPAddr{
+		IP:   net.ParseIP("123.0.0.1"),
+		Port: 8000,
+		Zone: "",
+	}
+	addr2 = net.TCPAddr{
+		IP:   net.ParseIP("123.0.0.2"),
+		Port: 8000,
+		Zone: "",
+	}
+	addr3 = net.TCPAddr{
+		IP:   net.ParseIP("123.0.0.3"),
+		Port: 8000,
+		Zone: "",
+	}
+)
+
+func TestGiveBack(t *testing.T) {
+	l := logrus.New()
+	edge := MockEdge(l, []*net.TCPAddr{&addr0, &addr1, &addr2, &addr3})
+
+	// Give this connection an address
+	assert.Equal(t, 4, edge.AvailableAddrs())
+	const connID = 0
+	addr, err := edge.GetAddr(connID)
+	assert.NoError(t, err)
+	assert.NotNil(t, addr)
+	assert.Equal(t, 3, edge.AvailableAddrs())
+
+	// Get it back
+	edge.GiveBack(addr)
+	assert.Equal(t, 4, edge.AvailableAddrs())
+}
+func TestGetAddrForRPC(t *testing.T) {
+	l := logrus.New()
+	edge := MockEdge(l, []*net.TCPAddr{&addr0, &addr1, &addr2, &addr3})
+
+	// Get a connection
+	assert.Equal(t, 4, edge.AvailableAddrs())
+	addr, err := edge.GetAddrForRPC()
+	assert.NoError(t, err)
+	assert.NotNil(t, addr)
+
+	// Using an address for RPC shouldn't consume it
+	assert.Equal(t, 4, edge.AvailableAddrs())
+
+	// Get it back
+	edge.GiveBack(addr)
+	assert.Equal(t, 4, edge.AvailableAddrs())
+}
+
+func TestOnlyOneAddrLeft(t *testing.T) {
+	l := logrus.New()
+
+	// Make an edge with only one address
+	edge := MockEdge(l, []*net.TCPAddr{&addr0})
+
+	// Use the only address
+	const connID = 0
+	addr, err := edge.GetAddr(connID)
+	assert.NoError(t, err)
+	assert.NotNil(t, addr)
+
+	// If that edge address is "bad", there's no alternative address.
+	_, err = edge.GetDifferentAddr(connID)
+	assert.Error(t, err)
+}
+
+func TestNoAddrsLeft(t *testing.T) {
+	l := logrus.New()
+
+	// Make an edge with no addresses
+	edge := MockEdge(l, []*net.TCPAddr{})
+
+	_, err := edge.GetAddr(2)
+	assert.Error(t, err)
+	_, err = edge.GetAddrForRPC()
+	assert.Error(t, err)
+}
+
+func TestGetAddr(t *testing.T) {
+	l := logrus.New()
+	edge := MockEdge(l, []*net.TCPAddr{&addr0, &addr1, &addr2, &addr3})
+
+	// Give this connection an address
+	const connID = 0
+	addr, err := edge.GetAddr(connID)
+	assert.NoError(t, err)
+	assert.NotNil(t, addr)
+
+	// If the same connection requests another address, it should get the same one.
+	addr2, err := edge.GetAddr(connID)
+	assert.NoError(t, err)
+	assert.Equal(t, addr, addr2)
+}
+
+func TestGetDifferentAddr(t *testing.T) {
+	l := logrus.New()
+	edge := MockEdge(l, []*net.TCPAddr{&addr0, &addr1, &addr2, &addr3})
+
+	// Give this connection an address
+	assert.Equal(t, 4, edge.AvailableAddrs())
+	const connID = 0
+	addr, err := edge.GetAddr(connID)
+	assert.NoError(t, err)
+	assert.NotNil(t, addr)
+	assert.Equal(t, 3, edge.AvailableAddrs())
+
+	// If the same connection requests another address, it should get the same one.
+	addr2, err := edge.GetDifferentAddr(connID)
+	assert.NoError(t, err)
+	assert.NotEqual(t, addr, addr2)
+	assert.Equal(t, 3, edge.AvailableAddrs())
+}

go.mod

@@ -25,9 +25,10 @@ require (
 	github.com/gliderlabs/ssh v0.0.0-20191009160644-63518b5243e0
 	github.com/go-sql-driver/mysql v1.4.1
 	github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3
+	github.com/google/certificate-transparency-go v1.1.0
 	github.com/google/uuid v1.1.1
 	github.com/gorilla/mux v1.7.3
-	github.com/gorilla/websocket v1.2.0
+	github.com/gorilla/websocket v1.4.0
 	github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 // indirect
 	github.com/jmoiron/sqlx v1.2.0
 	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
@@ -56,7 +57,6 @@ require (
 	golang.org/x/net v0.0.0-20191007182048-72f939374954
 	golang.org/x/sync v0.0.0-20190423024810-112230192c58
 	golang.org/x/sys v0.0.0-20191008105621-543471e840be
-	golang.org/x/text v0.3.2 // indirect
 	google.golang.org/genproto v0.0.0-20191007204434-a023cd5227bd // indirect
 	google.golang.org/grpc v1.24.0 // indirect
 	gopkg.in/coreos/go-oidc.v2 v2.1.0

go.sum

@@ -1,18 +1,28 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
+contrib.go.opencensus.io/exporter/stackdriver v0.12.1/go.mod h1:iwB6wGarfphGGe/e5CWqyUk/cLzKnWsOKPVW3no6OTw=
+contrib.go.opencensus.io/resource v0.1.1/go.mod h1:F361eGI91LCmW1I/Saf+rX0+OFcigGlFvXwEGEnkRLA=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/DATA-DOG/go-sqlmock v1.3.3 h1:CWUqKXe0s8A2z6qCgkP4Kru7wC11YoAnoupUKFDnH08=
 github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
+github.com/OpenPeeDeeP/depguard v1.0.0/go.mod h1:7/4sitnI9YlQgTLLk734QlzXT8DuHVnAyztLplQjk+o=
+github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
+github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
+github.com/aws/aws-sdk-go v1.19.18/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/aws/aws-sdk-go v1.25.8 h1:n7I+HUUXjun2CsX7JK+1hpRIkZrlKhd3nayeb+Xmavs=
 github.com/aws/aws-sdk-go v1.25.8/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
 github.com/bkaradzic/go-lz4 v1.0.0 h1:RXc4wYsyz985CkXXeX04y4VnZFGG8Rd43pRaHsOXAKk=
 github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4=
+github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261 h1:6/yVvBsKeAw05IUj4AzvrxaCnDjN4nUqKjW9+w5wixg=
 github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
@@ -24,15 +34,26 @@ github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 h1:F1EaeKL/ta07PY
 github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80=
 github.com/coredns/coredns v1.2.0 h1:YEI38K2BJYzL/SxO2tZFD727T/C68DqVWkBQjT0sWPU=
 github.com/coredns/coredns v1.2.0/go.mod h1:zASH/MVDgR6XZTbxvOnsZfffS+31vg6Ackf/wo1+AM0=
+github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
+github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
+github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
+github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
 github.com/coreos/go-oidc v0.0.0-20171002155002-a93f71fdfe73 h1:7CNPV0LWRCa1FNmqg700pbXhzvmoaXKyfxWRkjRym7Q=
 github.com/coreos/go-oidc v0.0.0-20171002155002-a93f71fdfe73/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
+github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/go-systemd v0.0.0-20190620071333-e64a0ec8b42a h1:W8b4lQ4tFF21aspRGoBuCNV6V2fFJBF+pm1J6OY8Lys=
 github.com/coreos/go-systemd v0.0.0-20190620071333-e64a0ec8b42a/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
+github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/denisenkom/go-mssqldb v0.0.0-20191001013358-cfbb681360f0 h1:epsH3lb7KVbXHYk7LYGN5EiE0MxcevHU85CKITJ0wUY=
 github.com/denisenkom/go-mssqldb v0.0.0-20191001013358-cfbb681360f0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/equinox-io/equinox v1.2.0 h1:bBS7Ou+Y7Jwgmy8TWSYxEh85WctuFn7FPlgbUzX4DBA=
 github.com/equinox-io/equinox v1.2.0/go.mod h1:6s3HJB0PYUNgs0mxmI8fHdfVl3TQ25ieA/PVfr+eyVo=
 github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 h1:0JZ+dUmQeA8IIVUMzysrX4/AKuQwWhV2dYQuPZdvdSQ=
@@ -45,73 +66,176 @@ github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojt
 github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg=
 github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870 h1:E2s37DuLxFhQDg5gKsWoLBOB0n+ZW8s599zru8FJ2/Y=
 github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0=
+github.com/fatih/color v1.6.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/getsentry/raven-go v0.0.0-20180517221441-ed7bcb39ff10 h1:YO10pIIBftO/kkTFdWhctH96grJ7qiy7bMdiZcIvPKs=
 github.com/getsentry/raven-go v0.0.0-20180517221441-ed7bcb39ff10/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ=
+github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/gliderlabs/ssh v0.0.0-20191009160644-63518b5243e0 h1:gF8ngtda767ddth2SH0YSAhswhz6qUkvyI9EZFYCWJA=
 github.com/gliderlabs/ssh v0.0.0-20191009160644-63518b5243e0/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
+github.com/go-critic/go-critic v0.3.5-0.20190526074819-1df300866540/go.mod h1:+sE8vrLDS2M0pZkBk0wy6+nLdKexVDrl/jBqQOTDThA=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-lintpack/lintpack v0.5.2/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTDqfpXvXAN0sXM=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
+github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
 github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
 github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
+github.com/go-toolsmith/astcast v1.0.0/go.mod h1:mt2OdQTeAQcY4DQgPSArJjHCcOwlX+Wl/kwN+LbLGQ4=
+github.com/go-toolsmith/astcopy v1.0.0/go.mod h1:vrgyG+5Bxrnz4MZWPF+pI4R8h3qKRjjyvV/DSez4WVQ=
+github.com/go-toolsmith/astequal v0.0.0-20180903214952-dcb477bfacd6/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY=
+github.com/go-toolsmith/astequal v1.0.0/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY=
+github.com/go-toolsmith/astfmt v0.0.0-20180903215011-8f8ee99c3086/go.mod h1:mP93XdblcopXwlyN4X4uodxXQhldPGZbcEJIimQHrkg=
+github.com/go-toolsmith/astfmt v1.0.0/go.mod h1:cnWmsOAuq4jJY6Ct5YWlVLmcmLMn1JUPuQIHCY7CJDw=
+github.com/go-toolsmith/astinfo v0.0.0-20180906194353-9809ff7efb21/go.mod h1:dDStQCHtmZpYOmjRP/8gHHnCCch3Zz3oEgCdZVdtweU=
+github.com/go-toolsmith/astp v0.0.0-20180903215135-0af7e3c24f30/go.mod h1:SV2ur98SGypH1UjcPpCatrV5hPazG6+IfNHbkDXBRrk=
+github.com/go-toolsmith/astp v1.0.0/go.mod h1:RSyrtpVlfTFGDYRbrjyWP1pYu//tSFcvdYrA8meBmLI=
+github.com/go-toolsmith/pkgload v0.0.0-20181119091011-e9e65178eee8/go.mod h1:WoMrjiy4zvdS+Bg6z9jZH82QXwkcgCBX6nOfnmdaHks=
+github.com/go-toolsmith/pkgload v1.0.0/go.mod h1:5eFArkbO80v7Z0kdngIxsRXRMTaX4Ilcwuh3clNrQJc=
+github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8=
+github.com/go-toolsmith/typep v1.0.0/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU=
+github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
+github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3 h1:zN2lZNZRflqFyxVaTIU61KNKQ9C0055u9CAfpmqUvo4=
 github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3/go.mod h1:nPpo7qLxd6XL3hWJG/O60sR8ZKfMCiIoNap5GvD12KU=
 github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY=
 github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.0.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2/go.mod h1:k9Qvh+8juN+UKMCS/3jFtGICgW8O96FVaZsaxdzDkR4=
+github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a/go.mod h1:ryS0uhF+x9jgbj/N71xsEqODy9BN81/GonCZiOzirOk=
+github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6/go.mod h1:DbHgvLiFKX1Sh2T1w8Q/h4NAI8MHIpzCdnBUDTXU3I0=
+github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613/go.mod h1:SyvUF2NxV+sN8upjjeVYr5W7tyxaT1JVtvhKhOn2ii8=
+github.com/golangci/go-tools v0.0.0-20190318055746-e32c54105b7c/go.mod h1:unzUULGw35sjyOYjUt0jMTXqHlZPpPc6e+xfO4cd6mM=
+github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3/go.mod h1:JXrF4TWy4tXYn62/9x8Wm/K/dm06p8tCKwFRDPZG/1o=
+github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU=
+github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU=
+github.com/golangci/golangci-lint v1.17.2-0.20190910081718-bad04bb7378f/go.mod h1:kaqo8l0OZKYPtjNmG4z4HrWLgcYNIJ9B9q3LWri9uLg=
+github.com/golangci/gosec v0.0.0-20190211064107-66fb7fc33547/go.mod h1:0qUabqiIQgfmlAmulqxyiGkkyF6/tOGSnY2cnPVwrzU=
+github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc/go.mod h1:e5tpTHCfVze+7EpLEozzMB3eafxo2KT5veNg1k6byQU=
+github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg=
+github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca/go.mod h1:tvlJhZqDe4LMs4ZHD0oMUlt9G2LWuDGoisJTBzLMV9o=
+github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA=
+github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21/go.mod h1:tf5+bzsHdTM0bsB7+8mt0GUMvjCgwLpTapNZHU8AajI=
+github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4=
+github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg8RkN3rCIMLGE9CyYmU9pY2Jer6DgANEnZ/L/cQ=
+github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/certificate-transparency-go v1.1.0 h1:10MlrYzh5wfkToxWI4yJzffsxLfxcEDlOATMx/V9Kzw=
+github.com/google/certificate-transparency-go v1.1.0/go.mod h1:i+Q7XY+ArBveOUT36jiHGfuSK1fHICIg6sUkRxPAbCs=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/monologue v0.0.0-20190606152607-4b11a32b5934/go.mod h1:6NTfaQoUpg5QmPsCUWLR3ig33FHrKXhTtWzF0DVdmuk=
+github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/trillian v1.2.2-0.20190612132142-05461f4df60a/go.mod h1:YPmUVn5NGwgnDUgqlVyFGMTgaWlnSvH7W5p+NdOG8UA=
+github.com/google/trillian-examples v0.0.0-20190603134952-4e75ba15216c/go.mod h1:WgL3XZ3pA8/9cm7yxqWrZE6iZkESB2ItGxy5Fo6k2lk=
 github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw=
 github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
 github.com/gorilla/websocket v1.2.0 h1:VJtLvh6VQym50czpZzx07z/kw9EgAxI3x1ZB8taTMQQ=
 github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
+github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q=
+github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
+github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE=
+github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
+github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
+github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
 github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 h1:MJG/KsmcqMwFAkh8mTnAwhyKoB+sTAnY4CACC110tbU=
 github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645/go.mod h1:6iZfnjpejD4L/4DwD7NryNaJyCQdzwWwH2MWhCA90Kw=
+github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/hcl v0.0.0-20180404174102-ef8a98b0bbce/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w=
+github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA=
 github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
+github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/juju/ratelimit v1.0.1/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSgWNm/qk=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/kisielk/gotool v0.0.0-20161130080628-0de1eaf82fa3/go.mod h1:jxZFDH7ILpTPQTk+E2s+z4CUas9lVNjIuKR4c5/zKgM=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
+github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
 github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kshvakov/clickhouse v1.3.11 h1:dtzTJY0fCA+MWkLyuKZaNPkmSwdX4gh8+Klic9NB1Lw=
 github.com/kshvakov/clickhouse v1.3.11/go.mod h1:/SVBAcqF3u7rxQ9sTWCZwf8jzzvxiZGeQvtmSF2BBEc=
 github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4=
 github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
+github.com/letsencrypt/pkcs11key v2.0.1-0.20170608213348-396559074696+incompatible/go.mod h1:iGYXKqDXt0cpBthCHdr9ZdsQwyGlYFh/+8xa4WzIQ34=
 github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0=
 github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
+github.com/magiconair/properties v1.7.6/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA=
 github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.10 h1:qxFzApOv4WsAL965uUPIsXzAKCZxN2p9UqdhFS4ZW10=
 github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84=
+github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
+github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
 github.com/mattn/go-sqlite3 v1.11.0 h1:LDdKkqtYlom37fkvqs8rMPFKAMe8+SgjbwZ6ex1/A/Q=
 github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
+github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/mholt/caddy v0.0.0-20180807230124-d3b731e9255b h1:/BbY4n99iMazlr2igipph+hj0MwlZIWpcsP8Iy+na+s=
 github.com/mholt/caddy v0.0.0-20180807230124-d3b731e9255b/go.mod h1:Wb1PlT4DAYSqOEd03MsqkdkXnTxA8v9pKjdpxbqM1kY=
 github.com/miekg/dns v1.1.8 h1:1QYRAKU3lN5cRfLCkPU08hwvLJFhvjP6MqNMmQz6ZVI=
 github.com/miekg/dns v1.1.8/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
+github.com/miekg/pkcs11 v1.0.2/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
+github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk=
+github.com/mitchellh/mapstructure v0.0.0-20180220230111-00c29f56e238/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
+github.com/mozilla/tls-observatory v0.0.0-20180409132520-8791a200eb40/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/nbutton23/zxcvbn-go v0.0.0-20160627004424-a22cb81b2ecd/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU=
+github.com/nbutton23/zxcvbn-go v0.0.0-20171102151520-eafdab6b0663/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU=
+github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
+github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
+github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU=
 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
+github.com/pelletier/go-toml v1.1.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
+github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/philhofer/fwd v1.0.0 h1:UbZqGr5Y38ApvM/V/jEljVxwocdweyH+vmYvRPBnbqQ=
 github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
 github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I=
@@ -131,80 +255,191 @@ github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLy
 github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA=
 github.com/prometheus/procfs v0.0.5 h1:3+auTFlqw+ZaQYJARz6ArODtkaIwtvBTx3N2NehQlL8=
 github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
+github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI=
 github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5 h1:mZHayPoR0lNmnHyvtYjDeq0zlVHn9K/ZXoy17ylucdo=
 github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5/go.mod h1:GEXHk5HgEKCvEIIrSpFI3ozzG5xOKA2DVlEX/gGnewM=
+github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
+github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rs/cors v1.6.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
+github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
+github.com/ryanuber/go-glob v0.0.0-20170128012129-256dc444b735/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
+github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
+github.com/shirou/gopsutil v0.0.0-20180427012116-c95755e4bcd7/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
+github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
+github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
+github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ=
+github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
 github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
+github.com/sourcegraph/go-diff v0.5.1/go.mod h1:j2dHj3m8aZgQO8lMTcTnBcXkRRRqi34cd2MNlA9u1mE=
+github.com/spf13/afero v1.1.0/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
+github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
+github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg=
+github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cobra v0.0.2/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
+github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
+github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
+github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
+github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/viper v1.0.2/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM=
+github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/timakin/bodyclose v0.0.0-20190721030226-87058b9bfcec/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk=
 github.com/tinylib/msgp v1.1.0 h1:9fQd+ICuRIu/ue4vxJZu6/LzxN0HwMds2nq/0cFvxHU=
 github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
+github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
+github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce/go.mod h1:o8v6yHRoik09Xen7gje4m9ERNah1d1PPsVq1VEx9vE4=
+github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
+github.com/ultraware/funlen v0.0.1/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA=
+github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
+github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
+github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s=
+github.com/valyala/quicktemplate v1.1.1/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOVRUAfrukLPuGJ4=
+github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
+github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xo/dburl v0.0.0-20191005012637-293c3298d6c0 h1:6DtWz8hNS4qbq0OCRPhdBMG9E2qKTSDKlwnP3dmZvuA=
 github.com/xo/dburl v0.0.0-20191005012637-293c3298d6c0/go.mod h1:A47W3pdWONaZmXuLZgfKLAVgUY0qvfTRM5vVDKS40S4=
+github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
+go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.etcd.io/etcd v3.3.13+incompatible/go.mod h1:yaeTdrJi5lOmYerz05bd8+V7KubZs8YSFZfzsF9A6aI=
+go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
+go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
+go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
+go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
+golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc h1:c0o/qxkaO2LF5t6fQrT4b5hzyggAkLLlCUjqfRxd8Q4=
 golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/net v0.0.0-20170915142106-8351a756f30f/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191007182048-72f939374954 h1:JGZucVF/L/TotR719NbujzadOZ2AgnYlqphQGHDCKaU=
 golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20171026204733-164713f0dfce/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191008105621-543471e840be h1:QAcqgptGM8IQBC9K/RC4o+O9YmqEm0diQn9QmZw/0mU=
 golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.0.0-20170915090833-1cbadb444a80/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20170915040203-e531a2a1c15f/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20181117154741-2ddaf7f79a09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190110163146-51295c7ec13a/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190121143147-24cd39ecf745/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190311215038-5c2858a9cfe5/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190521203540-521d6ed310dd/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190909030654-5b82db07426d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
+google.golang.org/api v0.5.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
+google.golang.org/api v0.6.0/go.mod h1:btoxGiFvQNVUZQ8W08zLtrVS08CNpINPEfxXxgJL1Q4=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190605220351-eb0b1bdb6ae6/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s=
 google.golang.org/genproto v0.0.0-20191007204434-a023cd5227bd h1:84VQPzup3IpKLxuIAZjHMhVjJ8fZ4/i3yUnj3k6fUdw=
 google.golang.org/genproto v0.0.0-20191007204434-a023cd5227bd/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
+google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
 google.golang.org/grpc v1.24.0 h1:vb/1TCsVn3DcJlQ0Gs1yB1pKI6Do2/QNwxdKqmc/b0s=
 google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA=
+gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
 gopkg.in/coreos/go-oidc.v2 v2.1.0 h1:E8PjVFdj/SLDKB0hvb70KTbMbYVHjqztiQdSkIg8E+I=
 gopkg.in/coreos/go-oidc.v2 v2.1.0/go.mod h1:fYaTe2FS96wZZwR17YTDHwG+Mw6fmyqJNxN2eNCGPCI=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
+gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
 gopkg.in/square/go-jose.v2 v2.4.0 h1:0kXPskUMGAXXWJlP05ktEMOV0vmzFQUWw6d+aZJQU8A=
 gopkg.in/square/go-jose.v2 v2.4.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/urfave/cli.v2 v2.0.0-20180128181224-d604b6ffeee8 h1:/pLAskKF+d5SawboKd8GB8ew4ClHDbt2c3K9EBFeRGU=
 gopkg.in/urfave/cli.v2 v2.0.0-20180128181224-d604b6ffeee8/go.mod h1:cKXr3E0k4aosgycml1b5z33BVV6hai1Kh7uDgFOkbcs=
+gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc=
+mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4=
+mvdan.cc/unparam v0.0.0-20190209190245-fbb59629db34/go.mod h1:H6SUd1XjIs+qQCyskXg5OFSrilMRUkD8ePJpHKDPaeY=
+sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0=
 zombiezen.com/go/capnproto2 v0.0.0-20180616160808-7cfd211c19c7 h1:CZoOFlTPbKfAShKYrMuUfYbnXexFT1rYRUX1SPnrdE4=
 zombiezen.com/go/capnproto2 v0.0.0-20180616160808-7cfd211c19c7/go.mod h1:TMGa8HWGJkXiq4nHe9Zu/JgRF5oUtg4XizFC+Vexbec=

h2mux/activestreammap.go

@@ -13,26 +13,28 @@ type activeStreamMap struct {
 	sync.RWMutex
 	// streams tracks open streams.
 	streams map[uint32]*MuxedStream
-	// streamsEmpty is a chan that should be closed when no more streams are open.
-	streamsEmpty chan struct{}
 	// nextStreamID is the next ID to use on our side of the connection.
 	// This is odd for clients, even for servers.
 	nextStreamID uint32
 	// maxPeerStreamID is the ID of the most recent stream opened by the peer.
 	maxPeerStreamID uint32
+	// activeStreams is a gauge shared by all muxers of this process to expose the total number of active streams
+	activeStreams prometheus.Gauge
+
 	// ignoreNewStreams is true when the connection is being shut down. New streams
 	// cannot be registered.
 	ignoreNewStreams bool
-	// activeStreams is a gauge shared by all muxers of this process to expose the total number of active streams
-	activeStreams prometheus.Gauge
+	// streamsEmpty is a chan that will be closed when no more streams are open.
+	streamsEmptyChan chan struct{}
+	closeOnce        sync.Once
 }
 
 func newActiveStreamMap(useClientStreamNumbers bool, activeStreams prometheus.Gauge) *activeStreamMap {
 	m := &activeStreamMap{
-		streams:       make(map[uint32]*MuxedStream),
-		streamsEmpty:  make(chan struct{}),
-		nextStreamID:  1,
-		activeStreams: activeStreams,
+		streams:          make(map[uint32]*MuxedStream),
+		streamsEmptyChan: make(chan struct{}),
+		nextStreamID:     1,
+		activeStreams:    activeStreams,
 	}
 	// Client initiated stream uses odd stream ID, server initiated stream uses even stream ID
 	if !useClientStreamNumbers {
@@ -41,6 +43,13 @@ func newActiveStreamMap(useClientStreamNumbers bool, activeStreams prometheus.Ga
 	return m
 }
 
+// This function should be called while `m` is locked.
+func (m *activeStreamMap) notifyStreamsEmpty() {
+	m.closeOnce.Do(func() {
+		close(m.streamsEmptyChan)
+	})
+}
+
 // Len returns the number of active streams.
 func (m *activeStreamMap) Len() int {
 	m.RLock()
@@ -79,30 +88,29 @@ func (m *activeStreamMap) Delete(streamID uint32) {
 		delete(m.streams, streamID)
 		m.activeStreams.Dec()
 	}
-	if len(m.streams) == 0 && m.streamsEmpty != nil {
-		close(m.streamsEmpty)
-		m.streamsEmpty = nil
+
+	// shutting down, and now the map is empty
+	if m.ignoreNewStreams && len(m.streams) == 0 {
+		m.notifyStreamsEmpty()
 	}
 }
 
-// Shutdown blocks new streams from being created. It returns a channel that receives an event
-// once the last stream has closed, or nil if a shutdown is in progress.
-func (m *activeStreamMap) Shutdown() <-chan struct{} {
+// Shutdown blocks new streams from being created.
+// It returns `done`, a channel that is closed once the last stream has closed
+// and `progress`, whether a shutdown was already in progress
+func (m *activeStreamMap) Shutdown() (done <-chan struct{}, alreadyInProgress bool) {
 	m.Lock()
 	defer m.Unlock()
 	if m.ignoreNewStreams {
 		// already shutting down
-		return nil
+		return m.streamsEmptyChan, true
 	}
 	m.ignoreNewStreams = true
-	done := make(chan struct{})
 	if len(m.streams) == 0 {
-		// nothing to shut down
-		close(done)
-		return done
+		// there are no streams to wait for
+		m.notifyStreamsEmpty()
 	}
-	m.streamsEmpty = done
-	return done
+	return m.streamsEmptyChan, false
 }
 
 // AcquireLocalID acquires a new stream ID for a stream you're opening.
@@ -170,4 +178,5 @@ func (m *activeStreamMap) Abort() {
 		stream.Close()
 	}
 	m.ignoreNewStreams = true
+	m.notifyStreamsEmpty()
 }

h2mux/activestreammap_test.go

@@ -0,0 +1,195 @@
+package h2mux
+
+import (
+	"sync"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestShutdown(t *testing.T) {
+	const numStreams = 1000
+	m := newActiveStreamMap(true, NewActiveStreamsMetrics("test", t.Name()))
+
+	// Add all the streams
+	{
+		var wg sync.WaitGroup
+		wg.Add(numStreams)
+		for i := 0; i < numStreams; i++ {
+			go func(streamID int) {
+				defer wg.Done()
+				stream := &MuxedStream{streamID: uint32(streamID)}
+				ok := m.Set(stream)
+				assert.True(t, ok)
+			}(i)
+		}
+		wg.Wait()
+	}
+	assert.Equal(t, numStreams, m.Len(), "All the streams should have been added")
+
+	shutdownChan, alreadyInProgress := m.Shutdown()
+	select {
+	case <-shutdownChan:
+		assert.Fail(t, "before Shutdown(), shutdownChan shouldn't be closed")
+	default:
+	}
+	assert.False(t, alreadyInProgress)
+
+	shutdownChan2, alreadyInProgress2 := m.Shutdown()
+	assert.Equal(t, shutdownChan, shutdownChan2, "repeated calls to Shutdown() should return the same channel")
+	assert.True(t, alreadyInProgress2, "repeated calls to Shutdown() should return true for 'in progress'")
+
+	// Delete all the streams
+	{
+		var wg sync.WaitGroup
+		wg.Add(numStreams)
+		for i := 0; i < numStreams; i++ {
+			go func(streamID int) {
+				defer wg.Done()
+				m.Delete(uint32(streamID))
+			}(i)
+		}
+		wg.Wait()
+	}
+	assert.Equal(t, 0, m.Len(), "All the streams should have been deleted")
+
+	select {
+	case <-shutdownChan:
+	default:
+		assert.Fail(t, "After all the streams are deleted, shutdownChan should have been closed")
+	}
+}
+
+func TestEmptyBeforeShutdown(t *testing.T) {
+	const numStreams = 1000
+	m := newActiveStreamMap(true, NewActiveStreamsMetrics("test", t.Name()))
+
+	// Add all the streams
+	{
+		var wg sync.WaitGroup
+		wg.Add(numStreams)
+		for i := 0; i < numStreams; i++ {
+			go func(streamID int) {
+				defer wg.Done()
+				stream := &MuxedStream{streamID: uint32(streamID)}
+				ok := m.Set(stream)
+				assert.True(t, ok)
+			}(i)
+		}
+		wg.Wait()
+	}
+	assert.Equal(t, numStreams, m.Len(), "All the streams should have been added")
+
+	// Delete all the streams, bringing m to size 0
+	{
+		var wg sync.WaitGroup
+		wg.Add(numStreams)
+		for i := 0; i < numStreams; i++ {
+			go func(streamID int) {
+				defer wg.Done()
+				m.Delete(uint32(streamID))
+			}(i)
+		}
+		wg.Wait()
+	}
+	assert.Equal(t, 0, m.Len(), "All the streams should have been deleted")
+
+	// Add one stream back
+	const soloStreamID = uint32(0)
+	ok := m.Set(&MuxedStream{streamID: soloStreamID})
+	assert.True(t, ok)
+
+	shutdownChan, alreadyInProgress := m.Shutdown()
+	select {
+	case <-shutdownChan:
+		assert.Fail(t, "before Shutdown(), shutdownChan shouldn't be closed")
+	default:
+	}
+	assert.False(t, alreadyInProgress)
+
+	shutdownChan2, alreadyInProgress2 := m.Shutdown()
+	assert.Equal(t, shutdownChan, shutdownChan2, "repeated calls to Shutdown() should return the same channel")
+	assert.True(t, alreadyInProgress2, "repeated calls to Shutdown() should return true for 'in progress'")
+
+	// Remove the remaining stream
+	m.Delete(soloStreamID)
+
+	select {
+	case <-shutdownChan:
+	default:
+		assert.Fail(t, "After all the streams are deleted, shutdownChan should have been closed")
+	}
+}
+
+type noopBuffer struct {
+	isClosed bool
+}
+
+func (t *noopBuffer) Read(p []byte) (n int, err error)  { return len(p), nil }
+func (t *noopBuffer) Write(p []byte) (n int, err error) { return len(p), nil }
+func (t *noopBuffer) Reset()                            {}
+func (t *noopBuffer) Len() int                          { return 0 }
+func (t *noopBuffer) Close() error                      { t.isClosed = true; return nil }
+func (t *noopBuffer) Closed() bool                      { return t.isClosed }
+
+type noopReadyList struct{}
+
+func (_ *noopReadyList) Signal(streamID uint32) {}
+
+func TestAbort(t *testing.T) {
+	const numStreams = 1000
+	m := newActiveStreamMap(true, NewActiveStreamsMetrics("test", t.Name()))
+
+	var openedStreams sync.Map
+
+	// Add all the streams
+	{
+		var wg sync.WaitGroup
+		wg.Add(numStreams)
+		for i := 0; i < numStreams; i++ {
+			go func(streamID int) {
+				defer wg.Done()
+				stream := &MuxedStream{
+					streamID:    uint32(streamID),
+					readBuffer:  &noopBuffer{},
+					writeBuffer: &noopBuffer{},
+					readyList:   &noopReadyList{},
+				}
+				ok := m.Set(stream)
+				assert.True(t, ok)
+
+				openedStreams.Store(stream.streamID, stream)
+			}(i)
+		}
+		wg.Wait()
+	}
+	assert.Equal(t, numStreams, m.Len(), "All the streams should have been added")
+
+	shutdownChan, alreadyInProgress := m.Shutdown()
+	select {
+	case <-shutdownChan:
+		assert.Fail(t, "before Abort(), shutdownChan shouldn't be closed")
+	default:
+	}
+	assert.False(t, alreadyInProgress)
+
+	m.Abort()
+	assert.Equal(t, numStreams, m.Len(), "Abort() shouldn't delete any streams")
+	openedStreams.Range(func(key interface{}, value interface{}) bool {
+		stream := value.(*MuxedStream)
+		readBuffer := stream.readBuffer.(*noopBuffer)
+		writeBuffer := stream.writeBuffer.(*noopBuffer)
+		return assert.True(t, readBuffer.isClosed && writeBuffer.isClosed, "Abort() should have closed all the streams")
+	})
+
+	select {
+	case <-shutdownChan:
+	default:
+		assert.Fail(t, "after Abort(), shutdownChan should have been closed")
+	}
+
+	// multiple aborts shouldn't cause any issues
+	m.Abort()
+	m.Abort()
+	m.Abort()
+}

h2mux/h2_dictionaries.go

@@ -542,7 +542,10 @@ func (w *h2DictWriter) Write(p []byte) (n int, err error) {
 }
 
 func (w *h2DictWriter) Close() error {
-	return w.comp.Close()
+	if w.comp != nil {
+		return w.comp.Close()
+	}
+	return nil
 }
 
 // From http2/hpack

h2mux/h2mux.go

@@ -90,10 +90,6 @@ type Muxer struct {
 	compressionQuality CompressionPreset
 }
 
-type Header struct {
-	Name, Value string
-}
-
 func RPCHeaders() []Header {
 	return []Header{
 		{Name: ":method", Value: "RPC"},
@@ -325,24 +321,51 @@ func joinErrorsWithTimeout(errChan <-chan error, receiveCount int, timeout time.
 func (m *Muxer) Serve(ctx context.Context) error {
 	errGroup, _ := errgroup.WithContext(ctx)
 	errGroup.Go(func() error {
-		err := m.muxReader.run(m.config.Logger)
-		m.explicitShutdown.Fuse(false)
-		m.r.Close()
-		m.abort()
-		return err
+		ch := make(chan error)
+		go func() {
+			err := m.muxReader.run(m.config.Logger)
+			m.explicitShutdown.Fuse(false)
+			m.r.Close()
+			m.abort()
+			ch <- err
+		}()
+		select {
+		case err := <-ch:
+			return err
+		case <-ctx.Done():
+			return ctx.Err()
+		}
 	})
 
 	errGroup.Go(func() error {
-		err := m.muxWriter.run(m.config.Logger)
-		m.explicitShutdown.Fuse(false)
-		m.w.Close()
-		m.abort()
-		return err
+		ch := make(chan error)
+		go func() {
+			err := m.muxWriter.run(m.config.Logger)
+			m.explicitShutdown.Fuse(false)
+			m.w.Close()
+			m.abort()
+			ch <- err
+		}()
+		select {
+		case err := <-ch:
+			return err
+		case <-ctx.Done():
+			return ctx.Err()
+		}
 	})
 
 	errGroup.Go(func() error {
-		err := m.muxMetricsUpdater.run(m.config.Logger)
-		return err
+		ch := make(chan error)
+		go func() {
+			err := m.muxMetricsUpdater.run(m.config.Logger)
+			ch <- err
+		}()
+		select {
+		case err := <-ch:
+			return err
+		case <-ctx.Done():
+			return ctx.Err()
+		}
 	})
 
 	err := errGroup.Wait()
@@ -353,9 +376,11 @@ func (m *Muxer) Serve(ctx context.Context) error {
 }
 
 // Shutdown is called to initiate the "happy path" of muxer termination.
-func (m *Muxer) Shutdown() {
+// It blocks new streams from being created.
+// It returns a channel that is closed when the last stream has been closed.
+func (m *Muxer) Shutdown() <-chan struct{} {
 	m.explicitShutdown.Fuse(true)
-	m.muxReader.Shutdown()
+	return m.muxReader.Shutdown()
 }
 
 // IsUnexpectedTunnelError identifies errors that are expected when shutting down the h2mux tunnel.

h2mux/h2mux_test.go

@@ -55,6 +55,8 @@ func NewDefaultMuxerPair(t assert.TestingT, testName string, f MuxedStreamFunc)
 			DefaultWindowSize:       (1 << 8) - 1,
 			MaxWindowSize:           (1 << 15) - 1,
 			StreamWriteBufferMaxLen: 1024,
+			HeartbeatInterval:       defaultTimeout,
+			MaxHeartbeats:           defaultRetries,
 		},
 		OriginConn: origin,
 		EdgeMuxConfig: MuxerConfig{
@@ -65,6 +67,8 @@ func NewDefaultMuxerPair(t assert.TestingT, testName string, f MuxedStreamFunc)
 			DefaultWindowSize:       (1 << 8) - 1,
 			MaxWindowSize:           (1 << 15) - 1,
 			StreamWriteBufferMaxLen: 1024,
+			HeartbeatInterval:       defaultTimeout,
+			MaxHeartbeats:           defaultRetries,
 		},
 		EdgeConn: edge,
 		doneC:    make(chan struct{}),
@@ -83,6 +87,8 @@ func NewCompressedMuxerPair(t assert.TestingT, testName string, quality Compress
 			Name:               "origin",
 			CompressionQuality: quality,
 			Logger:             log.NewEntry(log.New()),
+			HeartbeatInterval:  defaultTimeout,
+			MaxHeartbeats:      defaultRetries,
 		},
 		OriginConn: origin,
 		EdgeMuxConfig: MuxerConfig{
@@ -91,6 +97,8 @@ func NewCompressedMuxerPair(t assert.TestingT, testName string, quality Compress
 			Name:               "edge",
 			CompressionQuality: quality,
 			Logger:             log.NewEntry(log.New()),
+			HeartbeatInterval:  defaultTimeout,
+			MaxHeartbeats:      defaultRetries,
 		},
 		EdgeConn: edge,
 		doneC:    make(chan struct{}),

h2mux/header.go

@@ -0,0 +1,279 @@
+package h2mux
+
+import (
+	"encoding/base64"
+	"fmt"
+	"github.com/pkg/errors"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+)
+
+type Header struct {
+	Name, Value string
+}
+
+var headerEncoding = base64.RawStdEncoding
+
+const (
+	RequestUserHeadersField  = "cf-cloudflared-request-headers"
+	ResponseUserHeadersField = "cf-cloudflared-response-headers"
+)
+
+// H2RequestHeadersToH1Request converts the HTTP/2 headers coming from origintunneld
+// to an HTTP/1 Request object destined for the local origin web service.
+// This operation includes conversion of the pseudo-headers into their closest
+// HTTP/1 equivalents. See https://tools.ietf.org/html/rfc7540#section-8.1.2.3
+func H2RequestHeadersToH1Request(h2 []Header, h1 *http.Request) error {
+	for _, header := range h2 {
+		switch strings.ToLower(header.Name) {
+		case ":method":
+			h1.Method = header.Value
+		case ":scheme":
+			// noop - use the preexisting scheme from h1.URL
+		case ":authority":
+			// Otherwise the host header will be based on the origin URL
+			h1.Host = header.Value
+		case ":path":
+			// We don't want to be an "opinionated" proxy, so ideally we would use :path as-is.
+			// However, this HTTP/1 Request object belongs to the Go standard library,
+			// whose URL package makes some opinionated decisions about the encoding of
+			// URL characters: see the docs of https://godoc.org/net/url#URL,
+			// in particular the EscapedPath method https://godoc.org/net/url#URL.EscapedPath,
+			// which is always used when computing url.URL.String(), whether we'd like it or not.
+			//
+			// Well, not *always*. We could circumvent this by using url.URL.Opaque. But
+			// that would present unusual difficulties when using an HTTP proxy: url.URL.Opaque
+			// is treated differently when HTTP_PROXY is set!
+			// See https://github.com/golang/go/issues/5684#issuecomment-66080888
+			//
+			// This means we are subject to the behavior of net/url's function `shouldEscape`
+			// (as invoked with mode=encodePath): https://github.com/golang/go/blob/go1.12.7/src/net/url/url.go#L101
+
+			if header.Value == "*" {
+				h1.URL.Path = "*"
+				continue
+			}
+			// Due to the behavior of validation.ValidateUrl, h1.URL may
+			// already have a partial value, with or without a trailing slash.
+			base := h1.URL.String()
+			base = strings.TrimRight(base, "/")
+			// But we know :path begins with '/', because we handled '*' above - see RFC7540
+			requestURL, err := url.Parse(base + header.Value)
+			if err != nil {
+				return errors.Wrap(err, fmt.Sprintf("invalid path '%v'", header.Value))
+			}
+			h1.URL = requestURL
+		case "content-length":
+			contentLength, err := strconv.ParseInt(header.Value, 10, 64)
+			if err != nil {
+				return fmt.Errorf("unparseable content length")
+			}
+			h1.ContentLength = contentLength
+		default:
+			// Ignore any other header;
+			// User headers will be read from `RequestUserHeadersField`
+			continue
+		}
+	}
+
+	// Find and parse user headers serialized into a single one
+	userHeaders, err := ParseUserHeaders(RequestUserHeadersField, h2)
+	if err != nil {
+		return errors.Wrap(err, "Unable to parse user headers")
+	}
+	for _, userHeader := range userHeaders {
+		h1.Header.Add(http.CanonicalHeaderKey(userHeader.Name), userHeader.Value)
+	}
+
+	return nil
+}
+
+func ParseUserHeaders(headerNameToParseFrom string, headers []Header) ([]Header, error) {
+	for _, header := range headers {
+		if header.Name == headerNameToParseFrom {
+			return DeserializeHeaders(header.Value)
+		}
+	}
+
+	return nil, fmt.Errorf("%v header not found", RequestUserHeadersField)
+}
+
+func IsControlHeader(headerName string) bool {
+	headerName = strings.ToLower(headerName)
+
+	return strings.ToLower(headerName) == "content-length" ||
+		strings.HasPrefix(headerName, ":") ||
+		strings.HasPrefix(headerName, "cf-")
+}
+
+func H1ResponseToH2ResponseHeaders(h1 *http.Response) (h2 []Header) {
+	h2 = []Header{
+		{Name: ":status", Value: strconv.Itoa(h1.StatusCode)},
+	}
+	userHeaders := http.Header{}
+	for header, values := range h1.Header {
+		for _, value := range values {
+			if strings.ToLower(header) == "content-length" {
+				// This header has meaning in HTTP/2 and will be used by the edge,
+				// so it should be sent as an HTTP/2 response header.
+
+				// Since these are http2 headers, they're required to be lowercase
+				h2 = append(h2, Header{Name: strings.ToLower(header), Value: value})
+			} else if !IsControlHeader(header) {
+				// User headers, on the other hand, must all be serialized so that
+				// HTTP/2 header validation won't be applied to HTTP/1 header values
+				if _, ok := userHeaders[header]; ok {
+					userHeaders[header] = append(userHeaders[header], value)
+				} else {
+					userHeaders[header] = []string{value}
+				}
+			}
+		}
+	}
+
+	// Perform user header serialization and set them in the single header
+	h2 = append(h2, CreateSerializedHeaders(ResponseUserHeadersField, userHeaders)...)
+
+	return h2
+}
+
+// Obsolete version of H2RequestHeadersToH1Request
+func OldH2RequestHeadersToH1Request(h2 []Header, h1 *http.Request) error {
+	for _, header := range h2 {
+		switch header.Name {
+		case ":method":
+			h1.Method = header.Value
+		case ":scheme":
+			// noop - use the preexisting scheme from h1.URL
+		case ":authority":
+			// Otherwise the host header will be based on the origin URL
+			h1.Host = header.Value
+		case ":path":
+			// We don't want to be an "opinionated" proxy, so ideally we would use :path as-is.
+			// However, this HTTP/1 Request object belongs to the Go standard library,
+			// whose URL package makes some opinionated decisions about the encoding of
+			// URL characters: see the docs of https://godoc.org/net/url#URL,
+			// in particular the EscapedPath method https://godoc.org/net/url#URL.EscapedPath,
+			// which is always used when computing url.URL.String(), whether we'd like it or not.
+			//
+			// Well, not *always*. We could circumvent this by using url.URL.Opaque. But
+			// that would present unusual difficulties when using an HTTP proxy: url.URL.Opaque
+			// is treated differently when HTTP_PROXY is set!
+			// See https://github.com/golang/go/issues/5684#issuecomment-66080888
+			//
+			// This means we are subject to the behavior of net/url's function `shouldEscape`
+			// (as invoked with mode=encodePath): https://github.com/golang/go/blob/go1.12.7/src/net/url/url.go#L101
+
+			if header.Value == "*" {
+				h1.URL.Path = "*"
+				continue
+			}
+			// Due to the behavior of validation.ValidateUrl, h1.URL may
+			// already have a partial value, with or without a trailing slash.
+			base := h1.URL.String()
+			base = strings.TrimRight(base, "/")
+			// But we know :path begins with '/', because we handled '*' above - see RFC7540
+			url, err := url.Parse(base + header.Value)
+			if err != nil {
+				return errors.Wrap(err, fmt.Sprintf("invalid path '%v'", header.Value))
+			}
+			h1.URL = url
+		case "content-length":
+			contentLength, err := strconv.ParseInt(header.Value, 10, 64)
+			if err != nil {
+				return fmt.Errorf("unparseable content length")
+			}
+			h1.ContentLength = contentLength
+		default:
+			h1.Header.Add(http.CanonicalHeaderKey(header.Name), header.Value)
+		}
+	}
+	return nil
+}
+
+// Obsolete version of H1ResponseToH2ResponseHeaders
+func OldH1ResponseToH2ResponseHeaders(h1 *http.Response) (h2 []Header) {
+	h2 = []Header{{Name: ":status", Value: fmt.Sprintf("%d", h1.StatusCode)}}
+	for headerName, headerValues := range h1.Header {
+		for _, headerValue := range headerValues {
+			h2 = append(h2, Header{Name: strings.ToLower(headerName), Value: headerValue})
+		}
+	}
+
+	return h2
+}
+
+// Serialize HTTP1.x headers by base64-encoding each header name and value,
+// and then joining them in the format of [key:value;]
+func SerializeHeaders(h1Headers http.Header) string {
+	var serializedHeaders []string
+	for headerName, headerValues := range h1Headers {
+		for _, headerValue := range headerValues {
+			encodedName := make([]byte, headerEncoding.EncodedLen(len(headerName)))
+			headerEncoding.Encode(encodedName, []byte(headerName))
+
+			encodedValue := make([]byte, headerEncoding.EncodedLen(len(headerValue)))
+			headerEncoding.Encode(encodedValue, []byte(headerValue))
+
+			serializedHeaders = append(
+				serializedHeaders,
+				strings.Join(
+					[]string{string(encodedName), string(encodedValue)},
+					":",
+				),
+			)
+		}
+	}
+
+	return strings.Join(serializedHeaders, ";")
+}
+
+// Deserialize headers serialized by `SerializeHeader`
+func DeserializeHeaders(serializedHeaders string) ([]Header, error) {
+	const unableToDeserializeErr = "Unable to deserialize headers"
+
+	var deserialized []Header
+	for _, serializedPair := range strings.Split(serializedHeaders, ";") {
+		if len(serializedPair) == 0 {
+			continue
+		}
+
+		serializedHeaderParts := strings.Split(serializedPair, ":")
+		if len(serializedHeaderParts) != 2 {
+			return nil, errors.New(unableToDeserializeErr)
+		}
+
+		serializedName := serializedHeaderParts[0]
+		serializedValue := serializedHeaderParts[1]
+		deserializedName := make([]byte, headerEncoding.DecodedLen(len(serializedName)))
+		deserializedValue := make([]byte, headerEncoding.DecodedLen(len(serializedValue)))
+
+		if _, err := headerEncoding.Decode(deserializedName, []byte(serializedName)); err != nil {
+			return nil, errors.Wrap(err, unableToDeserializeErr)
+		}
+		if _, err := headerEncoding.Decode(deserializedValue, []byte(serializedValue)); err != nil {
+			return nil, errors.Wrap(err, unableToDeserializeErr)
+		}
+
+		deserialized = append(deserialized, Header{
+			Name:  string(deserializedName),
+			Value: string(deserializedValue),
+		})
+	}
+
+	return deserialized, nil
+}
+
+func CreateSerializedHeaders(headersField string, headers ...http.Header) []Header {
+	var serializedHeaderChunks []string
+	for _, headerChunk := range headers {
+		serializedHeaderChunks = append(serializedHeaderChunks, SerializeHeaders(headerChunk))
+	}
+
+	return []Header{{
+		headersField,
+		strings.Join(serializedHeaderChunks, ";"),
+	}}
+}

h2mux/header_test.go

@@ -1,4 +1,4 @@
-package streamhandler
+package h2mux
 
 import (
 	"fmt"
@@ -7,38 +7,39 @@ import (
 	"net/url"
 	"reflect"
 	"regexp"
+	"sort"
 	"strings"
 	"testing"
 	"testing/quick"
 
-	"github.com/cloudflare/cloudflared/h2mux"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
+type ByName []Header
+
+func (a ByName) Len() int      { return len(a) }
+func (a ByName) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+func (a ByName) Less(i, j int) bool {
+	if a[i].Name == a[j].Name {
+		return a[i].Value < a[j].Value
+	}
+
+	return a[i].Name < a[j].Name
+}
+
 func TestH2RequestHeadersToH1Request_RegularHeaders(t *testing.T) {
 	request, err := http.NewRequest(http.MethodGet, "http://example.com", nil)
 	assert.NoError(t, err)
 
-	headersConversionErr := H2RequestHeadersToH1Request(
-		[]h2mux.Header{
-			h2mux.Header{
-				Name:  "Mock header 1",
-				Value: "Mock value 1",
-			},
-			h2mux.Header{
-				Name:  "Mock header 2",
-				Value: "Mock value 2",
-			},
-		},
-		request,
-	)
+	mockHeaders := http.Header{
+		"Mock header 1": {"Mock value 1"},
+		"Mock header 2": {"Mock value 2"},
+	}
 
-	assert.Equal(t, http.Header{
-		"Mock header 1": []string{"Mock value 1"},
-		"Mock header 2": []string{"Mock value 2"},
-	}, request.Header)
+	headersConversionErr := H2RequestHeadersToH1Request(CreateSerializedHeaders(RequestUserHeadersField, mockHeaders), request)
 
+	assert.True(t, reflect.DeepEqual(mockHeaders, request.Header))
 	assert.NoError(t, headersConversionErr)
 }
 
@@ -47,12 +48,14 @@ func TestH2RequestHeadersToH1Request_NoHeaders(t *testing.T) {
 	assert.NoError(t, err)
 
 	headersConversionErr := H2RequestHeadersToH1Request(
-		[]h2mux.Header{},
+		[]Header{{
+			RequestUserHeadersField,
+			SerializeHeaders(http.Header{}),
+		}},
 		request,
 	)
 
-	assert.Equal(t, http.Header{}, request.Header)
-
+	assert.True(t, reflect.DeepEqual(http.Header{}, request.Header))
 	assert.NoError(t, headersConversionErr)
 }
 
@@ -60,19 +63,12 @@ func TestH2RequestHeadersToH1Request_InvalidHostPath(t *testing.T) {
 	request, err := http.NewRequest(http.MethodGet, "http://example.com", nil)
 	assert.NoError(t, err)
 
-	headersConversionErr := H2RequestHeadersToH1Request(
-		[]h2mux.Header{
-			h2mux.Header{
-				Name:  ":path",
-				Value: "//bad_path/",
-			},
-			h2mux.Header{
-				Name:  "Mock header",
-				Value: "Mock value",
-			},
-		},
-		request,
-	)
+	mockRequestHeaders := []Header{
+		{Name: ":path", Value: "//bad_path/"},
+		{Name: RequestUserHeadersField, Value: SerializeHeaders(http.Header{"Mock header": {"Mock value"}})},
+	}
+
+	headersConversionErr := H2RequestHeadersToH1Request(mockRequestHeaders, request)
 
 	assert.Equal(t, http.Header{
 		"Mock header": []string{"Mock value"},
@@ -87,19 +83,12 @@ func TestH2RequestHeadersToH1Request_HostPathWithQuery(t *testing.T) {
 	request, err := http.NewRequest(http.MethodGet, "http://example.com/", nil)
 	assert.NoError(t, err)
 
-	headersConversionErr := H2RequestHeadersToH1Request(
-		[]h2mux.Header{
-			h2mux.Header{
-				Name:  ":path",
-				Value: "/?query=mock%20value",
-			},
-			h2mux.Header{
-				Name:  "Mock header",
-				Value: "Mock value",
-			},
-		},
-		request,
-	)
+	mockRequestHeaders := []Header{
+		{Name: ":path", Value: "/?query=mock%20value"},
+		{Name: RequestUserHeadersField, Value: SerializeHeaders(http.Header{"Mock header": {"Mock value"}})},
+	}
+
+	headersConversionErr := H2RequestHeadersToH1Request(mockRequestHeaders, request)
 
 	assert.Equal(t, http.Header{
 		"Mock header": []string{"Mock value"},
@@ -114,19 +103,12 @@ func TestH2RequestHeadersToH1Request_HostPathWithURLEncoding(t *testing.T) {
 	request, err := http.NewRequest(http.MethodGet, "http://example.com/", nil)
 	assert.NoError(t, err)
 
-	headersConversionErr := H2RequestHeadersToH1Request(
-		[]h2mux.Header{
-			h2mux.Header{
-				Name:  ":path",
-				Value: "/mock%20path",
-			},
-			h2mux.Header{
-				Name:  "Mock header",
-				Value: "Mock value",
-			},
-		},
-		request,
-	)
+	mockRequestHeaders := []Header{
+		{Name: ":path", Value: "/mock%20path"},
+		{Name: RequestUserHeadersField, Value: SerializeHeaders(http.Header{"Mock header": {"Mock value"}})},
+	}
+
+	headersConversionErr := H2RequestHeadersToH1Request(mockRequestHeaders, request)
 
 	assert.Equal(t, http.Header{
 		"Mock header": []string{"Mock value"},
@@ -277,19 +259,13 @@ func TestH2RequestHeadersToH1Request_WeirdURLs(t *testing.T) {
 
 		request, err := http.NewRequest(http.MethodGet, requestURL, nil)
 		assert.NoError(t, err)
-		headersConversionErr := H2RequestHeadersToH1Request(
-			[]h2mux.Header{
-				h2mux.Header{
-					Name:  ":path",
-					Value: testCase.path,
-				},
-				h2mux.Header{
-					Name:  "Mock header",
-					Value: "Mock value",
-				},
-			},
-			request,
-		)
+
+		mockRequestHeaders := []Header{
+			{Name: ":path", Value: testCase.path},
+			{Name: RequestUserHeadersField, Value: SerializeHeaders(http.Header{"Mock header": {"Mock value"}})},
+		}
+
+		headersConversionErr := H2RequestHeadersToH1Request(mockRequestHeaders, request)
 		assert.NoError(t, headersConversionErr)
 
 		assert.Equal(t,
@@ -354,11 +330,12 @@ func TestH2RequestHeadersToH1Request_QuickCheck(t *testing.T) {
 				const expectedMethod = "POST"
 				const expectedHostname = "request.hostname.example.com"
 
-				h2 := []h2mux.Header{
-					h2mux.Header{Name: ":method", Value: expectedMethod},
-					h2mux.Header{Name: ":scheme", Value: testScheme},
-					h2mux.Header{Name: ":authority", Value: expectedHostname},
-					h2mux.Header{Name: ":path", Value: testPath},
+				h2 := []Header{
+					{Name: ":method", Value: expectedMethod},
+					{Name: ":scheme", Value: testScheme},
+					{Name: ":authority", Value: expectedHostname},
+					{Name: ":path", Value: testPath},
+					{Name: RequestUserHeadersField, Value: ""},
 				}
 				h1, err := http.NewRequest("GET", testOrigin.url, nil)
 				require.NoError(t, err)
@@ -406,14 +383,14 @@ func randomASCIIText(rand *rand.Rand, minLength int, maxLength int) string {
 // i.e. one that can pass unchanged through url.URL.String()
 func randomHTTP1Path(t *testing.T, rand *rand.Rand, minLength int, maxLength int) string {
 	text := randomASCIIText(rand, minLength, maxLength)
-	regexp, err := regexp.Compile("[^/;,]*")
+	re, err := regexp.Compile("[^/;,]*")
 	require.NoError(t, err)
-	return "/" + regexp.ReplaceAllStringFunc(text, url.PathEscape)
+	return "/" + re.ReplaceAllStringFunc(text, url.PathEscape)
 }
 
 // Calls `randomASCIIText` and ensures the result is a valid URL query,
 // i.e. one that can pass unchanged through url.URL.String()
-func randomHTTP1Query(t *testing.T, rand *rand.Rand, minLength int, maxLength int) string {
+func randomHTTP1Query(rand *rand.Rand, minLength int, maxLength int) string {
 	text := randomASCIIText(rand, minLength, maxLength)
 	return "?" + strings.ReplaceAll(text, "#", "%23")
 }
@@ -422,9 +399,9 @@ func randomHTTP1Query(t *testing.T, rand *rand.Rand, minLength int, maxLength in
 // i.e. one that can pass unchanged through url.URL.String()
 func randomHTTP1Fragment(t *testing.T, rand *rand.Rand, minLength int, maxLength int) string {
 	text := randomASCIIText(rand, minLength, maxLength)
-	url, err := url.Parse("#" + text)
+	u, err := url.Parse("#" + text)
 	require.NoError(t, err)
-	return url.String()
+	return u.String()
 }
 
 // Assemble a random :path pseudoheader that is legal by Go stdlib standards
@@ -432,10 +409,204 @@ func randomHTTP1Fragment(t *testing.T, rand *rand.Rand, minLength int, maxLength
 func randomHTTP2Path(t *testing.T, rand *rand.Rand) string {
 	result := randomHTTP1Path(t, rand, 1, 64)
 	if rand.Intn(2) == 1 {
-		result += randomHTTP1Query(t, rand, 1, 32)
+		result += randomHTTP1Query(rand, 1, 32)
 	}
 	if rand.Intn(2) == 1 {
 		result += randomHTTP1Fragment(t, rand, 1, 16)
 	}
 	return result
 }
+
+func stdlibHeaderToH2muxHeader(headers http.Header) (h2muxHeaders []Header) {
+	for name, values := range headers {
+		for _, value := range values {
+			h2muxHeaders = append(h2muxHeaders, Header{name, value})
+		}
+	}
+
+	return h2muxHeaders
+}
+
+func TestSerializeHeaders(t *testing.T) {
+	request, err := http.NewRequest(http.MethodGet, "http://example.com", nil)
+	assert.NoError(t, err)
+
+	mockHeaders := http.Header{
+		"Mock-Header-One":        {"Mock header one value", "three"},
+		"Mock-Header-Two-Long":   {"Mock header two value\nlong"},
+		":;":                     {":;", ";:"},
+		":":                      {":"},
+		";":                      {";"},
+		";;":                     {";;"},
+		"Empty values":           {"", ""},
+		"":                       {"Empty key"},
+		"control\tcharacter\b\n": {"value\n\b\t"},
+		";\v:":                   {":\v;"},
+	}
+
+	for header, values := range mockHeaders {
+		for _, value := range values {
+			// Note that Golang's http library is opinionated;
+			// at this point every header name will be title-cased in order to comply with the HTTP RFC
+			// This means our proxy is not completely transparent when it comes to proxying headers
+			request.Header.Add(header, value)
+		}
+	}
+
+	serializedHeaders := SerializeHeaders(request.Header)
+
+	// Sanity check: the headers serialized to something that's not an empty string
+	assert.NotEqual(t, "", serializedHeaders)
+
+	// Deserialize back, and ensure we get the same set of headers
+	deserializedHeaders, err := DeserializeHeaders(serializedHeaders)
+	assert.NoError(t, err)
+
+	assert.Equal(t, 13, len(deserializedHeaders))
+	h2muxExpectedHeaders := stdlibHeaderToH2muxHeader(mockHeaders)
+
+	sort.Sort(ByName(deserializedHeaders))
+	sort.Sort(ByName(h2muxExpectedHeaders))
+
+	assert.True(
+		t,
+		reflect.DeepEqual(h2muxExpectedHeaders, deserializedHeaders),
+		fmt.Sprintf("got = %#v, want = %#v\n", deserializedHeaders, h2muxExpectedHeaders),
+	)
+}
+
+func TestSerializeNoHeaders(t *testing.T) {
+	request, err := http.NewRequest(http.MethodGet, "http://example.com", nil)
+	assert.NoError(t, err)
+
+	serializedHeaders := SerializeHeaders(request.Header)
+	deserializedHeaders, err := DeserializeHeaders(serializedHeaders)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(deserializedHeaders))
+}
+
+func TestDeserializeMalformed(t *testing.T) {
+	var err error
+
+	malformedData := []string{
+		"malformed data",
+		"bW9jawo=",                   // "mock"
+		"bW9jawo=:ZGF0YQo=:bW9jawo=", // "mock:data:mock"
+		"::",
+	}
+
+	for _, malformedValue := range malformedData {
+		_, err = DeserializeHeaders(malformedValue)
+		assert.Error(t, err)
+	}
+}
+
+func TestParseHeaders(t *testing.T) {
+	mockUserHeadersToSerialize := http.Header{
+		"Mock-Header-One":   {"1", "1.5"},
+		"Mock-Header-Two":   {"2"},
+		"Mock-Header-Three": {"3"},
+	}
+
+	mockHeaders := []Header{
+		{Name: "One", Value: "1"},
+		{Name: "Cf-Two", Value: "cf-value-1"},
+		{Name: "Cf-Two", Value: "cf-value-2"},
+		{Name: RequestUserHeadersField, Value: SerializeHeaders(mockUserHeadersToSerialize)},
+	}
+
+	expectedHeaders := []Header{
+		{Name: "Mock-Header-One", Value: "1"},
+		{Name: "Mock-Header-One", Value: "1.5"},
+		{Name: "Mock-Header-Two", Value: "2"},
+		{Name: "Mock-Header-Three", Value: "3"},
+	}
+	parsedHeaders, err := ParseUserHeaders(RequestUserHeadersField, mockHeaders)
+	assert.NoError(t, err)
+	assert.ElementsMatch(t, expectedHeaders, parsedHeaders)
+}
+
+func TestParseHeadersNoSerializedHeader(t *testing.T) {
+	mockHeaders := []Header{
+		{Name: "One", Value: "1"},
+		{Name: "Cf-Two", Value: "cf-value-1"},
+		{Name: "Cf-Two", Value: "cf-value-2"},
+	}
+
+	_, err := ParseUserHeaders(RequestUserHeadersField, mockHeaders)
+	assert.EqualError(t, err, fmt.Sprintf("%s header not found", RequestUserHeadersField))
+}
+
+func TestIsControlHeader(t *testing.T) {
+	controlHeaders := []string{
+		// Anything that begins with cf-
+		"cf-sample-header",
+		"CF-SAMPLE-HEADER",
+		"Cf-Sample-Header",
+
+		// Any http2 pseudoheader
+		":sample-pseudo-header",
+
+		// content-length is a special case, it has to be there
+		// for some requests to work (per the HTTP2 spec)
+		"content-length",
+	}
+
+	for _, header := range controlHeaders {
+		assert.True(t, IsControlHeader(header))
+	}
+}
+
+func TestIsNotControlHeader(t *testing.T) {
+	notControlHeaders := []string{
+		"Mock-header",
+		"Another-sample-header",
+	}
+
+	for _, header := range notControlHeaders {
+		assert.False(t, IsControlHeader(header))
+	}
+}
+
+func TestH1ResponseToH2ResponseHeaders(t *testing.T) {
+	mockHeaders := http.Header{
+		"User-header-one": {""},
+		"User-header-two": {"1", "2"},
+		"cf-header":       {"cf-value"},
+		"Content-Length":  {"123"},
+	}
+	mockResponse := http.Response{
+		StatusCode: 200,
+		Header:     mockHeaders,
+	}
+
+	headers := H1ResponseToH2ResponseHeaders(&mockResponse)
+
+	serializedHeadersIndex := -1
+	for i, header := range headers {
+		if header.Name == ResponseUserHeadersField {
+			serializedHeadersIndex = i
+			break
+		}
+	}
+	assert.NotEqual(t, -1, serializedHeadersIndex)
+	actualControlHeaders := append(
+		headers[:serializedHeadersIndex],
+		headers[serializedHeadersIndex+1:]...,
+	)
+	expectedControlHeaders := []Header{
+		{Name: ":status", Value: "200"},
+		{Name: "content-length", Value: "123"},
+	}
+
+	assert.ElementsMatch(t, expectedControlHeaders, actualControlHeaders)
+
+	actualUserHeaders, err := DeserializeHeaders(headers[serializedHeadersIndex].Value)
+	expectedUserHeaders := []Header{
+		{Name: "User-header-one", Value: ""},
+		{Name: "User-header-two", Value: "1"},
+		{Name: "User-header-two", Value: "2"},
+	}
+	assert.NoError(t, err)
+	assert.ElementsMatch(t, expectedUserHeaders, actualUserHeaders)
+}

h2mux/muxedstream.go

@@ -17,6 +17,12 @@ type ReadWriteClosedCloser interface {
 	Closed() bool
 }
 
+// MuxedStreamDataSignaller is a write-only *ReadyList
+type MuxedStreamDataSignaller interface {
+	// Non-blocking: call this when data is ready to be sent for the given stream ID.
+	Signal(ID uint32)
+}
+
 // MuxedStream is logically an HTTP/2 stream, with an additional buffer for outgoing data.
 type MuxedStream struct {
 	streamID uint32
@@ -55,8 +61,8 @@ type MuxedStream struct {
 	// This is the amount of bytes that are in the peer's receive window
 	// (how much data we can send from this stream).
 	sendWindow uint32
-	// Reference to the muxer's readyList; signal this for stream data to be sent.
-	readyList *ReadyList
+	// The muxer's readyList
+	readyList MuxedStreamDataSignaller
 	// The headers that should be sent, and a flag so we only send them once.
 	headersSent  bool
 	writeHeaders []Header
@@ -88,7 +94,7 @@ func (th TunnelHostname) IsSet() bool {
 	return th != ""
 }
 
-func NewStream(config MuxerConfig, writeHeaders []Header, readyList *ReadyList, dictionaries h2Dictionaries) *MuxedStream {
+func NewStream(config MuxerConfig, writeHeaders []Header, readyList MuxedStreamDataSignaller, dictionaries h2Dictionaries) *MuxedStream {
 	return &MuxedStream{
 		responseHeadersReceived: make(chan struct{}),
 		readBuffer:              NewSharedBuffer(),

h2mux/muxreader.go

@@ -51,10 +51,12 @@ type MuxReader struct {
 	dictionaries h2Dictionaries
 }
 
-func (r *MuxReader) Shutdown() {
-	done := r.streams.Shutdown()
-	if done == nil {
-		return
+// Shutdown blocks new streams from being created.
+// It returns a channel that is closed once the last stream has closed.
+func (r *MuxReader) Shutdown() <-chan struct{} {
+	done, alreadyInProgress := r.streams.Shutdown()
+	if alreadyInProgress {
+		return done
 	}
 	r.sendGoAway(http2.ErrCodeNo)
 	go func() {
@@ -62,6 +64,7 @@ func (r *MuxReader) Shutdown() {
 		<-done
 		r.r.Close()
 	}()
+	return done
 }
 
 func (r *MuxReader) run(parentLogger *log.Entry) error {
@@ -94,7 +97,12 @@ func (r *MuxReader) run(parentLogger *log.Entry) error {
 			switch e := err.(type) {
 			case http2.StreamError:
 				errLogger.Warn("stream error")
-				r.streamError(e.StreamID, e.Code)
+				// Ideally we wouldn't return here, since that aborts the muxer.
+				// We should communicate the error to the relevant MuxedStream
+				// data structure, so that callers of MuxedStream.Read() and
+				// MuxedStream.Write() would see it. Then we could `continue`
+				// and keep the muxer going.
+				return r.streamError(e.StreamID, e.Code)
 			case http2.ConnectionError:
 				errLogger.Warn("connection error")
 				return r.connectionError(err)

origin/backoffhandler.go

@@ -92,3 +92,8 @@ func (b BackoffHandler) GetBaseTime() time.Duration {
 	}
 	return b.BaseTime
 }
+
+// Retries returns the number of retries consumed so far.
+func (b *BackoffHandler) Retries() int {
+	return int(b.retries)
+}

origin/metrics.go

@@ -58,9 +58,11 @@ type TunnelMetrics struct {
 	// oldServerLocations stores the last server the tunnel was connected to
 	oldServerLocations map[string]string
 
-	regSuccess prometheus.Counter
-	regFail    *prometheus.CounterVec
-	rpcFail    *prometheus.CounterVec
+	regSuccess  *prometheus.CounterVec
+	regFail     *prometheus.CounterVec
+	authSuccess prometheus.Counter
+	authFail    *prometheus.CounterVec
+	rpcFail     *prometheus.CounterVec
 
 	muxerMetrics        *muxerMetrics
 	tunnelsHA           tunnelsForHA
@@ -417,7 +419,7 @@ func NewTunnelMetrics() *TunnelMetrics {
 			Name:      "tunnel_rpc_fail",
 			Help:      "Count of RPC connection errors by type",
 		},
-		[]string{"error"},
+		[]string{"error", "rpcName"},
 	)
 	prometheus.MustRegister(rpcFail)
 
@@ -428,7 +430,7 @@ func NewTunnelMetrics() *TunnelMetrics {
 			Name:      "tunnel_register_fail",
 			Help:      "Count of tunnel registration errors by type",
 		},
-		[]string{"error"},
+		[]string{"error", "rpcName"},
 	)
 	prometheus.MustRegister(registerFail)
 
@@ -443,15 +445,38 @@ func NewTunnelMetrics() *TunnelMetrics {
 	)
 	prometheus.MustRegister(userHostnamesCounts)
 
-	registerSuccess := prometheus.NewCounter(
+	registerSuccess := prometheus.NewCounterVec(
 		prometheus.CounterOpts{
 			Namespace: metricsNamespace,
 			Subsystem: tunnelSubsystem,
 			Name:      "tunnel_register_success",
 			Help:      "Count of successful tunnel registrations",
-		})
+		},
+		[]string{"rpcName"},
+	)
 	prometheus.MustRegister(registerSuccess)
 
+	authSuccess := prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Namespace: metricsNamespace,
+			Subsystem: tunnelSubsystem,
+			Name:      "tunnel_authenticate_success",
+			Help:      "Count of successful tunnel authenticate",
+		},
+	)
+	prometheus.MustRegister(authSuccess)
+
+	authFail := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: metricsNamespace,
+			Subsystem: tunnelSubsystem,
+			Name:      "tunnel_authenticate_fail",
+			Help:      "Count of tunnel authenticate errors by type",
+		},
+		[]string{"error"},
+	)
+	prometheus.MustRegister(authFail)
+
 	return &TunnelMetrics{
 		haConnections:                  haConnections,
 		activeStreams:                  activeStreams,
@@ -472,6 +497,8 @@ func NewTunnelMetrics() *TunnelMetrics {
 		regFail:                        registerFail,
 		rpcFail:                        rpcFail,
 		userHostnamesCounts:            userHostnamesCounts,
+		authSuccess:                    authSuccess,
+		authFail:                       authFail,
 	}
 }
 

origin/supervisor.go

@@ -2,16 +2,21 @@ package origin
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"net"
+	"sync"
 	"time"
 
+	"github.com/google/uuid"
 	"github.com/sirupsen/logrus"
 
+	"github.com/cloudflare/cloudflared/buffer"
 	"github.com/cloudflare/cloudflared/connection"
+	"github.com/cloudflare/cloudflared/edgediscovery"
+	"github.com/cloudflare/cloudflared/h2mux"
 	"github.com/cloudflare/cloudflared/signal"
-
-	"github.com/google/uuid"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
 const (
@@ -21,13 +26,27 @@ const (
 	resolveTTL = time.Hour
 	// Interval between registering new tunnels
 	registrationInterval = time.Second
+
+	subsystemRefreshAuth = "refresh_auth"
+	// Maximum exponent for 'Authenticate' exponential backoff
+	refreshAuthMaxBackoff = 10
+	// Waiting time before retrying a failed 'Authenticate' connection
+	refreshAuthRetryDuration = time.Second * 10
+	// Maximum time to make an Authenticate RPC
+	authTokenTimeout = time.Second * 30
 )
 
+var (
+	errJWTUnset         = errors.New("JWT unset")
+	errEventDigestUnset = errors.New("event digest unset")
+)
+
+// Supervisor manages non-declarative tunnels. Establishes TCP connections with the edge, and
+// reconnects them if they disconnect.
 type Supervisor struct {
-	config  *TunnelConfig
-	edgeIPs []*net.TCPAddr
-	// nextUnusedEdgeIP is the index of the next addr k edgeIPs to try
-	nextUnusedEdgeIP  int
+	cloudflaredUUID   uuid.UUID
+	config            *TunnelConfig
+	edgeIPs           *edgediscovery.Edge
 	lastResolve       time.Time
 	resolverC         chan resolveResult
 	tunnelErrors      chan tunnelError
@@ -38,36 +57,74 @@ type Supervisor struct {
 	nextConnectedSignal chan struct{}
 
 	logger *logrus.Entry
+
+	jwtLock *sync.RWMutex
+	jwt     []byte
+
+	eventDigestLock *sync.RWMutex
+	eventDigest     []byte
+
+	bufferPool *buffer.Pool
 }
 
 type resolveResult struct {
-	edgeIPs []*net.TCPAddr
-	err     error
+	err error
 }
 
 type tunnelError struct {
 	index int
+	addr  *net.TCPAddr
 	err   error
 }
 
-func NewSupervisor(config *TunnelConfig) *Supervisor {
+func NewSupervisor(config *TunnelConfig, u uuid.UUID) (*Supervisor, error) {
+	var (
+		edgeIPs *edgediscovery.Edge
+		err     error
+	)
+	if len(config.EdgeAddrs) > 0 {
+		edgeIPs, err = edgediscovery.StaticEdge(config.Logger, config.EdgeAddrs)
+	} else {
+		edgeIPs, err = edgediscovery.ResolveEdge(config.Logger)
+	}
+	if err != nil {
+		return nil, err
+	}
 	return &Supervisor{
+		cloudflaredUUID:   u,
 		config:            config,
+		edgeIPs:           edgeIPs,
 		tunnelErrors:      make(chan tunnelError),
 		tunnelsConnecting: map[int]chan struct{}{},
 		logger:            config.Logger.WithField("subsystem", "supervisor"),
-	}
+		jwtLock:           &sync.RWMutex{},
+		eventDigestLock:   &sync.RWMutex{},
+		bufferPool:        buffer.NewPool(512 * 1024),
+	}, nil
 }
 
-func (s *Supervisor) Run(ctx context.Context, connectedSignal *signal.Signal, u uuid.UUID) error {
+func (s *Supervisor) Run(ctx context.Context, connectedSignal *signal.Signal) error {
 	logger := s.config.Logger
-	if err := s.initialize(ctx, connectedSignal, u); err != nil {
+	if err := s.initialize(ctx, connectedSignal); err != nil {
 		return err
 	}
 	var tunnelsWaiting []int
+	tunnelsActive := s.config.HAConnections
+
 	backoff := BackoffHandler{MaxRetries: s.config.Retries, BaseTime: tunnelRetryDuration, RetryForever: true}
 	var backoffTimer <-chan time.Time
-	tunnelsActive := s.config.HAConnections
+
+	refreshAuthBackoff := &BackoffHandler{MaxRetries: refreshAuthMaxBackoff, BaseTime: refreshAuthRetryDuration, RetryForever: true}
+	var refreshAuthBackoffTimer <-chan time.Time
+
+	if s.config.UseReconnectToken {
+		if timer, err := s.refreshAuth(ctx, refreshAuthBackoff, s.authenticate); err == nil {
+			refreshAuthBackoffTimer = timer
+		} else {
+			logger.WithError(err).Errorf("initial refreshAuth failed, retrying in %v", refreshAuthRetryDuration)
+			refreshAuthBackoffTimer = time.After(refreshAuthRetryDuration)
+		}
+	}
 
 	for {
 		select {
@@ -91,23 +148,27 @@ func (s *Supervisor) Run(ctx context.Context, connectedSignal *signal.Signal, u
 					backoffTimer = backoff.BackoffTimer()
 				}
 
-				// If the error is a dial error, the problem is likely to be network related
-				// try another addr before refreshing since we are likely to get back the
-				// same IPs in the same order. Same problem with duplicate connection error.
-				if s.unusedIPs() {
-					s.replaceEdgeIP(tunnelError.index)
-				} else {
-					s.refreshEdgeIPs()
-				}
+				// Previously we'd mark the edge address as bad here, but now we'll just silently use
+				// another.
 			}
 		// Backoff was set and its timer expired
 		case <-backoffTimer:
 			backoffTimer = nil
 			for _, index := range tunnelsWaiting {
-				go s.startTunnel(ctx, index, s.newConnectedTunnelSignal(index), u)
+				go s.startTunnel(ctx, index, s.newConnectedTunnelSignal(index))
 			}
 			tunnelsActive += len(tunnelsWaiting)
 			tunnelsWaiting = nil
+		// Time to call Authenticate
+		case <-refreshAuthBackoffTimer:
+			newTimer, err := s.refreshAuth(ctx, refreshAuthBackoff, s.authenticate)
+			if err != nil {
+				logger.WithError(err).Error("Authentication failed")
+				// Permanent failure. Leave the `select` without setting the
+				// channel to be non-null, so we'll never hit this case of the `select` again.
+				continue
+			}
+			refreshAuthBackoffTimer = newTimer
 		// Tunnel successfully connected
 		case <-s.nextConnectedSignal:
 			if !s.waitForNextTunnel(s.nextConnectedIndex) && len(tunnelsWaiting) == 0 {
@@ -120,7 +181,6 @@ func (s *Supervisor) Run(ctx context.Context, connectedSignal *signal.Signal, u
 			s.resolverC = nil
 			if result.err == nil {
 				logger.Debug("Service discovery refresh complete")
-				s.edgeIPs = result.edgeIPs
 			} else {
 				logger.WithError(result.err).Error("Service discovery error")
 			}
@@ -128,29 +188,22 @@ func (s *Supervisor) Run(ctx context.Context, connectedSignal *signal.Signal, u
 	}
 }
 
-func (s *Supervisor) initialize(ctx context.Context, connectedSignal *signal.Signal, u uuid.UUID) error {
+// Returns nil if initialization succeeded, else the initialization error.
+func (s *Supervisor) initialize(ctx context.Context, connectedSignal *signal.Signal) error {
 	logger := s.logger
 
-	edgeIPs, err := s.resolveEdgeIPs()
-
-	if err != nil {
-		logger.Infof("ResolveEdgeIPs err")
-		return err
-	}
-	s.edgeIPs = edgeIPs
-	if s.config.HAConnections > len(edgeIPs) {
-		logger.Warnf("You requested %d HA connections but I can give you at most %d.", s.config.HAConnections, len(edgeIPs))
-		s.config.HAConnections = len(edgeIPs)
-	}
 	s.lastResolve = time.Now()
-	// check entitlement and version too old error before attempting to register more tunnels
-	s.nextUnusedEdgeIP = s.config.HAConnections
-	go s.startFirstTunnel(ctx, connectedSignal, u)
+	availableAddrs := int(s.edgeIPs.AvailableAddrs())
+	if s.config.HAConnections > availableAddrs {
+		logger.Warnf("You requested %d HA connections but I can give you at most %d.", s.config.HAConnections, availableAddrs)
+		s.config.HAConnections = availableAddrs
+	}
+
+	go s.startFirstTunnel(ctx, connectedSignal)
 	select {
 	case <-ctx.Done():
 		<-s.tunnelErrors
-		// Error can't be nil. A nil error signals that initialization succeed
-		return fmt.Errorf("context was canceled")
+		return ctx.Err()
 	case tunnelError := <-s.tunnelErrors:
 		return tunnelError.err
 	case <-connectedSignal.Wait():
@@ -158,7 +211,7 @@ func (s *Supervisor) initialize(ctx context.Context, connectedSignal *signal.Sig
 	// At least one successful connection, so start the rest
 	for i := 1; i < s.config.HAConnections; i++ {
 		ch := signal.New(make(chan struct{}))
-		go s.startTunnel(ctx, i, ch, u)
+		go s.startTunnel(ctx, i, ch)
 		time.Sleep(registrationInterval)
 	}
 	return nil
@@ -166,37 +219,64 @@ func (s *Supervisor) initialize(ctx context.Context, connectedSignal *signal.Sig
 
 // startTunnel starts the first tunnel connection. The resulting error will be sent on
 // s.tunnelErrors. It will send a signal via connectedSignal if registration succeed
-func (s *Supervisor) startFirstTunnel(ctx context.Context, connectedSignal *signal.Signal, u uuid.UUID) {
-	err := ServeTunnelLoop(ctx, s.config, s.getEdgeIP(0), 0, connectedSignal, u)
+func (s *Supervisor) startFirstTunnel(ctx context.Context, connectedSignal *signal.Signal) {
+	var (
+		addr *net.TCPAddr
+		err  error
+	)
+	const thisConnID = 0
 	defer func() {
-		s.tunnelErrors <- tunnelError{index: 0, err: err}
+		s.tunnelErrors <- tunnelError{index: thisConnID, addr: addr, err: err}
 	}()
 
+	addr, err = s.edgeIPs.GetAddr(thisConnID)
+	if err != nil {
+		return
+	}
+
+	err = ServeTunnelLoop(ctx, s, s.config, addr, thisConnID, connectedSignal, s.cloudflaredUUID, s.bufferPool)
+	// If the first tunnel disconnects, keep restarting it.
+	edgeErrors := 0
 	for s.unusedIPs() {
-		select {
-		case <-ctx.Done():
+		if ctx.Err() != nil {
 			return
-		default:
 		}
 		switch err.(type) {
 		case nil:
 			return
 		// try the next address if it was a dialError(network problem) or
 		// dupConnRegisterTunnelError
-		case dialError, dupConnRegisterTunnelError:
-			s.replaceEdgeIP(0)
+		case connection.DialError, dupConnRegisterTunnelError:
+			edgeErrors++
 		default:
 			return
 		}
-		err = ServeTunnelLoop(ctx, s.config, s.getEdgeIP(0), 0, connectedSignal, u)
+		if edgeErrors >= 2 {
+			addr, err = s.edgeIPs.GetDifferentAddr(thisConnID)
+			if err != nil {
+				return
+			}
+		}
+		err = ServeTunnelLoop(ctx, s, s.config, addr, thisConnID, connectedSignal, s.cloudflaredUUID, s.bufferPool)
 	}
 }
 
 // startTunnel starts a new tunnel connection. The resulting error will be sent on
 // s.tunnelErrors.
-func (s *Supervisor) startTunnel(ctx context.Context, index int, connectedSignal *signal.Signal, u uuid.UUID) {
-	err := ServeTunnelLoop(ctx, s.config, s.getEdgeIP(index), uint8(index), connectedSignal, u)
-	s.tunnelErrors <- tunnelError{index: index, err: err}
+func (s *Supervisor) startTunnel(ctx context.Context, index int, connectedSignal *signal.Signal) {
+	var (
+		addr *net.TCPAddr
+		err  error
+	)
+	defer func() {
+		s.tunnelErrors <- tunnelError{index: index, addr: addr, err: err}
+	}()
+
+	addr, err = s.edgeIPs.GetAddr(index)
+	if err != nil {
+		return
+	}
+	err = ServeTunnelLoop(ctx, s, s.config, addr, uint8(index), connectedSignal, s.cloudflaredUUID, s.bufferPool)
 }
 
 func (s *Supervisor) newConnectedTunnelSignal(index int) *signal.Signal {
@@ -218,38 +298,124 @@ func (s *Supervisor) waitForNextTunnel(index int) bool {
 	return false
 }
 
-func (s *Supervisor) getEdgeIP(index int) *net.TCPAddr {
-	return s.edgeIPs[index%len(s.edgeIPs)]
-}
-
-func (s *Supervisor) resolveEdgeIPs() ([]*net.TCPAddr, error) {
-	// If --edge is specfied, resolve edge server addresses
-	if len(s.config.EdgeAddrs) > 0 {
-		return connection.ResolveAddrs(s.config.EdgeAddrs)
-	}
-	// Otherwise lookup edge server addresses through service discovery
-	return connection.EdgeDiscovery(s.logger)
-}
-
-func (s *Supervisor) refreshEdgeIPs() {
-	if s.resolverC != nil {
-		return
-	}
-	if time.Since(s.lastResolve) < resolveTTL {
-		return
-	}
-	s.resolverC = make(chan resolveResult)
-	go func() {
-		edgeIPs, err := s.resolveEdgeIPs()
-		s.resolverC <- resolveResult{edgeIPs: edgeIPs, err: err}
-	}()
-}
-
 func (s *Supervisor) unusedIPs() bool {
-	return s.nextUnusedEdgeIP < len(s.edgeIPs)
+	return s.edgeIPs.AvailableAddrs() > s.config.HAConnections
 }
 
-func (s *Supervisor) replaceEdgeIP(badIPIndex int) {
-	s.edgeIPs[badIPIndex] = s.edgeIPs[s.nextUnusedEdgeIP]
-	s.nextUnusedEdgeIP++
+func (s *Supervisor) ReconnectToken() ([]byte, error) {
+	s.jwtLock.RLock()
+	defer s.jwtLock.RUnlock()
+	if s.jwt == nil {
+		return nil, errJWTUnset
+	}
+	return s.jwt, nil
+}
+
+func (s *Supervisor) SetReconnectToken(jwt []byte) {
+	s.jwtLock.Lock()
+	defer s.jwtLock.Unlock()
+	s.jwt = jwt
+}
+
+func (s *Supervisor) EventDigest() ([]byte, error) {
+	s.eventDigestLock.RLock()
+	defer s.eventDigestLock.RUnlock()
+	if s.eventDigest == nil {
+		return nil, errEventDigestUnset
+	}
+	return s.eventDigest, nil
+}
+
+func (s *Supervisor) SetEventDigest(eventDigest []byte) {
+	s.eventDigestLock.Lock()
+	defer s.eventDigestLock.Unlock()
+	s.eventDigest = eventDigest
+}
+
+func (s *Supervisor) refreshAuth(
+	ctx context.Context,
+	backoff *BackoffHandler,
+	authenticate func(ctx context.Context, numPreviousAttempts int) (tunnelpogs.AuthOutcome, error),
+) (retryTimer <-chan time.Time, err error) {
+	logger := s.config.Logger.WithField("subsystem", subsystemRefreshAuth)
+	authOutcome, err := authenticate(ctx, backoff.Retries())
+	if err != nil {
+		s.config.Metrics.authFail.WithLabelValues(err.Error()).Inc()
+		if duration, ok := backoff.GetBackoffDuration(ctx); ok {
+			logger.WithError(err).Warnf("Retrying in %v", duration)
+			return backoff.BackoffTimer(), nil
+		}
+		return nil, err
+	}
+	// clear backoff timer
+	backoff.SetGracePeriod()
+
+	switch outcome := authOutcome.(type) {
+	case tunnelpogs.AuthSuccess:
+		s.SetReconnectToken(outcome.JWT())
+		s.config.Metrics.authSuccess.Inc()
+		return timeAfter(outcome.RefreshAfter()), nil
+	case tunnelpogs.AuthUnknown:
+		duration := outcome.RefreshAfter()
+		s.config.Metrics.authFail.WithLabelValues(outcome.Error()).Inc()
+		logger.WithError(outcome).Warnf("Retrying in %v", duration)
+		return timeAfter(duration), nil
+	case tunnelpogs.AuthFail:
+		s.config.Metrics.authFail.WithLabelValues(outcome.Error()).Inc()
+		return nil, outcome
+	default:
+		err := fmt.Errorf("Unexpected outcome type %T", authOutcome)
+		s.config.Metrics.authFail.WithLabelValues(err.Error()).Inc()
+		return nil, err
+	}
+}
+
+func (s *Supervisor) authenticate(ctx context.Context, numPreviousAttempts int) (tunnelpogs.AuthOutcome, error) {
+	arbitraryEdgeIP, err := s.edgeIPs.GetAddrForRPC()
+	if err != nil {
+		return nil, err
+	}
+
+	edgeConn, err := connection.DialEdge(ctx, dialTimeout, s.config.TlsConfig, arbitraryEdgeIP)
+	if err != nil {
+		return nil, err
+	}
+	defer edgeConn.Close()
+
+	handler := h2mux.MuxedStreamFunc(func(*h2mux.MuxedStream) error {
+		// This callback is invoked by h2mux when the edge initiates a stream.
+		return nil // noop
+	})
+	muxerConfig := s.config.muxerConfig(handler)
+	muxerConfig.Logger = muxerConfig.Logger.WithField("subsystem", subsystemRefreshAuth)
+	muxer, err := h2mux.Handshake(edgeConn, edgeConn, muxerConfig, s.config.Metrics.activeStreams)
+	if err != nil {
+		return nil, err
+	}
+	go muxer.Serve(ctx)
+	defer func() {
+		// If we don't wait for the muxer shutdown here, edgeConn.Close() runs before the muxer connections are done,
+		// and the user sees log noise: "error writing data", "connection closed unexpectedly"
+		<-muxer.Shutdown()
+	}()
+
+	tunnelServer, err := connection.NewRPCClient(ctx, muxer, s.logger.WithField("subsystem", subsystemRefreshAuth), openStreamTimeout)
+	if err != nil {
+		return nil, err
+	}
+	defer tunnelServer.Close()
+
+	const arbitraryConnectionID = uint8(0)
+	registrationOptions := s.config.RegistrationOptions(arbitraryConnectionID, edgeConn.LocalAddr().String(), s.cloudflaredUUID)
+	registrationOptions.NumPreviousAttempts = uint8(numPreviousAttempts)
+	authResponse, err := tunnelServer.Authenticate(
+		ctx,
+		s.config.OriginCert,
+		s.config.Hostname,
+		registrationOptions,
+	)
+	if err != nil {
+		return nil, err
+	}
+	return authResponse.Outcome(), nil
 }

origin/supervisor_test.go

@@ -0,0 +1,165 @@
+package origin
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+)
+
+func testConfig(logger *logrus.Logger) *TunnelConfig {
+	metrics := TunnelMetrics{}
+
+	metrics.authSuccess = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Namespace: metricsNamespace,
+			Subsystem: tunnelSubsystem,
+			Name:      "tunnel_authenticate_success",
+			Help:      "Count of successful tunnel authenticate",
+		},
+	)
+
+	metrics.authFail = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: metricsNamespace,
+			Subsystem: tunnelSubsystem,
+			Name:      "tunnel_authenticate_fail",
+			Help:      "Count of tunnel authenticate errors by type",
+		},
+		[]string{"error"},
+	)
+	return &TunnelConfig{Logger: logger, Metrics: &metrics}
+}
+
+func TestRefreshAuthBackoff(t *testing.T) {
+	logger := logrus.New()
+	logger.Level = logrus.ErrorLevel
+
+	var wait time.Duration
+	timeAfter = func(d time.Duration) <-chan time.Time {
+		wait = d
+		return time.After(d)
+	}
+
+	s, err := NewSupervisor(testConfig(logger), uuid.New())
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	backoff := &BackoffHandler{MaxRetries: 3}
+	auth := func(ctx context.Context, n int) (tunnelpogs.AuthOutcome, error) {
+		return nil, fmt.Errorf("authentication failure")
+	}
+
+	// authentication failures should consume the backoff
+	for i := uint(0); i < backoff.MaxRetries; i++ {
+		retryChan, err := s.refreshAuth(context.Background(), backoff, auth)
+		assert.NoError(t, err)
+		assert.NotNil(t, retryChan)
+		assert.Equal(t, (1<<i)*time.Second, wait)
+	}
+	retryChan, err := s.refreshAuth(context.Background(), backoff, auth)
+	assert.Error(t, err)
+	assert.Nil(t, retryChan)
+
+	// now we actually make contact with the remote server
+	_, _ = s.refreshAuth(context.Background(), backoff, func(ctx context.Context, n int) (tunnelpogs.AuthOutcome, error) {
+		return tunnelpogs.NewAuthUnknown(errors.New("auth unknown"), 19), nil
+	})
+
+	// The backoff timer should have been reset. To confirm this, make timeNow
+	// return a value after the backoff timer's grace period
+	timeNow = func() time.Time {
+		expectedGracePeriod := time.Duration(time.Second * 2 << backoff.MaxRetries)
+		return time.Now().Add(expectedGracePeriod * 2)
+	}
+	_, ok := backoff.GetBackoffDuration(context.Background())
+	assert.True(t, ok)
+}
+
+func TestRefreshAuthSuccess(t *testing.T) {
+	logger := logrus.New()
+	logger.Level = logrus.ErrorLevel
+
+	var wait time.Duration
+	timeAfter = func(d time.Duration) <-chan time.Time {
+		wait = d
+		return time.After(d)
+	}
+
+	s, err := NewSupervisor(testConfig(logger), uuid.New())
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	backoff := &BackoffHandler{MaxRetries: 3}
+	auth := func(ctx context.Context, n int) (tunnelpogs.AuthOutcome, error) {
+		return tunnelpogs.NewAuthSuccess([]byte("jwt"), 19), nil
+	}
+
+	retryChan, err := s.refreshAuth(context.Background(), backoff, auth)
+	assert.NoError(t, err)
+	assert.NotNil(t, retryChan)
+	assert.Equal(t, 19*time.Hour, wait)
+
+	token, err := s.ReconnectToken()
+	assert.NoError(t, err)
+	assert.Equal(t, []byte("jwt"), token)
+}
+
+func TestRefreshAuthUnknown(t *testing.T) {
+	logger := logrus.New()
+	logger.Level = logrus.ErrorLevel
+
+	var wait time.Duration
+	timeAfter = func(d time.Duration) <-chan time.Time {
+		wait = d
+		return time.After(d)
+	}
+
+	s, err := NewSupervisor(testConfig(logger), uuid.New())
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	backoff := &BackoffHandler{MaxRetries: 3}
+	auth := func(ctx context.Context, n int) (tunnelpogs.AuthOutcome, error) {
+		return tunnelpogs.NewAuthUnknown(errors.New("auth unknown"), 19), nil
+	}
+
+	retryChan, err := s.refreshAuth(context.Background(), backoff, auth)
+	assert.NoError(t, err)
+	assert.NotNil(t, retryChan)
+	assert.Equal(t, 19*time.Hour, wait)
+
+	token, err := s.ReconnectToken()
+	assert.Equal(t, errJWTUnset, err)
+	assert.Nil(t, token)
+}
+
+func TestRefreshAuthFail(t *testing.T) {
+	logger := logrus.New()
+	logger.Level = logrus.ErrorLevel
+
+	s, err := NewSupervisor(testConfig(logger), uuid.New())
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	backoff := &BackoffHandler{MaxRetries: 3}
+	auth := func(ctx context.Context, n int) (tunnelpogs.AuthOutcome, error) {
+		return tunnelpogs.NewAuthFail(errors.New("auth fail")), nil
+	}
+
+	retryChan, err := s.refreshAuth(context.Background(), backoff, auth)
+	assert.Error(t, err)
+	assert.Nil(t, retryChan)
+
+	token, err := s.ReconnectToken()
+	assert.Equal(t, errJWTUnset, err)
+	assert.Nil(t, token)
+}

origin/tunnel.go

@@ -14,7 +14,15 @@ import (
 	"sync"
 	"time"
 
+	"github.com/google/uuid"
+	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
+	log "github.com/sirupsen/logrus"
+	"golang.org/x/sync/errgroup"
+
+	"github.com/cloudflare/cloudflared/buffer"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
+	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/h2mux"
 	"github.com/cloudflare/cloudflared/signal"
 	"github.com/cloudflare/cloudflared/streamhandler"
@@ -22,24 +30,25 @@ import (
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 	"github.com/cloudflare/cloudflared/validation"
 	"github.com/cloudflare/cloudflared/websocket"
-
-	"github.com/google/uuid"
-	"github.com/pkg/errors"
-	"github.com/prometheus/client_golang/prometheus"
-	_ "github.com/prometheus/client_golang/prometheus"
-	log "github.com/sirupsen/logrus"
-	"golang.org/x/sync/errgroup"
-	rpc "zombiezen.com/go/capnproto2/rpc"
 )
 
 const (
 	dialTimeout              = 15 * time.Second
 	openStreamTimeout        = 30 * time.Second
+	muxerTimeout             = 5 * time.Second
 	lbProbeUserAgentPrefix   = "Mozilla/5.0 (compatible; Cloudflare-Traffic-Manager/1.0; +https://www.cloudflare.com/traffic-manager/;"
 	TagHeaderNamePrefix      = "Cf-Warp-Tag-"
 	DuplicateConnectionError = "EDUPCONN"
 )
 
+type registerRPCName string
+
+const (
+	register  registerRPCName = "register"
+	reconnect registerRPCName = "reconnect"
+	unknown   registerRPCName = "unknown"
+)
+
 type TunnelConfig struct {
 	BuildInfo            *buildinfo.BuildInfo
 	ClientID             string
@@ -73,14 +82,17 @@ type TunnelConfig struct {
 	WSGI                 bool
 	// OriginUrl may not be used if a user specifies a unix socket.
 	OriginUrl string
+
+	// feature-flag to use new edge reconnect tokens
+	UseReconnectToken bool
 }
 
-type dialError struct {
-	cause error
-}
-
-func (e dialError) Error() string {
-	return e.cause.Error()
+// ReconnectTunnelCredentialManager is invoked by functions in this file to
+// get/set parameters for ReconnectTunnel RPC calls.
+type ReconnectTunnelCredentialManager interface {
+	ReconnectToken() ([]byte, error)
+	EventDigest() ([]byte, error)
+	SetEventDigest(eventDigest []byte)
 }
 
 type dupConnRegisterTunnelError struct{}
@@ -110,8 +122,8 @@ type clientRegisterTunnelError struct {
 	cause error
 }
 
-func newClientRegisterTunnelError(cause error, counter *prometheus.CounterVec) clientRegisterTunnelError {
-	counter.WithLabelValues(cause.Error()).Inc()
+func newClientRegisterTunnelError(cause error, counter *prometheus.CounterVec, name registerRPCName) clientRegisterTunnelError {
+	counter.WithLabelValues(cause.Error(), string(name)).Inc()
 	return clientRegisterTunnelError{cause: cause}
 }
 
@@ -119,6 +131,18 @@ func (e clientRegisterTunnelError) Error() string {
 	return e.cause.Error()
 }
 
+func (c *TunnelConfig) muxerConfig(handler h2mux.MuxedStreamHandler) h2mux.MuxerConfig {
+	return h2mux.MuxerConfig{
+		Timeout:            muxerTimeout,
+		Handler:            handler,
+		IsClient:           true,
+		HeartbeatInterval:  c.HeartbeatInterval,
+		MaxHeartbeats:      c.MaxHeartbeats,
+		Logger:             c.TransportLogger.WithFields(log.Fields{}),
+		CompressionQuality: h2mux.CompressionSetting(c.CompressionQuality),
+	}
+}
+
 func (c *TunnelConfig) RegistrationOptions(connectionID uint8, OriginLocalIP string, uuid uuid.UUID) *tunnelpogs.RegistrationOptions {
 	policy := tunnelrpc.ExistingTunnelPolicy_balance
 	if c.HAConnections <= 1 && c.LBPool == "" {
@@ -141,15 +165,21 @@ func (c *TunnelConfig) RegistrationOptions(connectionID uint8, OriginLocalIP str
 }
 
 func StartTunnelDaemon(ctx context.Context, config *TunnelConfig, connectedSignal *signal.Signal, cloudflaredID uuid.UUID) error {
-	return NewSupervisor(config).Run(ctx, connectedSignal, cloudflaredID)
+	s, err := NewSupervisor(config, cloudflaredID)
+	if err != nil {
+		return err
+	}
+	return s.Run(ctx, connectedSignal)
 }
 
 func ServeTunnelLoop(ctx context.Context,
+	credentialManager ReconnectTunnelCredentialManager,
 	config *TunnelConfig,
 	addr *net.TCPAddr,
 	connectionID uint8,
 	connectedSignal *signal.Signal,
 	u uuid.UUID,
+	bufferPool *buffer.Pool,
 ) error {
 	connectionLogger := config.Logger.WithField("connectionID", connectionID)
 	config.Metrics.incrementHaConnections()
@@ -166,12 +196,14 @@ func ServeTunnelLoop(ctx context.Context,
 	for {
 		err, recoverable := ServeTunnel(
 			ctx,
+			credentialManager,
 			config,
 			connectionLogger,
 			addr, connectionID,
 			connectedFuse,
 			&backoff,
 			u,
+			bufferPool,
 		)
 		if recoverable {
 			if duration, ok := backoff.GetBackoffDuration(ctx); ok {
@@ -186,6 +218,7 @@ func ServeTunnelLoop(ctx context.Context,
 
 func ServeTunnel(
 	ctx context.Context,
+	credentialManager ReconnectTunnelCredentialManager,
 	config *TunnelConfig,
 	logger *log.Entry,
 	addr *net.TCPAddr,
@@ -193,6 +226,7 @@ func ServeTunnel(
 	connectedFuse *h2mux.BooleanFuse,
 	backoff *BackoffHandler,
 	u uuid.UUID,
+	bufferPool *buffer.Pool,
 ) (err error, recoverable bool) {
 	// Treat panics as recoverable errors
 	defer func() {
@@ -213,11 +247,11 @@ func ServeTunnel(
 	tags["ha"] = connectionTag
 
 	// Returns error from parsing the origin URL or handshake errors
-	handler, originLocalIP, err := NewTunnelHandler(ctx, config, addr.String(), connectionID)
+	handler, originLocalIP, err := NewTunnelHandler(ctx, config, addr, connectionID, bufferPool)
 	if err != nil {
 		errLog := logger.WithError(err)
 		switch err.(type) {
-		case dialError:
+		case connection.DialError:
 			errLog.Error("Unable to dial edge")
 		case h2mux.MuxerHandshakeError:
 			errLog.Error("Handshake failed with edge server")
@@ -230,13 +264,30 @@ func ServeTunnel(
 
 	errGroup, serveCtx := errgroup.WithContext(ctx)
 
-	errGroup.Go(func() error {
-		err := RegisterTunnel(serveCtx, handler.muxer, config, logger, connectionID, originLocalIP, u)
-		if err == nil {
-			connectedFuse.Fuse(true)
-			backoff.SetGracePeriod()
+	errGroup.Go(func() (err error) {
+		defer func() {
+			if err == nil {
+				connectedFuse.Fuse(true)
+				backoff.SetGracePeriod()
+			}
+		}()
+
+		if config.UseReconnectToken && connectedFuse.Value() {
+			token, tokenErr := credentialManager.ReconnectToken()
+			eventDigest, eventDigestErr := credentialManager.EventDigest()
+			// if we have both credentials, we can reconnect
+			if tokenErr == nil && eventDigestErr == nil {
+				return ReconnectTunnel(serveCtx, token, eventDigest, handler.muxer, config, logger, connectionID, originLocalIP, u)
+			}
+			// log errors and proceed to RegisterTunnel
+			if tokenErr != nil {
+				logger.WithError(tokenErr).Error("Couldn't get reconnect token")
+			}
+			if eventDigestErr != nil {
+				logger.WithError(eventDigestErr).Error("Couldn't get event digest")
+			}
 		}
-		return err
+		return RegisterTunnel(serveCtx, credentialManager, handler.muxer, config, logger, connectionID, originLocalIP, u)
 	})
 
 	errGroup.Go(func() error {
@@ -267,7 +318,7 @@ func ServeTunnel(
 
 	err = errGroup.Wait()
 	if err != nil {
-		_ = newClientRegisterTunnelError(err, config.Metrics.regFail)
+		_ = newClientRegisterTunnelError(err, config.Metrics.regFail, unknown)
 
 		switch castedErr := err.(type) {
 		case dupConnRegisterTunnelError:
@@ -295,18 +346,9 @@ func ServeTunnel(
 	return nil, true
 }
 
-func IsRPCStreamResponse(headers []h2mux.Header) bool {
-	if len(headers) != 1 {
-		return false
-	}
-	if headers[0].Name != ":status" || headers[0].Value != "200" {
-		return false
-	}
-	return true
-}
-
 func RegisterTunnel(
 	ctx context.Context,
+	credentialManager ReconnectTunnelCredentialManager,
 	muxer *h2mux.Muxer,
 	config *TunnelConfig,
 	logger *log.Entry,
@@ -315,45 +357,73 @@ func RegisterTunnel(
 	uuid uuid.UUID,
 ) error {
 	config.TransportLogger.Debug("initiating RPC stream to register")
-	stream, err := openStream(ctx, muxer)
+	tunnelServer, err := connection.NewRPCClient(ctx, muxer, config.TransportLogger.WithField("subsystem", "rpc-register"), openStreamTimeout)
 	if err != nil {
 		// RPC stream open error
-		return newClientRegisterTunnelError(err, config.Metrics.rpcFail)
+		return newClientRegisterTunnelError(err, config.Metrics.rpcFail, register)
 	}
-	if !IsRPCStreamResponse(stream.Headers) {
-		// stream response error
-		return newClientRegisterTunnelError(err, config.Metrics.rpcFail)
-	}
-	conn := rpc.NewConn(
-		tunnelrpc.NewTransportLogger(config.TransportLogger.WithField("subsystem", "rpc-register"), rpc.StreamTransport(stream)),
-		tunnelrpc.ConnLog(config.TransportLogger.WithField("subsystem", "rpc-transport")),
-	)
-	defer conn.Close()
-	ts := tunnelpogs.TunnelServer_PogsClient{Client: conn.Bootstrap(ctx)}
+	defer tunnelServer.Close()
 	// Request server info without blocking tunnel registration; must use capnp library directly.
-	tsClient := tunnelrpc.TunnelServer{Client: ts.Client}
-	serverInfoPromise := tsClient.GetServerInfo(ctx, func(tunnelrpc.TunnelServer_getServerInfo_Params) error {
+	serverInfoPromise := tunnelrpc.TunnelServer{Client: tunnelServer.Client}.GetServerInfo(ctx, func(tunnelrpc.TunnelServer_getServerInfo_Params) error {
 		return nil
 	})
-	registration, err := ts.RegisterTunnel(
+	LogServerInfo(serverInfoPromise.Result(), connectionID, config.Metrics, logger)
+	registration := tunnelServer.RegisterTunnel(
 		ctx,
 		config.OriginCert,
 		config.Hostname,
 		config.RegistrationOptions(connectionID, originLocalIP, uuid),
 	)
-	LogServerInfo(serverInfoPromise.Result(), connectionID, config.Metrics, logger)
-	if err != nil {
+	if registrationErr := registration.DeserializeError(); registrationErr != nil {
 		// RegisterTunnel RPC failure
-		return newClientRegisterTunnelError(err, config.Metrics.regFail)
+		return processRegisterTunnelError(registrationErr, config.Metrics, register)
 	}
+	credentialManager.SetEventDigest(registration.EventDigest)
+	return processRegistrationSuccess(config, logger, connectionID, registration, register)
+}
+
+func ReconnectTunnel(
+	ctx context.Context,
+	token []byte,
+	eventDigest []byte,
+	muxer *h2mux.Muxer,
+	config *TunnelConfig,
+	logger *log.Entry,
+	connectionID uint8,
+	originLocalIP string,
+	uuid uuid.UUID,
+) error {
+	config.TransportLogger.Debug("initiating RPC stream to reconnect")
+	tunnelServer, err := connection.NewRPCClient(ctx, muxer, config.TransportLogger.WithField("subsystem", "rpc-reconnect"), openStreamTimeout)
+	if err != nil {
+		// RPC stream open error
+		return newClientRegisterTunnelError(err, config.Metrics.rpcFail, reconnect)
+	}
+	defer tunnelServer.Close()
+	// Request server info without blocking tunnel registration; must use capnp library directly.
+	serverInfoPromise := tunnelrpc.TunnelServer{Client: tunnelServer.Client}.GetServerInfo(ctx, func(tunnelrpc.TunnelServer_getServerInfo_Params) error {
+		return nil
+	})
+	LogServerInfo(serverInfoPromise.Result(), connectionID, config.Metrics, logger)
+	registration := tunnelServer.ReconnectTunnel(
+		ctx,
+		token,
+		eventDigest,
+		config.Hostname,
+		config.RegistrationOptions(connectionID, originLocalIP, uuid),
+	)
+	if registrationErr := registration.DeserializeError(); registrationErr != nil {
+		// ReconnectTunnel RPC failure
+		return processRegisterTunnelError(registrationErr, config.Metrics, reconnect)
+	}
+	return processRegistrationSuccess(config, logger, connectionID, registration, reconnect)
+}
+
+func processRegistrationSuccess(config *TunnelConfig, logger *log.Entry, connectionID uint8, registration *tunnelpogs.TunnelRegistration, name registerRPCName) error {
 	for _, logLine := range registration.LogLines {
 		logger.Info(logLine)
 	}
 
-	if regErr := processRegisterTunnelError(registration.Err, registration.PermanentFailure, config.Metrics); regErr != nil {
-		return regErr
-	}
-
 	if registration.TunnelID != "" {
 		config.Metrics.tunnelsHA.AddTunnelID(connectionID, registration.TunnelID)
 		logger.Infof("Each HA connection's tunnel IDs: %v", config.Metrics.tunnelsHA.String())
@@ -374,55 +444,32 @@ func RegisterTunnel(
 	config.Metrics.userHostnamesCounts.WithLabelValues(registration.Url).Inc()
 
 	logger.Infof("Route propagating, it may take up to 1 minute for your new route to become functional")
+	config.Metrics.regSuccess.WithLabelValues(string(name)).Inc()
 	return nil
 }
 
-func processRegisterTunnelError(err string, permanentFailure bool, metrics *TunnelMetrics) error {
-	if err == "" {
-		metrics.regSuccess.Inc()
-		return nil
-	}
-
-	metrics.regFail.WithLabelValues(err).Inc()
-	if err == DuplicateConnectionError {
+func processRegisterTunnelError(err tunnelpogs.TunnelRegistrationError, metrics *TunnelMetrics, name registerRPCName) error {
+	if err.Error() == DuplicateConnectionError {
+		metrics.regFail.WithLabelValues("dup_edge_conn", string(name)).Inc()
 		return dupConnRegisterTunnelError{}
 	}
+	metrics.regFail.WithLabelValues("server_error", string(name)).Inc()
 	return serverRegisterTunnelError{
-		cause:     fmt.Errorf("Server error: %s", err),
-		permanent: permanentFailure,
+		cause:     fmt.Errorf("Server error: %s", err.Error()),
+		permanent: err.IsPermanent(),
 	}
 }
 
 func UnregisterTunnel(muxer *h2mux.Muxer, gracePeriod time.Duration, logger *log.Logger) error {
 	logger.Debug("initiating RPC stream to unregister")
 	ctx := context.Background()
-	stream, err := openStream(ctx, muxer)
+	tunnelServer, err := connection.NewRPCClient(ctx, muxer, logger.WithField("subsystem", "rpc-unregister"), openStreamTimeout)
 	if err != nil {
 		// RPC stream open error
 		return err
 	}
-	if !IsRPCStreamResponse(stream.Headers) {
-		// stream response error
-		return err
-	}
-	conn := rpc.NewConn(
-		tunnelrpc.NewTransportLogger(logger.WithField("subsystem", "rpc-unregister"), rpc.StreamTransport(stream)),
-		tunnelrpc.ConnLog(logger.WithField("subsystem", "rpc-transport")),
-	)
-	defer conn.Close()
-	ts := tunnelpogs.TunnelServer_PogsClient{Client: conn.Bootstrap(ctx)}
 	// gracePeriod is encoded in int64 using capnproto
-	return ts.UnregisterTunnel(ctx, gracePeriod.Nanoseconds())
-}
-
-func openStream(ctx context.Context, muxer *h2mux.Muxer) (*h2mux.MuxedStream, error) {
-	openStreamCtx, cancel := context.WithTimeout(ctx, openStreamTimeout)
-	defer cancel()
-	return muxer.OpenStream(openStreamCtx, []h2mux.Header{
-		{Name: ":method", Value: "RPC"},
-		{Name: ":scheme", Value: "capnp"},
-		{Name: ":path", Value: "*"},
-	}, nil)
+	return tunnelServer.UnregisterTunnel(ctx, gracePeriod.Nanoseconds())
 }
 
 func LogServerInfo(
@@ -445,16 +492,6 @@ func LogServerInfo(
 	metrics.registerServerLocation(uint8ToString(connectionID), serverInfo.LocationName)
 }
 
-func H1ResponseToH2Response(h1 *http.Response) (h2 []h2mux.Header) {
-	h2 = []h2mux.Header{{Name: ":status", Value: fmt.Sprintf("%d", h1.StatusCode)}}
-	for headerName, headerValues := range h1.Header {
-		for _, headerValue := range headerValues {
-			h2 = append(h2, h2mux.Header{Name: strings.ToLower(headerName), Value: headerValue})
-		}
-	}
-	return
-}
-
 type TunnelHandler struct {
 	originUrl      string
 	httpHostHeader string
@@ -467,15 +504,16 @@ type TunnelHandler struct {
 	connectionID      string
 	logger            *log.Logger
 	noChunkedEncoding bool
-}
 
-var dialer = net.Dialer{DualStack: true}
+	bufferPool *buffer.Pool
+}
 
 // NewTunnelHandler returns a TunnelHandler, origin LAN IP and error
 func NewTunnelHandler(ctx context.Context,
 	config *TunnelConfig,
-	addr string,
+	addr *net.TCPAddr,
 	connectionID uint8,
+	bufferPool *buffer.Pool,
 ) (*TunnelHandler, string, error) {
 	originURL, err := validation.ValidateUrl(config.OriginUrl)
 	if err != nil {
@@ -491,41 +529,23 @@ func NewTunnelHandler(ctx context.Context,
 		connectionID:      uint8ToString(connectionID),
 		logger:            config.Logger,
 		noChunkedEncoding: config.NoChunkedEncoding,
+		bufferPool:        bufferPool,
 	}
 	if h.httpClient == nil {
 		h.httpClient = http.DefaultTransport
 	}
-	// Inherit from parent context so we can cancel (Ctrl-C) while dialing
-	dialCtx, dialCancel := context.WithTimeout(ctx, dialTimeout)
-	// TUN-92: enforce a timeout on dial and handshake (as tls.Dial does not support one)
-	plaintextEdgeConn, err := dialer.DialContext(dialCtx, "tcp", addr)
-	dialCancel()
+
+	edgeConn, err := connection.DialEdge(ctx, dialTimeout, config.TlsConfig, addr)
 	if err != nil {
-		return nil, "", dialError{cause: errors.Wrap(err, "DialContext error")}
+		return nil, "", err
 	}
-	edgeConn := tls.Client(plaintextEdgeConn, config.TlsConfig)
-	edgeConn.SetDeadline(time.Now().Add(dialTimeout))
-	err = edgeConn.Handshake()
-	if err != nil {
-		return nil, "", dialError{cause: errors.Wrap(err, "Handshake with edge error")}
-	}
-	// clear the deadline on the conn; h2mux has its own timeouts
-	edgeConn.SetDeadline(time.Time{})
 	// Establish a muxed connection with the edge
 	// Client mux handshake with agent server
-	h.muxer, err = h2mux.Handshake(edgeConn, edgeConn, h2mux.MuxerConfig{
-		Timeout:            5 * time.Second,
-		Handler:            h,
-		IsClient:           true,
-		HeartbeatInterval:  config.HeartbeatInterval,
-		MaxHeartbeats:      config.MaxHeartbeats,
-		Logger:             config.TransportLogger.WithFields(log.Fields{}),
-		CompressionQuality: h2mux.CompressionSetting(config.CompressionQuality),
-	}, h.metrics.activeStreams)
+	h.muxer, err = h2mux.Handshake(edgeConn, edgeConn, config.muxerConfig(h), h.metrics.activeStreams)
 	if err != nil {
-		return h, "", errors.New("TLS handshake error")
+		return nil, "", errors.Wrap(err, "h2mux handshake with edge error")
 	}
-	return h, edgeConn.LocalAddr().String(), err
+	return h, edgeConn.LocalAddr().String(), nil
 }
 
 func (h *TunnelHandler) AppendTagHeaders(r *http.Request) {
@@ -568,7 +588,7 @@ func (h *TunnelHandler) createRequest(stream *h2mux.MuxedStream) (*http.Request,
 	if err != nil {
 		return nil, errors.Wrap(err, "Unexpected error from http.NewRequest")
 	}
-	err = streamhandler.H2RequestHeadersToH1Request(stream.Headers, req)
+	err = h2mux.OldH2RequestHeadersToH1Request(stream.Headers, req)
 	if err != nil {
 		return nil, errors.Wrap(err, "invalid request received")
 	}
@@ -587,7 +607,7 @@ func (h *TunnelHandler) serveWebsocket(stream *h2mux.MuxedStream, req *http.Requ
 		return nil, err
 	}
 	defer conn.Close()
-	err = stream.WriteHeaders(H1ResponseToH2Response(response))
+	err = stream.WriteHeaders(h2mux.OldH1ResponseToH2ResponseHeaders(response))
 	if err != nil {
 		return nil, errors.Wrap(err, "Error writing response header")
 	}
@@ -621,7 +641,7 @@ func (h *TunnelHandler) serveHTTP(stream *h2mux.MuxedStream, req *http.Request)
 	}
 	defer response.Body.Close()
 
-	err = stream.WriteHeaders(H1ResponseToH2Response(response))
+	err = stream.WriteHeaders(h2mux.OldH1ResponseToH2ResponseHeaders(response))
 	if err != nil {
 		return nil, errors.Wrap(err, "Error writing response header")
 	}
@@ -630,7 +650,9 @@ func (h *TunnelHandler) serveHTTP(stream *h2mux.MuxedStream, req *http.Request)
 	} else {
 		// Use CopyBuffer, because Copy only allocates a 32KiB buffer, and cross-stream
 		// compression generates dictionary on first write
-		io.CopyBuffer(stream, response.Body, make([]byte, 512*1024))
+		buf := h.bufferPool.Get()
+		defer h.bufferPool.Put(buf)
+		io.CopyBuffer(stream, response.Body, buf)
 	}
 	return response, nil
 }

originservice/originservice.go

@@ -12,6 +12,7 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/cloudflare/cloudflared/buffer"
 	"github.com/cloudflare/cloudflared/h2mux"
 	"github.com/cloudflare/cloudflared/hello"
 	"github.com/cloudflare/cloudflared/log"
@@ -33,6 +34,7 @@ type HTTPService struct {
 	client          http.RoundTripper
 	originURL       *url.URL
 	chunkedEncoding bool
+	bufferPool      *buffer.Pool
 }
 
 func NewHTTPService(transport http.RoundTripper, url *url.URL, chunkedEncoding bool) OriginService {
@@ -40,6 +42,7 @@ func NewHTTPService(transport http.RoundTripper, url *url.URL, chunkedEncoding b
 		client:          transport,
 		originURL:       url,
 		chunkedEncoding: chunkedEncoding,
+		bufferPool:      buffer.NewPool(512 * 1024),
 	}
 }
 
@@ -71,7 +74,9 @@ func (hc *HTTPService) Proxy(stream *h2mux.MuxedStream, req *http.Request) (*htt
 	} else {
 		// Use CopyBuffer, because Copy only allocates a 32KiB buffer, and cross-stream
 		// compression generates dictionary on first write
-		io.CopyBuffer(stream, resp.Body, make([]byte, 512*1024))
+		buf := hc.bufferPool.Get()
+		defer hc.bufferPool.Put(buf)
+		io.CopyBuffer(stream, resp.Body, buf)
 	}
 	return resp, nil
 }
@@ -142,10 +147,11 @@ func (wsc *WebsocketService) Shutdown() {
 
 // HelloWorldService talks to the hello world example origin
 type HelloWorldService struct {
-	client    http.RoundTripper
-	listener  net.Listener
-	originURL *url.URL
-	shutdownC chan struct{}
+	client     http.RoundTripper
+	listener   net.Listener
+	originURL  *url.URL
+	shutdownC  chan struct{}
+	bufferPool *buffer.Pool
 }
 
 func NewHelloWorldService(transport http.RoundTripper) (OriginService, error) {
@@ -164,7 +170,8 @@ func NewHelloWorldService(transport http.RoundTripper) (OriginService, error) {
 			Scheme: "https",
 			Host:   listener.Addr().String(),
 		},
-		shutdownC: shutdownC,
+		shutdownC:  shutdownC,
+		bufferPool: buffer.NewPool(512 * 1024),
 	}, nil
 }
 
@@ -184,7 +191,9 @@ func (hwc *HelloWorldService) Proxy(stream *h2mux.MuxedStream, req *http.Request
 
 	// Use CopyBuffer, because Copy only allocates a 32KiB buffer, and cross-stream
 	// compression generates dictionary on first write
-	io.CopyBuffer(stream, resp.Body, make([]byte, 512*1024))
+	buf := hwc.bufferPool.Get()
+	defer hwc.bufferPool.Put(buf)
+	io.CopyBuffer(stream, resp.Body, buf)
 
 	return resp, nil
 }

streamhandler/request.go

@@ -1,10 +1,8 @@
 package streamhandler
 
 import (
-	"fmt"
 	"net/http"
 	"net/url"
-	"strconv"
 	"strings"
 
 	"github.com/cloudflare/cloudflared/h2mux"
@@ -28,65 +26,9 @@ func createRequest(stream *h2mux.MuxedStream, url *url.URL) (*http.Request, erro
 	if err != nil {
 		return nil, errors.Wrap(err, "unexpected error from http.NewRequest")
 	}
-	err = H2RequestHeadersToH1Request(stream.Headers, req)
+	err = h2mux.OldH2RequestHeadersToH1Request(stream.Headers, req)
 	if err != nil {
 		return nil, errors.Wrap(err, "invalid request received")
 	}
 	return req, nil
 }
-
-// H2RequestHeadersToH1Request converts the HTTP/2 headers to an HTTP/1 Request
-// object. This includes conversion of the pseudo-headers into their closest
-// HTTP/1 equivalents. See https://tools.ietf.org/html/rfc7540#section-8.1.2.3
-func H2RequestHeadersToH1Request(h2 []h2mux.Header, h1 *http.Request) error {
-	for _, header := range h2 {
-		switch header.Name {
-		case ":method":
-			h1.Method = header.Value
-		case ":scheme":
-			// noop - use the preexisting scheme from h1.URL
-		case ":authority":
-			// Otherwise the host header will be based on the origin URL
-			h1.Host = header.Value
-		case ":path":
-			// We don't want to be an "opinionated" proxy, so ideally we would use :path as-is.
-			// However, this HTTP/1 Request object belongs to the Go standard library,
-			// whose URL package makes some opinionated decisions about the encoding of
-			// URL characters: see the docs of https://godoc.org/net/url#URL,
-			// in particular the EscapedPath method https://godoc.org/net/url#URL.EscapedPath,
-			// which is always used when computing url.URL.String(), whether we'd like it or not.
-			//
-			// Well, not *always*. We could circumvent this by using url.URL.Opaque. But
-			// that would present unusual difficulties when using an HTTP proxy: url.URL.Opaque
-			// is treated differently when HTTP_PROXY is set!
-			// See https://github.com/golang/go/issues/5684#issuecomment-66080888
-			//
-			// This means we are subject to the behavior of net/url's function `shouldEscape`
-			// (as invoked with mode=encodePath): https://github.com/golang/go/blob/go1.12.7/src/net/url/url.go#L101
-
-			if header.Value == "*" {
-				h1.URL.Path = "*"
-				continue
-			}
-			// Due to the behavior of validation.ValidateUrl, h1.URL may
-			// already have a partial value, with or without a trailing slash.
-			base := h1.URL.String()
-			base = strings.TrimRight(base, "/")
-			// But we know :path begins with '/', because we handled '*' above - see RFC7540
-			url, err := url.Parse(base + header.Value)
-			if err != nil {
-				return errors.Wrap(err, fmt.Sprintf("invalid path '%v'", header.Value))
-			}
-			h1.URL = url
-		case "content-length":
-			contentLength, err := strconv.ParseInt(header.Value, 10, 64)
-			if err != nil {
-				return fmt.Errorf("unparseable content length")
-			}
-			h1.ContentLength = contentLength
-		default:
-			h1.Header.Add(http.CanonicalHeaderKey(header.Name), header.Value)
-		}
-	}
-	return nil
-}

supervisor/supervisor.go

@@ -1,3 +1,4 @@
+// Package supervisor is used by declarative tunnels to get/apply new config from the edge.
 package supervisor
 
 import (
@@ -13,6 +14,7 @@ import (
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/updater"
 	"github.com/cloudflare/cloudflared/connection"
+	"github.com/cloudflare/cloudflared/edgediscovery"
 	"github.com/cloudflare/cloudflared/h2mux"
 	"github.com/cloudflare/cloudflared/streamhandler"
 	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
@@ -56,7 +58,7 @@ func NewSupervisor(
 	defaultClientConfig *pogs.ClientConfig,
 	userCredential []byte,
 	tlsConfig *tls.Config,
-	serviceDiscoverer connection.EdgeServiceDiscoverer,
+	serviceDiscoverer *edgediscovery.Edge,
 	cloudflaredConfig *connection.CloudflaredConfig,
 	autoupdater *updater.AutoUpdater,
 	supportAutoupdate bool,

tlsconfig/cloudflare_ca.go

@@ -26,28 +26,28 @@ mcifak4CQsr+DH4pn5SJD7JxtCG3YGswW8QZsw==
 -----END CERTIFICATE-----
 Issuer: C=US, O=CloudFlare, Inc., OU=CloudFlare Origin SSL Certificate Authority, L=San Francisco, ST=California
 -----BEGIN CERTIFICATE-----
-MIID/DCCAuagAwIBAgIID+rOSdTGfGcwCwYJKoZIhvcNAQELMIGLMQswCQYDVQQG
-EwJVUzEZMBcGA1UEChMQQ2xvdWRGbGFyZSwgSW5jLjE0MDIGA1UECxMrQ2xvdWRG
-bGFyZSBPcmlnaW4gU1NMIENlcnRpZmljYXRlIEF1dGhvcml0eTEWMBQGA1UEBxMN
-U2FuIEZyYW5jaXNjbzETMBEGA1UECBMKQ2FsaWZvcm5pYTAeFw0xNDExMTMyMDM4
-NTBaFw0xOTExMTQwMTQzNTBaMIGLMQswCQYDVQQGEwJVUzEZMBcGA1UEChMQQ2xv
-dWRGbGFyZSwgSW5jLjE0MDIGA1UECxMrQ2xvdWRGbGFyZSBPcmlnaW4gU1NMIENl
-cnRpZmljYXRlIEF1dGhvcml0eTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzETMBEG
-A1UECBMKQ2FsaWZvcm5pYTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB
-AMBIlWf1KEKR5hbB75OYrAcUXobpD/AxvSYRXr91mbRu+lqE7YbyyRUShQh15lem
-ef+umeEtPZoLFLhcLyczJxOhI+siLGDQm/a/UDkWvAXYa5DZ+pHU5ct5nZ8pGzqJ
-p8G1Hy5RMVYDXZT9F6EaHjMG0OOffH6Ih25TtgfyyrjXycwDH0u6GXt+G/rywcqz
-/9W4Aki3XNQMUHNQAtBLEEIYHMkyTYJxuL2tXO6ID5cCsoWw8meHufTeZW2DyUpl
-yP3AHt4149RQSyWZMJ6AyntL9d8Xhfpxd9rJkh9Kge2iV9rQTFuE1rRT5s7OSJcK
-xUsklgHcGHYMcNfNMilNHb8CAwEAAaNmMGQwDgYDVR0PAQH/BAQDAgAGMBIGA1Ud
-EwEB/wQIMAYBAf8CAQIwHQYDVR0OBBYEFCToU1ddfDRAh6nrlNu64RZ4/CmkMB8G
-A1UdIwQYMBaAFCToU1ddfDRAh6nrlNu64RZ4/CmkMAsGCSqGSIb3DQEBCwOCAQEA
-cQDBVAoRrhhsGegsSFsv1w8v27zzHKaJNv6ffLGIRvXK8VKKK0gKXh2zQtN9SnaD
-gYNe7Pr4C3I8ooYKRJJWLsmEHdGdnYYmj0OJfGrfQf6MLIc/11bQhLepZTxdhFYh
-QGgDl6gRmb8aDwk7Q92BPvek5nMzaWlP82ixavvYI+okoSY8pwdcVKobx6rWzMWz
-ZEC9M6H3F0dDYE23XcCFIdgNSAmmGyXPBstOe0aAJXwJTxOEPn36VWr0PKIQJy5Y
-4o1wpMpqCOIwWc8J9REV/REzN6Z1LXImdUgXIXOwrz56gKUJzPejtBQyIGj0mveX
-Fu6q54beR89jDc+oABmOgg==
+MIIEADCCAuigAwIBAgIID+rOSdTGfGcwDQYJKoZIhvcNAQELBQAwgYsxCzAJBgNV
+BAYTAlVTMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMTQwMgYDVQQLEytDbG91
+ZEZsYXJlIE9yaWdpbiBTU0wgQ2VydGlmaWNhdGUgQXV0aG9yaXR5MRYwFAYDVQQH
+Ew1TYW4gRnJhbmNpc2NvMRMwEQYDVQQIEwpDYWxpZm9ybmlhMB4XDTE5MDgyMzIx
+MDgwMFoXDTI5MDgxNTE3MDAwMFowgYsxCzAJBgNVBAYTAlVTMRkwFwYDVQQKExBD
+bG91ZEZsYXJlLCBJbmMuMTQwMgYDVQQLEytDbG91ZEZsYXJlIE9yaWdpbiBTU0wg
+Q2VydGlmaWNhdGUgQXV0aG9yaXR5MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRMw
+EQYDVQQIEwpDYWxpZm9ybmlhMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC
+AQEAwEiVZ/UoQpHmFsHvk5isBxRehukP8DG9JhFev3WZtG76WoTthvLJFRKFCHXm
+V6Z5/66Z4S09mgsUuFwvJzMnE6Ej6yIsYNCb9r9QORa8BdhrkNn6kdTly3mdnykb
+OomnwbUfLlExVgNdlP0XoRoeMwbQ4598foiHblO2B/LKuNfJzAMfS7oZe34b+vLB
+yrP/1bgCSLdc1AxQc1AC0EsQQhgcyTJNgnG4va1c7ogPlwKyhbDyZ4e59N5lbYPJ
+SmXI/cAe3jXj1FBLJZkwnoDKe0v13xeF+nF32smSH0qB7aJX2tBMW4TWtFPmzs5I
+lwrFSySWAdwYdgxw180yKU0dvwIDAQABo2YwZDAOBgNVHQ8BAf8EBAMCAQYwEgYD
+VR0TAQH/BAgwBgEB/wIBAjAdBgNVHQ4EFgQUJOhTV118NECHqeuU27rhFnj8KaQw
+HwYDVR0jBBgwFoAUJOhTV118NECHqeuU27rhFnj8KaQwDQYJKoZIhvcNAQELBQAD
+ggEBAHwOf9Ur1l0Ar5vFE6PNrZWrDfQIMyEfdgSKofCdTckbqXNTiXdgbHs+TWoQ
+wAB0pfJDAHJDXOTCWRyTeXOseeOi5Btj5CnEuw3P0oXqdqevM1/+uWp0CM35zgZ8
+VD4aITxity0djzE6Qnx3Syzz+ZkoBgTnNum7d9A66/V636x4vTeqbZFBr9erJzgz
+hhurjcoacvRNhnjtDRM0dPeiCJ50CP3wEYuvUzDHUaowOsnLCjQIkWbR7Ni6KEIk
+MOz2U0OBSif3FTkhCgZWQKOOLo1P42jHC3ssUZAtVNXrCk3fw9/E15k8NPkBazZ6
+0iykLhH1trywrKRMVw67F44IE8Y=
 -----END CERTIFICATE-----
 Issuer: C=US, O=CloudFlare, Inc., OU=Origin Pull, L=San Francisco, ST=California, CN=origin-pull.cloudflare.net
 -----BEGIN CERTIFICATE-----

tunneldns/https_upstream.go

@@ -83,7 +83,7 @@ func (u *UpstreamHTTPS) exchangeWireformat(msg []byte) ([]byte, error) {
 	}
 
 	req.Header.Add("Content-Type", "application/dns-message")
-	req.Host = u.endpoint.Hostname()
+	req.Host = u.endpoint.Host
 
 	resp, err := u.client.Do(req)
 	if err != nil {

tunnelrpc/pogs/auth_outcome.go

@@ -1,7 +1,7 @@
 package pogs
 
 import (
-	"fmt"
+	"errors"
 	"time"
 )
 
@@ -18,21 +18,21 @@ type AuthenticateResponse struct {
 
 // Outcome turns the deserialized response of Authenticate into a programmer-friendly sum type.
 func (ar AuthenticateResponse) Outcome() AuthOutcome {
-	// If there was a network error, then cloudflared should retry later,
-	// because origintunneld couldn't prove whether auth was correct or not.
-	if ar.RetryableErr != "" {
-		return &AuthUnknown{Err: fmt.Errorf(ar.RetryableErr), HoursUntilRefresh: ar.HoursUntilRefresh}
-	}
-
 	// If the user's authentication was unsuccessful, the server will return an error explaining why.
 	// cloudflared should fatal with this error.
 	if ar.PermanentErr != "" {
-		return &AuthFail{Err: fmt.Errorf(ar.PermanentErr)}
+		return NewAuthFail(errors.New(ar.PermanentErr))
+	}
+
+	// If there was a network error, then cloudflared should retry later,
+	// because origintunneld couldn't prove whether auth was correct or not.
+	if ar.RetryableErr != "" {
+		return NewAuthUnknown(errors.New(ar.RetryableErr), ar.HoursUntilRefresh)
 	}
 
 	// If auth succeeded, return the token and refresh it when instructed.
-	if ar.PermanentErr == "" && len(ar.Jwt) > 0 {
-		return &AuthSuccess{Jwt: ar.Jwt, HoursUntilRefresh: ar.HoursUntilRefresh}
+	if len(ar.Jwt) > 0 {
+		return NewAuthSuccess(ar.Jwt, ar.HoursUntilRefresh)
 	}
 
 	// Otherwise the state got messed up.
@@ -49,60 +49,84 @@ type AuthOutcome interface {
 
 // AuthSuccess means the backend successfully authenticated this cloudflared.
 type AuthSuccess struct {
-	Jwt               []byte
-	HoursUntilRefresh uint8
+	jwt               []byte
+	hoursUntilRefresh uint8
+}
+
+func NewAuthSuccess(jwt []byte, hoursUntilRefresh uint8) AuthSuccess {
+	return AuthSuccess{jwt: jwt, hoursUntilRefresh: hoursUntilRefresh}
+}
+
+func (ao AuthSuccess) JWT() []byte {
+	return ao.jwt
 }
 
 // RefreshAfter is how long cloudflared should wait before rerunning Authenticate.
-func (ao *AuthSuccess) RefreshAfter() time.Duration {
-	return hoursToTime(ao.HoursUntilRefresh)
+func (ao AuthSuccess) RefreshAfter() time.Duration {
+	return hoursToTime(ao.hoursUntilRefresh)
 }
 
 // Serialize into an AuthenticateResponse which can be sent via Capnp
-func (ao *AuthSuccess) Serialize() AuthenticateResponse {
+func (ao AuthSuccess) Serialize() AuthenticateResponse {
 	return AuthenticateResponse{
-		Jwt:               ao.Jwt,
-		HoursUntilRefresh: ao.HoursUntilRefresh,
+		Jwt:               ao.jwt,
+		HoursUntilRefresh: ao.hoursUntilRefresh,
 	}
 }
 
-func (ao *AuthSuccess) isAuthOutcome() {}
+func (ao AuthSuccess) isAuthOutcome() {}
 
 // AuthFail means this cloudflared has the wrong auth and should exit.
 type AuthFail struct {
-	Err error
+	err error
+}
+
+func NewAuthFail(err error) AuthFail {
+	return AuthFail{err: err}
+}
+
+func (ao AuthFail) Error() string {
+	return ao.err.Error()
 }
 
 // Serialize into an AuthenticateResponse which can be sent via Capnp
-func (ao *AuthFail) Serialize() AuthenticateResponse {
+func (ao AuthFail) Serialize() AuthenticateResponse {
 	return AuthenticateResponse{
-		PermanentErr: ao.Err.Error(),
+		PermanentErr: ao.err.Error(),
 	}
 }
 
-func (ao *AuthFail) isAuthOutcome() {}
+func (ao AuthFail) isAuthOutcome() {}
 
 // AuthUnknown means the backend couldn't finish checking authentication. Try again later.
 type AuthUnknown struct {
-	Err               error
-	HoursUntilRefresh uint8
+	err               error
+	hoursUntilRefresh uint8
+}
+
+func NewAuthUnknown(err error, hoursUntilRefresh uint8) AuthUnknown {
+	return AuthUnknown{err: err, hoursUntilRefresh: hoursUntilRefresh}
+}
+
+func (ao AuthUnknown) Error() string {
+	return ao.err.Error()
 }
 
 // RefreshAfter is how long cloudflared should wait before rerunning Authenticate.
-func (ao *AuthUnknown) RefreshAfter() time.Duration {
-	return hoursToTime(ao.HoursUntilRefresh)
+func (ao AuthUnknown) RefreshAfter() time.Duration {
+	return hoursToTime(ao.hoursUntilRefresh)
 }
 
 // Serialize into an AuthenticateResponse which can be sent via Capnp
-func (ao *AuthUnknown) Serialize() AuthenticateResponse {
+func (ao AuthUnknown) Serialize() AuthenticateResponse {
 	return AuthenticateResponse{
-		RetryableErr:      ao.Err.Error(),
-		HoursUntilRefresh: ao.HoursUntilRefresh,
+		RetryableErr:      ao.err.Error(),
+		HoursUntilRefresh: ao.hoursUntilRefresh,
 	}
 }
 
-func (ao *AuthUnknown) isAuthOutcome() {}
+func (ao AuthUnknown) isAuthOutcome() {}
 
 func hoursToTime(hours uint8) time.Duration {
 	return time.Duration(hours) * time.Hour
-}
+}

tunnelrpc/pogs/auth_test.go

@@ -31,15 +31,15 @@ func TestAuthenticateResponseOutcome(t *testing.T) {
 	}{
 		{"success",
 			fields{Jwt: []byte("asdf"), HoursUntilRefresh: 6},
-			&AuthSuccess{Jwt: []byte("asdf"), HoursUntilRefresh: 6},
+			AuthSuccess{jwt: []byte("asdf"), hoursUntilRefresh: 6},
 		},
 		{"fail",
 			fields{PermanentErr: "bad creds"},
-			&AuthFail{Err: fmt.Errorf("bad creds")},
+			AuthFail{err: fmt.Errorf("bad creds")},
 		},
 		{"error",
 			fields{RetryableErr: "bad conn", HoursUntilRefresh: 6},
-			&AuthUnknown{Err: fmt.Errorf("bad conn"), HoursUntilRefresh: 6},
+			AuthUnknown{err: fmt.Errorf("bad conn"), hoursUntilRefresh: 6},
 		},
 		{"nil (no fields are set)",
 			fields{},
@@ -69,6 +69,27 @@ func TestAuthenticateResponseOutcome(t *testing.T) {
 	}
 }
 
+func TestAuthSuccess(t *testing.T) {
+	input := NewAuthSuccess([]byte("asdf"), 6)
+	output, ok := input.Serialize().Outcome().(AuthSuccess)
+	assert.True(t, ok)
+	assert.Equal(t, input, output)
+}
+
+func TestAuthUnknown(t *testing.T) {
+	input := NewAuthUnknown(fmt.Errorf("pdx unreachable"), 6)
+	output, ok := input.Serialize().Outcome().(AuthUnknown)
+	assert.True(t, ok)
+	assert.Equal(t, input, output)
+}
+
+func TestAuthFail(t *testing.T) {
+	input := NewAuthFail(fmt.Errorf("wrong creds"))
+	output, ok := input.Serialize().Outcome().(AuthFail)
+	assert.True(t, ok)
+	assert.Equal(t, input, output)
+}
+
 func TestWhenToRefresh(t *testing.T) {
 	expected := 4 * time.Hour
 	actual := hoursToTime(4)

tunnelrpc/pogs/config.go

@@ -197,11 +197,14 @@ func (hc *HTTPOriginConfig) Service() (originservice.OriginService, error) {
 		return nil, err
 	}
 
-	dialContext := (&net.Dialer{
+	dialer := &net.Dialer{
 		Timeout:   hc.ProxyConnectionTimeout,
 		KeepAlive: hc.TCPKeepAlive,
-		DualStack: hc.DialDualStack,
-	}).DialContext
+	}
+	if !hc.DialDualStack {
+		dialer.FallbackDelay = -1
+	}
+	dialContext := dialer.DialContext
 	transport := &http.Transport{
 		Proxy:       http.ProxyFromEnvironment,
 		DialContext: dialContext,
@@ -270,7 +273,6 @@ func (*HelloWorldOriginConfig) Service() (originservice.OriginService, error) {
 		DialContext: (&net.Dialer{
 			Timeout:   30 * time.Second,
 			KeepAlive: 30 * time.Second,
-			DualStack: true,
 		}).DialContext,
 		TLSClientConfig: &tls.Config{
 			RootCAs: rootCAs,

tunnelrpc/pogs/reconnect_tunnel.go

@@ -12,6 +12,10 @@ func (i TunnelServer_PogsImpl) ReconnectTunnel(p tunnelrpc.TunnelServer_reconnec
 	if err != nil {
 		return err
 	}
+	eventDigest, err := p.Params.EventDigest()
+	if err != nil {
+		return err
+	}
 	hostname, err := p.Params.Hostname()
 	if err != nil {
 		return err
@@ -25,7 +29,7 @@ func (i TunnelServer_PogsImpl) ReconnectTunnel(p tunnelrpc.TunnelServer_reconnec
 		return err
 	}
 	server.Ack(p.Options)
-	registration, err := i.impl.ReconnectTunnel(p.Ctx, jwt, hostname, pogsOptions)
+	registration, err := i.impl.ReconnectTunnel(p.Ctx, jwt, eventDigest, hostname, pogsOptions)
 	if err != nil {
 		return err
 	}
@@ -38,16 +42,21 @@ func (i TunnelServer_PogsImpl) ReconnectTunnel(p tunnelrpc.TunnelServer_reconnec
 
 func (c TunnelServer_PogsClient) ReconnectTunnel(
 	ctx context.Context,
-	jwt []byte,
+	jwt,
+	eventDigest []byte,
 	hostname string,
 	options *RegistrationOptions,
-) (*TunnelRegistration, error) {
+) *TunnelRegistration {
 	client := tunnelrpc.TunnelServer{Client: c.Client}
 	promise := client.ReconnectTunnel(ctx, func(p tunnelrpc.TunnelServer_reconnectTunnel_Params) error {
 		err := p.SetJwt(jwt)
 		if err != nil {
 			return err
 		}
+		err = p.SetEventDigest(eventDigest)
+		if err != nil {
+			return err
+		}
 		err = p.SetHostname(hostname)
 		if err != nil {
 			return err
@@ -64,7 +73,11 @@ func (c TunnelServer_PogsClient) ReconnectTunnel(
 	})
 	retval, err := promise.Result().Struct()
 	if err != nil {
-		return nil, err
+		return NewRetryableRegistrationError(err, defaultRetryAfterSeconds).Serialize()
 	}
-	return UnmarshalTunnelRegistration(retval)
+	registration, err := UnmarshalTunnelRegistration(retval)
+	if err != nil {
+		return NewRetryableRegistrationError(err, defaultRetryAfterSeconds).Serialize()
+	}
+	return registration
 }

tunnelrpc/pogs/tunnelrpc.go

@@ -9,13 +9,16 @@ import (
 	"github.com/google/uuid"
 	"github.com/pkg/errors"
 
-	log "github.com/sirupsen/logrus"
 	capnp "zombiezen.com/go/capnproto2"
 	"zombiezen.com/go/capnproto2/pogs"
 	"zombiezen.com/go/capnproto2/rpc"
 	"zombiezen.com/go/capnproto2/server"
 )
 
+const (
+	defaultRetryAfterSeconds = 15
+)
+
 type Authentication struct {
 	Key         string
 	Email       string
@@ -33,14 +36,114 @@ func UnmarshalAuthentication(s tunnelrpc.Authentication) (*Authentication, error
 }
 
 type TunnelRegistration struct {
+	SuccessfulTunnelRegistration
 	Err               string
-	Url               string
-	LogLines          []string
 	PermanentFailure  bool
-	TunnelID          string `capnp:"tunnelID"`
 	RetryAfterSeconds uint16
 }
 
+type SuccessfulTunnelRegistration struct {
+	Url         string
+	LogLines    []string
+	TunnelID    string `capnp:"tunnelID"`
+	EventDigest []byte
+}
+
+func NewSuccessfulTunnelRegistration(
+	url string,
+	logLines []string,
+	tunnelID string,
+	eventDigest []byte,
+) *TunnelRegistration {
+	// Marshal nil will result in an error
+	if logLines == nil {
+		logLines = []string{}
+	}
+	return &TunnelRegistration{
+		SuccessfulTunnelRegistration: SuccessfulTunnelRegistration{
+			Url:         url,
+			LogLines:    logLines,
+			TunnelID:    tunnelID,
+			EventDigest: eventDigest,
+		},
+	}
+}
+
+// Not calling this function Error() to avoid confusion with implementing error interface
+func (tr TunnelRegistration) DeserializeError() TunnelRegistrationError {
+	if tr.Err != "" {
+		err := fmt.Errorf(tr.Err)
+		if tr.PermanentFailure {
+			return NewPermanentRegistrationError(err)
+		}
+		retryAfterSeconds := tr.RetryAfterSeconds
+		if retryAfterSeconds < defaultRetryAfterSeconds {
+			retryAfterSeconds = defaultRetryAfterSeconds
+		}
+		return NewRetryableRegistrationError(err, retryAfterSeconds)
+	}
+	return nil
+}
+
+type TunnelRegistrationError interface {
+	error
+	Serialize() *TunnelRegistration
+	IsPermanent() bool
+}
+
+type PermanentRegistrationError struct {
+	err string
+}
+
+func NewPermanentRegistrationError(err error) TunnelRegistrationError {
+	return &PermanentRegistrationError{
+		err: err.Error(),
+	}
+}
+
+func (pre *PermanentRegistrationError) Error() string {
+	return pre.err
+}
+
+func (pre *PermanentRegistrationError) Serialize() *TunnelRegistration {
+	return &TunnelRegistration{
+		Err:              pre.err,
+		PermanentFailure: true,
+	}
+}
+
+func (*PermanentRegistrationError) IsPermanent() bool {
+	return true
+}
+
+type RetryableRegistrationError struct {
+	err               string
+	retryAfterSeconds uint16
+}
+
+func NewRetryableRegistrationError(err error, retryAfterSeconds uint16) TunnelRegistrationError {
+	return &RetryableRegistrationError{
+		err:               err.Error(),
+		retryAfterSeconds: retryAfterSeconds,
+	}
+}
+
+func (rre *RetryableRegistrationError) Error() string {
+	return rre.err
+}
+
+func (rre *RetryableRegistrationError) Serialize() *TunnelRegistration {
+	return &TunnelRegistration{
+		Err:               rre.err,
+		PermanentFailure:  false,
+		RetryAfterSeconds: rre.retryAfterSeconds,
+	}
+}
+
+func (*RetryableRegistrationError) IsPermanent() bool {
+	return false
+}
+
 func MarshalTunnelRegistration(s tunnelrpc.TunnelRegistration, p *TunnelRegistration) error {
 	return pogs.Insert(tunnelrpc.TunnelRegistration_TypeID, s.Struct, p)
 }
@@ -325,12 +428,12 @@ func UnmarshalConnectParameters(s tunnelrpc.CapnpConnectParameters) (*ConnectPar
 }
 
 type TunnelServer interface {
-	RegisterTunnel(ctx context.Context, originCert []byte, hostname string, options *RegistrationOptions) (*TunnelRegistration, error)
+	RegisterTunnel(ctx context.Context, originCert []byte, hostname string, options *RegistrationOptions) *TunnelRegistration
 	GetServerInfo(ctx context.Context) (*ServerInfo, error)
 	UnregisterTunnel(ctx context.Context, gracePeriodNanoSec int64) error
 	Connect(ctx context.Context, parameters *ConnectParameters) (ConnectResult, error)
 	Authenticate(ctx context.Context, originCert []byte, hostname string, options *RegistrationOptions) (*AuthenticateResponse, error)
-	ReconnectTunnel(ctx context.Context, jwt []byte, hostname string, options *RegistrationOptions) (*TunnelRegistration, error)
+	ReconnectTunnel(ctx context.Context, jwt, eventDigest []byte, hostname string, options *RegistrationOptions) (*TunnelRegistration, error)
 }
 
 func TunnelServer_ServerToClient(s TunnelServer) tunnelrpc.TunnelServer {
@@ -359,15 +462,12 @@ func (i TunnelServer_PogsImpl) RegisterTunnel(p tunnelrpc.TunnelServer_registerT
 		return err
 	}
 	server.Ack(p.Options)
-	registration, err := i.impl.RegisterTunnel(p.Ctx, originCert, hostname, pogsOptions)
-	if err != nil {
-		return err
-	}
+	registration := i.impl.RegisterTunnel(p.Ctx, originCert, hostname, pogsOptions)
+
 	result, err := p.Results.NewResult()
 	if err != nil {
 		return err
 	}
-	log.Info(registration.TunnelID)
 	return MarshalTunnelRegistration(result, registration)
 }
 
@@ -420,7 +520,7 @@ func (c TunnelServer_PogsClient) Close() error {
 	return c.Conn.Close()
 }
 
-func (c TunnelServer_PogsClient) RegisterTunnel(ctx context.Context, originCert []byte, hostname string, options *RegistrationOptions) (*TunnelRegistration, error) {
+func (c TunnelServer_PogsClient) RegisterTunnel(ctx context.Context, originCert []byte, hostname string, options *RegistrationOptions) *TunnelRegistration {
 	client := tunnelrpc.TunnelServer{Client: c.Client}
 	promise := client.RegisterTunnel(ctx, func(p tunnelrpc.TunnelServer_registerTunnel_Params) error {
 		err := p.SetOriginCert(originCert)
@@ -443,9 +543,13 @@ func (c TunnelServer_PogsClient) RegisterTunnel(ctx context.Context, originCert
 	})
 	retval, err := promise.Result().Struct()
 	if err != nil {
-		return nil, err
+		return NewRetryableRegistrationError(err, defaultRetryAfterSeconds).Serialize()
 	}
-	return UnmarshalTunnelRegistration(retval)
+	registration, err := UnmarshalTunnelRegistration(retval)
+	if err != nil {
+		return NewRetryableRegistrationError(err, defaultRetryAfterSeconds).Serialize()
+	}
+	return registration
 }
 
 func (c TunnelServer_PogsClient) GetServerInfo(ctx context.Context) (*ServerInfo, error) {

tunnelrpc/pogs/tunnelrpc_test.go

@@ -1,6 +1,7 @@
 package pogs
 
 import (
+	"fmt"
 	"reflect"
 	"testing"
 	"time"
@@ -11,16 +12,30 @@ import (
 	capnp "zombiezen.com/go/capnproto2"
 )
 
+const (
+	testURL               = "tunnel.example.com"
+	testTunnelID          = "asdfghjkl;"
+	testRetryAfterSeconds = 19
+)
+
+var (
+	testErr         = fmt.Errorf("Invalid credential")
+	testLogLines    = []string{"all", "working"}
+	testEventDigest = []byte("asdf")
+)
+
+// *PermanentRegistrationError implements TunnelRegistrationError
+var _ TunnelRegistrationError = (*PermanentRegistrationError)(nil)
+
+// *RetryableRegistrationError implements TunnelRegistrationError
+var _ TunnelRegistrationError = (*RetryableRegistrationError)(nil)
+
 func TestTunnelRegistration(t *testing.T) {
 	testCases := []*TunnelRegistration{
-		&TunnelRegistration{
-			Err:               "it broke",
-			Url:               "asdf.cftunnel.com",
-			LogLines:          []string{"it", "was", "broken"},
-			PermanentFailure:  true,
-			TunnelID:          "asdfghjkl;",
-			RetryAfterSeconds: 19,
-		},
+		NewSuccessfulTunnelRegistration(testURL, testLogLines, testTunnelID, testEventDigest),
+		NewSuccessfulTunnelRegistration(testURL, nil, testTunnelID, testEventDigest),
+		NewPermanentRegistrationError(testErr).Serialize(),
+		NewRetryableRegistrationError(testErr, testRetryAfterSeconds).Serialize(),
 	}
 	for i, testCase := range testCases {
 		_, seg, err := capnp.NewMessage(capnp.SingleSegment(nil))

tunnelrpc/tunnelrpc.capnp

@@ -21,6 +21,8 @@ struct TunnelRegistration {
     tunnelID @4 :Text;
     # How long should this connection wait to retry in seconds, if the error wasn't permanent
     retryAfterSeconds @5 :UInt16;
+    # A unique ID used to reconnect this tunnel.
+    eventDigest @6 :Data;
 }
 
 struct RegistrationOptions {
@@ -48,6 +50,8 @@ struct RegistrationOptions {
     uuid @11 :Text;
     # number of previous attempts to send RegisterTunnel/ReconnectTunnel
     numPreviousAttempts @12 :UInt8;
+    # Set of features this cloudflared knows it supports
+    features @13 :List(Text);
 }
 
 struct CapnpConnectParameters {
@@ -291,7 +295,7 @@ interface TunnelServer {
     unregisterTunnel @2 (gracePeriodNanoSec :Int64) -> ();
     connect @3 (parameters :CapnpConnectParameters) -> (result :ConnectResult);
     authenticate @4 (originCert :Data, hostname :Text, options :RegistrationOptions) -> (result :AuthenticateResponse);
-    reconnectTunnel @5 (jwt :Data, hostname :Text, options :RegistrationOptions) -> (result :TunnelRegistration);
+    reconnectTunnel @5 (jwt :Data, eventDigest :Data, hostname :Text, options :RegistrationOptions) -> (result :TunnelRegistration);
 }
 
 interface ClientService {

validation/validation.go

@@ -190,10 +190,11 @@ func ValidateHTTPService(originURL string, hostname string, transport http.Round
 		_, secondErr := client.Do(secondRequest)
 		if secondErr == nil { // Worked this time--advise the user to switch protocols
 			return errors.Errorf(
-				"%s doesn't seem to work over %s, but does seem to work over %s. Consider changing the origin URL to %s",
+				"%s doesn't seem to work over %s, but does seem to work over %s. Reason: %v. Consider changing the origin URL to %s",
 				parsedURL.Host,
 				oldScheme,
 				parsedURL.Scheme,
+				initialErr,
 				parsedURL,
 			)
 		}