Release 2019.9.1

This reverts commit 817c3be9da5465043c2a2fda6c48f7ada760682e.

.gitignore

@@ -9,4 +9,8 @@ guide/public
 \#*\#
 cscope.*
 cloudflared
-!cmd/cloudflared/
+cloudflared.exe
+!cmd/cloudflared/
+.DS_Store
+*-session.log
+ssh_server_tests/.env

Dockerfile

@@ -0,0 +1,12 @@
+FROM golang:1.12 as builder
+WORKDIR /go/src/github.com/cloudflare/cloudflared/
+RUN apt-get update && apt-get install -y --no-install-recommends upx
+# Run after `apt-get update` to improve rebuild scenarios
+COPY . .
+RUN make cloudflared
+RUN upx --no-progress cloudflared
+
+FROM gcr.io/distroless/base
+COPY --from=builder /go/src/github.com/cloudflare/cloudflared/cloudflared /usr/local/bin/
+ENTRYPOINT ["cloudflared", "--no-autoupdate"]
+CMD ["version"]

Gopkg.lock

@@ -9,6 +9,59 @@
   revision = "3012a1dbe2e4bd1391d42b32f0577cb7bbc7f005"
   version = "v0.3.1"
 
+[[projects]]
+  branch = "master"
+  digest = "1:1a200e7e73293b75eb8e5c93d023b5472663432da0b663e1532624fcfede9ca8"
+  name = "github.com/anmitsu/go-shlex"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "648efa622239a2f6ff949fed78ee37b48d499ba4"
+
+[[projects]]
+  digest = "1:4a296b8f487dab7ea37c4558d6e84be8103eb6bd297202e870266ff99ebd769d"
+  name = "github.com/aws/aws-sdk-go"
+  packages = [
+    "aws",
+    "aws/awserr",
+    "aws/awsutil",
+    "aws/client",
+    "aws/client/metadata",
+    "aws/corehandlers",
+    "aws/credentials",
+    "aws/credentials/ec2rolecreds",
+    "aws/credentials/endpointcreds",
+    "aws/credentials/processcreds",
+    "aws/credentials/stscreds",
+    "aws/csm",
+    "aws/defaults",
+    "aws/ec2metadata",
+    "aws/endpoints",
+    "aws/request",
+    "aws/session",
+    "aws/signer/v4",
+    "internal/ini",
+    "internal/s3err",
+    "internal/sdkio",
+    "internal/sdkrand",
+    "internal/sdkuri",
+    "internal/shareddefaults",
+    "private/protocol",
+    "private/protocol/eventstream",
+    "private/protocol/eventstream/eventstreamapi",
+    "private/protocol/json/jsonutil",
+    "private/protocol/query",
+    "private/protocol/query/queryutil",
+    "private/protocol/rest",
+    "private/protocol/restxml",
+    "private/protocol/xml/xmlutil",
+    "service/s3",
+    "service/sts",
+    "service/sts/stsiface",
+  ]
+  pruneopts = "UT"
+  revision = "2eb9a651f22769d792f974e8a8efe16afea3370f"
+  version = "v1.23.9"
+
 [[projects]]
   digest = "1:d6afaeed1502aa28e80a4ed0981d570ad91b2579193404256ce672ed0a609e0d"
   name = "github.com/beorn7/perks"
@@ -73,15 +126,9 @@
   version = "v1.2.0"
 
 [[projects]]
-  digest = "1:6a503e232df389d94ebb97dfb22d4ae463b6e2f351660613e11d9e42f57ab6df"
+  digest = "1:6f70106e7bc1c803e8a0a4519e09c12d154771acfa2559206e97b033bbd1dd38"
   name = "github.com/coreos/go-oidc"
-  packages = [
-    "http",
-    "jose",
-    "key",
-    "oauth2",
-    "oidc",
-  ]
+  packages = ["jose"]
   pruneopts = "UT"
   revision = "a93f71fdfe73d2c0f5413c0565eea0af6523a6df"
 
@@ -94,16 +141,12 @@
   version = "v19"
 
 [[projects]]
-  digest = "1:6fda0d7f5e52b081e075775b1ecebf1ea0c923e7be33604ed0225ae078e701b5"
-  name = "github.com/coreos/pkg"
-  packages = [
-    "health",
-    "httputil",
-    "timeutil",
-  ]
+  digest = "1:385dff5bf83bf1c9d40ef2b0f2e697f0d1973fc9e24ae9401cf41698612bbebc"
+  name = "github.com/creack/pty"
+  packages = ["."]
   pruneopts = "UT"
-  revision = "97fdf19511ea361ae1c100dd393cc47f8dcfa1e1"
-  version = "v4"
+  revision = "2769f65a3a94eb8f876f44a0459d24ae7ad2e488"
+  version = "v1.1.7"
 
 [[projects]]
   digest = "1:ffe9824d294da03b391f44e1ae8281281b4afc1bdaa9588c9097785e3af10cec"
@@ -158,6 +201,14 @@
   pruneopts = "UT"
   revision = "ed7bcb39ff10f39ab08e317ce16df282845852fa"
 
+[[projects]]
+  digest = "1:0a01355005024757ae2a1f62e8fe68a30b4f5fde7a08ac88b3685e7999ac354b"
+  name = "github.com/gliderlabs/ssh"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "5b6cc7030f17095c0cf23bb063b0bfe824fe5f8b"
+  version = "v0.2.2"
+
 [[projects]]
   branch = "master"
   digest = "1:3e6afc3ed8a72949aa735c00fddc23427dc9384ccfd51cf0d91a412e668da632"
@@ -213,12 +264,30 @@
   revision = "8e809c8a86450a29b90dcc9efbf062d0fe6d9746"
 
 [[projects]]
-  digest = "1:75ab90ae3f5d876167e60f493beadfe66f0ed861a710f283fb06c86437a09538"
-  name = "github.com/jonboulle/clockwork"
+  digest = "1:bb81097a5b62634f3e9fec1014657855610c82d19b9a40c17612e32651e35dca"
+  name = "github.com/jmespath/go-jmespath"
   packages = ["."]
   pruneopts = "UT"
-  revision = "2eee05ed794112d45db504eb05aa693efd2b8b09"
-  version = "v0.1.0"
+  revision = "c2b33e84"
+
+[[projects]]
+  digest = "1:6c41d4f998a03b6604227ccad36edaed6126c397e5d78709ef4814a1145a6757"
+  name = "github.com/jmoiron/sqlx"
+  packages = [
+    ".",
+    "reflectx",
+  ]
+  pruneopts = "UT"
+  revision = "d161d7a76b5661016ad0b085869f77fd410f3e6a"
+  version = "v1.2.0"
+
+[[projects]]
+  digest = "1:31e761d97c76151dde79e9d28964a812c46efc5baee4085b86f68f0c654450de"
+  name = "github.com/konsorten/go-windows-terminal-sequences"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "f55edac94c9bbba5d6182a4be46d86a2c9b5b50e"
+  version = "v1.0.2"
 
 [[projects]]
   digest = "1:bc1c0be40c67b6b4aee09d7508d5a2a52c1c116b1fa43806dad2b0d6b4d4003b"
@@ -283,6 +352,14 @@
   revision = "af06845cf3004701891bf4fdb884bfe4920b3727"
   version = "v1.1.0"
 
+[[projects]]
+  digest = "1:53bc4cd4914cd7cd52139990d5170d6dc99067ae31c56530621b18b35fc30318"
+  name = "github.com/mitchellh/mapstructure"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "3536a929edddb9a5b34bd6861dc4a9647cb459fe"
+  version = "v1.1.2"
+
 [[projects]]
   digest = "1:11e62d6050198055e6cd87ed57e5d8c669e84f839c16e16f192374d913d1a70d"
   name = "github.com/opentracing/opentracing-go"
@@ -359,17 +436,23 @@
   version = "v2.4"
 
 [[projects]]
-  digest = "1:5f2aaa360f48d1711795bd88c7e45a38f86cf81e4bc01453d20983baa67e2d51"
+  digest = "1:31538f8d774e8bf2fb9380d2d2a82d69e206a7d0603946586926f1819c546c26"
   name = "github.com/sirupsen/logrus"
-  packages = ["."]
+  packages = [
+    ".",
+    "hooks/test",
+  ]
   pruneopts = "UT"
-  revision = "f006c2ac4710855cf0f916dd6b77acf6b048dc6e"
-  version = "v1.0.3"
+  revision = "839c75faf7f98a33d445d181f3018b5c3409a45e"
+  version = "v1.4.2"
 
 [[projects]]
-  digest = "1:f85e109eda8f6080877185d1c39e98dd8795e1780c08beca28304b87fd855a1c"
+  digest = "1:7e8d267900c7fa7f35129a2a37596e38ed0f11ca746d6d9ba727980ee138f9f6"
   name = "github.com/stretchr/testify"
-  packages = ["assert"]
+  packages = [
+    "assert",
+    "require",
+  ]
   pruneopts = "UT"
   revision = "12b6f73e6084dad08a7c6e575284b177ecafbc71"
   version = "v1.2.1"
@@ -559,6 +642,11 @@
   analyzer-name = "dep"
   analyzer-version = 1
   input-imports = [
+    "github.com/DATA-DOG/go-sqlmock",
+    "github.com/aws/aws-sdk-go/aws",
+    "github.com/aws/aws-sdk-go/aws/credentials",
+    "github.com/aws/aws-sdk-go/aws/session",
+    "github.com/aws/aws-sdk-go/service/s3",
     "github.com/cloudflare/brotli-go",
     "github.com/cloudflare/golibs/lrucache",
     "github.com/coredns/coredns/core/dnsserver",
@@ -569,12 +657,13 @@
     "github.com/coredns/coredns/plugin/pkg/rcode",
     "github.com/coredns/coredns/request",
     "github.com/coreos/go-oidc/jose",
-    "github.com/coreos/go-oidc/oidc",
     "github.com/coreos/go-systemd/daemon",
+    "github.com/creack/pty",
     "github.com/elgs/gosqljson",
     "github.com/equinox-io/equinox",
     "github.com/facebookgo/grace/gracenet",
     "github.com/getsentry/raven-go",
+    "github.com/gliderlabs/ssh",
     "github.com/golang-collections/collections/queue",
     "github.com/google/uuid",
     "github.com/gorilla/mux",
@@ -583,12 +672,15 @@
     "github.com/mattn/go-colorable",
     "github.com/miekg/dns",
     "github.com/mitchellh/go-homedir",
+    "github.com/mitchellh/mapstructure",
     "github.com/pkg/errors",
     "github.com/prometheus/client_golang/prometheus",
     "github.com/prometheus/client_golang/prometheus/promhttp",
     "github.com/rifflock/lfshook",
     "github.com/sirupsen/logrus",
+    "github.com/sirupsen/logrus/hooks/test",
     "github.com/stretchr/testify/assert",
+    "github.com/stretchr/testify/require",
     "golang.org/x/crypto/nacl/box",
     "golang.org/x/crypto/ssh",
     "golang.org/x/crypto/ssh/terminal",

Gopkg.toml

@@ -26,7 +26,7 @@
 
 [[constraint]]
   name = "github.com/sirupsen/logrus"
-  version = "=1.0.3"
+  version = "=1.4.2"
 
 [[constraint]]
   name = "github.com/stretchr/testify"
@@ -85,3 +85,19 @@
 [[constraint]]
   name = "github.com/google/uuid"
   version = "=1.1.1"
+
+[[constraint]]
+  name = "github.com/mitchellh/mapstructure"
+  version = "1.1.2"
+
+[[constraint]]
+  name = "github.com/gliderlabs/ssh"
+  version = "0.2.2"
+
+[[constraint]]
+  name = "github.com/creack/pty"
+  version = "1.1.7"
+
+[[constraint]]
+  name = "github.com/aws/aws-sdk-go"
+  version = "1.23.9"

Makefile

@@ -31,10 +31,18 @@ clean:
 cloudflared: tunnel-deps
 	go build -v $(VERSION_FLAGS) $(IMPORT_PATH)/cmd/cloudflared
 
+.PHONY: container
+container:
+	docker build -t cloudflare/cloudflared:"$(VERSION)" .
+
 .PHONY: test
 test: vet
 	go test -v -race $(VERSION_FLAGS) ./...
 
+.PHONY: test-ssh-server
+test-ssh-server:
+	docker-compose -f ssh_server_tests/docker-compose.yml up
+
 .PHONY: cloudflared-deb
 cloudflared-deb: cloudflared
 	mkdir -p $(PACKAGE_DIR)
@@ -70,7 +78,7 @@ tunnel-deps: tunnelrpc/tunnelrpc.capnp.go
 tunnelrpc/tunnelrpc.capnp.go: tunnelrpc/tunnelrpc.capnp
 	which capnp  # https://capnproto.org/install.html
 	which capnpc-go  # go get zombiezen.com/go/capnproto2/capnpc-go
-	capnp compile -ogo -I ./tunnelrpc tunnelrpc/tunnelrpc.capnp
+	capnp compile -ogo tunnelrpc/tunnelrpc.capnp
 
 .PHONY: vet
 vet:

RELEASE_NOTES

@@ -1,3 +1,87 @@
+2019.9.1
+- 2019-09-23 TUN-2334: remove tlsConfig.ServerName special case
+- 2019-09-23 AUTH-2077: Quotes open browser command in windows
+- 2019-09-11 AUTH-2050: Adds time.sleep to temporarily avoid hitting tunnel muxer dealock issue
+- 2019-09-10 AUTH-2056: Writes stderr to its own stream for non-pty connections
+- 2019-09-16 TUN-2307: Capnp is the only serialization format used in tunnelpogs
+- 2019-09-18 TUN-2315: Replace Scope with IntentLabel
+- 2019-09-17 TUN-2309: Split ConnectResult into ConnectError and ConnectSuccess, each implementing its own capnp serialization logic
+- 2019-09-18 AUTH-2052: Adds tests for SSH server
+- 2019-09-18 AUTH-2067: Log commands correctly
+- 2019-09-19 AUTH-2055: Verifies token at edge on access login
+- 2019-09-04 TUN-2201: change SRV records used by cloudflared
+- 2019-09-06 TUN-2280: Revert "TUN-2260: add name/group to CapnpConnectParameters, remove Scope"
+- 2019-09-03 AUTH-1943 hooked up uploader to logger, added timestamp to session logs, add tests
+- 2019-09-04 AUTH-2036: Refactor user retrieval, shutdown after ssh server stops, add custom version string
+- 2019-09-06 AUTH-1942 added event log to ssh server
+- 2019-09-04 AUTH-2037: Adds support for ssh port forwarding
+- 2019-09-05 TUN-2276: Path encoding broken
+
+2019.9.0
+- 2019-09-05 TUN-2279: Revert path encoding fix
+- 2019-08-30 AUTH-2021 - check error for failing tests
+- 2019-08-29 AUTH-2030: Support both authorized_key and short lived cert authentication simultaniously without specifiying at start time
+- 2019-08-29 AUTH-2026: Adds support for non-pty sessions and inline command exec
+- 2019-08-26 AUTH-1943: Adds session logging
+- 2019-08-26 TUN-2162: Decomplect OpenStream to allow finer-grained timeouts
+- 2019-08-29 TUN-2260: add name/group to CapnpConnectParameters, remove Scope
+
+2019.8.4
+- 2019-08-30 Fix #111: Add support for specifying a specific HTTP Host: header on the origin. (#114)
+- 2019-08-22 TUN-2165: Add ClientConfig to tunnelrpc.ConnectResult
+- 2019-08-20 AUTH-2014: Checks users login shell
+- 2019-08-26 TUN-2243: Revert "STOR-519: Add db-connect, a SQL over HTTPS server"
+- 2019-08-27 TUN-2244: Add NO_AUTOUPDATE env var
+- 2019-08-22 AUTH-2018: Adds support for authorized keys and short lived certs
+- 2019-08-28 AUTH-2022: Adds ssh timeout configuration
+- 2019-08-28 TUN-1968: Gracefully diff StreamHandler.UpdateConfig
+- 2019-08-26 AUTH-2021 - s3 bucket uploading for SSH logs
+- 2019-08-19 AUTH-2004: Adds static host key support
+- 2019-07-18 AUTH-1941: Adds initial SSH server implementation
+
+2019.8.3
+- 2019-08-20 STOR-519: Add db-connect, a SQL over HTTPS server
+- 2019-08-20 Release 2019.8.2
+- 2019-08-20 Revert "AUTH-1941: Adds initial SSH server implementation"
+- 2019-08-11 TUN-2163: Add GrapQLType method to Scope interface
+- 2019-08-06 TUN-2152: Requests with a query in the URL are erroneously escaped
+- 2019-07-18 AUTH-1941: Adds initial SSH server implementation
+
+2019.8.1
+- 2019-08-05 TUN-2111: Implement custom serialization logic for FallibleConfig and OriginConfig
+- 2019-08-06 Revert "TUN-1736: Missing headers when passing an invalid path"
+
+2019.8.0
+- 2019-07-11 TUN-1956: Go 1.12 update
+- 2019-07-24 TUN-1736: Missing headers when passing an invalid path
+- 2019-07-30 TUN-2117: read group/system-name from CLI, send it to edge
+- 2019-08-02 TUN-2125: Add PostgresType() to Scope
+- 2019-08-05 TUN-2147: Implemented ScopeUnmarshaler
+- 2019-07-31 TUN-2110: Implement custom deserialization logic for OriginConfig
+- 2019-07-31 AUTH-1972: Deletes token lock file if backoff retry attempts exceeded and intercepts signals until lock is released
+
+2019.7.0
+- 2019-05-28 TUN-1913: Define OriginService for each type of origin
+- 2019-04-29 Build a docker container
+- 2019-06-12 TUN-1952: Group ClientConfig fields by the component that uses the config, and return the part of the config that failed to be applied
+- 2019-06-05 TUN-1893: Proxy requests to the origin based on tunnel hostname
+- 2019-06-17 TUN-1961: Create EdgeConnectionManager to maintain outbound connections to the edge
+- 2019-06-18 TUN-1885: Reconfigure cloudflared on receiving new ClientConfig
+- 2019-06-19 TUN-1976: Pass tunnel hostname through header
+- 2019-06-20 TUN-1982: Load custom origin CA when OriginCAPool is specified
+- 2019-06-26 TUN-2005: Upgrade logrus
+- 2019-06-20 TUN-1981: Write response header & body on proxy error to notify eyeballs of failure category
+- 2019-06-20 TUN-1977: Validate OriginConfig has valid URL, and use scheme to determine if a HTTPOriginService is expecting HTTP or Unix
+- 2019-06-13 DoH: change the media type to application/dns-message
+- 2019-06-26 AUTH-1736: Better handling of token revocation
+
+2019.6.0
+- 2019-05-17 TUN-1828: Update declarative tunnel config struct
+- 2019-05-29 Handle exit code on err
+- 2019-05-29 AUTH-1802: Fixed ssh-config templating
+- 2019-05-30 TUN-1914: Conflate HTTP and Unix OriginConfig, and add TLS config to WebSocketOriginConfig
+- 2019-06-03 AUTH-1811: ssh-gen config fixes
+
 2019.5.0
 - 2019-04-25 TUN-1781: ServeStream should return early on error
 - 2019-04-30 TUN-1786: Remove low-level Windows service logging

awsuploader/directory_upload_manager.go

@@ -0,0 +1,106 @@
+package awsuploader
+
+import (
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/sirupsen/logrus"
+)
+
+// DirectoryUploadManager is used to manage file uploads on an interval from a directory
+type DirectoryUploadManager struct {
+	logger        *logrus.Logger
+	uploader      Uploader
+	rootDirectory string
+	sweepInterval time.Duration
+	ticker        *time.Ticker
+	shutdownC     chan struct{}
+	workQueue     chan string
+}
+
+// NewDirectoryUploadManager create a new DirectoryUploadManager
+// uploader is an Uploader to use as an actual uploading engine
+// directory is the directory to sweep for files to upload
+// sweepInterval is how often to iterate the directory and upload the files within
+func NewDirectoryUploadManager(logger *logrus.Logger, uploader Uploader, directory string, sweepInterval time.Duration, shutdownC chan struct{}) *DirectoryUploadManager {
+	workerCount := 10
+	manager := &DirectoryUploadManager{
+		logger:        logger,
+		uploader:      uploader,
+		rootDirectory: directory,
+		sweepInterval: sweepInterval,
+		shutdownC:     shutdownC,
+		workQueue:     make(chan string, workerCount),
+	}
+
+	//start workers
+	for i := 0; i < workerCount; i++ {
+		go manager.worker()
+	}
+
+	return manager
+}
+
+// Upload a file using the uploader
+// This is useful for "out of band" uploads that need to be triggered immediately instead of waiting for the sweep
+func (m *DirectoryUploadManager) Upload(filepath string) error {
+	return m.uploader.Upload(filepath)
+}
+
+// Start the upload ticker to walk the directories
+func (m *DirectoryUploadManager) Start() {
+	m.ticker = time.NewTicker(m.sweepInterval)
+	go m.run()
+}
+
+func (m *DirectoryUploadManager) run() {
+	for {
+		select {
+		case <-m.shutdownC:
+			m.ticker.Stop()
+			return
+		case <-m.ticker.C:
+			m.sweep()
+		}
+	}
+}
+
+// sweep the directory and kick off uploads
+func (m *DirectoryUploadManager) sweep() {
+	filepath.Walk(m.rootDirectory, func(path string, info os.FileInfo, err error) error {
+		if err != nil || info.IsDir() {
+			return nil
+		}
+		//30 days ago
+		retentionTime := 30 * (time.Hour * 24)
+		checkTime := time.Now().Add(-time.Duration(retentionTime))
+
+		//delete the file it is stale
+		if info.ModTime().Before(checkTime) {
+			os.Remove(path)
+			return nil
+		}
+		//add the upload to the work queue
+		go func() {
+			m.workQueue <- path
+		}()
+		return nil
+	})
+}
+
+// worker handles upload requests
+func (m *DirectoryUploadManager) worker() {
+	for {
+		select {
+		case <-m.shutdownC:
+			return
+		case filepath := <-m.workQueue:
+			if err := m.Upload(filepath); err != nil {
+				m.logger.WithError(err).Error("Cannot upload file to s3 bucket")
+			} else {
+				os.Remove(filepath)
+			}
+		}
+	}
+}

awsuploader/directory_upload_manager_test.go

@@ -0,0 +1,137 @@
+package awsuploader
+
+import (
+	"errors"
+	"io/ioutil"
+	"math/rand"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/sirupsen/logrus"
+)
+
+type MockUploader struct {
+	shouldFail bool
+}
+
+func (m *MockUploader) Upload(filepath string) error {
+	if m.shouldFail {
+		return errors.New("upload set to fail")
+	}
+	return nil
+}
+
+func NewMockUploader(shouldFail bool) Uploader {
+	return &MockUploader{shouldFail: shouldFail}
+}
+
+func getDirectoryPath(t *testing.T) string {
+	dir, err := os.Getwd()
+	if err != nil {
+		t.Fatal("couldn't create the test directory!", err)
+	}
+	return filepath.Join(dir, "uploads")
+}
+
+func setupTestDirectory(t *testing.T) string {
+	path := getDirectoryPath(t)
+	os.RemoveAll(path)
+	time.Sleep(100 * time.Millisecond) //short way to wait for the OS to delete the folder
+	err := os.MkdirAll(path, os.ModePerm)
+	if err != nil {
+		t.Fatal("couldn't create the test directory!", err)
+	}
+	return path
+}
+
+func createUploadManager(t *testing.T, shouldFailUpload bool) *DirectoryUploadManager {
+	rootDirectory := setupTestDirectory(t)
+	uploader := NewMockUploader(shouldFailUpload)
+	logger := logrus.New()
+	shutdownC := make(chan struct{})
+	return NewDirectoryUploadManager(logger, uploader, rootDirectory, 1*time.Second, shutdownC)
+}
+
+func createFile(t *testing.T, fileName string) (*os.File, string) {
+	path := filepath.Join(getDirectoryPath(t), fileName)
+	f, err := os.Create(path)
+	if err != nil {
+		t.Fatal("upload to create file for sweep test", err)
+	}
+	return f, path
+}
+
+func TestUploadSuccess(t *testing.T) {
+	manager := createUploadManager(t, false)
+	path := filepath.Join(getDirectoryPath(t), "test_file")
+	if err := manager.Upload(path); err != nil {
+		t.Fatal("the upload request method failed", err)
+	}
+}
+
+func TestUploadFailure(t *testing.T) {
+	manager := createUploadManager(t, true)
+	path := filepath.Join(getDirectoryPath(t), "test_file")
+	if err := manager.Upload(path); err == nil {
+		t.Fatal("the upload request method should have failed and didn't", err)
+	}
+}
+
+func TestSweepSuccess(t *testing.T) {
+	manager := createUploadManager(t, false)
+	f, path := createFile(t, "test_file")
+	defer f.Close()
+
+	manager.Start()
+	time.Sleep(2 * time.Second)
+	if _, err := os.Stat(path); os.IsExist(err) {
+		//the file should have been deleted
+		t.Fatal("the manager failed to delete the file", err)
+	}
+}
+
+func TestSweepFailure(t *testing.T) {
+	manager := createUploadManager(t, true)
+	f, path := createFile(t, "test_file")
+	defer f.Close()
+
+	manager.Start()
+	time.Sleep(2 * time.Second)
+	_, serr := f.Stat()
+	if serr != nil {
+		//the file should still exist
+		os.Remove(path)
+		t.Fatal("the manager failed to delete the file", serr)
+	}
+}
+
+func TestHighLoad(t *testing.T) {
+	manager := createUploadManager(t, false)
+	for i := 0; i < 30; i++ {
+		f, _ := createFile(t, randomString(6))
+		defer f.Close()
+	}
+	manager.Start()
+	time.Sleep(4 * time.Second)
+
+	directory := getDirectoryPath(t)
+	files, err := ioutil.ReadDir(directory)
+	if err != nil || len(files) > 0 {
+		t.Fatalf("the manager failed to upload all the files: %s files left: %d", err, len(files))
+	}
+}
+
+// LowerCase [a-z]
+const randSet = "abcdefghijklmnopqrstuvwxyz"
+
+// String returns a string of length 'n' from a set of letters 'lset'
+func randomString(n int) string {
+	b := make([]byte, n)
+	lsetLen := len(randSet)
+	for i := range b {
+		b[i] = randSet[rand.Intn(lsetLen)]
+	}
+	return string(b)
+}

awsuploader/file_uploader.go

@@ -0,0 +1,62 @@
+package awsuploader
+
+import (
+	"context"
+	"os"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+)
+
+//FileUploader aws compliant bucket upload
+type FileUploader struct {
+	storage    *s3.S3
+	bucketName string
+	clientID   string
+	secretID   string
+}
+
+// NewFileUploader creates a new S3 compliant bucket uploader
+func NewFileUploader(bucketName, region, accessKeyID, secretID, token, s3Host string) (*FileUploader, error) {
+	sess, err := session.NewSession(&aws.Config{
+		Region:      aws.String(region),
+		Credentials: credentials.NewStaticCredentials(accessKeyID, secretID, token),
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	var storage *s3.S3
+	if s3Host != "" {
+		storage = s3.New(sess, &aws.Config{Endpoint: aws.String(s3Host)})
+	} else {
+		storage = s3.New(sess)
+	}
+
+	return &FileUploader{
+		storage:    storage,
+		bucketName: bucketName,
+	}, nil
+}
+
+// Upload a file to the bucket
+func (u *FileUploader) Upload(filepath string) error {
+	info, err := os.Stat(filepath)
+	if err != nil {
+		return err
+	}
+	file, err := os.Open(filepath)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	_, serr := u.storage.PutObjectWithContext(context.Background(), &s3.PutObjectInput{
+		Bucket: aws.String(u.bucketName),
+		Key:    aws.String(info.Name()),
+		Body:   file,
+	})
+	return serr
+}

awsuploader/upload_manager.go

@@ -0,0 +1,7 @@
+package awsuploader
+
+// UploadManager is used to manage file uploads on an interval
+type UploadManager interface {
+	Upload(string) error
+	Start()
+}

awsuploader/uploader.go

@@ -0,0 +1,7 @@
+package awsuploader
+
+// Uploader the functions required to upload to a bucket
+type Uploader interface {
+	//Upload a file to the bucket
+	Upload(string) error
+}

carrier/carrier.go

@@ -4,7 +4,6 @@
 package carrier
 
 import (
-	"errors"
 	"io"
 	"net"
 	"net/http"
@@ -12,7 +11,8 @@ import (
 	"strings"
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/token"
-	"github.com/cloudflare/cloudflared/websocket"
+	cloudflaredWebsocket "github.com/cloudflare/cloudflared/websocket"
+	"github.com/gorilla/websocket"
 	"github.com/sirupsen/logrus"
 )
 
@@ -37,6 +37,13 @@ func (c *StdinoutStream) Write(p []byte) (int, error) {
 	return os.Stdout.Write(p)
 }
 
+// Helper to allow defering the response close with a check that the resp is not nil
+func closeRespBody(resp *http.Response) {
+	if resp != nil {
+		resp.Body.Close()
+	}
+}
+
 // StartClient will copy the data from stdin/stdout over a WebSocket connection
 // to the edge (originURL)
 func StartClient(logger *logrus.Logger, stream io.ReadWriter, options *StartOptions) error {
@@ -90,7 +97,7 @@ func serveStream(logger *logrus.Logger, conn io.ReadWriter, options *StartOption
 	}
 	defer wsConn.Close()
 
-	websocket.Stream(wsConn, conn)
+	cloudflaredWebsocket.Stream(wsConn, conn)
 
 	return nil
 }
@@ -98,28 +105,17 @@ func serveStream(logger *logrus.Logger, conn io.ReadWriter, options *StartOption
 // createWebsocketStream will create a WebSocket connection to stream data over
 // It also handles redirects from Access and will present that flow if
 // the token is not present on the request
-func createWebsocketStream(options *StartOptions) (*websocket.Conn, error) {
+func createWebsocketStream(options *StartOptions) (*cloudflaredWebsocket.Conn, error) {
 	req, err := http.NewRequest(http.MethodGet, options.OriginURL, nil)
 	if err != nil {
 		return nil, err
 	}
 	req.Header = options.Headers
 
-	wsConn, resp, err := websocket.ClientConnect(req, nil)
-	if err != nil && resp != nil && resp.StatusCode > 300 {
-		location, err := resp.Location()
-		if err != nil {
-			return nil, err
-		}
-		if !strings.Contains(location.String(), "cdn-cgi/access/login") {
-			return nil, errors.New("not an Access redirect")
-		}
-		req, err := buildAccessRequest(options.OriginURL)
-		if err != nil {
-			return nil, err
-		}
-
-		wsConn, _, err = websocket.ClientConnect(req, nil)
+	wsConn, resp, err := cloudflaredWebsocket.ClientConnect(req, nil)
+	defer closeRespBody(resp)
+	if err != nil && IsAccessResponse(resp) {
+		wsConn, err = createAccessAuthenticatedStream(options)
 		if err != nil {
 			return nil, err
 		}
@@ -127,12 +123,72 @@ func createWebsocketStream(options *StartOptions) (*websocket.Conn, error) {
 		return nil, err
 	}
 
-	return &websocket.Conn{Conn: wsConn}, nil
+	return &cloudflaredWebsocket.Conn{Conn: wsConn}, nil
+}
+
+// IsAccessResponse checks the http Response to see if the url location
+// contains the Access structure.
+func IsAccessResponse(resp *http.Response) bool {
+	if resp == nil || resp.StatusCode != http.StatusFound {
+		return false
+	}
+
+	location, err := resp.Location()
+	if err != nil || location == nil {
+		return false
+	}
+	if strings.HasPrefix(location.Path, "/cdn-cgi/access/login") {
+		return true
+	}
+
+	return false
+}
+
+// createAccessAuthenticatedStream will try load a token from storage and make
+// a connection with the token set on the request. If it still get redirect,
+// this probably means the token in storage is invalid (expired/revoked). If that
+// happens it deletes the token and runs the connection again, so the user can
+// login again and generate a new one.
+func createAccessAuthenticatedStream(options *StartOptions) (*websocket.Conn, error) {
+	wsConn, resp, err := createAccessWebSocketStream(options)
+	defer closeRespBody(resp)
+	if err == nil {
+		return wsConn, nil
+	}
+
+	if !IsAccessResponse(resp) {
+		return nil, err
+	}
+
+	// Access Token is invalid for some reason. Go through regen flow
+	originReq, err := http.NewRequest(http.MethodGet, options.OriginURL, nil)
+	if err != nil {
+		return nil, err
+	}
+	if err := token.RemoveTokenIfExists(originReq.URL); err != nil {
+		return nil, err
+	}
+	wsConn, resp, err = createAccessWebSocketStream(options)
+	defer closeRespBody(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	return wsConn, nil
+}
+
+// createAccessWebSocketStream builds an Access request and makes a connection
+func createAccessWebSocketStream(options *StartOptions) (*websocket.Conn, *http.Response, error) {
+	req, err := BuildAccessRequest(options)
+	if err != nil {
+		return nil, nil, err
+	}
+	return cloudflaredWebsocket.ClientConnect(req, nil)
 }
 
 // buildAccessRequest builds an HTTP request with the Access token set
-func buildAccessRequest(originURL string) (*http.Request, error) {
-	req, err := http.NewRequest(http.MethodGet, originURL, nil)
+func BuildAccessRequest(options *StartOptions) (*http.Request, error) {
+	req, err := http.NewRequest(http.MethodGet, options.OriginURL, nil)
 	if err != nil {
 		return nil, err
 	}
@@ -144,11 +200,17 @@ func buildAccessRequest(originURL string) (*http.Request, error) {
 
 	// We need to create a new request as FetchToken will modify req (boo mutable)
 	// as it has to follow redirect on the API and such, so here we init a new one
-	originRequest, err := http.NewRequest(http.MethodGet, originURL, nil)
+	originRequest, err := http.NewRequest(http.MethodGet, options.OriginURL, nil)
 	if err != nil {
 		return nil, err
 	}
 	originRequest.Header.Set("cf-access-token", token)
 
+	for k, v := range options.Headers {
+		if len(v) >= 1 {
+			originRequest.Header.Set(k, v[0])
+		}
+	}
+
 	return originRequest, nil
 }

carrier/carrier_test.go

@@ -91,6 +91,31 @@ func TestStartServer(t *testing.T) {
 	assert.Equal(t, string(readBuffer), message)
 }
 
+func TestIsAccessResponse(t *testing.T) {
+	validLocationHeader := http.Header{}
+	validLocationHeader.Add("location", "https://test.cloudflareaccess.com/cdn-cgi/access/login/blahblah")
+	invalidLocationHeader := http.Header{}
+	invalidLocationHeader.Add("location", "https://google.com")
+	testCases := []struct {
+		Description string
+		In          *http.Response
+		ExpectedOut bool
+	}{
+		{"nil response", nil, false},
+		{"redirect with no location", &http.Response{StatusCode: http.StatusFound}, false},
+		{"200 ok", &http.Response{StatusCode: http.StatusOK}, false},
+		{"redirect with location", &http.Response{StatusCode: http.StatusFound, Header: validLocationHeader}, true},
+		{"redirect with invalid location", &http.Response{StatusCode: http.StatusFound, Header: invalidLocationHeader}, false},
+	}
+
+	for i, tc := range testCases {
+		if IsAccessResponse(tc.In) != tc.ExpectedOut {
+			t.Fatalf("Failed case %d -- %s", i, tc.Description)
+		}
+	}
+
+}
+
 func newTestWebSocketServer() *httptest.Server {
 	upgrader := ws.Upgrader{
 		ReadBufferSize:  1024,

cfsetup.yaml

@@ -1,4 +1,4 @@
-pinned_go: &pinned_go go=1.11.5-1
+pinned_go: &pinned_go go=1.12.7-1
 build_dir: &build_dir /cfsetup_build/src/github.com/cloudflare/cloudflared/
 default-flavor: stretch
 stretch: &stretch

cmd/cloudflared/access/cmd.go

@@ -1,17 +1,20 @@
 package access
 
 import (
-	"errors"
 	"fmt"
-	"html/template"
+	"net/http"
 	"net/url"
 	"os"
 	"strings"
+	"text/template"
+	"time"
 
+	"github.com/cloudflare/cloudflared/carrier"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/shell"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/token"
 	"github.com/cloudflare/cloudflared/sshgen"
 	"github.com/cloudflare/cloudflared/validation"
+	"github.com/pkg/errors"
 	"golang.org/x/net/idna"
 
 	"github.com/cloudflare/cloudflared/log"
@@ -27,19 +30,19 @@ const (
 	sshTokenSecretFlag = "service-token-secret"
 	sshGenCertFlag     = "short-lived-cert"
 	sshConfigTemplate  = `
-Add this configuration block to your {{.Home}}/.ssh/config:
+Add to your {{.Home}}/.ssh/config:
 
 Host {{.Hostname}}
 {{- if .ShortLivedCerts}}
-	ProxyCommand bash -c '{{.Cloudflared}} access ssh-gen --hostname %h; ssh -tt cfpipe >&2 <&1' 
+  ProxyCommand bash -c '{{.Cloudflared}} access ssh-gen --hostname %h; ssh -tt %r@cfpipe-{{.Hostname}} >&2 <&1' 
 
 Host cfpipe-{{.Hostname}}
-	HostName {{.Hostname}}
-	ProxyCommand {{.Cloudflared}} access ssh --hostname %h
-	IdentityFile ~/.cloudflared/{{.Hostname}}.me-cf_key
-	CertificateFile ~/.cloudflared/{{.Hostname}}-cf_key-cert.pub
+  HostName {{.Hostname}}
+  ProxyCommand {{.Cloudflared}} access ssh --hostname %h
+  IdentityFile ~/.cloudflared/{{.Hostname}}-cf_key
+  CertificateFile ~/.cloudflared/{{.Hostname}}-cf_key-cert.pub
 {{- else}}
-	ProxyCommand {{.Cloudflared}} access ssh --hostname %h
+  ProxyCommand {{.Cloudflared}} access ssh --hostname %h
 {{end}}
 `
 )
@@ -188,9 +191,14 @@ func login(c *cli.Context) error {
 		logger.Errorf("Please provide the url of the Access application\n")
 		return err
 	}
-	token, err := token.FetchToken(appURL)
-	if err != nil {
-		logger.Errorf("Failed to fetch token: %s\n", err)
+	if err := verifyTokenAtEdge(appURL, c); err != nil {
+		logger.WithError(err).Error("Could not verify token")
+		return err
+	}
+
+	token, err := token.GetTokenIfExists(appURL)
+	if err != nil || token == "" {
+		fmt.Fprintln(os.Stderr, "Unable to find token for provided application.")
 		return err
 	}
 	fmt.Fprintf(os.Stdout, "Successfully fetched your token:\n\n%s\n\n", string(token))
@@ -372,3 +380,59 @@ func isFileThere(candidate string) bool {
 	}
 	return true
 }
+
+// verifyTokenAtEdge checks for a token on disk, or generates a new one.
+// Then makes a request to to the origin with the token to ensure it is valid.
+// Returns nil if token is valid.
+func verifyTokenAtEdge(appUrl *url.URL, c *cli.Context) error {
+	headers := buildRequestHeaders(c.StringSlice(sshHeaderFlag))
+	if c.IsSet(sshTokenIDFlag) {
+		headers.Add("CF-Access-Client-Id", c.String(sshTokenIDFlag))
+	}
+	if c.IsSet(sshTokenSecretFlag) {
+		headers.Add("CF-Access-Client-Secret", c.String(sshTokenSecretFlag))
+	}
+	options := &carrier.StartOptions{OriginURL: appUrl.String(), Headers: headers}
+
+	if valid, err := isTokenValid(options); err != nil {
+		return err
+	} else if valid {
+		return nil
+	}
+
+	if err := token.RemoveTokenIfExists(appUrl); err != nil {
+		return err
+	}
+
+	if valid, err := isTokenValid(options); err != nil {
+		return err
+	} else if !valid {
+		return errors.New("failed to verify token")
+	}
+
+	return nil
+}
+
+// isTokenValid makes a request to the origin and returns true if the response was not a 302.
+func isTokenValid(options *carrier.StartOptions) (bool, error) {
+	req, err := carrier.BuildAccessRequest(options)
+	if err != nil {
+		return false, errors.Wrap(err, "Could not create access request")
+	}
+
+	// Do not follow redirects
+	client := &http.Client{
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			return http.ErrUseLastResponse
+		},
+		Timeout: time.Second * 5,
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		return false, err
+	}
+	defer resp.Body.Close()
+
+	// A redirect to login means the token was invalid.
+	return !carrier.IsAccessResponse(resp), nil
+}

cmd/cloudflared/buildinfo/build_info.go

@@ -0,0 +1,28 @@
+package buildinfo
+
+import (
+	"runtime"
+
+	"github.com/sirupsen/logrus"
+)
+
+type BuildInfo struct {
+	GoOS               string `json:"go_os"`
+	GoVersion          string `json:"go_version"`
+	GoArch             string `json:"go_arch"`
+	CloudflaredVersion string `json:"cloudflared_version"`
+}
+
+func GetBuildInfo(cloudflaredVersion string) *BuildInfo {
+	return &BuildInfo{
+		GoOS:               runtime.GOOS,
+		GoVersion:          runtime.Version(),
+		GoArch:             runtime.GOARCH,
+		CloudflaredVersion: cloudflaredVersion,
+	}
+}
+
+func (bi *BuildInfo) Log(logger *logrus.Logger) {
+	logger.Infof("Version %s", bi.CloudflaredVersion)
+	logger.Infof("GOOS: %s, GOVersion: %s, GoArch: %s", bi.GoOS, bi.GoVersion, bi.GoArch)
+}

cmd/cloudflared/main.go

@@ -127,10 +127,14 @@ func action(version string, shutdownC, graceShutdownC chan struct{}) cli.ActionF
 		tags["hostname"] = c.String("hostname")
 		raven.SetTagsContext(tags)
 		raven.CapturePanic(func() { err = tunnel.StartServer(c, version, shutdownC, graceShutdownC) }, nil)
+		exitCode := 0
 		if err != nil {
 			handleError(err)
+			exitCode = 1
 		}
-		return err
+		// we already handle error printing, so we pass an empty string so we
+		// don't have to print again.
+		return cli.Exit("", exitCode)
 	}
 }
 

cmd/cloudflared/shell/launch_browser_darwin.go

@@ -0,0 +1,11 @@
+//+build darwin
+
+package shell
+
+import (
+	"os/exec"
+)
+
+func getBrowserCmd(url string) *exec.Cmd {
+	return exec.Command("open", url)
+}

cmd/cloudflared/shell/launch_browser_other.go

@@ -0,0 +1,11 @@
+//+build !windows,!darwin,!linux,!netbsd,!freebsd,!openbsd
+
+package shell
+
+import (
+	"os/exec"
+)
+
+func getBrowserCmd(url string) *exec.Cmd {
+	return nil
+}

cmd/cloudflared/shell/launch_browser_unix.go

@@ -0,0 +1,11 @@
+//+build linux freebsd openbsd netbsd
+
+package shell
+
+import (
+	"os/exec"
+)
+
+func getBrowserCmd(url string) *exec.Cmd {
+	return exec.Command("xdg-open", url)
+}

cmd/cloudflared/shell/launch_browser_windows.go

@@ -0,0 +1,18 @@
+//+build windows
+
+package shell
+
+import (
+	"fmt"
+	"os/exec"
+	"syscall"
+)
+
+func getBrowserCmd(url string) *exec.Cmd {
+	cmd := exec.Command("cmd")
+	// CmdLine is only defined when compiling for windows.
+	// Empty string is the cmd proc "Title". Needs to be included because the start command will interpret the first
+	// quoted string as that field and we want to quote the URL.
+	cmd.SysProcAttr = &syscall.SysProcAttr{CmdLine: fmt.Sprintf(`/c start "" "%s"`, url)}
+	return cmd
+}

cmd/cloudflared/shell/shell.go

@@ -4,25 +4,11 @@ import (
 	"io"
 	"os"
 	"os/exec"
-	"runtime"
 )
 
 // OpenBrowser opens the specified URL in the default browser of the user
 func OpenBrowser(url string) error {
-	var cmd string
-	var args []string
-
-	switch runtime.GOOS {
-	case "windows":
-		cmd = "cmd"
-		args = []string{"/c", "start"}
-	case "darwin":
-		cmd = "open"
-	default: // "linux", "freebsd", "openbsd", "netbsd"
-		cmd = "xdg-open"
-	}
-	args = append(args, url)
-	return exec.Command(cmd, args...).Start()
+	return getBrowserCmd(url).Start()
 }
 
 // Run will kick off a shell task and pipe the results to the respective std pipes

cmd/cloudflared/token/token.go

@@ -1,15 +1,20 @@
 package token
 
 import (
+	"context"
+	"fmt"
 	"io/ioutil"
 	"net/url"
-	"time"
+	"os"
+	"os/signal"
+	"syscall"
 
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/path"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/transfer"
 	"github.com/cloudflare/cloudflared/log"
+	"github.com/cloudflare/cloudflared/origin"
 	"github.com/coreos/go-oidc/jose"
-	"github.com/coreos/go-oidc/oidc"
 )
 
 const (
@@ -18,6 +23,95 @@ const (
 
 var logger = log.CreateLogger()
 
+type lock struct {
+	lockFilePath string
+	backoff      *origin.BackoffHandler
+	sigHandler 	 *signalHandler
+}
+
+type signalHandler struct {
+	sigChannel 		 chan os.Signal
+	signals		     []os.Signal
+}
+
+func (s *signalHandler) register(handler func()){
+	s.sigChannel = make(chan os.Signal, 1)
+	signal.Notify(s.sigChannel, s.signals...)
+	go func(s *signalHandler) {
+		for range s.sigChannel {
+			handler()
+		}
+	}(s)
+}
+
+func (s *signalHandler) deregister() {
+	signal.Stop(s.sigChannel)
+	close(s.sigChannel)
+}
+
+func errDeleteTokenFailed(lockFilePath string) error {
+	return fmt.Errorf("failed to acquire a new Access token. Please try to delete %s", lockFilePath)
+}
+
+// newLock will get a new file lock
+func newLock(path string) *lock {
+	lockPath := path + ".lock"
+	return &lock{
+		lockFilePath: lockPath,
+		backoff:      &origin.BackoffHandler{MaxRetries: 7},
+		sigHandler:   &signalHandler{
+			signals: 	[]os.Signal{syscall.SIGINT, syscall.SIGTERM},
+		},
+	}
+}
+
+func (l *lock) Acquire() error {
+	// Intercept SIGINT and SIGTERM to release lock before exiting
+	l.sigHandler.register(func() {
+        l.deleteLockFile()
+        os.Exit(0)
+	})
+
+	// Check for a path.lock file
+	// if the lock file exists; start polling
+	// if not, create the lock file and go through the normal flow.
+	// See AUTH-1736 for the reason why we do all this
+	for isTokenLocked(l.lockFilePath) {
+		if l.backoff.Backoff(context.Background()) {
+			continue
+		}
+
+		if err := l.deleteLockFile(); err != nil {
+			return err
+		}
+	}
+
+	// Create a lock file so other processes won't also try to get the token at
+	// the same time
+	if err := ioutil.WriteFile(l.lockFilePath, []byte{}, 0600); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (l *lock) deleteLockFile() error {
+	if err := os.Remove(l.lockFilePath); err != nil && !os.IsNotExist(err) {
+		return errDeleteTokenFailed(l.lockFilePath)
+	}
+	return nil
+}
+
+func (l *lock) Release() error {
+	defer l.sigHandler.deregister()
+	return l.deleteLockFile()
+}
+
+// isTokenLocked checks to see if there is another process attempting to get the token already
+func isTokenLocked(lockFilePath string) bool {
+	exists, err := config.FileExists(lockFilePath)
+	return exists && err == nil
+}
+
 // FetchToken will either load a stored token or generate a new one
 func FetchToken(appURL *url.URL) (string, error) {
 	if token, err := GetTokenIfExists(appURL); token != "" && err == nil {
@@ -29,6 +123,19 @@ func FetchToken(appURL *url.URL) (string, error) {
 		return "", err
 	}
 
+	fileLock := newLock(path)
+
+	err = fileLock.Acquire()
+	if err != nil {
+		return "", err
+	}
+	defer fileLock.Release()
+
+	// check to see if another process has gotten a token while we waited for the lock
+	if token, err := GetTokenIfExists(appURL); token != "" && err == nil {
+		return token, nil
+	}
+
 	// this weird parameter is the resource name (token) and the key/value
 	// we want to send to the transfer service. the key is token and the value
 	// is blank (basically just the id generated in the transfer service)
@@ -55,14 +162,18 @@ func GetTokenIfExists(url *url.URL) (string, error) {
 		return "", err
 	}
 
-	claims, err := token.Claims()
-	if err != nil {
-		return "", err
-	}
-	ident, err := oidc.IdentityFromClaims(claims)
-	// AUTH-1404, reauth if the token is about to expire within 15 minutes
-	if err == nil && ident.ExpiresAt.After(time.Now().Add(time.Minute*15)) {
-		return token.Encode(), nil
-	}
-	return "", err
+	return token.Encode(), nil
+}
+
+// RemoveTokenIfExists removes the a token from local storage if it exists
+func RemoveTokenIfExists(url *url.URL) error {
+	path, err := path.GenerateFilePathFromURL(url, keyName)
+	if err != nil {
+		return err
+	}
+	if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
+		return err
+	}
+
+	return nil
 }

cmd/cloudflared/token/token_test.go

@@ -0,0 +1,52 @@
+package token
+
+import (
+	"os"
+	"syscall"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestSignalHandler(t *testing.T) {
+	sigHandler := signalHandler{signals: []os.Signal{syscall.SIGUSR1}}
+	handlerRan := false
+	done := make(chan struct{})
+	timer := time.NewTimer(time.Second)
+	sigHandler.register(func(){
+		handlerRan = true
+		done <- struct{}{}
+	})
+
+	p, err := os.FindProcess(os.Getpid())
+	require.Nil(t, err)
+	p.Signal(syscall.SIGUSR1)
+
+	// Blocks for up to one second to make sure the handler callback runs before the assert.
+	select {
+		case <- done:
+			assert.True(t, handlerRan)
+		case <- timer.C:
+			t.Fail()
+	}
+	sigHandler.deregister()
+}
+
+func TestSignalHandlerClose(t *testing.T) {
+	sigHandler := signalHandler{signals: []os.Signal{syscall.SIGUSR1}}
+	done := make(chan struct{})
+	timer := time.NewTimer(time.Second)
+	sigHandler.register(func(){done <- struct{}{}})
+	sigHandler.deregister()
+
+	p, err := os.FindProcess(os.Getpid())
+	require.Nil(t, err)
+	p.Signal(syscall.SIGUSR1)
+	select {
+	case <- done:
+		t.Fail()
+	case <- timer.C:
+	}
+}

cmd/cloudflared/transfer/transfer.go

@@ -18,7 +18,7 @@ import (
 )
 
 const (
-	baseStoreURL  = "https://login.cloudflarewarp.com/"
+	baseStoreURL  = "https://login.argotunnel.com/"
 	clientTimeout = time.Second * 60
 )
 

cmd/cloudflared/tunnel/cmd.go

@@ -1,36 +1,84 @@
 package tunnel
 
 import (
+	"context"
 	"fmt"
 	"io/ioutil"
 	"net"
 	"net/url"
 	"os"
+	"runtime"
 	"runtime/trace"
 	"sync"
 	"syscall"
 	"time"
 
-	"github.com/getsentry/raven-go"
-	"golang.org/x/crypto/ssh/terminal"
-
+	"github.com/cloudflare/cloudflared/awsuploader"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/updater"
 	"github.com/cloudflare/cloudflared/cmd/sqlgateway"
+	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/hello"
 	"github.com/cloudflare/cloudflared/metrics"
 	"github.com/cloudflare/cloudflared/origin"
 	"github.com/cloudflare/cloudflared/signal"
+	"github.com/cloudflare/cloudflared/sshlog"
+	"github.com/cloudflare/cloudflared/sshserver"
+	"github.com/cloudflare/cloudflared/supervisor"
+	"github.com/cloudflare/cloudflared/tlsconfig"
 	"github.com/cloudflare/cloudflared/tunneldns"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 	"github.com/cloudflare/cloudflared/websocket"
+
 	"github.com/coreos/go-systemd/daemon"
 	"github.com/facebookgo/grace/gracenet"
+	"github.com/getsentry/raven-go"
+	"github.com/gliderlabs/ssh"
+	"github.com/google/uuid"
 	"github.com/pkg/errors"
+	"golang.org/x/crypto/ssh/terminal"
 	"gopkg.in/urfave/cli.v2"
 	"gopkg.in/urfave/cli.v2/altsrc"
 )
 
-const sentryDSN = "https://56a9c9fa5c364ab28f34b14f35ea0f1b:3e8827f6f9f740738eb11138f7bebb68@sentry.io/189878"
+const (
+	sentryDSN = "https://56a9c9fa5c364ab28f34b14f35ea0f1b:3e8827f6f9f740738eb11138f7bebb68@sentry.io/189878"
+
+	sshLogFileDirectory = "/var/log/cloudflared/"
+
+	// sshPortFlag is the port on localhost the cloudflared ssh server will run on
+	sshPortFlag = "local-ssh-port"
+
+	// sshIdleTimeoutFlag defines the duration a SSH session can remain idle before being closed
+	sshIdleTimeoutFlag = "ssh-idle-timeout"
+
+	// sshMaxTimeoutFlag defines the max duration a SSH session can remain open for
+	sshMaxTimeoutFlag = "ssh-max-timeout"
+
+	// bucketNameFlag is the bucket name to use for the SSH log uploader
+	bucketNameFlag = "bucket-name"
+
+	// regionNameFlag is the AWS region name to use for the SSH log uploader
+	regionNameFlag = "region-name"
+
+	// secretIDFlag is the Secret id of SSH log uploader
+	secretIDFlag = "secret-id"
+
+	// accessKeyIDFlag is the Access key id of SSH log uploader
+	accessKeyIDFlag = "access-key-id"
+
+	// sessionTokenIDFlag is the Session token of SSH log uploader
+	sessionTokenIDFlag = "session-token"
+
+	// s3URLFlag is the S3 URL of SSH log uploader (e.g. don't use AWS s3 and use google storage bucket instead)
+	s3URLFlag = "s3-url-host"
+
+	// disablePortForwarding disables both remote and local ssh port forwarding
+	enablePortForwardingFlag = "enable-port-forwarding"
+
+	noIntentMsg = "The --intent argument is required. Cloudflared looks up an Intent to determine what configuration to use (i.e. which tunnels to start). If you don't have any Intents yet, you can use a placeholder Intent Label for now. Then, when you make an Intent with that label, cloudflared will get notified and open the tunnels you specified in that Intent."
+)
 
 var (
 	shutdownC      chan struct{}
@@ -234,9 +282,8 @@ func StartServer(c *cli.Context, version string, shutdownC, graceShutdownC chan
 		return err
 	}
 
-	buildInfo := origin.GetBuildInfo()
-	logger.Infof("Build info: %+v", *buildInfo)
-	logger.Infof("Version %s", version)
+	buildInfo := buildinfo.GetBuildInfo(version)
+	buildInfo.Log(logger)
 	logClientOptions(c)
 
 	if c.IsSet("proxy-dns") {
@@ -252,16 +299,6 @@ func StartServer(c *cli.Context, version string, shutdownC, graceShutdownC chan
 	// Wait for proxy-dns to come up (if used)
 	<-dnsReadySignal
 
-	// update needs to be after DNS proxy is up to resolve equinox server address
-	if updater.IsAutoupdateEnabled(c) {
-		logger.Infof("Autoupdate frequency is set to %v", c.Duration("autoupdate-freq"))
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			errC <- updater.Autoupdate(c.Duration("autoupdate-freq"), &listeners, shutdownC)
-		}()
-	}
-
 	metricsListener, err := listeners.Listen("tcp", c.String("metrics"))
 	if err != nil {
 		logger.WithError(err).Error("Error opening metrics server listener")
@@ -279,6 +316,33 @@ func StartServer(c *cli.Context, version string, shutdownC, graceShutdownC chan
 		go writePidFile(connectedSignal, c.String("pidfile"))
 	}
 
+	cloudflaredID, err := uuid.NewRandom()
+	if err != nil {
+		logger.WithError(err).Error("Cannot generate cloudflared ID")
+		return err
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	go func() {
+		<-shutdownC
+		cancel()
+	}()
+
+	if c.IsSet("use-declarative-tunnels") {
+		return startDeclarativeTunnel(ctx, c, cloudflaredID, buildInfo, &listeners)
+	}
+
+	// update needs to be after DNS proxy is up to resolve equinox server address
+	if updater.IsAutoupdateEnabled(c) {
+		logger.Infof("Autoupdate frequency is set to %v", c.Duration("autoupdate-freq"))
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			autoupdater := updater.NewAutoUpdater(c.Duration("autoupdate-freq"), &listeners)
+			errC <- autoupdater.Run(ctx)
+		}()
+	}
+
 	// Serve DNS proxy stand-alone if no hostname or tag or app is going to run
 	if dnsProxyStandAlone(c) {
 		connectedSignal.Notify()
@@ -287,6 +351,7 @@ func StartServer(c *cli.Context, version string, shutdownC, graceShutdownC chan
 	}
 
 	if c.IsSet("hello-world") {
+		logger.Infof("hello-world set")
 		helloListener, err := hello.CreateTLSListener("127.0.0.1:")
 		if err != nil {
 			logger.WithError(err).Error("Cannot start Hello World Server")
@@ -301,6 +366,57 @@ func StartServer(c *cli.Context, version string, shutdownC, graceShutdownC chan
 		c.Set("url", "https://"+helloListener.Addr().String())
 	}
 
+	if c.IsSet("ssh-server") {
+		if runtime.GOOS != "darwin" && runtime.GOOS != "linux" {
+			msg := fmt.Sprintf("--ssh-server is not supported on %s", runtime.GOOS)
+			logger.Error(msg)
+			return errors.New(msg)
+
+		}
+
+		logger.Infof("ssh-server set")
+
+		logManager := sshlog.NewEmptyManager()
+		if c.IsSet(bucketNameFlag) && c.IsSet(regionNameFlag) && c.IsSet(accessKeyIDFlag) && c.IsSet(secretIDFlag) {
+			uploader, err := awsuploader.NewFileUploader(c.String(bucketNameFlag), c.String(regionNameFlag),
+				c.String(accessKeyIDFlag), c.String(secretIDFlag), c.String(sessionTokenIDFlag), c.String(s3URLFlag))
+			if err != nil {
+				msg := "Cannot create uploader for SSH Server"
+				logger.WithError(err).Error(msg)
+				return errors.Wrap(err, msg)
+			}
+
+			if err := os.MkdirAll(sshLogFileDirectory, 0600); err != nil {
+				msg := fmt.Sprintf("Cannot create SSH log file directory %s", sshLogFileDirectory)
+				logger.WithError(err).Errorf(msg)
+				return errors.Wrap(err, msg)
+			}
+
+			logManager = sshlog.New(sshLogFileDirectory)
+
+			uploadManager := awsuploader.NewDirectoryUploadManager(logger, uploader, sshLogFileDirectory, 30*time.Minute, shutdownC)
+			uploadManager.Start()
+		}
+
+		sshServerAddress := "127.0.0.1:" + c.String(sshPortFlag)
+		server, err := sshserver.New(logManager, logger, version, sshServerAddress, shutdownC, c.Duration(sshIdleTimeoutFlag), c.Duration(sshMaxTimeoutFlag), c.Bool(enablePortForwardingFlag))
+		if err != nil {
+			msg := "Cannot create new SSH Server"
+			logger.WithError(err).Error(msg)
+			return errors.Wrap(err, msg)
+		}
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			if err = server.Start(); err != nil && err != ssh.ErrServerClosed {
+				logger.WithError(err).Error("SSH server error")
+			}
+			// TODO: remove when declarative tunnels are implemented.
+			close(shutdownC)
+		}()
+		c.Set("url", "ssh://"+sshServerAddress)
+	}
+
 	if host := hostnameFromURI(c.String("url")); host != "" {
 		listener, err := net.Listen("tcp", "127.0.0.1:")
 		if err != nil {
@@ -323,7 +439,7 @@ func StartServer(c *cli.Context, version string, shutdownC, graceShutdownC chan
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
-		errC <- origin.StartTunnelDaemon(tunnelConfig, graceShutdownC, connectedSignal)
+		errC <- origin.StartTunnelDaemon(ctx, tunnelConfig, connectedSignal, cloudflaredID)
 	}()
 
 	return waitToShutdown(&wg, errC, shutdownC, graceShutdownC, c.Duration("grace-period"))
@@ -348,6 +464,123 @@ func Before(c *cli.Context) error {
 	return nil
 }
 
+func startDeclarativeTunnel(ctx context.Context,
+	c *cli.Context,
+	cloudflaredID uuid.UUID,
+	buildInfo *buildinfo.BuildInfo,
+	listeners *gracenet.Net,
+) error {
+	reverseProxyOrigin, err := defaultOriginConfig(c)
+	if err != nil {
+		logger.WithError(err)
+		return err
+	}
+	reverseProxyConfig, err := pogs.NewReverseProxyConfig(
+		c.String("hostname"),
+		reverseProxyOrigin,
+		c.Uint64("retries"),
+		c.Duration("proxy-connection-timeout"),
+		c.Uint64("compression-quality"),
+	)
+	if err != nil {
+		logger.WithError(err).Error("Cannot initialize default client config because reverse proxy config is invalid")
+		return err
+	}
+	defaultClientConfig := &pogs.ClientConfig{
+		Version: pogs.InitVersion(),
+		SupervisorConfig: &pogs.SupervisorConfig{
+			AutoUpdateFrequency:    c.Duration("autoupdate-freq"),
+			MetricsUpdateFrequency: c.Duration("metrics-update-freq"),
+			GracePeriod:            c.Duration("grace-period"),
+		},
+		EdgeConnectionConfig: &pogs.EdgeConnectionConfig{
+			NumHAConnections:    uint8(c.Int("ha-connections")),
+			HeartbeatInterval:   c.Duration("heartbeat-interval"),
+			Timeout:             c.Duration("dial-edge-timeout"),
+			MaxFailedHeartbeats: c.Uint64("heartbeat-count"),
+		},
+		DoHProxyConfigs:     []*pogs.DoHProxyConfig{},
+		ReverseProxyConfigs: []*pogs.ReverseProxyConfig{reverseProxyConfig},
+	}
+
+	autoupdater := updater.NewAutoUpdater(defaultClientConfig.SupervisorConfig.AutoUpdateFrequency, listeners)
+
+	originCert, err := getOriginCert(c)
+	if err != nil {
+		logger.WithError(err).Error("error getting origin cert")
+		return err
+	}
+	toEdgeTLSConfig, err := tlsconfig.CreateTunnelConfig(c)
+	if err != nil {
+		logger.WithError(err).Error("unable to create TLS config to connect with edge")
+		return err
+	}
+
+	tags, err := NewTagSliceFromCLI(c.StringSlice("tag"))
+	if err != nil {
+		logger.WithError(err).Error("unable to parse tag")
+		return err
+	}
+
+	intentLabel := c.String("intent")
+	if intentLabel == "" {
+		logger.Error("--intent was empty")
+		return fmt.Errorf(noIntentMsg)
+	}
+
+	cloudflaredConfig := &connection.CloudflaredConfig{
+		BuildInfo:     buildInfo,
+		CloudflaredID: cloudflaredID,
+		IntentLabel:   intentLabel,
+		Tags:          tags,
+	}
+
+	serviceDiscoverer, err := serviceDiscoverer(c, logger)
+	if err != nil {
+		logger.WithError(err).Error("unable to create service discoverer")
+		return err
+	}
+	supervisor, err := supervisor.NewSupervisor(defaultClientConfig, originCert, toEdgeTLSConfig,
+		serviceDiscoverer, cloudflaredConfig, autoupdater, updater.SupportAutoUpdate(), logger)
+	if err != nil {
+		logger.WithError(err).Error("unable to create Supervisor")
+		return err
+	}
+	return supervisor.Run(ctx)
+}
+
+func defaultOriginConfig(c *cli.Context) (pogs.OriginConfig, error) {
+	if c.IsSet("hello-world") {
+		return &pogs.HelloWorldOriginConfig{}, nil
+	}
+	originConfig := &pogs.HTTPOriginConfig{
+		TCPKeepAlive:           c.Duration("proxy-tcp-keepalive"),
+		DialDualStack:          !c.Bool("proxy-no-happy-eyeballs"),
+		TLSHandshakeTimeout:    c.Duration("proxy-tls-timeout"),
+		TLSVerify:              !c.Bool("no-tls-verify"),
+		OriginCAPool:           c.String("origin-ca-pool"),
+		OriginServerName:       c.String("origin-server-name"),
+		MaxIdleConnections:     c.Uint64("proxy-keepalive-connections"),
+		IdleConnectionTimeout:  c.Duration("proxy-keepalive-timeout"),
+		ProxyConnectionTimeout: c.Duration("proxy-connection-timeout"),
+		ExpectContinueTimeout:  c.Duration("proxy-expect-continue-timeout"),
+		ChunkedEncoding:        c.Bool("no-chunked-encoding"),
+	}
+	if c.IsSet("unix-socket") {
+		unixSocket, err := config.ValidateUnixSocket(c)
+		if err != nil {
+			return nil, errors.Wrap(err, "error validating --unix-socket")
+		}
+		originConfig.URLString = unixSocket
+	}
+	originAddr, err := config.ValidateUrl(c)
+	if err != nil {
+		return nil, errors.Wrap(err, "error validating origin URL")
+	}
+	originConfig.URLString = originAddr
+	return originConfig, nil
+}
+
 func waitToShutdown(wg *sync.WaitGroup,
 	errC chan error,
 	shutdownC, graceShutdownC chan struct{},
@@ -421,15 +654,16 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 		},
 		altsrc.NewDurationFlag(&cli.DurationFlag{
 			Name:   "autoupdate-freq",
-			Usage:  "Autoupdate frequency. Default is 24h.",
-			Value:  time.Hour * 24,
+			Usage:  fmt.Sprintf("Autoupdate frequency. Default is %v.", updater.DefaultCheckUpdateFreq),
+			Value:  updater.DefaultCheckUpdateFreq,
 			Hidden: shouldHide,
 		}),
 		altsrc.NewBoolFlag(&cli.BoolFlag{
-			Name:   "no-autoupdate",
-			Usage:  "Disable periodic check for updates, restarting the server with the new version.",
-			Value:  false,
-			Hidden: shouldHide,
+			Name:    "no-autoupdate",
+			Usage:   "Disable periodic check for updates, restarting the server with the new version.",
+			EnvVars: []string{"NO_AUTOUPDATE"},
+			Value:   false,
+			Hidden:  shouldHide,
 		}),
 		altsrc.NewBoolFlag(&cli.BoolFlag{
 			Name:   "is-autoupdated",
@@ -444,7 +678,7 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			Hidden:  true,
 		}),
 		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:    "cacert",
+			Name:    tlsconfig.CaCertFlag,
 			Usage:   "Certificate Authority authenticating connections with Cloudflare's edge network.",
 			EnvVars: []string{"TUNNEL_CACERT"},
 			Hidden:  true,
@@ -463,7 +697,7 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			Hidden:  shouldHide,
 		}),
 		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:    "origin-ca-pool",
+			Name:    tlsconfig.OriginCAPoolFlag,
 			Usage:   "Path to the CA for the certificate of your origin. This option should be used only if your certificate is not signed by Cloudflare.",
 			EnvVars: []string{"TUNNEL_ORIGIN_CA_POOL"},
 			Hidden:  shouldHide,
@@ -487,6 +721,12 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			EnvVars: []string{"TUNNEL_HOSTNAME"},
 			Hidden:  shouldHide,
 		}),
+		altsrc.NewStringFlag(&cli.StringFlag{
+			Name:    "http-host-header",
+			Usage:   "Sets the HTTP Host header for the local webserver.",
+			EnvVars: []string{"TUNNEL_HTTP_HOST_HEADER"},
+			Hidden:  shouldHide,
+		}),
 		altsrc.NewStringFlag(&cli.StringFlag{
 			Name:    "origin-server-name",
 			Usage:   "Hostname on the origin server certificate.",
@@ -584,6 +824,13 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			EnvVars: []string{"TUNNEL_HELLO_WORLD"},
 			Hidden:  shouldHide,
 		}),
+		altsrc.NewBoolFlag(&cli.BoolFlag{
+			Name:    "ssh-server",
+			Value:   false,
+			Usage:   "Run an SSH Server",
+			EnvVars: []string{"TUNNEL_SSH_SERVER"},
+			Hidden:  true, // TODO: remove when feature is complete
+		}),
 		altsrc.NewStringFlag(&cli.StringFlag{
 			Name:    "pidfile",
 			Usage:   "Write the application's PID to this file after first successful connection.",
@@ -636,6 +883,18 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			Value:  time.Second * 90,
 			Hidden: shouldHide,
 		}),
+		altsrc.NewDurationFlag(&cli.DurationFlag{
+			Name:   "proxy-connection-timeout",
+			Usage:  "HTTP proxy timeout for closing an idle connection",
+			Value:  time.Second * 90,
+			Hidden: shouldHide,
+		}),
+		altsrc.NewDurationFlag(&cli.DurationFlag{
+			Name:   "proxy-expect-continue-timeout",
+			Usage:  "HTTP proxy timeout for closing an idle connection",
+			Value:  time.Second * 90,
+			Hidden: shouldHide,
+		}),
 		altsrc.NewBoolFlag(&cli.BoolFlag{
 			Name:    "proxy-dns",
 			Usage:   "Run a DNS over HTTPS proxy server.",
@@ -695,5 +954,79 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			EnvVars: []string{"TUNNEL_USE_DECLARATIVE"},
 			Hidden:  true,
 		}),
+		altsrc.NewStringFlag(&cli.StringFlag{
+			Name:    "intent",
+			Usage:   "The label of an Intent from which `cloudflared` should gets its tunnels from. Intents can be created in the Origin Registry UI.",
+			EnvVars: []string{"TUNNEL_INTENT"},
+			Hidden:  true,
+		}),
+		altsrc.NewDurationFlag(&cli.DurationFlag{
+			Name:    "dial-edge-timeout",
+			Usage:   "Maximum wait time to set up a connection with the edge",
+			Value:   time.Second * 15,
+			EnvVars: []string{"DIAL_EDGE_TIMEOUT"},
+			Hidden:  true,
+		}),
+		altsrc.NewStringFlag(&cli.StringFlag{
+			Name:    sshPortFlag,
+			Usage:   "Localhost port that cloudflared SSH server will run on",
+			Value:   "2222",
+			EnvVars: []string{"LOCAL_SSH_PORT"},
+			Hidden:  true,
+		}),
+		altsrc.NewDurationFlag(&cli.DurationFlag{
+			Name:    sshIdleTimeoutFlag,
+			Usage:   "Connection timeout after no activity",
+			EnvVars: []string{"SSH_IDLE_TIMEOUT"},
+			Hidden:  true,
+		}),
+		altsrc.NewDurationFlag(&cli.DurationFlag{
+			Name:    sshMaxTimeoutFlag,
+			Usage:   "Absolute connection timeout",
+			EnvVars: []string{"SSH_MAX_TIMEOUT"},
+			Hidden:  true,
+		}),
+		altsrc.NewStringFlag(&cli.StringFlag{
+			Name:    bucketNameFlag,
+			Usage:   "Bucket name of where to upload SSH logs",
+			EnvVars: []string{"BUCKET_ID"},
+			Hidden:  true,
+		}),
+		altsrc.NewStringFlag(&cli.StringFlag{
+			Name:    regionNameFlag,
+			Usage:   "Region name of where to upload SSH logs",
+			EnvVars: []string{"REGION_ID"},
+			Hidden:  true,
+		}),
+		altsrc.NewStringFlag(&cli.StringFlag{
+			Name:    accessKeyIDFlag,
+			Usage:   "Access Key ID of where to upload SSH logs",
+			EnvVars: []string{"ACCESS_CLIENT_ID"},
+			Hidden:  true,
+		}),
+		altsrc.NewStringFlag(&cli.StringFlag{
+			Name:    secretIDFlag,
+			Usage:   "Secret ID of where to upload SSH logs",
+			EnvVars: []string{"SECRET_ID"},
+			Hidden:  true,
+		}),
+		altsrc.NewStringFlag(&cli.StringFlag{
+			Name:    sessionTokenIDFlag,
+			Usage:   "Session Token to use in the configuration of SSH logs uploading",
+			EnvVars: []string{"SESSION_TOKEN_ID"},
+			Hidden:  true,
+		}),
+		altsrc.NewStringFlag(&cli.StringFlag{
+			Name:    s3URLFlag,
+			Usage:   "S3 url of where to upload SSH logs",
+			EnvVars: []string{"S3_URL"},
+			Hidden:  true,
+		}),
+		altsrc.NewBoolFlag(&cli.BoolFlag{
+			Name:    enablePortForwardingFlag,
+			Usage:   "Enables remote and local SSH port forwarding",
+			EnvVars: []string{"ENABLE_PORT_FORWARDING"},
+			Hidden:  true,
+		}),
 	}
 }

cmd/cloudflared/tunnel/configuration.go

@@ -3,18 +3,18 @@ package tunnel
 import (
 	"context"
 	"crypto/tls"
-	"crypto/x509"
 	"fmt"
 	"io/ioutil"
 	"net"
 	"net/http"
 	"os"
 	"path/filepath"
-	"runtime"
 	"strings"
 	"time"
 
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/origin"
 	"github.com/cloudflare/cloudflared/tlsconfig"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
@@ -147,7 +147,7 @@ If you don't have a certificate signed by Cloudflare, run the command:
 
 func prepareTunnelConfig(
 	c *cli.Context,
-	buildInfo *origin.BuildInfo,
+	buildInfo *buildinfo.BuildInfo,
 	version string, logger,
 	transportLogger *logrus.Logger,
 ) (*origin.TunnelConfig, error) {
@@ -187,7 +187,7 @@ func prepareTunnelConfig(
 		}
 	}
 
-	originCertPool, err := loadCertPool(c, logger)
+	originCertPool, err := tlsconfig.LoadOriginCA(c, logger)
 	if err != nil {
 		logger.WithError(err).Error("Error loading cert pool")
 		return nil, errors.Wrap(err, "Error loading cert pool")
@@ -236,7 +236,7 @@ func prepareTunnelConfig(
 		return nil, errors.Wrap(err, "unable to connect to the origin")
 	}
 
-	toEdgeTLSConfig, err := createTunnelConfig(c)
+	toEdgeTLSConfig, err := tlsconfig.CreateTunnelConfig(c)
 	if err != nil {
 		logger.WithError(err).Error("unable to create TLS config to connect with edge")
 		return nil, errors.Wrap(err, "unable to create TLS config to connect with edge")
@@ -253,6 +253,7 @@ func prepareTunnelConfig(
 		HTTPTransport:        httpTransport,
 		HeartbeatInterval:    c.Duration("heartbeat-interval"),
 		Hostname:             hostname,
+		HTTPHostHeader:       c.String("http-host-header"),
 		IncidentLookup:       origin.NewIncidentLookup(),
 		IsAutoupdated:        c.Bool("is-autoupdated"),
 		IsFreeTunnel:         isFreeTunnel,
@@ -274,110 +275,13 @@ func prepareTunnelConfig(
 	}, nil
 }
 
-func loadCertPool(c *cli.Context, logger *logrus.Logger) (*x509.CertPool, error) {
-	const originCAPoolFlag = "origin-ca-pool"
-	originCAPoolFilename := c.String(originCAPoolFlag)
-	var originCustomCAPool []byte
-
-	if originCAPoolFilename != "" {
-		var err error
-		originCustomCAPool, err = ioutil.ReadFile(originCAPoolFilename)
-		if err != nil {
-			return nil, errors.Wrap(err, fmt.Sprintf("unable to read the file %s for --%s", originCAPoolFilename, originCAPoolFlag))
-		}
+func serviceDiscoverer(c *cli.Context, logger *logrus.Logger) (connection.EdgeServiceDiscoverer, error) {
+	// If --edge is specfied, resolve edge server addresses
+	if len(c.StringSlice("edge")) > 0 {
+		return connection.NewEdgeHostnameResolver(c.StringSlice("edge"))
 	}
-
-	originCertPool, err := loadOriginCertPool(originCustomCAPool)
-	if err != nil {
-		return nil, errors.Wrap(err, "error loading the certificate pool")
-	}
-
-	// Windows users should be notified that they can use the flag
-	if runtime.GOOS == "windows" && originCAPoolFilename == "" {
-		logger.Infof("cloudflared does not support loading the system root certificate pool on Windows. Please use the --%s to specify it", originCAPoolFlag)
-	}
-
-	return originCertPool, nil
-}
-
-func loadOriginCertPool(originCAPoolPEM []byte) (*x509.CertPool, error) {
-	// Get the global pool
-	certPool, err := loadGlobalCertPool()
-	if err != nil {
-		return nil, err
-	}
-
-	// Then, add any custom origin CA pool the user may have passed
-	if originCAPoolPEM != nil {
-		if !certPool.AppendCertsFromPEM(originCAPoolPEM) {
-			logger.Warn("could not append the provided origin CA to the cloudflared certificate pool")
-		}
-	}
-
-	return certPool, nil
-}
-
-func loadGlobalCertPool() (*x509.CertPool, error) {
-	// First, obtain the system certificate pool
-	certPool, err := x509.SystemCertPool()
-	if err != nil {
-		if runtime.GOOS != "windows" {
-			logger.WithError(err).Warn("error obtaining the system certificates")
-		}
-		certPool = x509.NewCertPool()
-	}
-
-	// Next, append the Cloudflare CAs into the system pool
-	cfRootCA, err := tlsconfig.GetCloudflareRootCA()
-	if err != nil {
-		return nil, errors.Wrap(err, "could not append Cloudflare Root CAs to cloudflared certificate pool")
-	}
-	for _, cert := range cfRootCA {
-		certPool.AddCert(cert)
-	}
-
-	// Finally, add the Hello certificate into the pool (since it's self-signed)
-	helloCert, err := tlsconfig.GetHelloCertificateX509()
-	if err != nil {
-		return nil, errors.Wrap(err, "could not append Hello server certificate to cloudflared certificate pool")
-	}
-	certPool.AddCert(helloCert)
-
-	return certPool, nil
-}
-
-func createTunnelConfig(c *cli.Context) (*tls.Config, error) {
-	var rootCAs []string
-	if c.String("cacert") != "" {
-		rootCAs = append(rootCAs, c.String("cacert"))
-	}
-	edgeAddrs := c.StringSlice("edge")
-
-	userConfig := &tlsconfig.TLSParameters{RootCAs: rootCAs}
-	tlsConfig, err := tlsconfig.GetConfig(userConfig)
-	if err != nil {
-		return nil, err
-	}
-	if tlsConfig.RootCAs == nil {
-		rootCAPool := x509.NewCertPool()
-		cfRootCA, err := tlsconfig.GetCloudflareRootCA()
-		if err != nil {
-			return nil, errors.Wrap(err, "could not append Cloudflare Root CAs to cloudflared certificate pool")
-		}
-		for _, cert := range cfRootCA {
-			rootCAPool.AddCert(cert)
-		}
-		tlsConfig.RootCAs = rootCAPool
-		tlsConfig.ServerName = "cftunnel.com"
-	} else if len(edgeAddrs) > 0 {
-		// Set for development environments and for testing specific origintunneld instances
-		tlsConfig.ServerName, _, _ = net.SplitHostPort(edgeAddrs[0])
-	}
-
-	if tlsConfig.ServerName == "" && !tlsConfig.InsecureSkipVerify {
-		return nil, fmt.Errorf("either ServerName or InsecureSkipVerify must be specified in the tls.Config")
-	}
-	return tlsConfig, nil
+	// Otherwise lookup edge server addresses through service discovery
+	return connection.NewEdgeAddrResolver(logger)
 }
 
 func isRunningFromTerminal() bool {

cmd/cloudflared/tunnel/login.go

@@ -15,7 +15,7 @@ import (
 
 const (
 	baseLoginURL     = "https://dash.cloudflare.com/argotunnel"
-	callbackStoreURL = "https://login.cloudflarewarp.com/"
+	callbackStoreURL = "https://login.argotunnel.com/"
 )
 
 func login(c *cli.Context) error {

cmd/cloudflared/updater/update.go

@@ -1,6 +1,7 @@
 package updater
 
 import (
+	"context"
 	"os"
 	"runtime"
 	"time"
@@ -14,6 +15,7 @@ import (
 )
 
 const (
+	DefaultCheckUpdateFreq   = time.Hour * 24
 	appID                    = "app_idCzgxYerVD"
 	noUpdateInShellMessage   = "cloudflared will not automatically update when run from the shell. To enable auto-updates, run cloudflared as a service: https://developers.cloudflare.com/argo-tunnel/reference/service/"
 	noUpdateOnWindowsMessage = "cloudflared will not automatically update on Windows systems."
@@ -75,30 +77,6 @@ func Update(_ *cli.Context) error {
 	return updateOutcome.Error
 }
 
-func Autoupdate(freq time.Duration, listeners *gracenet.Net, shutdownC chan struct{}) error {
-	tickC := time.Tick(freq)
-	for {
-		updateOutcome := loggedUpdate()
-		if updateOutcome.Updated {
-			os.Args = append(os.Args, "--is-autoupdated=true")
-			pid, err := listeners.StartProcess()
-			if err != nil {
-				logger.WithError(err).Error("Unable to restart server automatically")
-				return err
-			}
-			// stop old process after autoupdate. Otherwise we create a new process
-			// after each update
-			logger.Infof("PID of the new process is %d", pid)
-			return nil
-		}
-		select {
-		case <-tickC:
-		case <-shutdownC:
-			return nil
-		}
-	}
-}
-
 // Checks for an update and applies it if one is available
 func loggedUpdate() UpdateOutcome {
 	updateOutcome := checkForUpdateAndApply()
@@ -112,7 +90,88 @@ func loggedUpdate() UpdateOutcome {
 	return updateOutcome
 }
 
+// AutoUpdater periodically checks for new version of cloudflared.
+type AutoUpdater struct {
+	configurable     *configurable
+	listeners        *gracenet.Net
+	updateConfigChan chan *configurable
+}
+
+// AutoUpdaterConfigurable is the attributes of AutoUpdater that can be reconfigured during runtime
+type configurable struct {
+	enabled bool
+	freq    time.Duration
+}
+
+func NewAutoUpdater(freq time.Duration, listeners *gracenet.Net) *AutoUpdater {
+	updaterConfigurable := &configurable{
+		enabled: true,
+		freq:    freq,
+	}
+	if freq == 0 {
+		updaterConfigurable.enabled = false
+		updaterConfigurable.freq = DefaultCheckUpdateFreq
+	}
+	return &AutoUpdater{
+		configurable:     updaterConfigurable,
+		listeners:        listeners,
+		updateConfigChan: make(chan *configurable),
+	}
+}
+
+func (a *AutoUpdater) Run(ctx context.Context) error {
+	ticker := time.NewTicker(a.configurable.freq)
+	for {
+		if a.configurable.enabled {
+			updateOutcome := loggedUpdate()
+			if updateOutcome.Updated {
+				os.Args = append(os.Args, "--is-autoupdated=true")
+				pid, err := a.listeners.StartProcess()
+				if err != nil {
+					logger.WithError(err).Error("Unable to restart server automatically")
+					return err
+				}
+				// stop old process after autoupdate. Otherwise we create a new process
+				// after each update
+				logger.Infof("PID of the new process is %d", pid)
+				return nil
+			}
+		}
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case newConfigurable := <-a.updateConfigChan:
+			ticker.Stop()
+			a.configurable = newConfigurable
+			ticker = time.NewTicker(a.configurable.freq)
+			// Check if there is new version of cloudflared after receiving new AutoUpdaterConfigurable
+		case <-ticker.C:
+		}
+	}
+}
+
+// Update is the method to pass new AutoUpdaterConfigurable to a running AutoUpdater. It is safe to be called concurrently
+func (a *AutoUpdater) Update(newFreq time.Duration) {
+	newConfigurable := &configurable{
+		enabled: true,
+		freq:    newFreq,
+	}
+	// A ero duration means autoupdate is disabled
+	if newFreq == 0 {
+		newConfigurable.enabled = false
+		newConfigurable.freq = DefaultCheckUpdateFreq
+	}
+	a.updateConfigChan <- newConfigurable
+}
+
 func IsAutoupdateEnabled(c *cli.Context) bool {
+	if !SupportAutoUpdate() {
+		return false
+	}
+	return !c.Bool("no-autoupdate") && c.Duration("autoupdate-freq") != 0
+}
+
+func SupportAutoUpdate() bool {
 	if runtime.GOOS == "windows" {
 		logger.Info(noUpdateOnWindowsMessage)
 		return false
@@ -122,8 +181,7 @@ func IsAutoupdateEnabled(c *cli.Context) bool {
 		logger.Info(noUpdateInShellMessage)
 		return false
 	}
-
-	return !c.Bool("no-autoupdate") && c.Duration("autoupdate-freq") != 0
+	return true
 }
 
 func isRunningFromTerminal() bool {

cmd/cloudflared/updater/update_test.go

@@ -0,0 +1,26 @@
+package updater
+
+import (
+	"context"
+	"testing"
+
+	"github.com/facebookgo/grace/gracenet"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDisabledAutoUpdater(t *testing.T) {
+	listeners := &gracenet.Net{}
+	autoupdater := NewAutoUpdater(0, listeners)
+	ctx, cancel := context.WithCancel(context.Background())
+	errC := make(chan error)
+	go func() {
+		errC <- autoupdater.Run(ctx)
+	}()
+
+	assert.False(t, autoupdater.configurable.enabled)
+	assert.Equal(t, DefaultCheckUpdateFreq, autoupdater.configurable.freq)
+
+	cancel()
+	// Make sure that autoupdater terminates after canceling the context
+	assert.Equal(t, context.Canceled, <-errC)
+}

connection/connection.go

@@ -2,14 +2,14 @@ package connection
 
 import (
 	"context"
-	"crypto/tls"
 	"net"
-	"sync"
 	"time"
 
 	"github.com/cloudflare/cloudflared/h2mux"
 	"github.com/cloudflare/cloudflared/tunnelrpc"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/google/uuid"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 
@@ -17,7 +17,6 @@ import (
 )
 
 const (
-	dialTimeout       = 5 * time.Second
 	openStreamTimeout = 30 * time.Second
 )
 
@@ -29,134 +28,54 @@ func (e dialError) Error() string {
 	return e.cause.Error()
 }
 
-type muxerShutdownError struct{}
-
-func (e muxerShutdownError) Error() string {
-	return "muxer shutdown"
+type Connection struct {
+	id    uuid.UUID
+	muxer *h2mux.Muxer
 }
 
-type ConnectionConfig struct {
-	TLSConfig         *tls.Config
-	HeartbeatInterval time.Duration
-	MaxHeartbeats     uint64
-	Logger            *logrus.Entry
-}
-
-type connectionHandler interface {
-	serve(ctx context.Context) error
-	connect(ctx context.Context, parameters *tunnelpogs.ConnectParameters) (*tunnelpogs.ConnectResult, error)
-	shutdown()
-}
-
-type h2muxHandler struct {
-	muxer  *h2mux.Muxer
-	logger *logrus.Entry
-}
-
-type muxedStreamHandler struct {
-}
-
-// Implements MuxedStreamHandler interface
-func (h *muxedStreamHandler) ServeStream(stream *h2mux.MuxedStream) error {
-	return nil
-}
-
-func (h *h2muxHandler) serve(ctx context.Context) error {
-	// Serve doesn't return until h2mux is shutdown
-	if err := h.muxer.Serve(ctx); err != nil {
-		return err
+func newConnection(muxer *h2mux.Muxer, edgeIP *net.TCPAddr) (*Connection, error) {
+	id, err := uuid.NewRandom()
+	if err != nil {
+		return nil, err
 	}
-	return muxerShutdownError{}
+	return &Connection{
+		id:    id,
+		muxer: muxer,
+	}, nil
+}
+
+func (c *Connection) Serve(ctx context.Context) error {
+	// Serve doesn't return until h2mux is shutdown
+	return c.muxer.Serve(ctx)
 }
 
 // Connect is used to establish connections with cloudflare's edge network
-func (h *h2muxHandler) connect(ctx context.Context, parameters *tunnelpogs.ConnectParameters) (*tunnelpogs.ConnectResult, error) {
+func (c *Connection) Connect(ctx context.Context, parameters *tunnelpogs.ConnectParameters, logger *logrus.Entry) (pogs.ConnectResult, error) {
 	openStreamCtx, cancel := context.WithTimeout(ctx, openStreamTimeout)
 	defer cancel()
-	conn, err := h.newRPConn(openStreamCtx)
+
+	rpcConn, err := c.newRPConn(openStreamCtx, logger)
 	if err != nil {
-		return nil, errors.Wrap(err, "Failed to create new RPC connection")
+		return nil, errors.Wrap(err, "cannot create new RPC connection")
 	}
-	defer conn.Close()
-	tsClient := tunnelpogs.TunnelServer_PogsClient{Client: conn.Bootstrap(ctx)}
+	defer rpcConn.Close()
+
+	tsClient := tunnelpogs.TunnelServer_PogsClient{Client: rpcConn.Bootstrap(ctx)}
+
 	return tsClient.Connect(ctx, parameters)
 }
 
-func (h *h2muxHandler) shutdown() {
-	h.muxer.Shutdown()
+func (c *Connection) Shutdown() {
+	c.muxer.Shutdown()
 }
 
-func (h *h2muxHandler) newRPConn(ctx context.Context) (*rpc.Conn, error) {
-	stream, err := h.muxer.OpenStream(ctx, []h2mux.Header{
-		{Name: ":method", Value: "RPC"},
-		{Name: ":scheme", Value: "capnp"},
-		{Name: ":path", Value: "*"},
-	}, nil)
+func (c *Connection) newRPConn(ctx context.Context, logger *logrus.Entry) (*rpc.Conn, error) {
+	stream, err := c.muxer.OpenRPCStream(ctx)
 	if err != nil {
 		return nil, err
 	}
 	return rpc.NewConn(
-		tunnelrpc.NewTransportLogger(h.logger.WithField("subsystem", "rpc-register"), rpc.StreamTransport(stream)),
-		tunnelrpc.ConnLog(h.logger.WithField("subsystem", "rpc-transport")),
+		tunnelrpc.NewTransportLogger(logger.WithField("rpc", "connect"), rpc.StreamTransport(stream)),
+		tunnelrpc.ConnLog(logger.WithField("rpc", "connect")),
 	), nil
 }
-
-// NewConnectionHandler returns a connectionHandler, wrapping h2mux to make RPC calls
-func newH2MuxHandler(ctx context.Context,
-	config *ConnectionConfig,
-	edgeIP *net.TCPAddr,
-) (connectionHandler, error) {
-	// Inherit from parent context so we can cancel (Ctrl-C) while dialing
-	dialCtx, dialCancel := context.WithTimeout(ctx, dialTimeout)
-	defer dialCancel()
-	dialer := net.Dialer{DualStack: true}
-	plaintextEdgeConn, err := dialer.DialContext(dialCtx, "tcp", edgeIP.String())
-	if err != nil {
-		return nil, dialError{cause: errors.Wrap(err, "DialContext error")}
-	}
-	edgeConn := tls.Client(plaintextEdgeConn, config.TLSConfig)
-	edgeConn.SetDeadline(time.Now().Add(dialTimeout))
-	err = edgeConn.Handshake()
-	if err != nil {
-		return nil, dialError{cause: errors.Wrap(err, "Handshake with edge error")}
-	}
-	// clear the deadline on the conn; h2mux has its own timeouts
-	edgeConn.SetDeadline(time.Time{})
-	// Establish a muxed connection with the edge
-	// Client mux handshake with agent server
-	muxer, err := h2mux.Handshake(edgeConn, edgeConn, h2mux.MuxerConfig{
-		Timeout:           dialTimeout,
-		Handler:           &muxedStreamHandler{},
-		IsClient:          true,
-		HeartbeatInterval: config.HeartbeatInterval,
-		MaxHeartbeats:     config.MaxHeartbeats,
-		Logger:            config.Logger,
-	})
-	if err != nil {
-		return nil, err
-	}
-	return &h2muxHandler{
-		muxer:  muxer,
-		logger: config.Logger,
-	}, nil
-}
-
-// connectionPool is a pool of connection handlers
-type connectionPool struct {
-	sync.Mutex
-	connectionHandlers []connectionHandler
-}
-
-func (cp *connectionPool) put(h connectionHandler) {
-	cp.Lock()
-	defer cp.Unlock()
-	cp.connectionHandlers = append(cp.connectionHandlers, h)
-}
-
-func (cp *connectionPool) close() {
-	cp.Lock()
-	defer cp.Unlock()
-	for _, h := range cp.connectionHandlers {
-		h.shutdown()
-	}
-}

connection/discovery.go

@@ -5,55 +5,104 @@ import (
 	"crypto/tls"
 	"fmt"
 	"net"
+	"sync"
 	"time"
 
 	"github.com/pkg/errors"
-	log "github.com/sirupsen/logrus"
+	"github.com/sirupsen/logrus"
 )
 
 const (
-	// Used to discover HA Warp servers
-	srvService = "warp"
+	// Used to discover HA origintunneld servers
+	srvService = "origintunneld"
 	srvProto   = "tcp"
-	srvName    = "cloudflarewarp.com"
+	srvName    = "argotunnel.com"
 
 	// Used to fallback to DoT when we can't use the default resolver to
-	// discover HA Warp servers (GitHub issue #75).
+	// discover HA origintunneld servers (GitHub issue #75).
 	dotServerName = "cloudflare-dns.com"
 	dotServerAddr = "1.1.1.1:853"
 	dotTimeout    = time.Duration(15 * time.Second)
+
+	// SRV record resolution TTL
+	resolveEdgeAddrTTL = 1 * time.Hour
 )
 
 var friendlyDNSErrorLines = []string{
 	`Please try the following things to diagnose this issue:`,
-	`  1. ensure that cloudflarewarp.com is returning "warp" service records.`,
-	`     Run your system's equivalent of: dig srv _warp._tcp.cloudflarewarp.com`,
+	`  1. ensure that argotunnel.com is returning "origintunneld" service records.`,
+	`     Run your system's equivalent of: dig srv _origintunneld._tcp.argotunnel.com`,
 	`  2. ensure that your DNS resolver is not returning compressed SRV records.`,
 	`     See GitHub issue https://github.com/golang/go/issues/27546`,
 	`     For example, you could use Cloudflare's 1.1.1.1 as your resolver:`,
 	`     https://developers.cloudflare.com/1.1.1.1/setting-up-1.1.1.1/`,
 }
 
-func ResolveEdgeIPs(logger *log.Logger, addresses []string) ([]*net.TCPAddr, error) {
-	if len(addresses) > 0 {
-		var tcpAddrs []*net.TCPAddr
-		for _, address := range addresses {
-			// Addresses specified (for testing, usually)
-			tcpAddr, err := net.ResolveTCPAddr("tcp", address)
-			if err != nil {
-				return nil, err
-			}
-			tcpAddrs = append(tcpAddrs, tcpAddr)
-		}
-		return tcpAddrs, nil
+// EdgeServiceDiscoverer is an interface for looking up Cloudflare's edge network addresses
+type EdgeServiceDiscoverer interface {
+	// Addr returns an address to connect to cloudflare's edge network
+	Addr() *net.TCPAddr
+	// AvailableAddrs returns the number of unique addresses
+	AvailableAddrs() uint8
+	// Refresh rediscover Cloudflare's edge network addresses
+	Refresh() error
+}
+
+// EdgeAddrResolver discovers the addresses of Cloudflare's edge network through SRV record.
+// It implements EdgeServiceDiscoverer interface
+type EdgeAddrResolver struct {
+	sync.Mutex
+	// Addrs to connect to cloudflare's edge network
+	addrs []*net.TCPAddr
+	// index of the next element to use in addrs
+	nextAddrIndex int
+	logger        *logrus.Entry
+}
+
+func NewEdgeAddrResolver(logger *logrus.Logger) (EdgeServiceDiscoverer, error) {
+	r := &EdgeAddrResolver{
+		logger: logger.WithField("subsystem", " edgeAddrResolver"),
 	}
-	// HA service discovery lookup
+	if err := r.Refresh(); err != nil {
+		return nil, err
+	}
+	return r, nil
+}
+
+func (r *EdgeAddrResolver) Addr() *net.TCPAddr {
+	r.Lock()
+	defer r.Unlock()
+	addr := r.addrs[r.nextAddrIndex]
+	r.nextAddrIndex = (r.nextAddrIndex + 1) % len(r.addrs)
+	return addr
+}
+
+func (r *EdgeAddrResolver) AvailableAddrs() uint8 {
+	r.Lock()
+	defer r.Unlock()
+	return uint8(len(r.addrs))
+}
+
+func (r *EdgeAddrResolver) Refresh() error {
+	newAddrs, err := EdgeDiscovery(r.logger)
+	if err != nil {
+		return err
+	}
+	r.Lock()
+	defer r.Unlock()
+	r.addrs = newAddrs
+	r.nextAddrIndex = 0
+	return nil
+}
+
+// HA service discovery lookup
+func EdgeDiscovery(logger *logrus.Entry) ([]*net.TCPAddr, error) {
 	_, addrs, err := net.LookupSRV(srvService, srvProto, srvName)
 	if err != nil {
 		// Try to fall back to DoT from Cloudflare directly.
 		//
 		// Note: Instead of DoT, we could also have used DoH. Either of these:
-		//     - directly via the JSON API (https://1.1.1.1/dns-query?ct=application/dns-json&name=_warp._tcp.cloudflarewarp.com&type=srv)
+		//     - directly via the JSON API (https://1.1.1.1/dns-query?ct=application/dns-json&name=_origintunneld._tcp.argotunnel.com&type=srv)
 		//     - indirectly via `tunneldns.NewUpstreamHTTPS()`
 		// But both of these cases miss out on a key feature from the stdlib:
 		//     "The returned records are sorted by priority and randomized by weight within a priority."
@@ -70,7 +119,7 @@ func ResolveEdgeIPs(logger *log.Logger, addresses []string) ([]*net.TCPAddr, err
 			for _, s := range friendlyDNSErrorLines {
 				logger.Errorln(s)
 			}
-			return nil, errors.Wrap(err, "Could not lookup srv records on _warp._tcp.cloudflarewarp.com")
+			return nil, errors.Wrap(err, "Could not lookup srv records on _origintunneld._tcp.argotunnel.com")
 		}
 		// Accept the fallback results and keep going
 		addrs = fallbackAddrs
@@ -78,7 +127,7 @@ func ResolveEdgeIPs(logger *log.Logger, addresses []string) ([]*net.TCPAddr, err
 	var resolvedIPsPerCNAME [][]*net.TCPAddr
 	var lookupErr error
 	for _, addr := range addrs {
-		ips, err := ResolveSRVToTCP(addr)
+		ips, err := resolveSRVToTCP(addr)
 		if err != nil || len(ips) == 0 {
 			// don't return early, we might be able to resolve other addresses
 			lookupErr = err
@@ -86,14 +135,14 @@ func ResolveEdgeIPs(logger *log.Logger, addresses []string) ([]*net.TCPAddr, err
 		}
 		resolvedIPsPerCNAME = append(resolvedIPsPerCNAME, ips)
 	}
-	ips := FlattenServiceIPs(resolvedIPsPerCNAME)
+	ips := flattenServiceIPs(resolvedIPsPerCNAME)
 	if lookupErr == nil && len(ips) == 0 {
 		return nil, fmt.Errorf("Unknown service discovery error")
 	}
 	return ips, lookupErr
 }
 
-func ResolveSRVToTCP(srv *net.SRV) ([]*net.TCPAddr, error) {
+func resolveSRVToTCP(srv *net.SRV) ([]*net.TCPAddr, error) {
 	ips, err := net.LookupIP(srv.Target)
 	if err != nil {
 		return nil, err
@@ -107,7 +156,7 @@ func ResolveSRVToTCP(srv *net.SRV) ([]*net.TCPAddr, error) {
 
 // FlattenServiceIPs transposes and flattens the input slices such that the
 // first element of the n inner slices are the first n elements of the result.
-func FlattenServiceIPs(ipsByService [][]*net.TCPAddr) []*net.TCPAddr {
+func flattenServiceIPs(ipsByService [][]*net.TCPAddr) []*net.TCPAddr {
 	var result []*net.TCPAddr
 	for len(ipsByService) > 0 {
 		filtered := ipsByService[:0]
@@ -141,3 +190,65 @@ func fallbackResolver(serverName, serverAddress string) *net.Resolver {
 		},
 	}
 }
+
+// EdgeHostnameResolver discovers the addresses of Cloudflare's edge network via a list of server hostnames.
+// It implements EdgeServiceDiscoverer interface, and is used mainly for testing connectivity.
+type EdgeHostnameResolver struct {
+	sync.Mutex
+	// hostnames of edge servers
+	hostnames []string
+	// Addrs to connect to cloudflare's edge network
+	addrs []*net.TCPAddr
+	// index of the next element to use in addrs
+	nextAddrIndex int
+}
+
+func NewEdgeHostnameResolver(edgeHostnames []string) (EdgeServiceDiscoverer, error) {
+	r := &EdgeHostnameResolver{
+		hostnames: edgeHostnames,
+	}
+	if err := r.Refresh(); err != nil {
+		return nil, err
+	}
+	return r, nil
+}
+
+func (r *EdgeHostnameResolver) Addr() *net.TCPAddr {
+	r.Lock()
+	defer r.Unlock()
+	addr := r.addrs[r.nextAddrIndex]
+	r.nextAddrIndex = (r.nextAddrIndex + 1) % len(r.addrs)
+	return addr
+}
+
+func (r *EdgeHostnameResolver) AvailableAddrs() uint8 {
+	r.Lock()
+	defer r.Unlock()
+	return uint8(len(r.addrs))
+}
+
+func (r *EdgeHostnameResolver) Refresh() error {
+	newAddrs, err := ResolveAddrs(r.hostnames)
+	if err != nil {
+		return err
+	}
+	r.Lock()
+	defer r.Unlock()
+	r.addrs = newAddrs
+	r.nextAddrIndex = 0
+	return nil
+}
+
+// Resolve TCP address given a list of addresses. Address can be a hostname, however, it will return at most one
+// of the hostname's IP addresses
+func ResolveAddrs(addrs []string) ([]*net.TCPAddr, error) {
+	var tcpAddrs []*net.TCPAddr
+	for _, addr := range addrs {
+		tcpAddr, err := net.ResolveTCPAddr("tcp", addr)
+		if err != nil {
+			return nil, err
+		}
+		tcpAddrs = append(tcpAddrs, tcpAddr)
+	}
+	return tcpAddrs, nil
+}

connection/discovery_test.go

@@ -7,8 +7,26 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
+type mockEdgeServiceDiscoverer struct {
+}
+
+func (mr *mockEdgeServiceDiscoverer) Addr() *net.TCPAddr {
+	return &net.TCPAddr{
+		IP:   net.ParseIP("127.0.0.1"),
+		Port: 63102,
+	}
+}
+
+func (mr *mockEdgeServiceDiscoverer) AvailableAddrs() uint8 {
+	return 1
+}
+
+func (mr *mockEdgeServiceDiscoverer) Refresh() error {
+	return nil
+}
+
 func TestFlattenServiceIPs(t *testing.T) {
-	result := FlattenServiceIPs([][]*net.TCPAddr{
+	result := flattenServiceIPs([][]*net.TCPAddr{
 		[]*net.TCPAddr{
 			&net.TCPAddr{Port: 1},
 			&net.TCPAddr{Port: 2},

connection/manager.go

@@ -0,0 +1,294 @@
+package connection
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"net"
+	"sync"
+	"time"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
+	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/google/uuid"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	quickStartLink    = "https://developers.cloudflare.com/argo-tunnel/quickstart/"
+	faqLink           = "https://developers.cloudflare.com/argo-tunnel/faq/"
+	defaultRetryAfter = time.Second * 5
+)
+
+// EdgeManager manages connections with the edge
+type EdgeManager struct {
+	// streamHandler handles stream opened by the edge
+	streamHandler h2mux.MuxedStreamHandler
+	// TLSConfig is the TLS configuration to connect with edge
+	tlsConfig *tls.Config
+	// cloudflaredConfig is the cloudflared configuration that is determined when the process first starts
+	cloudflaredConfig *CloudflaredConfig
+	// serviceDiscoverer returns the next edge addr to connect to
+	serviceDiscoverer EdgeServiceDiscoverer
+	// state is attributes of ConnectionManager that can change during runtime.
+	state *edgeManagerState
+
+	logger *logrus.Entry
+}
+
+// EdgeConnectionManagerConfigurable is the configurable attributes of a EdgeConnectionManager
+type EdgeManagerConfigurable struct {
+	TunnelHostnames []h2mux.TunnelHostname
+	*pogs.EdgeConnectionConfig
+}
+
+type CloudflaredConfig struct {
+	CloudflaredID uuid.UUID
+	Tags          []pogs.Tag
+	BuildInfo     *buildinfo.BuildInfo
+	IntentLabel   string
+}
+
+func NewEdgeManager(
+	streamHandler h2mux.MuxedStreamHandler,
+	edgeConnMgrConfigurable *EdgeManagerConfigurable,
+	userCredential []byte,
+	tlsConfig *tls.Config,
+	serviceDiscoverer EdgeServiceDiscoverer,
+	cloudflaredConfig *CloudflaredConfig,
+	logger *logrus.Logger,
+) *EdgeManager {
+	return &EdgeManager{
+		streamHandler:     streamHandler,
+		tlsConfig:         tlsConfig,
+		cloudflaredConfig: cloudflaredConfig,
+		serviceDiscoverer: serviceDiscoverer,
+		state:             newEdgeConnectionManagerState(edgeConnMgrConfigurable, userCredential),
+		logger:            logger.WithField("subsystem", "connectionManager"),
+	}
+}
+
+func (em *EdgeManager) Run(ctx context.Context) error {
+	defer em.shutdown()
+
+	resolveEdgeIPTicker := time.Tick(resolveEdgeAddrTTL)
+	for {
+		select {
+		case <-ctx.Done():
+			return errors.Wrap(ctx.Err(), "EdgeConnectionManager terminated")
+		case <-resolveEdgeIPTicker:
+			if err := em.serviceDiscoverer.Refresh(); err != nil {
+				em.logger.WithError(err).Warn("Cannot refresh Cloudflare edge addresses")
+			}
+		default:
+			time.Sleep(1 * time.Second)
+		}
+		// Create/delete connection one at a time, so we don't need to adjust for connections that are being created/deleted
+		// in shouldCreateConnection or shouldReduceConnection calculation
+		if em.state.shouldCreateConnection(em.serviceDiscoverer.AvailableAddrs()) {
+			if connErr := em.newConnection(ctx); connErr != nil {
+				if !connErr.ShouldRetry {
+					em.logger.WithError(connErr).Error(em.noRetryMessage())
+					return connErr
+				}
+				em.logger.WithError(connErr).Error("cannot create new connection")
+			}
+		} else if em.state.shouldReduceConnection() {
+			if err := em.closeConnection(ctx); err != nil {
+				em.logger.WithError(err).Error("cannot close connection")
+			}
+		}
+	}
+}
+
+func (em *EdgeManager) UpdateConfigurable(newConfigurable *EdgeManagerConfigurable) {
+	em.logger.Infof("New edge connection manager configuration %+v", newConfigurable)
+	em.state.updateConfigurable(newConfigurable)
+}
+
+func (em *EdgeManager) newConnection(ctx context.Context) *pogs.ConnectError {
+	edgeIP := em.serviceDiscoverer.Addr()
+	edgeConn, err := em.dialEdge(ctx, edgeIP)
+	if err != nil {
+		return retryConnection(fmt.Sprintf("dial edge error: %v", err))
+	}
+	configurable := em.state.getConfigurable()
+	// Establish a muxed connection with the edge
+	// Client mux handshake with agent server
+	muxer, err := h2mux.Handshake(edgeConn, edgeConn, h2mux.MuxerConfig{
+		Timeout:           configurable.Timeout,
+		Handler:           em.streamHandler,
+		IsClient:          true,
+		HeartbeatInterval: configurable.HeartbeatInterval,
+		MaxHeartbeats:     configurable.MaxFailedHeartbeats,
+		Logger:            em.logger.WithField("subsystem", "muxer"),
+	})
+	if err != nil {
+		retryConnection(fmt.Sprintf("couldn't perform handshake with edge: %v", err))
+	}
+
+	h2muxConn, err := newConnection(muxer, edgeIP)
+	if err != nil {
+		return retryConnection(fmt.Sprintf("couldn't create h2mux connection: %v", err))
+	}
+
+	go em.serveConn(ctx, h2muxConn)
+
+	connResult, err := h2muxConn.Connect(ctx, &pogs.ConnectParameters{
+		CloudflaredID:       em.cloudflaredConfig.CloudflaredID,
+		CloudflaredVersion:  em.cloudflaredConfig.BuildInfo.CloudflaredVersion,
+		NumPreviousAttempts: 0,
+		OriginCert:          em.state.getUserCredential(),
+		IntentLabel:         em.cloudflaredConfig.IntentLabel,
+		Tags:                em.cloudflaredConfig.Tags,
+	}, em.logger)
+	if err != nil {
+		h2muxConn.Shutdown()
+		return retryConnection(fmt.Sprintf("couldn't connect to edge: %v", err))
+	}
+
+	if connErr := connResult.ConnectError(); connErr != nil {
+		return connErr
+	}
+
+	em.state.newConnection(h2muxConn)
+	em.logger.Infof("connected to %s", connResult.ConnectedTo())
+	return nil
+}
+
+func (em *EdgeManager) closeConnection(ctx context.Context) error {
+	conn := em.state.getFirstConnection()
+	if conn == nil {
+		return fmt.Errorf("no connection to close")
+	}
+	conn.Shutdown()
+	return nil
+}
+
+func (em *EdgeManager) serveConn(ctx context.Context, conn *Connection) {
+	err := conn.Serve(ctx)
+	em.logger.WithError(err).Warn("Connection closed")
+	em.state.closeConnection(conn)
+}
+
+func (em *EdgeManager) dialEdge(ctx context.Context, edgeIP *net.TCPAddr) (*tls.Conn, error) {
+	timeout := em.state.getConfigurable().Timeout
+	// Inherit from parent context so we can cancel (Ctrl-C) while dialing
+	dialCtx, dialCancel := context.WithTimeout(ctx, timeout)
+	defer dialCancel()
+
+	dialer := net.Dialer{DualStack: true}
+	edgeConn, err := dialer.DialContext(dialCtx, "tcp", edgeIP.String())
+	if err != nil {
+		return nil, dialError{cause: errors.Wrap(err, "DialContext error")}
+	}
+	tlsEdgeConn := tls.Client(edgeConn, em.tlsConfig)
+	tlsEdgeConn.SetDeadline(time.Now().Add(timeout))
+
+	if err = tlsEdgeConn.Handshake(); err != nil {
+		return nil, dialError{cause: errors.Wrap(err, "Handshake with edge error")}
+	}
+	// clear the deadline on the conn; h2mux has its own timeouts
+	tlsEdgeConn.SetDeadline(time.Time{})
+	return tlsEdgeConn, nil
+}
+
+func (em *EdgeManager) noRetryMessage() string {
+	messageTemplate := "cloudflared could not register an Argo Tunnel on your account. Please confirm the following before trying again:" +
+		"1. You have Argo Smart Routing enabled in your account, See Enable Argo section of %s." +
+		"2. Your credential at %s is still valid. See %s."
+	return fmt.Sprintf(messageTemplate, quickStartLink, em.state.getConfigurable().UserCredentialPath, faqLink)
+}
+
+func (em *EdgeManager) shutdown() {
+	em.state.shutdown()
+}
+
+type edgeManagerState struct {
+	sync.RWMutex
+	configurable   *EdgeManagerConfigurable
+	userCredential []byte
+	conns          map[uuid.UUID]*Connection
+}
+
+func newEdgeConnectionManagerState(configurable *EdgeManagerConfigurable, userCredential []byte) *edgeManagerState {
+	return &edgeManagerState{
+		configurable:   configurable,
+		userCredential: userCredential,
+		conns:          make(map[uuid.UUID]*Connection),
+	}
+}
+
+func (ems *edgeManagerState) shouldCreateConnection(availableEdgeAddrs uint8) bool {
+	ems.RLock()
+	defer ems.RUnlock()
+	expectedHAConns := ems.configurable.NumHAConnections
+	if availableEdgeAddrs < expectedHAConns {
+		expectedHAConns = availableEdgeAddrs
+	}
+	return uint8(len(ems.conns)) < expectedHAConns
+}
+
+func (ems *edgeManagerState) shouldReduceConnection() bool {
+	ems.RLock()
+	defer ems.RUnlock()
+	return uint8(len(ems.conns)) > ems.configurable.NumHAConnections
+}
+
+func (ems *edgeManagerState) newConnection(conn *Connection) {
+	ems.Lock()
+	defer ems.Unlock()
+	ems.conns[conn.id] = conn
+}
+
+func (ems *edgeManagerState) closeConnection(conn *Connection) {
+	ems.Lock()
+	defer ems.Unlock()
+	delete(ems.conns, conn.id)
+}
+
+func (ems *edgeManagerState) getFirstConnection() *Connection {
+	ems.RLock()
+	defer ems.RUnlock()
+
+	for _, conn := range ems.conns {
+		return conn
+	}
+	return nil
+}
+
+func (ems *edgeManagerState) shutdown() {
+	ems.Lock()
+	defer ems.Unlock()
+	for _, conn := range ems.conns {
+		conn.Shutdown()
+	}
+}
+
+func (ems *edgeManagerState) getConfigurable() *EdgeManagerConfigurable {
+	ems.Lock()
+	defer ems.Unlock()
+	return ems.configurable
+}
+
+func (ems *edgeManagerState) updateConfigurable(newConfigurable *EdgeManagerConfigurable) {
+	ems.Lock()
+	defer ems.Unlock()
+	ems.configurable = newConfigurable
+}
+
+func (ems *edgeManagerState) getUserCredential() []byte {
+	ems.RLock()
+	defer ems.RUnlock()
+	return ems.userCredential
+}
+
+func retryConnection(cause string) *pogs.ConnectError {
+	return &pogs.ConnectError{
+		Cause:       cause,
+		RetryAfter:  defaultRetryAfter,
+		ShouldRetry: true,
+	}
+}

connection/manager_test.go

@@ -0,0 +1,77 @@
+package connection
+
+import (
+	"testing"
+	"time"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/google/uuid"
+	"github.com/sirupsen/logrus"
+)
+
+var (
+	configurable = &EdgeManagerConfigurable{
+		[]h2mux.TunnelHostname{
+			"http.example.com",
+			"ws.example.com",
+			"hello.example.com",
+		},
+		&pogs.EdgeConnectionConfig{
+			NumHAConnections:    1,
+			HeartbeatInterval:   1 * time.Second,
+			Timeout:             5 * time.Second,
+			MaxFailedHeartbeats: 3,
+			UserCredentialPath:  "/etc/cloudflared/cert.pem",
+		},
+	}
+	cloudflaredConfig = &CloudflaredConfig{
+		CloudflaredID: uuid.New(),
+		Tags: []pogs.Tag{
+			{Name: "pool", Value: "east-6"},
+		},
+		BuildInfo: &buildinfo.BuildInfo{
+			GoOS:               "linux",
+			GoVersion:          "1.12",
+			GoArch:             "amd64",
+			CloudflaredVersion: "2019.6.0",
+		},
+	}
+)
+
+type mockStreamHandler struct {
+}
+
+func (msh *mockStreamHandler) ServeStream(*h2mux.MuxedStream) error {
+	return nil
+}
+
+func mockEdgeManager() *EdgeManager {
+	return NewEdgeManager(
+		&mockStreamHandler{},
+		configurable,
+		[]byte{},
+		nil,
+		&mockEdgeServiceDiscoverer{},
+		cloudflaredConfig,
+		logrus.New(),
+	)
+}
+
+func TestUpdateConfigurable(t *testing.T) {
+	m := mockEdgeManager()
+	newConfigurable := &EdgeManagerConfigurable{
+		[]h2mux.TunnelHostname{
+			"second.example.com",
+		},
+		&pogs.EdgeConnectionConfig{
+			NumHAConnections: 2,
+		},
+	}
+	m.UpdateConfigurable(newConfigurable)
+
+	assert.Equal(t, newConfigurable, m.state.getConfigurable())
+}

connection/supervisor.go

@@ -1,147 +0,0 @@
-package connection
-
-import (
-	"context"
-	"net"
-	"time"
-
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
-	"github.com/google/uuid"
-	"github.com/pkg/errors"
-	"github.com/sirupsen/logrus"
-)
-
-const (
-	// Waiting time before retrying a failed tunnel connection
-	reconnectDuration = time.Second * 10
-	// SRV record resolution TTL
-	resolveTTL = time.Hour
-	// Interval between establishing new connection
-	connectionInterval = time.Second
-)
-
-type CloudflaredConfig struct {
-	ConnectionConfig   *ConnectionConfig
-	OriginCert         []byte
-	Tags               []tunnelpogs.Tag
-	EdgeAddrs          []string
-	HAConnections      uint
-	Logger             *logrus.Logger
-	CloudflaredVersion string
-}
-
-// Supervisor is a stateful object that manages connections with the edge
-type Supervisor struct {
-	config     *CloudflaredConfig
-	state      *supervisorState
-	connErrors chan error
-}
-
-type supervisorState struct {
-	// IPs to connect to cloudflare's edge network
-	edgeIPs []*net.TCPAddr
-	// index of the next element to use in edgeIPs
-	nextEdgeIPIndex int
-	// last time edgeIPs were refreshed
-	lastResolveTime time.Time
-	// ID of this cloudflared instance
-	cloudflaredID uuid.UUID
-	// connectionPool is a pool of connectionHandlers that can be used to make RPCs
-	connectionPool *connectionPool
-}
-
-func (s *supervisorState) getNextEdgeIP() *net.TCPAddr {
-	ip := s.edgeIPs[s.nextEdgeIPIndex%len(s.edgeIPs)]
-	s.nextEdgeIPIndex++
-	return ip
-}
-
-func NewSupervisor(config *CloudflaredConfig) *Supervisor {
-	return &Supervisor{
-		config: config,
-		state: &supervisorState{
-			connectionPool: &connectionPool{},
-		},
-		connErrors: make(chan error),
-	}
-}
-
-func (s *Supervisor) Run(ctx context.Context) error {
-	logger := s.config.Logger
-	if err := s.initialize(); err != nil {
-		logger.WithError(err).Error("Failed to get edge IPs")
-		return err
-	}
-	defer s.state.connectionPool.close()
-
-	var currentConnectionCount uint
-	expectedConnectionCount := s.config.HAConnections
-	if uint(len(s.state.edgeIPs)) < s.config.HAConnections {
-		logger.Warnf("You requested %d HA connections but I can give you at most %d.", s.config.HAConnections, len(s.state.edgeIPs))
-		expectedConnectionCount = uint(len(s.state.edgeIPs))
-	}
-	for {
-		select {
-		case <-ctx.Done():
-			return nil
-		case connErr := <-s.connErrors:
-			logger.WithError(connErr).Warnf("Connection dropped unexpectedly")
-			currentConnectionCount--
-		default:
-			time.Sleep(5 * time.Second)
-		}
-		if currentConnectionCount < expectedConnectionCount {
-			h, err := newH2MuxHandler(ctx, s.config.ConnectionConfig, s.state.getNextEdgeIP())
-			if err != nil {
-				logger.WithError(err).Error("Failed to create new connection handler")
-				continue
-			}
-			go func() {
-				s.connErrors <- h.serve(ctx)
-			}()
-			connResult, err := s.connect(ctx, s.config, s.state.cloudflaredID, h)
-			if err != nil {
-				logger.WithError(err).Errorf("Failed to connect to cloudflared's edge network")
-				h.shutdown()
-				continue
-			}
-			if connErr := connResult.Err; connErr != nil && !connErr.ShouldRetry {
-				logger.WithError(connErr).Errorf("Server respond with don't retry to connect")
-				h.shutdown()
-				return err
-			}
-			logger.Infof("Connected to %s", connResult.ServerInfo.LocationName)
-			s.state.connectionPool.put(h)
-			currentConnectionCount++
-		}
-	}
-}
-
-func (s *Supervisor) initialize() error {
-	edgeIPs, err := ResolveEdgeIPs(s.config.Logger, s.config.EdgeAddrs)
-	if err != nil {
-		return errors.Wrapf(err, "Failed to resolve cloudflare edge network address")
-	}
-	s.state.edgeIPs = edgeIPs
-	s.state.lastResolveTime = time.Now()
-	cloudflaredID, err := uuid.NewRandom()
-	if err != nil {
-		return errors.Wrap(err, "Failed to generate cloudflared ID")
-	}
-	s.state.cloudflaredID = cloudflaredID
-	return nil
-}
-
-func (s *Supervisor) connect(ctx context.Context,
-	config *CloudflaredConfig,
-	cloudflaredID uuid.UUID,
-	h connectionHandler,
-) (*tunnelpogs.ConnectResult, error) {
-	connectParameters := &tunnelpogs.ConnectParameters{
-		OriginCert:          config.OriginCert,
-		CloudflaredID:       cloudflaredID,
-		NumPreviousAttempts: 0,
-		CloudflaredVersion:  config.CloudflaredVersion,
-	}
-	return h.connect(ctx, connectParameters)
-}

h2mux/error.go

@@ -20,11 +20,12 @@ var (
 	ErrUnknownStream       = MuxerProtocolError{"2002 unknown stream", http2.ErrCodeProtocol}
 	ErrInvalidStream       = MuxerProtocolError{"2003 invalid stream", http2.ErrCodeProtocol}
 
-	ErrStreamHeadersSent      = MuxerApplicationError{"3000 headers already sent"}
-	ErrConnectionClosed       = MuxerApplicationError{"3001 connection closed"}
-	ErrConnectionDropped      = MuxerApplicationError{"3002 connection dropped"}
-	ErrOpenStreamTimeout      = MuxerApplicationError{"3003 open stream timeout"}
-	ErrResponseHeadersTimeout = MuxerApplicationError{"3004 timeout waiting for initial response headers"}
+	ErrStreamHeadersSent               = MuxerApplicationError{"3000 headers already sent"}
+	ErrStreamRequestConnectionClosed   = MuxerApplicationError{"3001 connection closed while opening stream"}
+	ErrConnectionDropped               = MuxerApplicationError{"3002 connection dropped"}
+	ErrStreamRequestTimeout            = MuxerApplicationError{"3003 open stream timeout"}
+	ErrResponseHeadersTimeout          = MuxerApplicationError{"3004 timeout waiting for initial response headers"}
+	ErrResponseHeadersConnectionClosed = MuxerApplicationError{"3005 connection closed while waiting for initial response headers"}
 
 	ErrClosedStream = MuxerStreamError{"4000 stream closed", http2.ErrCodeStreamClosed}
 )

h2mux/h2mux.go

@@ -1,7 +1,6 @@
 package h2mux
 
 import (
-	"bytes"
 	"context"
 	"io"
 	"strings"
@@ -94,6 +93,14 @@ type Header struct {
 	Name, Value string
 }
 
+func RPCHeaders() []Header {
+	return []Header{
+		{Name: ":method", Value: "RPC"},
+		{Name: ":scheme", Value: "capnp"},
+		{Name: ":path", Value: "*"},
+	}
+}
+
 // Handshake establishes a muxed connection with the peer.
 // After the handshake completes, it is possible to open and accept streams.
 func Handshake(
@@ -380,37 +387,52 @@ func isConnectionClosedError(err error) bool {
 // OpenStream opens a new data stream with the given headers.
 // Called by proxy server and tunnel
 func (m *Muxer) OpenStream(ctx context.Context, headers []Header, body io.Reader) (*MuxedStream, error) {
-	stream := &MuxedStream{
-		responseHeadersReceived: make(chan struct{}),
-		readBuffer:              NewSharedBuffer(),
-		writeBuffer:             &bytes.Buffer{},
-		writeBufferMaxLen:       m.config.StreamWriteBufferMaxLen,
-		writeBufferHasSpace:     make(chan struct{}, 1),
-		receiveWindow:           m.config.DefaultWindowSize,
-		receiveWindowCurrentMax: m.config.DefaultWindowSize,
-		receiveWindowMax:        m.config.MaxWindowSize,
-		sendWindow:              m.config.DefaultWindowSize,
-		readyList:               m.readyList,
-		writeHeaders:            headers,
-		dictionaries:            m.muxReader.dictionaries,
+	stream := m.NewStream(headers)
+	if err := m.MakeMuxedStreamRequest(ctx, MuxedStreamRequest{stream, body}); err != nil {
+		return nil, err
 	}
+	if err := m.AwaitResponseHeaders(ctx, stream); err != nil {
+		return nil, err
+	}
+	return stream, nil
+}
 
+
+func (m *Muxer) OpenRPCStream(ctx context.Context) (*MuxedStream, error) {
+	stream := m.NewStream(RPCHeaders())
+	if err := m.MakeMuxedStreamRequest(ctx, MuxedStreamRequest{stream: stream, body: nil}); err != nil {
+		return nil, err
+	}
+	if err := m.AwaitResponseHeaders(ctx, stream); err != nil {
+		return nil, err
+	}
+	return stream, nil
+}
+
+func (m *Muxer) NewStream(headers []Header) *MuxedStream {
+	return NewStream(m.config, headers, m.readyList, m.muxReader.dictionaries)
+}
+
+func (m *Muxer) MakeMuxedStreamRequest(ctx context.Context, request MuxedStreamRequest) error {
 	select {
+	case <-ctx.Done():
+		return ErrStreamRequestTimeout
+	case <-m.abortChan:
+		return ErrStreamRequestConnectionClosed
 	// Will be received by mux writer
-	case <-ctx.Done():
-		return nil, ErrOpenStreamTimeout
-	case <-m.abortChan:
-		return nil, ErrConnectionClosed
-	case m.newStreamChan <- MuxedStreamRequest{stream: stream, body: body}:
+	case m.newStreamChan <- request:
+		return nil
 	}
+}
 
+func (m *Muxer) AwaitResponseHeaders(ctx context.Context, stream *MuxedStream) error {
 	select {
 	case <-ctx.Done():
-		return nil, ErrResponseHeadersTimeout
+		return ErrResponseHeadersTimeout
 	case <-m.abortChan:
-		return nil, ErrConnectionClosed
+		return ErrResponseHeadersConnectionClosed
 	case <-stream.responseHeadersReceived:
-		return stream, nil
+		return nil
 	}
 }
 

h2mux/h2mux_test.go

@@ -584,8 +584,8 @@ func TestOpenAfterDisconnect(t *testing.T) {
 			[]Header{{Name: "test-header", Value: "headerValue"}},
 			nil,
 		)
-		if err != ErrConnectionClosed {
-			t.Fatalf("unexpected error in OpenStream: %s", err)
+		if err != ErrStreamRequestConnectionClosed && err != ErrResponseHeadersConnectionClosed {
+			t.Fatalf("case %v: unexpected error in OpenStream: %v", i, err)
 		}
 	}
 }

h2mux/muxedstream.go

@@ -68,7 +68,8 @@ type MuxedStream struct {
 	sentEOF bool
 	// true if the peer sent us an EOF
 	receivedEOF bool
-
+	// If valid, tunnelHostname is used to identify which origin service is the intended recipient of the request
+	tunnelHostname TunnelHostname
 	// Compression-related fields
 	receivedUseDict bool
 	method          string
@@ -77,6 +78,33 @@ type MuxedStream struct {
 	dictionaries    h2Dictionaries
 }
 
+type TunnelHostname string
+
+func (th TunnelHostname) String() string {
+	return string(th)
+}
+
+func (th TunnelHostname) IsSet() bool {
+	return th != ""
+}
+
+func NewStream(config MuxerConfig, writeHeaders []Header, readyList *ReadyList, dictionaries h2Dictionaries) *MuxedStream {
+	return &MuxedStream{
+		responseHeadersReceived: make(chan struct{}),
+		readBuffer:              NewSharedBuffer(),
+		writeBuffer:             &bytes.Buffer{},
+		writeBufferMaxLen:       config.StreamWriteBufferMaxLen,
+		writeBufferHasSpace:     make(chan struct{}, 1),
+		receiveWindow:           config.DefaultWindowSize,
+		receiveWindowCurrentMax: config.DefaultWindowSize,
+		receiveWindowMax:        config.MaxWindowSize,
+		sendWindow:              config.DefaultWindowSize,
+		readyList:               readyList,
+		writeHeaders:            writeHeaders,
+		dictionaries:            dictionaries,
+	}
+}
+
 func (s *MuxedStream) Read(p []byte) (n int, err error) {
 	var readBuffer ReadWriteClosedCloser
 	if s.dictionaries.read != nil {
@@ -185,6 +213,25 @@ func (s *MuxedStream) WriteHeaders(headers []Header) error {
 	return nil
 }
 
+// IsRPCStream returns if the stream is used to transport RPC.
+func (s *MuxedStream) IsRPCStream() bool {
+	rpcHeaders := RPCHeaders()
+	if len(s.Headers) != len(rpcHeaders) {
+		return false
+	}
+	// The headers order matters, so RPC stream should be opened with OpenRPCStream method and let MuxWriter serializes the headers.
+	for i, rpcHeader := range rpcHeaders {
+		if s.Headers[i] != rpcHeader {
+			return false
+		}
+	}
+	return true
+}
+
+func (s *MuxedStream) TunnelHostname() TunnelHostname {
+	return s.tunnelHostname
+}
+
 func (s *MuxedStream) getReceiveWindow() uint32 {
 	s.writeLock.Lock()
 	defer s.writeLock.Unlock()

h2mux/muxedstream_test.go

@@ -98,3 +98,30 @@ func TestMuxedStreamEOF(t *testing.T) {
 		assert.Equal(t, 0, n)
 	}
 }
+
+func TestIsRPCStream(t *testing.T) {
+	tests := []struct {
+		stream      *MuxedStream
+		isRPCStream bool
+	}{
+		{
+			stream:      &MuxedStream{},
+			isRPCStream: false,
+		},
+		{
+			stream:      &MuxedStream{Headers: RPCHeaders()},
+			isRPCStream: true,
+		},
+		{
+			stream: &MuxedStream{Headers: []Header{
+				{Name: ":method", Value: "rpc"},
+				{Name: ":scheme", Value: "Capnp"},
+				{Name: ":path", Value: "/"},
+			}},
+			isRPCStream: false,
+		},
+	}
+	for _, test := range tests {
+		assert.Equal(t, test.isRPCStream, test.stream.IsRPCStream())
+	}
+}

h2mux/muxreader.go

@@ -11,6 +11,10 @@ import (
 	"golang.org/x/net/http2"
 )
 
+const (
+	CloudflaredProxyTunnelHostnameHeader = "cf-cloudflared-proxy-tunnel-hostname"
+)
+
 type MuxReader struct {
 	// f is used to read HTTP2 frames.
 	f *http2.Framer
@@ -235,6 +239,8 @@ func (r *MuxReader) receiveHeaderData(frame *http2.MetaHeadersFrame) error {
 			if r.dictionaries.write != nil {
 				continue
 			}
+		case CloudflaredProxyTunnelHostnameHeader:
+			stream.tunnelHostname = TunnelHostname(header.Value)
 		}
 		headers = append(headers, Header{Name: header.Name, Value: header.Value})
 	}

h2mux/muxreader_test.go

@@ -0,0 +1,107 @@
+package h2mux
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+var (
+	methodHeader = Header{
+		Name:  ":method",
+		Value: "GET",
+	}
+	schemeHeader = Header{
+		Name:  ":scheme",
+		Value: "https",
+	}
+	pathHeader = Header{
+		Name:  ":path",
+		Value: "/api/tunnels",
+	}
+	tunnelHostnameHeader = Header{
+		Name:  CloudflaredProxyTunnelHostnameHeader,
+		Value: "tunnel.example.com",
+	}
+	respStatusHeader = Header{
+		Name:  ":status",
+		Value: "200",
+	}
+)
+
+type mockOriginStreamHandler struct {
+	stream *MuxedStream
+}
+
+func (mosh *mockOriginStreamHandler) ServeStream(stream *MuxedStream) error {
+	mosh.stream = stream
+	// Echo tunnel hostname in header
+	stream.WriteHeaders([]Header{respStatusHeader})
+	return nil
+}
+
+func getCloudflaredProxyTunnelHostnameHeader(stream *MuxedStream) string {
+	for _, header := range stream.Headers {
+		if header.Name == CloudflaredProxyTunnelHostnameHeader {
+			return header.Value
+		}
+	}
+	return ""
+}
+
+func assertOpenStreamSucceed(t *testing.T, stream *MuxedStream, err error) {
+	assert.NoError(t, err)
+	assert.Len(t, stream.Headers, 1)
+	assert.Equal(t, respStatusHeader, stream.Headers[0])
+}
+
+func TestMissingHeaders(t *testing.T) {
+	originHandler := &mockOriginStreamHandler{}
+	muxPair := NewDefaultMuxerPair(t, originHandler.ServeStream)
+	muxPair.Serve(t)
+
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+	defer cancel()
+
+	reqHeaders := []Header{
+		{
+			Name:  "content-type",
+			Value: "application/json",
+		},
+	}
+
+	// Request doesn't contain CloudflaredProxyTunnelHostnameHeader
+	stream, err := muxPair.EdgeMux.OpenStream(ctx, reqHeaders, nil)
+	assertOpenStreamSucceed(t, stream, err)
+
+	assert.Empty(t, originHandler.stream.method)
+	assert.Empty(t, originHandler.stream.path)
+	assert.False(t, originHandler.stream.TunnelHostname().IsSet())
+}
+
+func TestReceiveHeaderData(t *testing.T) {
+	originHandler := &mockOriginStreamHandler{}
+	muxPair := NewDefaultMuxerPair(t, originHandler.ServeStream)
+	muxPair.Serve(t)
+
+	reqHeaders := []Header{
+		methodHeader,
+		schemeHeader,
+		pathHeader,
+		tunnelHostnameHeader,
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+	defer cancel()
+
+	reqHeaders = append(reqHeaders, tunnelHostnameHeader)
+	stream, err := muxPair.EdgeMux.OpenStream(ctx, reqHeaders, nil)
+	assertOpenStreamSucceed(t, stream, err)
+
+	assert.Equal(t, methodHeader.Value, originHandler.stream.method)
+	assert.Equal(t, pathHeader.Value, originHandler.stream.path)
+	assert.True(t, originHandler.stream.TunnelHostname().IsSet())
+	assert.Equal(t, tunnelHostnameHeader.Value, originHandler.stream.TunnelHostname().String())
+}

origin/build_info.go

@@ -1,19 +0,0 @@
-package origin
-
-import (
-	"runtime"
-)
-
-type BuildInfo struct {
-	GoOS               string `json:"go_os"`
-	GoVersion          string `json:"go_version"`
-	GoArch             string `json:"go_arch"`
-}
-
-func GetBuildInfo() *BuildInfo {
-	return &BuildInfo{
-		GoOS:      runtime.GOOS,
-		GoVersion: runtime.Version(),
-		GoArch:    runtime.GOARCH,
-	}
-}

origin/supervisor.go

@@ -6,6 +6,8 @@ import (
 	"net"
 	"time"
 
+	"github.com/sirupsen/logrus"
+
 	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/signal"
 
@@ -34,6 +36,8 @@ type Supervisor struct {
 	// currently-connecting tunnels to finish connecting so we can reset backoff timer
 	nextConnectedIndex  int
 	nextConnectedSignal chan struct{}
+
+	logger *logrus.Entry
 }
 
 type resolveResult struct {
@@ -51,6 +55,7 @@ func NewSupervisor(config *TunnelConfig) *Supervisor {
 		config:            config,
 		tunnelErrors:      make(chan tunnelError),
 		tunnelsConnecting: map[int]chan struct{}{},
+		logger:            config.Logger.WithField("subsystem", "supervisor"),
 	}
 }
 
@@ -124,8 +129,10 @@ func (s *Supervisor) Run(ctx context.Context, connectedSignal *signal.Signal, u
 }
 
 func (s *Supervisor) initialize(ctx context.Context, connectedSignal *signal.Signal, u uuid.UUID) error {
-	logger := s.config.Logger
-	edgeIPs, err := connection.ResolveEdgeIPs(logger, s.config.EdgeAddrs)
+	logger := s.logger
+
+	edgeIPs, err := s.resolveEdgeIPs()
+
 	if err != nil {
 		logger.Infof("ResolveEdgeIPs err")
 		return err
@@ -215,6 +222,15 @@ func (s *Supervisor) getEdgeIP(index int) *net.TCPAddr {
 	return s.edgeIPs[index%len(s.edgeIPs)]
 }
 
+func (s *Supervisor) resolveEdgeIPs() ([]*net.TCPAddr, error) {
+	// If --edge is specfied, resolve edge server addresses
+	if len(s.config.EdgeAddrs) > 0 {
+		return connection.ResolveAddrs(s.config.EdgeAddrs)
+	}
+	// Otherwise lookup edge server addresses through service discovery
+	return connection.EdgeDiscovery(s.logger)
+}
+
 func (s *Supervisor) refreshEdgeIPs() {
 	if s.resolverC != nil {
 		return
@@ -224,7 +240,7 @@ func (s *Supervisor) refreshEdgeIPs() {
 	}
 	s.resolverC = make(chan resolveResult)
 	go func() {
-		edgeIPs, err := connection.ResolveEdgeIPs(s.config.Logger, s.config.EdgeAddrs)
+		edgeIPs, err := s.resolveEdgeIPs()
 		s.resolverC <- resolveResult{edgeIPs: edgeIPs, err: err}
 	}()
 }

origin/tunnel.go

@@ -14,9 +14,10 @@ import (
 	"sync"
 	"time"
 
-	"github.com/cloudflare/cloudflared/connection"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
 	"github.com/cloudflare/cloudflared/h2mux"
 	"github.com/cloudflare/cloudflared/signal"
+	"github.com/cloudflare/cloudflared/streamhandler"
 	"github.com/cloudflare/cloudflared/tunnelrpc"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 	"github.com/cloudflare/cloudflared/validation"
@@ -41,7 +42,7 @@ const (
 )
 
 type TunnelConfig struct {
-	BuildInfo            *BuildInfo
+	BuildInfo            *buildinfo.BuildInfo
 	ClientID             string
 	ClientTlsConfig      *tls.Config
 	CloseConnOnce        *sync.Once // Used to close connectedSignal no more than once
@@ -52,6 +53,7 @@ type TunnelConfig struct {
 	HTTPTransport        http.RoundTripper
 	HeartbeatInterval    time.Duration
 	Hostname             string
+	HTTPHostHeader       string
 	IncidentLookup       IncidentLookup
 	IsAutoupdated        bool
 	IsFreeTunnel         bool
@@ -139,44 +141,8 @@ func (c *TunnelConfig) RegistrationOptions(connectionID uint8, OriginLocalIP str
 	}
 }
 
-func StartTunnelDaemon(config *TunnelConfig, shutdownC <-chan struct{}, connectedSignal *signal.Signal) error {
-	ctx, cancel := context.WithCancel(context.Background())
-	go func() {
-		<-shutdownC
-		cancel()
-	}()
-
-	u, err := uuid.NewRandom()
-	if err != nil {
-		return err
-	}
-
-	// If a user specified negative HAConnections, we will treat it as requesting 1 connection
-	if config.HAConnections > 1 {
-		if config.UseDeclarativeTunnel {
-			return connection.NewSupervisor(&connection.CloudflaredConfig{
-				ConnectionConfig: &connection.ConnectionConfig{
-					TLSConfig:         config.TlsConfig,
-					HeartbeatInterval: config.HeartbeatInterval,
-					MaxHeartbeats:     config.MaxHeartbeats,
-					Logger:            config.Logger.WithField("subsystem", "connection_supervisor"),
-				},
-				OriginCert:         config.OriginCert,
-				Tags:               config.Tags,
-				EdgeAddrs:          config.EdgeAddrs,
-				HAConnections:      uint(config.HAConnections),
-				Logger:             config.Logger,
-				CloudflaredVersion: config.ReportedVersion,
-			}).Run(ctx)
-		}
-		return NewSupervisor(config).Run(ctx, connectedSignal, u)
-	} else {
-		addrs, err := connection.ResolveEdgeIPs(config.Logger, config.EdgeAddrs)
-		if err != nil {
-			return err
-		}
-		return ServeTunnelLoop(ctx, config, addrs[0], 0, connectedSignal, u)
-	}
+func StartTunnelDaemon(ctx context.Context, config *TunnelConfig, connectedSignal *signal.Signal, cloudflaredID uuid.UUID) error {
+	return NewSupervisor(config).Run(ctx, connectedSignal, cloudflaredID)
 }
 
 func ServeTunnelLoop(ctx context.Context,
@@ -471,39 +437,6 @@ func LogServerInfo(
 	metrics.registerServerLocation(uint8ToString(connectionID), serverInfo.LocationName)
 }
 
-func H2RequestHeadersToH1Request(h2 []h2mux.Header, h1 *http.Request) error {
-	for _, header := range h2 {
-		switch header.Name {
-		case ":method":
-			h1.Method = header.Value
-		case ":scheme":
-		case ":authority":
-			// Otherwise the host header will be based on the origin URL
-			h1.Host = header.Value
-		case ":path":
-			u, err := url.Parse(header.Value)
-			if err != nil {
-				return fmt.Errorf("unparseable path")
-			}
-			resolved := h1.URL.ResolveReference(u)
-			// prevent escaping base URL
-			if !strings.HasPrefix(resolved.String(), h1.URL.String()) {
-				return fmt.Errorf("invalid path")
-			}
-			h1.URL = resolved
-		case "content-length":
-			contentLength, err := strconv.ParseInt(header.Value, 10, 64)
-			if err != nil {
-				return fmt.Errorf("unparseable content length")
-			}
-			h1.ContentLength = contentLength
-		default:
-			h1.Header.Add(http.CanonicalHeaderKey(header.Name), header.Value)
-		}
-	}
-	return nil
-}
-
 func H1ResponseToH2Response(h1 *http.Response) (h2 []h2mux.Header) {
 	h2 = []h2mux.Header{{Name: ":status", Value: fmt.Sprintf("%d", h1.StatusCode)}}
 	for headerName, headerValues := range h1.Header {
@@ -514,17 +447,14 @@ func H1ResponseToH2Response(h1 *http.Response) (h2 []h2mux.Header) {
 	return
 }
 
-func FindCfRayHeader(h1 *http.Request) string {
-	return h1.Header.Get("Cf-Ray")
-}
-
 type TunnelHandler struct {
-	originUrl  string
-	muxer      *h2mux.Muxer
-	httpClient http.RoundTripper
-	tlsConfig  *tls.Config
-	tags       []tunnelpogs.Tag
-	metrics    *TunnelMetrics
+	originUrl      string
+	httpHostHeader string
+	muxer          *h2mux.Muxer
+	httpClient     http.RoundTripper
+	tlsConfig      *tls.Config
+	tags           []tunnelpogs.Tag
+	metrics        *TunnelMetrics
 	// connectionID is only used by metrics, and prometheus requires labels to be string
 	connectionID      string
 	logger            *log.Logger
@@ -545,6 +475,7 @@ func NewTunnelHandler(ctx context.Context,
 	}
 	h := &TunnelHandler{
 		originUrl:         originURL,
+		httpHostHeader:    config.HTTPHostHeader,
 		httpClient:        config.HTTPTransport,
 		tlsConfig:         config.ClientTlsConfig,
 		tags:              config.Tags,
@@ -605,8 +536,8 @@ func (h *TunnelHandler) ServeStream(stream *h2mux.MuxedStream) error {
 		return reqErr
 	}
 
-	cfRay := FindCfRayHeader(req)
-	lbProbe := isLBProbeRequest(req)
+	cfRay := streamhandler.FindCfRayHeader(req)
+	lbProbe := streamhandler.IsLBProbeRequest(req)
 	h.logRequest(req, cfRay, lbProbe)
 
 	var resp *http.Response
@@ -629,7 +560,7 @@ func (h *TunnelHandler) createRequest(stream *h2mux.MuxedStream) (*http.Request,
 	if err != nil {
 		return nil, errors.Wrap(err, "Unexpected error from http.NewRequest")
 	}
-	err = H2RequestHeadersToH1Request(stream.Headers, req)
+	err = streamhandler.H2RequestHeadersToH1Request(stream.Headers, req)
 	if err != nil {
 		return nil, errors.Wrap(err, "invalid request received")
 	}
@@ -638,6 +569,11 @@ func (h *TunnelHandler) createRequest(stream *h2mux.MuxedStream) (*http.Request,
 }
 
 func (h *TunnelHandler) serveWebsocket(stream *h2mux.MuxedStream, req *http.Request) (*http.Response, error) {
+	if h.httpHostHeader != "" {
+		req.Header.Set("Host", h.httpHostHeader)
+		req.Host = h.httpHostHeader
+	}
+
 	conn, response, err := websocket.ClientConnect(req, h.tlsConfig)
 	if err != nil {
 		return nil, err
@@ -666,6 +602,11 @@ func (h *TunnelHandler) serveHTTP(stream *h2mux.MuxedStream, req *http.Request)
 	// Request origin to keep connection alive to improve performance
 	req.Header.Set("Connection", "keep-alive")
 
+	if h.httpHostHeader != "" {
+		req.Header.Set("Host", h.httpHostHeader)
+		req.Host = h.httpHostHeader
+	}
+
 	response, err := h.httpClient.RoundTrip(req)
 	if err != nil {
 		return nil, errors.Wrap(err, "Error proxying request to origin")
@@ -759,10 +700,6 @@ func uint8ToString(input uint8) string {
 	return strconv.FormatUint(uint64(input), 10)
 }
 
-func isLBProbeRequest(req *http.Request) bool {
-	return strings.HasPrefix(req.UserAgent(), lbProbeUserAgentPrefix)
-}
-
 // Print out the given lines in a nice ASCII box.
 func asciiBox(lines []string, padding int) (box []string) {
 	maxLen := maxLen(lines)

originservice/originservice.go

@@ -0,0 +1,235 @@
+// Package client defines and implements interface to proxy to HTTP, websocket and hello world origins
+package originservice
+
+import (
+	"bufio"
+	"crypto/tls"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+
+	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/hello"
+	"github.com/cloudflare/cloudflared/log"
+	"github.com/cloudflare/cloudflared/websocket"
+
+	"github.com/pkg/errors"
+)
+
+// OriginService is an interface to proxy requests to different type of origins
+type OriginService interface {
+	Proxy(stream *h2mux.MuxedStream, req *http.Request) (resp *http.Response, err error)
+	URL() *url.URL
+	Summary() string
+	Shutdown()
+}
+
+// HTTPService talks to origin using HTTP/HTTPS
+type HTTPService struct {
+	client          http.RoundTripper
+	originURL       *url.URL
+	chunkedEncoding bool
+}
+
+func NewHTTPService(transport http.RoundTripper, url *url.URL, chunkedEncoding bool) OriginService {
+	return &HTTPService{
+		client:          transport,
+		originURL:       url,
+		chunkedEncoding: chunkedEncoding,
+	}
+}
+
+func (hc *HTTPService) Proxy(stream *h2mux.MuxedStream, req *http.Request) (*http.Response, error) {
+	// Support for WSGI Servers by switching transfer encoding from chunked to gzip/deflate
+	if !hc.chunkedEncoding {
+		req.TransferEncoding = []string{"gzip", "deflate"}
+		cLength, err := strconv.Atoi(req.Header.Get("Content-Length"))
+		if err == nil {
+			req.ContentLength = int64(cLength)
+		}
+	}
+
+	// Request origin to keep connection alive to improve performance
+	req.Header.Set("Connection", "keep-alive")
+
+	resp, err := hc.client.RoundTrip(req)
+	if err != nil {
+		return nil, errors.Wrap(err, "error proxying request to HTTP origin")
+	}
+	defer resp.Body.Close()
+
+	err = stream.WriteHeaders(h1ResponseToH2Response(resp))
+	if err != nil {
+		return nil, errors.Wrap(err, "error writing response header to HTTP origin")
+	}
+	if isEventStream(resp) {
+		writeEventStream(stream, resp.Body)
+	} else {
+		// Use CopyBuffer, because Copy only allocates a 32KiB buffer, and cross-stream
+		// compression generates dictionary on first write
+		io.CopyBuffer(stream, resp.Body, make([]byte, 512*1024))
+	}
+	return resp, nil
+}
+
+func (hc *HTTPService) URL() *url.URL {
+	return hc.originURL
+}
+
+func (hc *HTTPService) Summary() string {
+	return fmt.Sprintf("HTTP service listening on %s", hc.originURL)
+}
+
+func (hc *HTTPService) Shutdown() {}
+
+// WebsocketService talks to origin using WS/WSS
+type WebsocketService struct {
+	tlsConfig *tls.Config
+	originURL *url.URL
+	shutdownC chan struct{}
+}
+
+func NewWebSocketService(tlsConfig *tls.Config, url *url.URL) (OriginService, error) {
+	listener, err := net.Listen("tcp", "127.0.0.1:")
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot start Websocket Proxy Server")
+	}
+	shutdownC := make(chan struct{})
+	go func() {
+		websocket.StartProxyServer(log.CreateLogger(), listener, url.String(), shutdownC)
+	}()
+	return &WebsocketService{
+		tlsConfig: tlsConfig,
+		originURL: url,
+		shutdownC: shutdownC,
+	}, nil
+}
+
+func (wsc *WebsocketService) Proxy(stream *h2mux.MuxedStream, req *http.Request) (*http.Response, error) {
+	if !websocket.IsWebSocketUpgrade(req) {
+		return nil, fmt.Errorf("request is not a websocket connection")
+	}
+	conn, response, err := websocket.ClientConnect(req, wsc.tlsConfig)
+	if err != nil {
+		return nil, err
+	}
+	defer conn.Close()
+	err = stream.WriteHeaders(h1ResponseToH2Response(response))
+	if err != nil {
+		return nil, errors.Wrap(err, "error writing response header to websocket origin")
+	}
+	// Copy to/from stream to the undelying connection. Use the underlying
+	// connection because cloudflared doesn't operate on the message themselves
+	websocket.Stream(conn.UnderlyingConn(), stream)
+	return response, nil
+}
+
+func (wsc *WebsocketService) URL() *url.URL {
+	return wsc.originURL
+}
+
+func (wsc *WebsocketService) Summary() string {
+	return fmt.Sprintf("Websocket listening on %s", wsc.originURL)
+}
+
+func (wsc *WebsocketService) Shutdown() {
+	close(wsc.shutdownC)
+}
+
+// HelloWorldService talks to the hello world example origin
+type HelloWorldService struct {
+	client    http.RoundTripper
+	listener  net.Listener
+	originURL *url.URL
+	shutdownC chan struct{}
+}
+
+func NewHelloWorldService(transport http.RoundTripper) (OriginService, error) {
+	listener, err := hello.CreateTLSListener("127.0.0.1:")
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot start Hello World Server")
+	}
+	shutdownC := make(chan struct{})
+	go func() {
+		hello.StartHelloWorldServer(log.CreateLogger(), listener, shutdownC)
+	}()
+	return &HelloWorldService{
+		client:   transport,
+		listener: listener,
+		originURL: &url.URL{
+			Scheme: "https",
+			Host:   listener.Addr().String(),
+		},
+		shutdownC: shutdownC,
+	}, nil
+}
+
+func (hwc *HelloWorldService) Proxy(stream *h2mux.MuxedStream, req *http.Request) (*http.Response, error) {
+	// Request origin to keep connection alive to improve performance
+	req.Header.Set("Connection", "keep-alive")
+	resp, err := hwc.client.RoundTrip(req)
+	if err != nil {
+		return nil, errors.Wrap(err, "error proxying request to Hello World origin")
+	}
+	defer resp.Body.Close()
+
+	err = stream.WriteHeaders(h1ResponseToH2Response(resp))
+	if err != nil {
+		return nil, errors.Wrap(err, "error writing response header to Hello World origin")
+	}
+
+	// Use CopyBuffer, because Copy only allocates a 32KiB buffer, and cross-stream
+	// compression generates dictionary on first write
+	io.CopyBuffer(stream, resp.Body, make([]byte, 512*1024))
+
+	return resp, nil
+}
+
+func (hwc *HelloWorldService) URL() *url.URL {
+	return hwc.originURL
+}
+
+func (hwc *HelloWorldService) Summary() string {
+	return fmt.Sprintf("Hello World service listening on %s", hwc.originURL)
+}
+
+func (hwc *HelloWorldService) Shutdown() {
+	hwc.listener.Close()
+}
+
+func isEventStream(resp *http.Response) bool {
+	// Check if content-type is text/event-stream. We need to check if the header value starts with text/event-stream
+	// because text/event-stream; charset=UTF-8 is also valid
+	// Ref: https://tools.ietf.org/html/rfc7231#section-3.1.1.1
+	for _, contentType := range resp.Header["content-type"] {
+		if strings.HasPrefix(strings.ToLower(contentType), "text/event-stream") {
+			return true
+		}
+	}
+	return false
+}
+
+func writeEventStream(stream *h2mux.MuxedStream, respBody io.ReadCloser) {
+	reader := bufio.NewReader(respBody)
+	for {
+		line, err := reader.ReadBytes('\n')
+		if err != nil {
+			break
+		}
+		stream.Write(line)
+	}
+}
+
+func h1ResponseToH2Response(h1 *http.Response) (h2 []h2mux.Header) {
+	h2 = []h2mux.Header{{Name: ":status", Value: fmt.Sprintf("%d", h1.StatusCode)}}
+	for headerName, headerValues := range h1.Header {
+		for _, headerValue := range headerValues {
+			h2 = append(h2, h2mux.Header{Name: strings.ToLower(headerName), Value: headerValue})
+		}
+	}
+	return
+}

originservice/originservice_test.go

@@ -0,0 +1,60 @@
+package originservice
+
+import (
+	"net/http"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestIsEventStream(t *testing.T) {
+	tests := []struct {
+		resp          *http.Response
+		isEventStream bool
+	}{
+		{
+			resp:          &http.Response{},
+			isEventStream: false,
+		},
+		{
+			// isEventStream checks all headers
+			resp: &http.Response{
+				Header: http.Header{
+					"accept":       []string{"text/html"},
+					"content-type": []string{"text/event-stream"},
+				},
+			},
+			isEventStream: true,
+		},
+		{
+			// Content-Type and text/event-stream are case-insensitive. text/event-stream can be followed by OWS parameter
+			resp: &http.Response{
+				Header: http.Header{
+					"content-type": []string{"Text/event-stream;charset=utf-8"},
+				},
+			},
+			isEventStream: true,
+		},
+		{
+			// Content-Type and text/event-stream are case-insensitive. text/event-stream can be followed by OWS parameter
+			resp: &http.Response{
+				Header: http.Header{
+					"content-type": []string{"appication/json", "text/html", "Text/event-stream;charset=utf-8"},
+				},
+			},
+			isEventStream: true,
+		},
+		{
+			// Not an event stream because the content-type value doesn't start with text/event-stream
+			resp: &http.Response{
+				Header: http.Header{
+					"content-type": []string{" text/event-stream"},
+				},
+			},
+			isEventStream: false,
+		},
+	}
+	for _, test := range tests {
+		assert.Equal(t, test.isEventStream, isEventStream(test.resp), "Header: %v", test.resp.Header)
+	}
+}

ssh_server_tests/Dockerfile

@@ -0,0 +1,14 @@
+FROM python:3-buster
+
+RUN wget https://bin.equinox.io/c/VdrWdbjqyF/cloudflared-stable-linux-amd64.deb \
+  && dpkg -i cloudflared-stable-linux-amd64.deb
+
+RUN pip install pexpect
+
+COPY tests.py .
+COPY ssh /root/.ssh
+RUN chmod 600 /root/.ssh/id_rsa
+
+ARG SSH_HOSTNAME
+RUN bash -c 'sed -i "s/{{hostname}}/${SSH_HOSTNAME}/g" /root/.ssh/authorized_keys_config'
+RUN bash -c 'sed -i "s/{{hostname}}/${SSH_HOSTNAME}/g" /root/.ssh/short_lived_cert_config'

ssh_server_tests/README.md

@@ -0,0 +1,23 @@
+# Cloudflared SSH server smoke tests
+
+Runs several tests in a docker container against a server that is started out of band of these tests.
+Cloudflared token also needs to be retrieved out of band.
+SSH server hostname and user need to be configured in a docker environment file
+
+
+## Running tests
+
+* Build cloudflared:
+make cloudflared
+
+* Start server:
+sudo ./cloudflared tunnel --hostname HOSTNAME --ssh-server
+
+* Fetch token: 
+./cloudflared access login HOSTNAME
+
+* Create docker env file:
+echo "SSH_HOSTNAME=HOSTNAME\nSSH_USER=USERNAME\n" > ssh_server_tests/.env
+
+* Run tests:
+make test-ssh-server

ssh_server_tests/docker-compose.yml

@@ -0,0 +1,19 @@
+version: "3.1"
+
+services:
+  ssh_test:
+    build:
+      context: .
+      args:
+        - SSH_HOSTNAME=${SSH_HOSTNAME}
+    volumes:
+      - "~/.cloudflared/:/root/.cloudflared"
+    env_file:
+     - .env
+    environment:
+      - AUTHORIZED_KEYS_SSH_CONFIG=/root/.ssh/authorized_keys_config
+      - SHORT_LIVED_CERT_SSH_CONFIG=/root/.ssh/short_lived_cert_config
+      - REMOTE_SCP_FILENAME=scp_test.txt
+      - ROOT_ONLY_TEST_FILE_PATH=~/permission_test.txt
+
+    entrypoint: "python tests.py"

ssh_server_tests/ssh/authorized_keys_config

@@ -0,0 +1,5 @@
+Host *
+    AddressFamily inet
+
+Host {{hostname}}
+  ProxyCommand /usr/local/bin/cloudflared access ssh --hostname %h

ssh_server_tests/ssh/id_rsa

@@ -0,0 +1,49 @@
+-----BEGIN OPENSSH PRIVATE KEY-----
+b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAACFwAAAAdzc2gtcn
+NhAAAAAwEAAQAAAgEAvi26NDQ8cYTTztqPe9ZgF5HR/rIo5FoDgL5NbbZKW6h0txP9Fd8s
+id9Bgmo+aGkeM327tPVVMQ6UFmdRksOCIDWQDjNLF8b6S+Fu95tvMKSbGreRoR32OvgZKV
+I6KmOsF4z4GIv9naPplZswtKEUhSSI+/gPdAs9wfwalqZ77e82QJ727bYMeC3lzuoT+KBI
+dYufJ4OQhLtpHrqhB5sn7s6+oCv/u85GSln5SIC18Hi2t9lW4tgb5tH8P0kEDDWGfPS5ok
+qGi4kFTvwBXOCS2r4dhi5hRkpP7PqG4np0OCfvK5IRRJ27fCnj0loc+puZJAxnPMbuJr64
+vwxRx78PM/V0PDUsl0P6aR/vbe0XmF9FGqbWf2Tar1p4r6C9/bMzcDz8seYT8hzLIHP3+R
+l1hdlsTLm+1EzhaExKId+tjXegKGG4nU24h6qHEnRxLQDMwEsdkfj4E1pVypZJXVyNj99D
+o84vi0EUnu7R4HmQb/C+Pu7qMDtLT3Zk7O5Mg4LQ+cTz9V0noYEAyG46nAB4U/nJzBnV1J
++aAdpioHmUAYhLYlQ9Kiy7LCJi92g9Wqa4wxMKxBbO5ZeH++p2p2lUi/oQNqx/2dLYFmy0
+wxvJHbZIhAaFbOeCvHg1ucIAQznli2jOr2qoB+yKRRPAp/3NXnZg1v7ce2CkwiAD52wjtC
+kAAAdILMJUeyzCVHsAAAAHc3NoLXJzYQAAAgEAvi26NDQ8cYTTztqPe9ZgF5HR/rIo5FoD
+gL5NbbZKW6h0txP9Fd8sid9Bgmo+aGkeM327tPVVMQ6UFmdRksOCIDWQDjNLF8b6S+Fu95
+tvMKSbGreRoR32OvgZKVI6KmOsF4z4GIv9naPplZswtKEUhSSI+/gPdAs9wfwalqZ77e82
+QJ727bYMeC3lzuoT+KBIdYufJ4OQhLtpHrqhB5sn7s6+oCv/u85GSln5SIC18Hi2t9lW4t
+gb5tH8P0kEDDWGfPS5okqGi4kFTvwBXOCS2r4dhi5hRkpP7PqG4np0OCfvK5IRRJ27fCnj
+0loc+puZJAxnPMbuJr64vwxRx78PM/V0PDUsl0P6aR/vbe0XmF9FGqbWf2Tar1p4r6C9/b
+MzcDz8seYT8hzLIHP3+Rl1hdlsTLm+1EzhaExKId+tjXegKGG4nU24h6qHEnRxLQDMwEsd
+kfj4E1pVypZJXVyNj99Do84vi0EUnu7R4HmQb/C+Pu7qMDtLT3Zk7O5Mg4LQ+cTz9V0noY
+EAyG46nAB4U/nJzBnV1J+aAdpioHmUAYhLYlQ9Kiy7LCJi92g9Wqa4wxMKxBbO5ZeH++p2
+p2lUi/oQNqx/2dLYFmy0wxvJHbZIhAaFbOeCvHg1ucIAQznli2jOr2qoB+yKRRPAp/3NXn
+Zg1v7ce2CkwiAD52wjtCkAAAADAQABAAACAQCbnVsyAFQ9J00Rg/HIiUATyTQlzq57O9SF
+8jH1RiZOHedzLx32WaleH5rBFiJ+2RTnWUjQ57aP77fpJR2wk93UcT+w/vPBPwXsNUjRvx
+Qan3ZzRCYbyiKDWiNslmYV7X0RwD36CAK8jTVDP7t48h2SXLTiSLaMY+5i3uD6yLu7k/O2
+qNyw4jgN1rCmwQ8acD0aQec3NAZ7NcbsaBX/3Uutsup0scwOZtlJWZoLY5Z8cKpCgcsAz4
+j1NHnNZvey7dFgSffj/ktdvf7kBH0w/GnuJ4aNF0Jte70u0kiw5TZYBQVFh74tgUu6a6SJ
+qUbxIYUL5EJNjxGsDn+phHEemw3aMv0CwZG6Tqaionlna7bLsl9Bg1HTGclczVWx8uqC+M
+6agLmkhYCHG0rVj8h5smjXAQXtmvIDVYDOlJZZoF9VAOCj6QfmJUH1NAGpCs1HDHbeOxGA
+OLCh4d3F4rScPqhGdtSt4W13VFIvXn2Qqoz9ufepZsee1SZqpcerxywx2wN9ZAzu+X8lTN
+i+TA2B3vWpqqucOEsp4JwDN+VMKZqKUGUDWcm/eHSaG6wq0q734LUlgM85TjaIg8QsNtWV
+giB1nWwsYIuH4rsFNFGEwURYdGBcw6idH0GZ7I4RaIB5F9oOza1d601E0APHYrtnx9yOiK
+nOtJ+5ZmVZovaDRfu1aQAAAQBU/EFaNUzoVhO04pS2L6BlByt963bOIsSJhdlEzek5AAli
+eaf1S/PD6xWCc0IGY+GZE0HPbhsKYanjqOpWldcA2T7fzf4oz4vFBfUkPYo/MLSlLCYsDd
+IH3wBkCssnfR5EkzNgxnOvq646Nl64BMvxwSIXGPktdq9ZALxViwricSRzCFURnh5vLHWU
+wBzSgAA0UlZ9E64GtAv066+AoZCp83GhTLRC4o0naE2e/K4op4BCFHLrZ8eXmDRK3NJj80
+Vkn+uhrk+SHmbjIhmS57Pv9p8TWyRvemph/nMUuZGKBUu2X+JQxggck0KigIrXjsmciCsM
+BIM3mYDDfjYbyVhTAAABAQDkV8O1bWUsAIqk7RU+iDZojN5kaO+zUvj1TafX8QX1sY6pu4
+Z2cfSEka1532BaehM95bQm7BCPw4cYg56XidmCQTZ9WaWqxVrOo48EKXUtZMZx6nKFOKlq
+MT2XTMnGT9n7kFCfEjSVkAjuJ9ZTFLOaoXAaVRnxeHQwOKaup5KKP9GSzNIw328U+96s3V
+WKHeT4pMjHBccgW/qX/tRRidZw5in5uBC9Ew5y3UACFTkNOnhUwVfyUNbBZJ2W36msQ3KD
+AN7nOrQHqhd3NFyCEy2ovIAKVBacr/VEX6EsRUshIehJzz8EY9f3kXL7WT2QDoz2giPeBJ
+HJdEpXt43UpszjAAABAQDVNpqNdHUlCs9XnbIvc6ZRrNh79wt65YFfvh/QEuA33KnA6Ri6
+EgnV5IdUWXS/UFaYcm2udydrBpVIVifSYl3sioHBylpri23BEy38PKwVXvghUtfpN6dWGn
+NZUG25fQPtIzqi+lo953ZjIj+Adi17AeVv4P4NiLrZeM9lXfWf2pEPOecxXs1IwAf9IiDQ
+WepAwRLsu42eEnHA+DSJPZUkSbISfM5X345k0g6EVATX/yLL3CsqClPzPtsqjh6rbEfFg3
+2OfIMcWV77gOlGWGQ+bUHc8kV6xJqV9QVacLWzfLvIqHF0wQMf8WLOVHEzkfiq4VjwhVqr
+/+FFvljm5nSDAAAAEW1pa2VAQzAyWTUwVEdKR0g4AQ==
+-----END OPENSSH PRIVATE KEY-----

ssh_server_tests/ssh/id_rsa.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC+Lbo0NDxxhNPO2o971mAXkdH+sijkWgOAvk1ttkpbqHS3E/0V3yyJ30GCaj5oaR4zfbu09VUxDpQWZ1GSw4IgNZAOM0sXxvpL4W73m28wpJsat5GhHfY6+BkpUjoqY6wXjPgYi/2do+mVmzC0oRSFJIj7+A90Cz3B/BqWpnvt7zZAnvbttgx4LeXO6hP4oEh1i58ng5CEu2keuqEHmyfuzr6gK/+7zkZKWflIgLXweLa32Vbi2Bvm0fw/SQQMNYZ89LmiSoaLiQVO/AFc4JLavh2GLmFGSk/s+obienQ4J+8rkhFEnbt8KePSWhz6m5kkDGc8xu4mvri/DFHHvw8z9XQ8NSyXQ/ppH+9t7ReYX0UaptZ/ZNqvWnivoL39szNwPPyx5hPyHMsgc/f5GXWF2WxMub7UTOFoTEoh362Nd6AoYbidTbiHqocSdHEtAMzASx2R+PgTWlXKlkldXI2P30Ojzi+LQRSe7tHgeZBv8L4+7uowO0tPdmTs7kyDgtD5xPP1XSehgQDIbjqcAHhT+cnMGdXUn5oB2mKgeZQBiEtiVD0qLLssImL3aD1aprjDEwrEFs7ll4f76nanaVSL+hA2rH/Z0tgWbLTDG8kdtkiEBoVs54K8eDW5wgBDOeWLaM6vaqgH7IpFE8Cn/c1edmDW/tx7YKTCIAPnbCO0KQ== mike@C02Y50TGJGH8

ssh_server_tests/ssh/short_lived_cert_config

@@ -0,0 +1,11 @@
+Host *
+    AddressFamily inet
+
+Host {{hostname}}
+  ProxyCommand bash -c '/usr/local/bin/cloudflared access ssh-gen --hostname %h; ssh -F /root/.ssh/short_lived_cert_config -tt %r@cfpipe-{{hostname}} >&2 <&1'
+
+Host cfpipe-{{hostname}}
+  HostName {{hostname}}
+  ProxyCommand /usr/local/bin/cloudflared access ssh --hostname %h
+  IdentityFile ~/.cloudflared/{{hostname}}-cf_key
+  CertificateFile ~/.cloudflared/{{hostname}}-cf_key-cert.pub

ssh_server_tests/tests.py

@@ -0,0 +1,195 @@
+"""
+Cloudflared Integration tests
+"""
+
+import unittest
+import subprocess
+import os
+import tempfile
+from contextlib import contextmanager
+
+from pexpect import pxssh
+
+
+class TestSSHBase(unittest.TestCase):
+    """
+    SSH test base class containing constants and helper funcs
+    """
+
+    HOSTNAME = os.environ["SSH_HOSTNAME"]
+    SSH_USER = os.environ["SSH_USER"]
+    SSH_TARGET = f"{SSH_USER}@{HOSTNAME}"
+    AUTHORIZED_KEYS_SSH_CONFIG = os.environ["AUTHORIZED_KEYS_SSH_CONFIG"]
+    SHORT_LIVED_CERT_SSH_CONFIG = os.environ["SHORT_LIVED_CERT_SSH_CONFIG"]
+    SSH_OPTIONS = {"StrictHostKeyChecking": "no"}
+
+    @classmethod
+    def get_ssh_command(cls, pty=True):
+        """
+        Return ssh command arg list. If pty is true, a PTY is forced for the session.
+        """
+        cmd = [
+            "ssh",
+            "-o",
+            "StrictHostKeyChecking=no",
+            "-F",
+            cls.AUTHORIZED_KEYS_SSH_CONFIG,
+            cls.SSH_TARGET,
+        ]
+        if not pty:
+            cmd += ["-T"]
+        else:
+            cmd += ["-tt"]
+
+        return cmd
+
+    @classmethod
+    @contextmanager
+    def ssh_session_manager(cls, *args, **kwargs):
+        """
+        Context manager for interacting with a pxssh session.
+        Disables pty echo on the remote server and ensures session is terminated afterward.
+        """
+        session = pxssh.pxssh(options=cls.SSH_OPTIONS)
+
+        session.login(
+            cls.HOSTNAME,
+            username=cls.SSH_USER,
+            original_prompt=r"[#@$]",
+            ssh_config=kwargs.get("ssh_config", cls.AUTHORIZED_KEYS_SSH_CONFIG),
+            ssh_tunnels=kwargs.get("ssh_tunnels", {}),
+        )
+        try:
+            session.sendline("stty -echo")
+            session.prompt()
+            yield session
+        finally:
+            session.logout()
+
+    @staticmethod
+    def get_command_output(session, cmd):
+        """
+        Executes command on remote ssh server and waits for prompt.
+        Returns command output
+        """
+        session.sendline(cmd)
+        session.prompt()
+        return session.before.decode().strip()
+
+    def exec_command(self, cmd, shell=False):
+        """
+        Executes command locally. Raises Assertion error for non-zero return code.
+        Returns stdout and stderr
+        """
+        proc = subprocess.Popen(
+            cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=shell
+        )
+        raw_out, raw_err = proc.communicate()
+
+        out = raw_out.decode()
+        err = raw_err.decode()
+        self.assertEqual(proc.returncode, 0, msg=f"stdout: {out} stderr: {err}")
+        return out.strip(), err.strip()
+
+
+class TestSSHCommandExec(TestSSHBase):
+    """
+    Tests inline ssh command exec
+    """
+
+    # Name of file to be downloaded over SCP on remote server.
+    REMOTE_SCP_FILENAME = os.environ["REMOTE_SCP_FILENAME"]
+
+    @classmethod
+    def get_scp_base_command(cls):
+        return [
+            "scp",
+            "-o",
+            "StrictHostKeyChecking=no",
+            "-v",
+            "-F",
+            cls.AUTHORIZED_KEYS_SSH_CONFIG,
+        ]
+
+    @unittest.skip(
+        "This creates files on the remote. Should be skipped until server is dockerized."
+    )
+    def test_verbose_scp_sink_mode(self):
+        with tempfile.NamedTemporaryFile() as fl:
+            self.exec_command(
+                self.get_scp_base_command() + [fl.name, f"{self.SSH_TARGET}:"]
+            )
+
+    def test_verbose_scp_source_mode(self):
+        with tempfile.TemporaryDirectory() as tmpdirname:
+            self.exec_command(
+                self.get_scp_base_command()
+                + [f"{self.SSH_TARGET}:{self.REMOTE_SCP_FILENAME}", tmpdirname]
+            )
+            local_filename = os.path.join(tmpdirname, self.REMOTE_SCP_FILENAME)
+
+            self.assertTrue(os.path.exists(local_filename))
+            self.assertTrue(os.path.getsize(local_filename) > 0)
+
+    def test_pty_command(self):
+        base_cmd = self.get_ssh_command()
+
+        out, _ = self.exec_command(base_cmd + ["whoami"])
+        self.assertEqual(out.strip().lower(), self.SSH_USER.lower())
+
+        out, _ = self.exec_command(base_cmd + ["tty"])
+        self.assertNotEqual(out, "not a tty")
+
+    def test_non_pty_command(self):
+        base_cmd = self.get_ssh_command(pty=False)
+
+        out, _ = self.exec_command(base_cmd + ["whoami"])
+        self.assertEqual(out.strip().lower(), self.SSH_USER.lower())
+
+        out, _ = self.exec_command(base_cmd + ["tty"])
+        self.assertEqual(out, "not a tty")
+
+
+class TestSSHShell(TestSSHBase):
+    """
+    Tests interactive SSH shell
+    """
+
+    # File path to a file on the remote server with root only read privileges.
+    ROOT_ONLY_TEST_FILE_PATH = os.environ["ROOT_ONLY_TEST_FILE_PATH"]
+
+    def test_ssh_pty(self):
+        with self.ssh_session_manager() as session:
+
+            # Test shell launched as correct user
+            username = self.get_command_output(session, "whoami")
+            self.assertEqual(username.lower(), self.SSH_USER.lower())
+
+            # Test USER env variable set
+            user_var = self.get_command_output(session, "echo $USER")
+            self.assertEqual(user_var.lower(), self.SSH_USER.lower())
+
+            # Test HOME env variable set to true user home.
+            home_env = self.get_command_output(session, "echo $HOME")
+            pwd = self.get_command_output(session, "pwd")
+            self.assertEqual(pwd, home_env)
+
+            # Test shell launched in correct user home dir.
+            self.assertIn(username, pwd)
+
+            # Ensure shell launched with correct user's permissions and privs.
+            # Cant read root owned 0700 files.
+            output = self.get_command_output(
+                session, f"cat {self.ROOT_ONLY_TEST_FILE_PATH}"
+            )
+            self.assertIn("Permission denied", output)
+
+    def test_short_lived_cert_auth(self):
+        with self.ssh_session_manager(
+            ssh_config=self.SHORT_LIVED_CERT_SSH_CONFIG
+        ) as session:
+            username = self.get_command_output(session, "whoami")
+            self.assertEqual(username.lower(), self.SSH_USER.lower())
+
+
+unittest.main()

sshlog/empty_manager.go

@@ -0,0 +1,37 @@
+package sshlog
+
+import (
+	"io"
+
+	"github.com/sirupsen/logrus"
+)
+
+//empty manager implements the Manager but does nothing (for testing and to disable logging unless the logs are set)
+type emptyManager struct {
+}
+
+type emptyWriteCloser struct {
+}
+
+// NewEmptyManager creates a new instance of a log empty log manager that does nothing
+func NewEmptyManager() Manager {
+	return &emptyManager{}
+}
+
+func (m *emptyManager) NewLogger(name string, logger *logrus.Logger) (io.WriteCloser, error) {
+	return &emptyWriteCloser{}, nil
+}
+
+func (m *emptyManager) NewSessionLogger(name string, logger *logrus.Logger) (io.WriteCloser, error) {
+	return &emptyWriteCloser{}, nil
+}
+
+// emptyWriteCloser
+
+func (w *emptyWriteCloser) Write(p []byte) (n int, err error) {
+	return len(p), nil
+}
+
+func (w *emptyWriteCloser) Close() error {
+	return nil
+}

sshlog/go.capnp

@@ -0,0 +1,15 @@
+# Generate go.capnp.out with:
+# capnp compile -o- go.capnp > go.capnp.out
+# Must run inside this directory to preserve paths.
+
+@0xd12a1c51fedd6c88;
+
+annotation package(file) :Text;
+annotation import(file) :Text;
+annotation doc(struct, field, enum) :Text;
+annotation tag(enumerant) :Text;
+annotation notag(enumerant) :Void;
+annotation customtype(field) :Text;
+annotation name(struct, field, union, enum, enumerant, interface, method, param, annotation, const, group) :Text;
+
+$package("capnp");

sshlog/logger.go

@@ -0,0 +1,167 @@
+package sshlog
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sync"
+	"time"
+
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	logTimeFormat        = "2006-01-02T15-04-05.000"
+	megabyte             = 1024 * 1024
+	defaultFileSizeLimit = 100 * megabyte
+)
+
+// Logger will buffer and write events to disk
+type Logger struct {
+	sync.Mutex
+	filename      string
+	file          *os.File
+	writeBuffer   *bufio.Writer
+	logger        *logrus.Logger
+	flushInterval time.Duration
+	maxFileSize   int64
+	done          chan struct{}
+	once          sync.Once
+}
+
+// NewLogger creates a Logger instance. A buffer is created that needs to be
+// drained and closed when the caller is finished, so instances should call
+// Close when finished with this Logger instance. Writes will be flushed to disk
+// every second (fsync). filename is the name of the logfile to be created. The
+// logger variable is a logrus that will log all i/o, filesystem error etc, that
+// that shouldn't end execution of the logger, but are useful to report to the
+// caller.
+func NewLogger(filename string, logger *logrus.Logger, flushInterval time.Duration, maxFileSize int64) (*Logger, error) {
+	if logger == nil {
+		return nil, errors.New("logger can't be nil")
+	}
+	f, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(0600))
+	if err != nil {
+		return nil, err
+	}
+	l := &Logger{filename: filename,
+		file:          f,
+		writeBuffer:   bufio.NewWriter(f),
+		logger:        logger,
+		flushInterval: flushInterval,
+		maxFileSize:   maxFileSize,
+		done:          make(chan struct{}),
+	}
+
+	go l.writer()
+	return l, nil
+}
+
+// Writes to a log buffer. Implements the io.Writer interface.
+func (l *Logger) Write(p []byte) (n int, err error) {
+	l.Lock()
+	defer l.Unlock()
+	return l.writeBuffer.Write(p)
+}
+
+// Close drains anything left in the buffer and cleans up any resources still
+// in use.
+func (l *Logger) Close() error {
+	l.once.Do(func() {
+		close(l.done)
+	})
+	if err := l.write(); err != nil {
+		return err
+	}
+	return l.file.Close()
+}
+
+// writer is the run loop that handles draining the write buffer and syncing
+// data to disk.
+func (l *Logger) writer() {
+	ticker := time.NewTicker(l.flushInterval)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ticker.C:
+			if err := l.write(); err != nil {
+				l.logger.Errorln(err)
+			}
+		case <-l.done:
+			return
+		}
+	}
+}
+
+// write does the actual system write calls to disk and does a rotation if the
+// file size limit has been reached. Since the rotation happens at the end,
+// the rotation is a soft limit (aka the file can be bigger than the max limit
+// because of the final buffer flush)
+func (l *Logger) write() error {
+	l.Lock()
+	defer l.Unlock()
+
+	if l.writeBuffer.Buffered() <= 0 {
+		return nil
+	}
+
+	if err := l.writeBuffer.Flush(); err != nil {
+		return err
+	}
+
+	if err := l.file.Sync(); err != nil {
+		return err
+	}
+
+	if l.shouldRotate() {
+		return l.rotate()
+	}
+	return nil
+}
+
+// shouldRotate checks to see if the current file should be rotated to a new
+// logfile.
+func (l *Logger) shouldRotate() bool {
+	info, err := l.file.Stat()
+	if err != nil {
+		return false
+	}
+
+	return info.Size() >= l.maxFileSize
+}
+
+// rotate creates a new logfile with the existing filename and renames the
+// existing file with a current timestamp.
+func (l *Logger) rotate() error {
+	if err := l.file.Close(); err != nil {
+		return err
+	}
+
+	// move the existing file
+	newname := rotationName(l.filename)
+	if err := os.Rename(l.filename, newname); err != nil {
+		return fmt.Errorf("can't rename log file: %s", err)
+	}
+
+	f, err := os.OpenFile(l.filename, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(0600))
+	if err != nil {
+		return fmt.Errorf("failed to open new logfile %s", err)
+	}
+	l.file = f
+	l.writeBuffer = bufio.NewWriter(f)
+	return nil
+}
+
+// rotationName creates a new filename from the given name, inserting a timestamp
+// between the filename and the extension.
+func rotationName(name string) string {
+	dir := filepath.Dir(name)
+	filename := filepath.Base(name)
+	ext := filepath.Ext(filename)
+	prefix := filename[:len(filename)-len(ext)]
+	t := time.Now()
+	timestamp := t.Format(logTimeFormat)
+	return filepath.Join(dir, fmt.Sprintf("%s-%s%s", prefix, timestamp, ext))
+}

sshlog/logger_test.go

@@ -0,0 +1,90 @@
+package sshlog
+
+import (
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/sirupsen/logrus"
+)
+
+const logFileName = "test-logger.log"
+
+func createLogger(t *testing.T) *Logger {
+	os.Remove(logFileName)
+	l := logrus.New()
+	logger, err := NewLogger(logFileName, l, time.Millisecond, 1024)
+	if err != nil {
+		t.Fatal("couldn't create the logger!", err)
+	}
+	return logger
+}
+
+func TestWrite(t *testing.T) {
+	testStr := "hi"
+	logger := createLogger(t)
+	defer func() {
+		logger.Close()
+		os.Remove(logFileName)
+	}()
+
+	logger.Write([]byte(testStr))
+	time.Sleep(2 * time.Millisecond)
+	data, err := ioutil.ReadFile(logFileName)
+	if err != nil {
+		t.Fatal("couldn't read the log file!", err)
+	}
+	checkStr := string(data)
+	if checkStr != testStr {
+		t.Fatal("file data doesn't match!")
+	}
+}
+
+func TestFilenameRotation(t *testing.T) {
+	newName := rotationName("dir/bob/acoolloggername.log")
+
+	dir := filepath.Dir(newName)
+	if dir != "dir/bob" {
+		t.Fatal("rotation name doesn't respect the directory filepath:", newName)
+	}
+
+	filename := filepath.Base(newName)
+	if !strings.HasPrefix(filename, "acoolloggername") {
+		t.Fatal("rotation filename is wrong:", filename)
+	}
+
+	ext := filepath.Ext(newName)
+	if ext != ".log" {
+		t.Fatal("rotation file extension is wrong:", ext)
+	}
+}
+
+func TestRotation(t *testing.T) {
+	logger := createLogger(t)
+
+	for i := 0; i < 2000; i++ {
+		logger.Write([]byte("a string for testing rotation\n"))
+	}
+	logger.Close()
+
+	count := 0
+	filepath.Walk(".", func(path string, info os.FileInfo, err error) error {
+		if err != nil || info.IsDir() {
+			return nil
+		}
+		if strings.HasPrefix(info.Name(), "test-logger") {
+			log.Println("deleting: ", path)
+			os.Remove(path)
+			count++
+		}
+		return nil
+	})
+	if count < 2 {
+		t.Fatal("rotation didn't roll files:", count)
+	}
+
+}

sshlog/manager.go

@@ -0,0 +1,34 @@
+package sshlog
+
+import (
+	"io"
+	"path/filepath"
+	"time"
+
+	"github.com/sirupsen/logrus"
+)
+
+// Manager be managing logs bruh
+type Manager interface {
+	NewLogger(string, *logrus.Logger) (io.WriteCloser, error)
+	NewSessionLogger(string, *logrus.Logger) (io.WriteCloser, error)
+}
+
+type manager struct {
+	baseDirectory string
+}
+
+// New creates a new instance of a log manager
+func New(baseDirectory string) Manager {
+	return &manager{
+		baseDirectory: baseDirectory,
+	}
+}
+
+func (m *manager) NewLogger(name string, logger *logrus.Logger) (io.WriteCloser, error) {
+	return NewLogger(filepath.Join(m.baseDirectory, name), logger, time.Second, defaultFileSizeLimit)
+}
+
+func (m *manager) NewSessionLogger(name string, logger *logrus.Logger) (io.WriteCloser, error) {
+	return NewSessionLogger(filepath.Join(m.baseDirectory, name), logger, time.Second, defaultFileSizeLimit)
+}

sshlog/session_log.capnp

@@ -0,0 +1,9 @@
+using Go = import "go.capnp";
+@0x8f43375162194466;
+$Go.package("sshlog");
+$Go.import("github.com/cloudflare/cloudflared/sshlog");
+
+struct SessionLog {
+    timestamp @0 :Text;
+    content @1 :Data;
+}

sshlog/session_log.capnp.go

@@ -0,0 +1,110 @@
+// Code generated by capnpc-go. DO NOT EDIT.
+
+package sshlog
+
+import (
+	capnp "zombiezen.com/go/capnproto2"
+	text "zombiezen.com/go/capnproto2/encoding/text"
+	schemas "zombiezen.com/go/capnproto2/schemas"
+)
+
+type SessionLog struct{ capnp.Struct }
+
+// SessionLog_TypeID is the unique identifier for the type SessionLog.
+const SessionLog_TypeID = 0xa13a07c504a5ab64
+
+func NewSessionLog(s *capnp.Segment) (SessionLog, error) {
+	st, err := capnp.NewStruct(s, capnp.ObjectSize{DataSize: 0, PointerCount: 2})
+	return SessionLog{st}, err
+}
+
+func NewRootSessionLog(s *capnp.Segment) (SessionLog, error) {
+	st, err := capnp.NewRootStruct(s, capnp.ObjectSize{DataSize: 0, PointerCount: 2})
+	return SessionLog{st}, err
+}
+
+func ReadRootSessionLog(msg *capnp.Message) (SessionLog, error) {
+	root, err := msg.RootPtr()
+	return SessionLog{root.Struct()}, err
+}
+
+func (s SessionLog) String() string {
+	str, _ := text.Marshal(0xa13a07c504a5ab64, s.Struct)
+	return str
+}
+
+func (s SessionLog) Timestamp() (string, error) {
+	p, err := s.Struct.Ptr(0)
+	return p.Text(), err
+}
+
+func (s SessionLog) HasTimestamp() bool {
+	p, err := s.Struct.Ptr(0)
+	return p.IsValid() || err != nil
+}
+
+func (s SessionLog) TimestampBytes() ([]byte, error) {
+	p, err := s.Struct.Ptr(0)
+	return p.TextBytes(), err
+}
+
+func (s SessionLog) SetTimestamp(v string) error {
+	return s.Struct.SetText(0, v)
+}
+
+func (s SessionLog) Content() ([]byte, error) {
+	p, err := s.Struct.Ptr(1)
+	return []byte(p.Data()), err
+}
+
+func (s SessionLog) HasContent() bool {
+	p, err := s.Struct.Ptr(1)
+	return p.IsValid() || err != nil
+}
+
+func (s SessionLog) SetContent(v []byte) error {
+	return s.Struct.SetData(1, v)
+}
+
+// SessionLog_List is a list of SessionLog.
+type SessionLog_List struct{ capnp.List }
+
+// NewSessionLog creates a new list of SessionLog.
+func NewSessionLog_List(s *capnp.Segment, sz int32) (SessionLog_List, error) {
+	l, err := capnp.NewCompositeList(s, capnp.ObjectSize{DataSize: 0, PointerCount: 2}, sz)
+	return SessionLog_List{l}, err
+}
+
+func (s SessionLog_List) At(i int) SessionLog { return SessionLog{s.List.Struct(i)} }
+
+func (s SessionLog_List) Set(i int, v SessionLog) error { return s.List.SetStruct(i, v.Struct) }
+
+func (s SessionLog_List) String() string {
+	str, _ := text.MarshalList(0xa13a07c504a5ab64, s.List)
+	return str
+}
+
+// SessionLog_Promise is a wrapper for a SessionLog promised by a client call.
+type SessionLog_Promise struct{ *capnp.Pipeline }
+
+func (p SessionLog_Promise) Struct() (SessionLog, error) {
+	s, err := p.Pipeline.Struct()
+	return SessionLog{s}, err
+}
+
+const schema_8f43375162194466 = "x\xda\x120q`\x12d\x8dg`\x08dae\xfb" +
+	"\x9f\xb2z)\xcbQv\xab\x85\x0c\x82B\x8c\xff\xd3\\" +
+	"$\x93\x02\xcd\x9d\xfb\x19X\x99\xd8\x19\x18\x04E_\x09" +
+	"*\x82h\xd9r\x06\xc6\xff\xc5\xa9\xc5\xc5\x99\xf9y\xf1" +
+	"L9\xf9\xe9z\xc9\x89\x05y\x05V\xc1`!\xfe<" +
+	"\x9f\xfc\xf4\x00F\xc6@\x0ef\x16\x06\x06\x16F\x06\x06" +
+	"A\xcd \x06\x86@\x0df\xc6@\x13&FAFF" +
+	"\x11F\x90\xa0\xa1\x13\x03C\xa0\x0e3c\xa0\x05\x13\xe3" +
+	"\xff\x92\xcc\xdc\xd4\xe2\x92\xc4\\\x06\xc6\x02F\x1e\x06&" +
+	"F\x1e\x06\xc6\xfa\xe4\xfc\xbc\x92\xd4\xbc\x12F^\x06&" +
+	"F^\x06F@\x00\x00\x00\xff\xff\xdaK$\x1a"
+
+func init() {
+	schemas.Register(schema_8f43375162194466,
+		0xa13a07c504a5ab64)
+}

sshlog/session_logger.go

@@ -0,0 +1,71 @@
+package sshlog
+
+import (
+	"time"
+
+	"github.com/sirupsen/logrus"
+	capnp "zombiezen.com/go/capnproto2"
+	"zombiezen.com/go/capnproto2/pogs"
+)
+
+// SessionLogger will buffer and write events to disk using capnp proto for session replay
+type SessionLogger struct {
+	logger  *Logger
+	encoder *capnp.Encoder
+}
+
+type sessionLogData struct {
+	Timestamp string // The UTC timestamp of when the log occurred
+	Content   []byte // The shell output
+}
+
+// NewSessionLogger creates a new session logger by encapsulating a Logger object and writing capnp encoded messages to it
+func NewSessionLogger(filename string, logger *logrus.Logger, flushInterval time.Duration, maxFileSize int64) (*SessionLogger, error) {
+	l, err := NewLogger(filename, logger, flushInterval, maxFileSize)
+	if err != nil {
+		return nil, err
+	}
+	sessionLogger := &SessionLogger{
+		logger:  l,
+		encoder: capnp.NewEncoder(l),
+	}
+	return sessionLogger, nil
+}
+
+// Writes to a log buffer. Implements the io.Writer interface.
+func (l *SessionLogger) Write(p []byte) (n int, err error) {
+	return l.writeSessionLog(&sessionLogData{
+		Timestamp: time.Now().UTC().Format(time.RFC3339),
+		Content:   p,
+	})
+}
+
+// Close drains anything left in the buffer and cleans up any resources still
+// in use.
+func (l *SessionLogger) Close() error {
+	return l.logger.Close()
+}
+
+func (l *SessionLogger) writeSessionLog(p *sessionLogData) (int, error) {
+	msg, seg, err := capnp.NewMessage(capnp.SingleSegment(nil))
+	if err != nil {
+		return 0, err
+	}
+	log, err := NewRootSessionLog(seg)
+	if err != nil {
+		return 0, err
+	}
+	log.SetTimestamp(p.Timestamp)
+	log.SetContent(p.Content)
+
+	if err := l.encoder.Encode(msg); err != nil {
+		return 0, err
+	}
+	return len(p.Content), nil
+}
+
+func unmarshalSessionLog(s SessionLog) (*sessionLogData, error) {
+	p := new(sessionLogData)
+	err := pogs.Extract(p, SessionLog_TypeID, s.Struct)
+	return p, err
+}

sshlog/session_logger_test.go

@@ -0,0 +1,69 @@
+package sshlog
+
+import (
+	"os"
+	"testing"
+	"time"
+
+	"github.com/sirupsen/logrus"
+	capnp "zombiezen.com/go/capnproto2"
+)
+
+const sessionLogFileName = "test-session-logger.log"
+
+func createSessionLogger(t *testing.T) *SessionLogger {
+	os.Remove(sessionLogFileName)
+	l := logrus.New()
+	logger, err := NewSessionLogger(sessionLogFileName, l, time.Millisecond, 1024)
+	if err != nil {
+		t.Fatal("couldn't create the logger!", err)
+	}
+	return logger
+}
+
+func TestSessionLogWrite(t *testing.T) {
+	testStr := "hi"
+	logger := createSessionLogger(t)
+	defer func() {
+		logger.Close()
+		os.Remove(sessionLogFileName)
+	}()
+
+	logger.Write([]byte(testStr))
+	time.Sleep(2 * time.Millisecond)
+	f, err := os.Open(sessionLogFileName)
+	if err != nil {
+		t.Fatal("couldn't read the log file!", err)
+	}
+	defer f.Close()
+
+	msg, err := capnp.NewDecoder(f).Decode()
+	if err != nil {
+		t.Fatal("couldn't read the capnp msg file!", err)
+	}
+
+	sessionLog, err := ReadRootSessionLog(msg)
+	if err != nil {
+		t.Fatal("couldn't read the session log from the msg!", err)
+	}
+
+	timeStr, err := sessionLog.Timestamp()
+	if err != nil {
+		t.Fatal("couldn't read the Timestamp field!", err)
+	}
+
+	_, terr := time.Parse(time.RFC3339, timeStr)
+	if terr != nil {
+		t.Fatal("couldn't parse the Timestamp into the expected RFC3339 format", terr)
+	}
+
+	data, err := sessionLog.Content()
+	if err != nil {
+		t.Fatal("couldn't read the Content field!", err)
+	}
+
+	checkStr := string(data)
+	if checkStr != testStr {
+		t.Fatal("file data doesn't match!")
+	}
+}

sshserver/authentication.go

@@ -0,0 +1,108 @@
+//+build !windows
+
+package sshserver
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+
+	"github.com/gliderlabs/ssh"
+	"github.com/pkg/errors"
+	gossh "golang.org/x/crypto/ssh"
+)
+
+var (
+	systemConfigPath  = "/etc/cloudflared/"
+	authorizedKeysDir = ".cloudflared/authorized_keys"
+)
+
+func (s *SSHServer) configureAuthentication() {
+	caCert, err := getCACert()
+	if err != nil {
+		s.logger.Info(err)
+	}
+	s.caCert = caCert
+	s.PublicKeyHandler = s.authenticationHandler
+}
+
+// authenticationHandler is a callback that returns true if the user attempting to connect is authenticated.
+func (s *SSHServer) authenticationHandler(ctx ssh.Context, key ssh.PublicKey) bool {
+	sshUser, err := lookupUser(ctx.User())
+	if err != nil {
+		s.logger.Debugf("Invalid user: %s", ctx.User())
+		return false
+	}
+	ctx.SetValue("sshUser", sshUser)
+
+	cert, ok := key.(*gossh.Certificate)
+	if !ok {
+		return s.authorizedKeyHandler(ctx, key)
+	}
+	return s.shortLivedCertHandler(ctx, cert)
+}
+
+func (s *SSHServer) authorizedKeyHandler(ctx ssh.Context, key ssh.PublicKey) bool {
+	sshUser, ok := ctx.Value("sshUser").(*User)
+	if !ok {
+		s.logger.Error("Failed to retrieve user from context")
+		return false
+	}
+
+	authorizedKeysPath := path.Join(sshUser.HomeDir, authorizedKeysDir)
+	if _, err := os.Stat(authorizedKeysPath); os.IsNotExist(err) {
+		s.logger.Debugf("authorized_keys file %s not found", authorizedKeysPath)
+		return false
+	}
+
+	authorizedKeysBytes, err := ioutil.ReadFile(authorizedKeysPath)
+	if err != nil {
+		s.logger.WithError(err).Errorf("Failed to load authorized_keys %s", authorizedKeysPath)
+		return false
+	}
+
+	for len(authorizedKeysBytes) > 0 {
+		// Skips invalid keys. Returns error if no valid keys remain.
+		pubKey, _, _, rest, err := ssh.ParseAuthorizedKey(authorizedKeysBytes)
+		authorizedKeysBytes = rest
+		if err != nil {
+			s.logger.Errorf("Invalid key(s) found in %s", authorizedKeysPath)
+			return false
+		}
+
+		if ssh.KeysEqual(pubKey, key) {
+			return true
+		}
+	}
+	s.logger.Debugf("Matching public key not found in %s", authorizedKeysPath)
+	return false
+}
+
+func (s *SSHServer) shortLivedCertHandler(ctx ssh.Context, cert *gossh.Certificate) bool {
+	if !ssh.KeysEqual(s.caCert, cert.SignatureKey) {
+		s.logger.Debug("CA certificate does not match user certificate signer")
+		return false
+	}
+
+	checker := gossh.CertChecker{}
+	if err := checker.CheckCert(ctx.User(), cert); err != nil {
+		s.logger.Debug(err)
+		return false
+	}
+	return true
+}
+
+func getCACert() (ssh.PublicKey, error) {
+	caCertPath := path.Join(systemConfigPath, "ca.pub")
+	caCertBytes, err := ioutil.ReadFile(caCertPath)
+	if err != nil {
+		return nil, errors.Wrap(err, fmt.Sprintf("Failed to load CA certificate %s", caCertPath))
+	}
+	caCert, _, _, _, err := ssh.ParseAuthorizedKey(caCertBytes)
+	if err != nil {
+		return nil, errors.Wrap(err, "Failed to parse CA Certificate")
+	}
+
+	return caCert, nil
+}

sshserver/authentication_test.go

@@ -0,0 +1,185 @@
+//+build !windows
+
+package sshserver
+
+import (
+	"context"
+	"io/ioutil"
+	"net"
+	"os"
+	"os/user"
+	"path"
+	"sync"
+	"testing"
+
+	"github.com/cloudflare/cloudflared/log"
+	"github.com/gliderlabs/ssh"
+	"github.com/sirupsen/logrus"
+	"github.com/sirupsen/logrus/hooks/test"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	gossh "golang.org/x/crypto/ssh"
+)
+
+const (
+	testDir              = "testdata"
+	testUserKeyFilename  = "id_rsa.pub"
+	testCAFilename       = "ca.pub"
+	testOtherCAFilename  = "other_ca.pub"
+	testUserCertFilename = "id_rsa-cert.pub"
+)
+
+var (
+	logger, hook = test.NewNullLogger()
+	mockUser     = &User{Username: "testUser", HomeDir: testDir}
+)
+
+func TestMain(m *testing.M) {
+	authorizedKeysDir = testUserKeyFilename
+	logger.SetLevel(logrus.DebugLevel)
+	code := m.Run()
+	os.Exit(code)
+}
+
+func TestPublicKeyAuth_Success(t *testing.T) {
+	context, cancel := newMockContext(mockUser)
+	defer cancel()
+
+	sshServer := SSHServer{logger: logger}
+
+	pubKey := getKey(t, testUserKeyFilename)
+	assert.True(t, sshServer.authorizedKeyHandler(context, pubKey))
+}
+
+func TestPublicKeyAuth_MissingKey(t *testing.T) {
+	context, cancel := newMockContext(mockUser)
+	defer cancel()
+
+	sshServer := SSHServer{logger: logger}
+
+	pubKey := getKey(t, testOtherCAFilename)
+	assert.False(t, sshServer.authorizedKeyHandler(context, pubKey))
+	assert.Contains(t, hook.LastEntry().Message, "Matching public key not found in")
+}
+
+func TestPublicKeyAuth_InvalidUser(t *testing.T) {
+	context, cancel := newMockContext(&User{Username: "notAUser"})
+	defer cancel()
+
+	sshServer := SSHServer{logger: logger}
+
+	pubKey := getKey(t, testUserKeyFilename)
+	assert.False(t, sshServer.authenticationHandler(context, pubKey))
+	assert.Contains(t, hook.LastEntry().Message, "Invalid user")
+}
+
+func TestPublicKeyAuth_MissingFile(t *testing.T) {
+	tempUser, err := user.Current()
+	require.Nil(t, err)
+	currentUser, err := lookupUser(tempUser.Username)
+	require.Nil(t, err)
+
+	require.Nil(t, err)
+	context, cancel := newMockContext(currentUser)
+	defer cancel()
+
+	sshServer := SSHServer{Server: ssh.Server{}, logger: logger}
+
+	pubKey := getKey(t, testUserKeyFilename)
+	assert.False(t, sshServer.authorizedKeyHandler(context, pubKey))
+	assert.Contains(t, hook.LastEntry().Message, "not found")
+}
+
+func TestShortLivedCerts_Success(t *testing.T) {
+	context, cancel := newMockContext(mockUser)
+	defer cancel()
+
+	caCert := getKey(t, testCAFilename)
+	sshServer := SSHServer{logger: log.CreateLogger(), caCert: caCert}
+
+	userCert, ok := getKey(t, testUserCertFilename).(*gossh.Certificate)
+	require.True(t, ok)
+	assert.True(t, sshServer.shortLivedCertHandler(context, userCert))
+}
+
+func TestShortLivedCerts_CAsDontMatch(t *testing.T) {
+	context, cancel := newMockContext(mockUser)
+	defer cancel()
+
+	caCert := getKey(t, testOtherCAFilename)
+	sshServer := SSHServer{logger: logger, caCert: caCert}
+
+	userCert, ok := getKey(t, testUserCertFilename).(*gossh.Certificate)
+	require.True(t, ok)
+	assert.False(t, sshServer.shortLivedCertHandler(context, userCert))
+	assert.Equal(t, "CA certificate does not match user certificate signer", hook.LastEntry().Message)
+}
+
+func TestShortLivedCerts_InvalidPrincipal(t *testing.T) {
+	context, cancel := newMockContext(&User{Username: "NotAUser"})
+	defer cancel()
+
+	caCert := getKey(t, testCAFilename)
+	sshServer := SSHServer{logger: logger, caCert: caCert}
+
+	userCert, ok := getKey(t, testUserCertFilename).(*gossh.Certificate)
+	require.True(t, ok)
+	assert.False(t, sshServer.shortLivedCertHandler(context, userCert))
+	assert.Contains(t, hook.LastEntry().Message, "not in the set of valid principals for given certificate")
+}
+
+func getKey(t *testing.T, filename string) ssh.PublicKey {
+	path := path.Join(testDir, filename)
+	bytes, err := ioutil.ReadFile(path)
+	require.Nil(t, err)
+	pubKey, _, _, _, err := ssh.ParseAuthorizedKey(bytes)
+	require.Nil(t, err)
+	return pubKey
+}
+
+type mockSSHContext struct {
+	context.Context
+	*sync.Mutex
+}
+
+func newMockContext(user *User) (*mockSSHContext, context.CancelFunc) {
+	innerCtx, cancel := context.WithCancel(context.Background())
+	mockCtx := &mockSSHContext{innerCtx, &sync.Mutex{}}
+	mockCtx.SetValue("sshUser", user)
+
+	// This naming is confusing but we cant change it because this mocks the SSHContext struct in gliderlabs/ssh
+	mockCtx.SetValue("user", user.Username)
+	return mockCtx, cancel
+}
+
+func (ctx *mockSSHContext) SetValue(key, value interface{}) {
+	ctx.Context = context.WithValue(ctx.Context, key, value)
+}
+
+func (ctx *mockSSHContext) User() string {
+	return ctx.Value("user").(string)
+}
+
+func (ctx *mockSSHContext) SessionID() string {
+	return ""
+}
+
+func (ctx *mockSSHContext) ClientVersion() string {
+	return ""
+}
+
+func (ctx *mockSSHContext) ServerVersion() string {
+	return ""
+}
+
+func (ctx *mockSSHContext) RemoteAddr() net.Addr {
+	return nil
+}
+
+func (ctx *mockSSHContext) LocalAddr() net.Addr {
+	return nil
+}
+
+func (ctx *mockSSHContext) Permissions() *ssh.Permissions {
+	return nil
+}

sshserver/get_user.go

@@ -0,0 +1,226 @@
+// Taken from https://github.com/golang/go/blob/ad644d2e86bab85787879d41c2d2aebbd7c57db8/src/os/user/user.go
+// and modified to return login shell in User struct. cloudflared requires cgo for compilation because of this addition.
+
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build aix darwin dragonfly freebsd !android,linux netbsd openbsd solaris
+// +build cgo,!osusergo
+
+package sshserver
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+	"syscall"
+	"unsafe"
+)
+
+/*
+#cgo solaris CFLAGS: -D_POSIX_PTHREAD_SEMANTICS
+#include <unistd.h>
+#include <sys/types.h>
+#include <pwd.h>
+#include <grp.h>
+#include <stdlib.h>
+
+static int mygetpwuid_r(int uid, struct passwd *pwd,
+	char *buf, size_t buflen, struct passwd **result) {
+	return getpwuid_r(uid, pwd, buf, buflen, result);
+}
+
+static int mygetpwnam_r(const char *name, struct passwd *pwd,
+	char *buf, size_t buflen, struct passwd **result) {
+	return getpwnam_r(name, pwd, buf, buflen, result);
+}
+
+static int mygetgrgid_r(int gid, struct group *grp,
+	char *buf, size_t buflen, struct group **result) {
+ return getgrgid_r(gid, grp, buf, buflen, result);
+}
+
+static int mygetgrnam_r(const char *name, struct group *grp,
+	char *buf, size_t buflen, struct group **result) {
+ return getgrnam_r(name, grp, buf, buflen, result);
+}
+*/
+import "C"
+
+type UnknownUserIdError int
+
+func (e UnknownUserIdError) Error() string {
+	return "user: unknown userid " + strconv.Itoa(int(e))
+}
+
+// UnknownUserError is returned by Lookup when
+// a user cannot be found.
+type UnknownUserError string
+
+func (e UnknownUserError) Error() string {
+	return "user: unknown user " + string(e)
+}
+
+// UnknownGroupIdError is returned by LookupGroupId when
+// a group cannot be found.
+type UnknownGroupIdError string
+
+func (e UnknownGroupIdError) Error() string {
+	return "group: unknown groupid " + string(e)
+}
+
+// UnknownGroupError is returned by LookupGroup when
+// a group cannot be found.
+type UnknownGroupError string
+
+func (e UnknownGroupError) Error() string {
+	return "group: unknown group " + string(e)
+}
+
+type User struct {
+	// Uid is the user ID.
+	// On POSIX systems, this is a decimal number representing the uid.
+	// On Windows, this is a security identifier (SID) in a string format.
+	// On Plan 9, this is the contents of /dev/user.
+	Uid string
+	// Gid is the primary group ID.
+	// On POSIX systems, this is a decimal number representing the gid.
+	// On Windows, this is a SID in a string format.
+	// On Plan 9, this is the contents of /dev/user.
+	Gid string
+	// Username is the login name.
+	Username string
+	// Name is the user's real or display name.
+	// It might be blank.
+	// On POSIX systems, this is the first (or only) entry in the GECOS field
+	// list.
+	// On Windows, this is the user's display name.
+	// On Plan 9, this is the contents of /dev/user.
+	Name string
+	// HomeDir is the path to the user's home directory (if they have one).
+	HomeDir string
+
+	/****************** Begin added code ******************/
+	// Login shell
+	Shell string
+	/****************** End added code ******************/
+}
+
+func lookupUser(username string) (*User, error) {
+	var pwd C.struct_passwd
+	var result *C.struct_passwd
+	nameC := make([]byte, len(username)+1)
+	copy(nameC, username)
+
+	buf := alloc(userBuffer)
+	defer buf.free()
+
+	err := retryWithBuffer(buf, func() syscall.Errno {
+		// mygetpwnam_r is a wrapper around getpwnam_r to avoid
+		// passing a size_t to getpwnam_r, because for unknown
+		// reasons passing a size_t to getpwnam_r doesn't work on
+		// Solaris.
+		return syscall.Errno(C.mygetpwnam_r((*C.char)(unsafe.Pointer(&nameC[0])),
+			&pwd,
+			(*C.char)(buf.ptr),
+			C.size_t(buf.size),
+			&result))
+	})
+	if err != nil {
+		return nil, fmt.Errorf("user: lookup username %s: %v", username, err)
+	}
+	if result == nil {
+		return nil, UnknownUserError(username)
+	}
+	return buildUser(&pwd), err
+}
+
+func buildUser(pwd *C.struct_passwd) *User {
+	u := &User{
+		Uid:      strconv.FormatUint(uint64(pwd.pw_uid), 10),
+		Gid:      strconv.FormatUint(uint64(pwd.pw_gid), 10),
+		Username: C.GoString(pwd.pw_name),
+		Name:     C.GoString(pwd.pw_gecos),
+		HomeDir:  C.GoString(pwd.pw_dir),
+		/****************** Begin added code ******************/
+		Shell: C.GoString(pwd.pw_shell),
+		/****************** End added code ******************/
+	}
+	// The pw_gecos field isn't quite standardized. Some docs
+	// say: "It is expected to be a comma separated list of
+	// personal data where the first item is the full name of the
+	// user."
+	if i := strings.Index(u.Name, ","); i >= 0 {
+		u.Name = u.Name[:i]
+	}
+	return u
+}
+
+type bufferKind C.int
+
+const (
+	userBuffer = bufferKind(C._SC_GETPW_R_SIZE_MAX)
+)
+
+func (k bufferKind) initialSize() C.size_t {
+	sz := C.sysconf(C.int(k))
+	if sz == -1 {
+		// DragonFly and FreeBSD do not have _SC_GETPW_R_SIZE_MAX.
+		// Additionally, not all Linux systems have it, either. For
+		// example, the musl libc returns -1.
+		return 1024
+	}
+	if !isSizeReasonable(int64(sz)) {
+		// Truncate.  If this truly isn't enough, retryWithBuffer will error on the first run.
+		return maxBufferSize
+	}
+	return C.size_t(sz)
+}
+
+type memBuffer struct {
+	ptr  unsafe.Pointer
+	size C.size_t
+}
+
+func alloc(kind bufferKind) *memBuffer {
+	sz := kind.initialSize()
+	return &memBuffer{
+		ptr:  C.malloc(sz),
+		size: sz,
+	}
+}
+
+func (mb *memBuffer) resize(newSize C.size_t) {
+	mb.ptr = C.realloc(mb.ptr, newSize)
+	mb.size = newSize
+}
+
+func (mb *memBuffer) free() {
+	C.free(mb.ptr)
+}
+
+// retryWithBuffer repeatedly calls f(), increasing the size of the
+// buffer each time, until f succeeds, fails with a non-ERANGE error,
+// or the buffer exceeds a reasonable limit.
+func retryWithBuffer(buf *memBuffer, f func() syscall.Errno) error {
+	for {
+		errno := f()
+		if errno == 0 {
+			return nil
+		} else if errno != syscall.ERANGE {
+			return errno
+		}
+		newSize := buf.size * 2
+		if !isSizeReasonable(int64(newSize)) {
+			return fmt.Errorf("internal buffer exceeds %d bytes", maxBufferSize)
+		}
+		buf.resize(newSize)
+	}
+}
+
+const maxBufferSize = 1 << 20
+
+func isSizeReasonable(sz int64) bool {
+	return sz > 0 && sz <= maxBufferSize
+}

sshserver/host_keys.go

@@ -0,0 +1,110 @@
+//+build !windows
+
+package sshserver
+
+import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"encoding/pem"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/gliderlabs/ssh"
+	"github.com/pkg/errors"
+)
+
+const (
+	rsaFilename   = "ssh_host_rsa_key"
+	ecdsaFilename = "ssh_host_ecdsa_key"
+)
+
+func (s *SSHServer) configureHostKeys() error {
+	if _, err := os.Stat(systemConfigPath); os.IsNotExist(err) {
+		if err := os.MkdirAll(systemConfigPath, 0755); err != nil {
+			return errors.Wrap(err, fmt.Sprintf("Error creating %s directory", systemConfigPath))
+		}
+	}
+
+	if err := s.configureHostKey(s.ensureECDSAKeyExists); err != nil {
+		return err
+	}
+
+	if err := s.configureHostKey(s.ensureRSAKeyExists); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (s *SSHServer) configureHostKey(keyFunc func() (string, error)) error {
+	path, err := keyFunc()
+	if err != nil {
+		return err
+	}
+
+	if err := s.SetOption(ssh.HostKeyFile(path)); err != nil {
+		return errors.Wrap(err, "Could not set SSH host key")
+	}
+	return nil
+}
+
+func (s *SSHServer) ensureRSAKeyExists() (string, error) {
+	keyPath := filepath.Join(systemConfigPath, rsaFilename)
+	if _, err := os.Stat(keyPath); os.IsNotExist(err) {
+		key, err := rsa.GenerateKey(rand.Reader, 2048)
+		if err != nil {
+			return "", errors.Wrap(err, "Error generating RSA host key")
+		}
+
+		privateKey := &pem.Block{
+			Type:  "RSA PRIVATE KEY",
+			Bytes: x509.MarshalPKCS1PrivateKey(key),
+		}
+
+		if err = writePrivateKey(keyPath, privateKey); err != nil {
+			return "", err
+		}
+
+		s.logger.Debug("Created new RSA SSH host key: ", keyPath)
+	}
+	return keyPath, nil
+}
+
+func (s *SSHServer) ensureECDSAKeyExists() (string, error) {
+	keyPath := filepath.Join(systemConfigPath, ecdsaFilename)
+	if _, err := os.Stat(keyPath); os.IsNotExist(err) {
+		key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+		if err != nil {
+			return "", errors.Wrap(err, "Error generating ECDSA host key")
+		}
+
+		keyBytes, err := x509.MarshalECPrivateKey(key)
+		if err != nil {
+			return "", errors.Wrap(err, "Error marshalling ECDSA key")
+		}
+
+		privateKey := &pem.Block{
+			Type:  "EC PRIVATE KEY",
+			Bytes: keyBytes,
+		}
+
+		if err = writePrivateKey(keyPath, privateKey); err != nil {
+			return "", err
+		}
+
+		s.logger.Debug("Created new ECDSA SSH host key: ", keyPath)
+	}
+	return keyPath, nil
+}
+
+func writePrivateKey(keyPath string, privateKey *pem.Block) error {
+	if err := ioutil.WriteFile(keyPath, pem.EncodeToMemory(privateKey), 0600); err != nil {
+		return errors.Wrap(err, fmt.Sprintf("Error writing host key to %s", keyPath))
+	}
+	return nil
+}

sshserver/sshserver_unix.go

@@ -0,0 +1,399 @@
+//+build !windows
+
+package sshserver
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"os"
+	"os/exec"
+	"os/user"
+	"runtime"
+	"strconv"
+	"strings"
+	"syscall"
+	"time"
+	"unsafe"
+
+	"github.com/cloudflare/cloudflared/sshlog"
+
+	"github.com/creack/pty"
+	"github.com/gliderlabs/ssh"
+	"github.com/google/uuid"
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	auditEventAuth   = "auth"
+	auditEventStart  = "session_start"
+	auditEventStop   = "session_stop"
+	auditEventExec   = "exec"
+	auditEventScp    = "scp"
+	auditEventResize = "resize"
+	sshContextSessionID = "sessionID"
+	sshContextEventLogger = "eventLogger"
+)
+
+type auditEvent struct {
+	Event     string `json:"event,omitempty"`
+	EventType string `json:"event_type,omitempty"`
+	SessionID string `json:"session_id,omitempty"`
+	User      string `json:"user,omitempty"`
+	Login     string `json:"login,omitempty"`
+	Datetime  string `json:"datetime,omitempty"`
+	IPAddress string `json:"ip_address,omitempty"`
+}
+
+// SSHServer adds on to the ssh.Server of the gliderlabs package
+type SSHServer struct {
+	ssh.Server
+	logger     *logrus.Logger
+	shutdownC  chan struct{}
+	caCert     ssh.PublicKey
+	logManager sshlog.Manager
+}
+
+// New creates a new SSHServer and configures its host keys and authenication by the data provided
+func New(logManager sshlog.Manager, logger *logrus.Logger, version, address string, shutdownC chan struct{}, idleTimeout, maxTimeout time.Duration, enablePortForwarding bool) (*SSHServer, error) {
+	currentUser, err := user.Current()
+	if err != nil {
+		return nil, err
+	}
+	if currentUser.Uid != "0" {
+		return nil, errors.New("cloudflared SSH server needs to run as root")
+	}
+
+	forwardHandler := &ssh.ForwardedTCPHandler{}
+	sshServer := SSHServer{
+		Server: ssh.Server{
+			Addr:        address,
+			MaxTimeout:  maxTimeout,
+			IdleTimeout: idleTimeout,
+			Version:     fmt.Sprintf("SSH-2.0-Cloudflare-Access_%s_%s", version, runtime.GOOS),
+			// Register SSH global Request handlers to respond to tcpip forwarding
+			RequestHandlers: map[string]ssh.RequestHandler{
+				"tcpip-forward":        forwardHandler.HandleSSHRequest,
+				"cancel-tcpip-forward": forwardHandler.HandleSSHRequest,
+			},
+			// Register SSH channel types
+			ChannelHandlers: map[string]ssh.ChannelHandler{
+				"session":      ssh.DefaultSessionHandler,
+				"direct-tcpip": ssh.DirectTCPIPHandler,
+			},
+		},
+		logger:     logger,
+		shutdownC:  shutdownC,
+		logManager: logManager,
+	}
+
+	// AUTH-2050: This is a temporary workaround of a timing issue in the tunnel muxer to allow further testing.
+	// TODO: Remove this
+	sshServer.ConnCallback = func(conn net.Conn) net.Conn {
+		time.Sleep(10 * time.Millisecond)
+		return conn
+	}
+
+	if enablePortForwarding {
+		sshServer.LocalPortForwardingCallback = allowForward
+		sshServer.ReversePortForwardingCallback = allowForward
+	}
+
+	if err := sshServer.configureHostKeys(); err != nil {
+		return nil, err
+	}
+
+	sshServer.configureAuthentication()
+
+	return &sshServer, nil
+}
+
+// Start the SSH server listener to start handling SSH connections from clients
+func (s *SSHServer) Start() error {
+	s.logger.Infof("Starting SSH server at %s", s.Addr)
+
+	go func() {
+		<-s.shutdownC
+		if err := s.Close(); err != nil {
+			s.logger.WithError(err).Error("Cannot close SSH server")
+		}
+	}()
+
+	s.Handle(s.connectionHandler)
+	return s.ListenAndServe()
+}
+
+func (s *SSHServer) connectionHandler(session ssh.Session) {
+	sessionUUID, err := uuid.NewRandom()
+
+	if err != nil {
+		if _, err := io.WriteString(session, "Failed to generate session ID\n"); err != nil {
+			s.logger.WithError(err).Error("Failed to generate session ID: Failed to write to SSH session")
+		}
+		s.errorAndExit(session, "", nil)
+		return
+	}
+	sessionID := sessionUUID.String()
+
+	eventLogger, err := s.logManager.NewLogger(fmt.Sprintf("%s-event.log", sessionID), s.logger)
+	if err != nil {
+		if _, err := io.WriteString(session, "Failed to create event log\n"); err != nil {
+			s.logger.WithError(err).Error("Failed to create event log: Failed to write to create event logger")
+		}
+		s.errorAndExit(session, "", nil)
+		return
+	}
+
+	sshContext, ok := session.Context().(ssh.Context)
+	if !ok {
+		s.logger.Error("Could not retrieve session context")
+		s.errorAndExit(session, "", nil)
+	}
+
+	sshContext.SetValue(sshContextSessionID, sessionID)
+	sshContext.SetValue(sshContextEventLogger, eventLogger)
+
+	// Get uid and gid of user attempting to login
+	sshUser, uidInt, gidInt, success := s.getSSHUser(session, eventLogger)
+	if !success {
+		return
+	}
+
+	// Spawn shell under user
+	var cmd *exec.Cmd
+	if session.RawCommand() != "" {
+		cmd = exec.Command(sshUser.Shell, "-c", session.RawCommand())
+
+		event := auditEventExec
+		if strings.HasPrefix(session.RawCommand(), "scp") {
+			event = auditEventScp
+		}
+		s.logAuditEvent(session, event)
+	} else {
+		cmd = exec.Command(sshUser.Shell)
+		s.logAuditEvent(session, auditEventStart)
+		defer s.logAuditEvent(session, auditEventStop)
+	}
+	// Supplementary groups are not explicitly specified. They seem to be inherited by default.
+	cmd.SysProcAttr = &syscall.SysProcAttr{Credential: &syscall.Credential{Uid: uidInt, Gid: gidInt}, Setsid: true}
+	cmd.Env = append(cmd.Env, session.Environ()...)
+	cmd.Env = append(cmd.Env, fmt.Sprintf("USER=%s", sshUser.Username))
+	cmd.Env = append(cmd.Env, fmt.Sprintf("HOME=%s", sshUser.HomeDir))
+	cmd.Dir = sshUser.HomeDir
+
+	var shellInput io.WriteCloser
+	var shellOutput io.ReadCloser
+	pr, pw := io.Pipe()
+	defer pw.Close()
+
+	ptyReq, winCh, isPty := session.Pty()
+
+	if isPty {
+		cmd.Env = append(cmd.Env, fmt.Sprintf("TERM=%s", ptyReq.Term))
+		tty, err := s.startPtySession(cmd, winCh, func() {
+			s.logAuditEvent(session, auditEventResize)
+		})
+		shellInput = tty
+		shellOutput = tty
+		if err != nil {
+			s.logger.WithError(err).Error("Failed to start pty session")
+			close(s.shutdownC)
+			return
+		}
+	} else {
+		var shellError io.ReadCloser
+		shellInput, shellOutput, shellError, err = s.startNonPtySession(cmd)
+		if err != nil {
+			s.logger.WithError(err).Error("Failed to start non-pty session")
+			close(s.shutdownC)
+			return
+		}
+
+		// Write stderr to both the command recorder, and remote user
+		go func() {
+			mw := io.MultiWriter(pw, session.Stderr())
+			if _, err := io.Copy(mw, shellError); err != nil {
+				s.logger.WithError(err).Error("Failed to write stderr to user")
+			}
+		}()
+	}
+
+	sessionLogger, err := s.logManager.NewSessionLogger(fmt.Sprintf("%s-session.log", sessionID), s.logger)
+	if err != nil {
+		if _, err := io.WriteString(session, "Failed to create log\n"); err != nil {
+			s.logger.WithError(err).Error("Failed to create log: Failed to write to SSH session")
+		}
+		s.errorAndExit(session, "", nil)
+		return
+	}
+	go func() {
+		defer sessionLogger.Close()
+		defer pr.Close()
+		_, err := io.Copy(sessionLogger, pr)
+		if err != nil {
+			s.logger.WithError(err).Error("Failed to write session log")
+		}
+	}()
+
+	// Write stdin to shell
+	go func() {
+
+		/*
+			Only close shell stdin for non-pty sessions because they have distinct stdin, stdout, and stderr.
+			This is done to prevent commands like SCP from hanging after all data has been sent.
+			PTY sessions share one file for all three streams and the shell process closes it.
+			Closing it here also closes shellOutput and causes an error on copy().
+		*/
+		if !isPty {
+			defer shellInput.Close()
+		}
+		if _, err := io.Copy(shellInput, session); err != nil {
+			s.logger.WithError(err).Error("Failed to write incoming command to pty")
+		}
+	}()
+
+	// Write stdout to both the command recorder, and remote user
+	mw := io.MultiWriter(pw, session)
+	if _, err := io.Copy(mw, shellOutput); err != nil {
+		s.logger.WithError(err).Error("Failed to write stdout to user")
+	}
+
+	// Wait for all resources associated with cmd to be released
+	// Returns error if shell exited with a non-zero status or received a signal
+	if err := cmd.Wait(); err != nil {
+		s.logger.WithError(err).Debug("Shell did not close correctly")
+	}
+}
+
+// getSSHUser gets the ssh user, uid, and gid of the user attempting to login
+func (s *SSHServer) getSSHUser(session ssh.Session, eventLogger io.WriteCloser) (*User, uint32, uint32, bool) {
+	// Get uid and gid of user attempting to login
+	sshUser, ok := session.Context().Value("sshUser").(*User)
+	if !ok || sshUser == nil {
+		s.errorAndExit(session, "Error retrieving credentials from session", nil)
+		return nil, 0, 0, false
+	}
+	s.logAuditEvent(session, auditEventAuth)
+
+	uidInt, err := stringToUint32(sshUser.Uid)
+	if err != nil {
+		s.errorAndExit(session, "Invalid user", err)
+		return sshUser, 0, 0, false
+	}
+	gidInt, err := stringToUint32(sshUser.Gid)
+	if err != nil {
+		s.errorAndExit(session, "Invalid user group", err)
+		return sshUser, 0, 0, false
+	}
+	return sshUser, uidInt, gidInt, true
+}
+
+// errorAndExit reports an error with the session and exits
+func (s *SSHServer) errorAndExit(session ssh.Session, errText string, err error) {
+	if exitError := session.Exit(1); exitError != nil {
+		s.logger.WithError(exitError).Error("Failed to close SSH session")
+	} else if err != nil {
+		s.logger.WithError(err).Error(errText)
+	} else if errText != "" {
+		s.logger.Error(errText)
+	}
+}
+
+func (s *SSHServer) startNonPtySession(cmd *exec.Cmd) (stdin io.WriteCloser, stdout io.ReadCloser, stderr io.ReadCloser, err error) {
+	stdin, err = cmd.StdinPipe()
+	if err != nil {
+		return
+	}
+	stdout, err = cmd.StdoutPipe()
+	if err != nil {
+		return
+	}
+
+	stderr, err = cmd.StderrPipe()
+	if err != nil {
+		return
+	}
+
+	if err = cmd.Start(); err != nil {
+		return
+	}
+	return
+}
+
+func (s *SSHServer) startPtySession(cmd *exec.Cmd, winCh <-chan ssh.Window, logCallback func()) (io.ReadWriteCloser, error) {
+	tty, err := pty.Start(cmd)
+	if err != nil {
+		return nil, err
+	}
+
+	// Handle terminal window size changes
+	go func() {
+		for win := range winCh {
+			if errNo := setWinsize(tty, win.Width, win.Height); errNo != 0 {
+				s.logger.WithError(err).Error("Failed to set pty window size")
+				close(s.shutdownC)
+				return
+			}
+			logCallback()
+		}
+	}()
+
+	return tty, nil
+}
+
+func (s *SSHServer) logAuditEvent(session ssh.Session, eventType string) {
+	username := "unknown"
+	sshUser, ok := session.Context().Value("sshUser").(*User)
+	if ok && sshUser != nil {
+		username = sshUser.Username
+	}
+
+	sessionID, ok := session.Context().Value(sshContextSessionID).(string)
+	if !ok {
+		s.logger.Error("Failed to retrieve sessionID from context")
+		return
+	}
+	writer, ok := session.Context().Value(sshContextEventLogger).(io.WriteCloser)
+	if !ok {
+		s.logger.Error("Failed to retrieve eventLogger from context")
+		return
+	}
+
+	event := auditEvent{
+		Event:     session.RawCommand(),
+		EventType: eventType,
+		SessionID: sessionID,
+		User:      username,
+		Login:     username,
+		Datetime:  time.Now().UTC().Format(time.RFC3339),
+		IPAddress: session.RemoteAddr().String(),
+	}
+	data, err := json.Marshal(&event)
+	if err != nil {
+		s.logger.WithError(err).Error("Failed to marshal audit event. malformed audit object")
+		return
+	}
+	line := string(data) + "\n"
+	if _, err := writer.Write([]byte(line)); err != nil {
+		s.logger.WithError(err).Error("Failed to write audit event.")
+	}
+
+}
+
+// Sets PTY window size for terminal
+func setWinsize(f *os.File, w, h int) syscall.Errno {
+	_, _, errNo := syscall.Syscall(syscall.SYS_IOCTL, f.Fd(), uintptr(syscall.TIOCSWINSZ),
+		uintptr(unsafe.Pointer(&struct{ h, w, x, y uint16 }{uint16(h), uint16(w), 0, 0})))
+	return errNo
+}
+
+func stringToUint32(str string) (uint32, error) {
+	uid, err := strconv.ParseUint(str, 10, 32)
+	return uint32(uid), err
+}
+
+func allowForward(_ ssh.Context, _ string, _ uint32) bool {
+	return true
+}

sshserver/sshserver_windows.go

@@ -0,0 +1,22 @@
+//+build windows
+
+package sshserver
+
+import (
+	"errors"
+
+	"time"
+
+	"github.com/cloudflare/cloudflared/sshlog"
+	"github.com/sirupsen/logrus"
+)
+
+type SSHServer struct{}
+
+func New(_ sshlog.Manager, _ *logrus.Logger, _, _ string, _ chan struct{}, _, _ time.Duration, _ bool) (*SSHServer, error) {
+	return nil, errors.New("cloudflared ssh server is not supported on windows")
+}
+
+func (s *SSHServer) Start() error {
+	return errors.New("cloudflared ssh server is not supported on windows")
+}

sshserver/testdata/ca

@@ -0,0 +1,27 @@
+-----BEGIN OPENSSH PRIVATE KEY-----
+b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABFwAAAAdzc2gtcn
+NhAAAAAwEAAQAAAQEA0c6EklYvC9B041qEGWDNuot6G4tTVm9LCQC0vA+v2n25ru9CINV6
+8IljmXBORXBwfG6PdLhg0SEabZUbsNX5WrIVbGovcghKS6GRsqI5+Quhm+o8eG042JE/hB
+oYdZ19TcMEyPOGzHsx0U/BSN9ZJWVCxqN51iI6qyhz9f6jlX2LQBFEvXlhxgF3owBEf8UC
+Zt/UvbZdmeeyKNQElPmiVLIJEAPCueECp7a2mjCiP3zqjDvSeeGk4CelB/1qZZ4V2n7fvb
+HZjAB5JJs4KXs5o8KgvQnqgQMxiLFZ4PATt4+mxEzh4JymppbqJOo2rYwOA3TAIEWWtYRV
+/ZKJ0AyhhQAAA8gciO8XHIjvFwAAAAdzc2gtcnNhAAABAQDRzoSSVi8L0HTjWoQZYM26i3
+obi1NWb0sJALS8D6/afbmu70Ig1XrwiWOZcE5FcHB8bo90uGDRIRptlRuw1flashVsai9y
+CEpLoZGyojn5C6Gb6jx4bTjYkT+EGhh1nX1NwwTI84bMezHRT8FI31klZULGo3nWIjqrKH
+P1/qOVfYtAEUS9eWHGAXejAER/xQJm39S9tl2Z57Io1ASU+aJUsgkQA8K54QKntraaMKI/
+fOqMO9J54aTgJ6UH/WplnhXaft+9sdmMAHkkmzgpezmjwqC9CeqBAzGIsVng8BO3j6bETO
+HgnKamluok6jatjA4DdMAgRZa1hFX9konQDKGFAAAAAwEAAQAAAQEApVzGdKhk8ETevAst
+rurze6JPHcKUbr3NQE1EJi2fBvCtF0oQrtxTx54h2GAB8Q0MO6bQfsiL1ojm0ZQCfUBJBs
+jxxb9zoccS98Vilo7ybm5SdBcMjkZX1am1jCMdQCZfCpk4/kGi7yvyOe1IhG01UBodpX5X
+mwTjhN+fdjW7LSiW6cKPClN49CZKgmtvI27FCt+/TtMzdCXOiJxJ4yZCzCRhSgssV0gWI1
+0VJr/MHirKUvv/qCLAuOBxIr9UgdduRZUpNX+KS2rfhFEbjnUqc/57aAakpQmuPB5I+s9G
+DnrF0HSHpq7u1XC1SvYlnFBN/0A7Hw/MX2SaBFH7mc9AAQAAAIAFuTHr6O8tCvWEawfxC0
+qiAPQ+Yy1vthq5uewmuQujMutUnc9JAUl32PdU1DbS7APC1Dg9XL7SyAB6A+ZpRJRAKgCY
+SneAKE6hOytH+yM206aekrz6VuZiSpBqpfEqDibVAaZIO8sv/9dtZd6kWemxNErPQoKJey
+Z7/cuWUWQovAAAAIEA6ugIlVj1irPmElyCCt5YfPv2x8Dl54ELoP/WsffsrPHNQog64hFd
+ahD7Wq63TA566bN85fkx8OVU5TbbEQmkHgOEV6nDRY2YsBSqIOblA/KehtfdUIqZB0iNBh
+Gn6TV/z6HwnSR3gKv4b66Gveek6LfRAG3mbsLCgyRAbYgn6YUAAACBAOSlf+n1eh6yjtvF
+Zecq3Zslj7O8cUs17PQx4vQ7sXNCFrIZdevWPIn9sVrt7/hsTrXunDz6eXCeclB35KZe3H
+WPVjRoD+xnr5+sXx2qXOnKCR0LdFybso6IR5bXAI6DNSNfP7D9LPEQ+R73Jk0jPuLYzocS
+iM89KZiuGpzr01gBAAAAEW1pa2VAQzAyWTUwVEdKR0g4AQ==
+-----END OPENSSH PRIVATE KEY-----

sshserver/testdata/ca.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDRzoSSVi8L0HTjWoQZYM26i3obi1NWb0sJALS8D6/afbmu70Ig1XrwiWOZcE5FcHB8bo90uGDRIRptlRuw1flashVsai9yCEpLoZGyojn5C6Gb6jx4bTjYkT+EGhh1nX1NwwTI84bMezHRT8FI31klZULGo3nWIjqrKHP1/qOVfYtAEUS9eWHGAXejAER/xQJm39S9tl2Z57Io1ASU+aJUsgkQA8K54QKntraaMKI/fOqMO9J54aTgJ6UH/WplnhXaft+9sdmMAHkkmzgpezmjwqC9CeqBAzGIsVng8BO3j6bETOHgnKamluok6jatjA4DdMAgRZa1hFX9konQDKGF mike@C02Y50TGJGH8

sshserver/testdata/id_rsa

@@ -0,0 +1,49 @@
+-----BEGIN OPENSSH PRIVATE KEY-----
+b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAACFwAAAAdzc2gtcn
+NhAAAAAwEAAQAAAgEA60Kneo87qPsh+zErWFl7vx93c7fyTxbZ9lUNqafgXy/BLOCc/nQS
+McosVSLsQrbHlhYzfmZEhTiubmuYUrHchmsn1ml1HIqP8T5aDgtNbLqYnS4H5oO4Sj1+XH
+lQtU7n7zHXgca9SnMWt1Fhkx1mvkeiOKs0eq7hV2TuIZxfmbYfIVvJGwrL0uWzbSEE1gvx
+gTXZHxEChIQyrNviljgi4u2MD/cIi6KMeYUnaTL1FxO9G4GIFiy7ueHRwOZPIFHgYm+Vrt
+X7XafSF0///zCrC63zzWt/6A06hFepOz2VXvm7SdckaR7qMXAb7kipsc0+dKk9ggU7Fqpx
+ZY5cVeZo9RlRVhRXGDy7mABA/FMwvv+qYCgJ3nlZbdKbaiPLQu8ScTlJ9sMI06/ZiEY04b
+meZ0ASM52gaDGjrFbbnuHNf5XV/oreEUhtCrryFnoIxmKgHznGjZ55q77FtTHnrAKFmKFP
+11s3MLIX9o4RgtriOtl4KenkIfUumgtrwY/UGjOaOQUOrVH1am54wkUiVEF0Qd3AD8KCl/
+l/xT5+t6cOspZ9GIhwa2NBmRjN/wVGp+Yrb08Re3kxPCX9bs5iLe+kHN0vuFr7RDo+eUoi
+SPhWl6FUqx2W9NZqekmEgKn3oKrfbGaMH1VLkaKWlzQ4xJzP0iadQbIXGryLEYASydemZt
+sAAAdQ/ovjxf6L48UAAAAHc3NoLXJzYQAAAgEA60Kneo87qPsh+zErWFl7vx93c7fyTxbZ
+9lUNqafgXy/BLOCc/nQSMcosVSLsQrbHlhYzfmZEhTiubmuYUrHchmsn1ml1HIqP8T5aDg
+tNbLqYnS4H5oO4Sj1+XHlQtU7n7zHXgca9SnMWt1Fhkx1mvkeiOKs0eq7hV2TuIZxfmbYf
+IVvJGwrL0uWzbSEE1gvxgTXZHxEChIQyrNviljgi4u2MD/cIi6KMeYUnaTL1FxO9G4GIFi
+y7ueHRwOZPIFHgYm+VrtX7XafSF0///zCrC63zzWt/6A06hFepOz2VXvm7SdckaR7qMXAb
+7kipsc0+dKk9ggU7FqpxZY5cVeZo9RlRVhRXGDy7mABA/FMwvv+qYCgJ3nlZbdKbaiPLQu
+8ScTlJ9sMI06/ZiEY04bmeZ0ASM52gaDGjrFbbnuHNf5XV/oreEUhtCrryFnoIxmKgHznG
+jZ55q77FtTHnrAKFmKFP11s3MLIX9o4RgtriOtl4KenkIfUumgtrwY/UGjOaOQUOrVH1am
+54wkUiVEF0Qd3AD8KCl/l/xT5+t6cOspZ9GIhwa2NBmRjN/wVGp+Yrb08Re3kxPCX9bs5i
+Le+kHN0vuFr7RDo+eUoiSPhWl6FUqx2W9NZqekmEgKn3oKrfbGaMH1VLkaKWlzQ4xJzP0i
+adQbIXGryLEYASydemZtsAAAADAQABAAACABUYzBYEhDAaHSj+dsmcdKll8/tPko4fGXqq
+k+gT4t4GVUdl+Q4kcIFAhQs5b4BoDava39FE8H4V4CaMxYMc6g6vy0nB+TuO/Wt/0OmTf+
+TxMsBdoV29kCgwLYWzZ1Zq9geQK6g6nzzu5ymXRa3ApDcKC3UTfUhHKHQC3AvtjvEk0NPX
+/EfNhwuph5aQsHNVbNnOb2MGznf9tuGjckVQUWiSLs47s+t5rykylJ8tb6cbIQk3a3G5nz
+gDFSE8Rfo6/Wk2YnDkRX9XjlKC3Q0QWzZX6hYQvs6baRT3G3jxg9SZhn8PqPc4S34VdJvA
+rl8AbcpeZuKi/3J/5F1cD9GwMNcl4gM87piF20/r9mMvC4zBAEgyF8WBi4OjSu0+ccsEsb
+GSpxKK04OPTB7p8mLJ8hQUiREg5OuPEEcAoDSuHgdliE7nDHzuImbpTcAZcWhkJaUdBWI6
+qcnGPARzxAOmuzkY8Gq0MtcWge5QxnLWJyrfy43M984Cvxql/maLUij4eTbMDDwV7Qx30V
+P2tJp5+hOnitRwB6cQIg5N7/cTQdJ6eiFYuw0v3IfHjYmaolY8F3u38Zv2PPk50CorPRDG
+esx0a9Elm2UKPb145MtHGZtLH2mayRnDjnxr25iLwgokI06tCLCNvbkYLA7wVpJn81eKmZ
+tQBtbfqBSiDiLjCrehAAABAQDh8vmgPR95Kx1DeBxzMSja7TStP5V58JcUvLP4dAN6zqtt
+rMuKDfJRSIVYGkU8vXlME5E6phWP7R5lYY7+kLDbeZrYQg1AxkK8y4fUYkCLBuEcCjzWDK
+oqZQNskk4urbCdBIP6AR7d/LMCHBb9rk2iOuUeos6JHRKbPGP1lvH3hLkbH9CA0F41sz86
+JFg6u/XaRQ2CyhS7y7SQ8dmaANGz9LGdIRqIoZ8Hfht8t1VRbM9fzSb3xoxUItbHpk9R9g
+GZsHSryi7AtRmHt0uBrWIv6RbIY0epCbjdCLvHflbkPgwBM7UndgkOSIwQ4SQF8Fs+e9/r
+hV05h0Y81vd1RZvOAAABAQD5EgW3SpmYzeMmiP7MKkfIlbZtwVmRu4anTzWxlk5pJ9GXnC
+QoInULCipWAOeJbuLIgRWLU4VzhOUbYLNKQPXECARfgoto2VXoXZZ2q2O4aXaCpeyU6nE8
+VKbp4nU1jEg5hWB3PRwZ8Pzs4A93/9mrpVzLmCT+LW9Rlnp6tTpqcUKGugg8vr64SSgqnV
+ZFyQgHgw+ZGOG9w714urS3U97WNTeHXAs0p2YBOu5XW3JQ3jkRo7YyZF3+TtBxbgfHRZfH
+O2mFcMBD3Sn4t+LAbgnLye3S2/WZf/gQwdVB7BgrVqguzQ2hGoOxNiwadkIDsxb6r/u3n6
+2lScpHFDS0WnpRAAABAQDxzkV52VX6wAWkQe/2KFH9wTG0XFANmZUnnTPR8wd+b9E7HIr0
+Mdd8iAHOhLRvTy8mih53GGBptXK7GdABMZtkqDErbXhuC8xbi9uRLEHiRe/oBfWr8vYIZY
+awiw3/EqxaTv0HBMicdr2S31Bs2/mjrVuJH0wAaI9ueQnZizzjgWuzeNZMWq1qk0akUUdm
+PDVd58yBkt8lKlkOG0LJAn6JEG9oH9XiTFShHzu1dQmoC2bKVHdxL8WCcYFVtmyoMRcLZq
+u6d4nyKha02cYZB5hM3VcizJI5HY/A+H3fBkRR0hXgkU5R89w+8x9VSJkNVx+JGC7ziK4a
+kUjfOmR5WBdrAAAAE3Rlc3RAY2xvdWRmbGFyZS5jb20BAgMEBQYH
+-----END OPENSSH PRIVATE KEY-----

sshserver/testdata/id_rsa-cert.pub

@@ -0,0 +1 @@
+ssh-rsa-cert-v01@openssh.com AAAAHHNzaC1yc2EtY2VydC12MDFAb3BlbnNzaC5jb20AAAAgOsuFqKdzp/nC3wQfKVJBdHa8axtGryKplPkDjdSXT4kAAAADAQABAAACAQDrQqd6jzuo+yH7MStYWXu/H3dzt/JPFtn2VQ2pp+BfL8Es4Jz+dBIxyixVIuxCtseWFjN+ZkSFOK5ua5hSsdyGayfWaXUcio/xPloOC01supidLgfmg7hKPX5ceVC1TufvMdeBxr1Kcxa3UWGTHWa+R6I4qzR6ruFXZO4hnF+Zth8hW8kbCsvS5bNtIQTWC/GBNdkfEQKEhDKs2+KWOCLi7YwP9wiLoox5hSdpMvUXE70bgYgWLLu54dHA5k8gUeBib5Wu1ftdp9IXT///MKsLrfPNa3/oDTqEV6k7PZVe+btJ1yRpHuoxcBvuSKmxzT50qT2CBTsWqnFljlxV5mj1GVFWFFcYPLuYAED8UzC+/6pgKAneeVlt0ptqI8tC7xJxOUn2wwjTr9mIRjThuZ5nQBIznaBoMaOsVtue4c1/ldX+it4RSG0KuvIWegjGYqAfOcaNnnmrvsW1MeesAoWYoU/XWzcwshf2jhGC2uI62Xgp6eQh9S6aC2vBj9QaM5o5BQ6tUfVqbnjCRSJUQXRB3cAPwoKX+X/FPn63pw6yln0YiHBrY0GZGM3/BUan5itvTxF7eTE8Jf1uzmIt76Qc3S+4WvtEOj55SiJI+FaXoVSrHZb01mp6SYSAqfegqt9sZowfVUuRopaXNDjEnM/SJp1BshcavIsRgBLJ16Zm2wAAAAAAAAAAAAAAAQAAAA10ZXN0VXNlckB0ZXN0AAAADAAAAAh0ZXN0VXNlcgAAAAAAAAAA//////////8AAAAAAAAAggAAABVwZXJtaXQtWDExLWZvcndhcmRpbmcAAAAAAAAAF3Blcm1pdC1hZ2VudC1mb3J3YXJkaW5nAAAAAAAAABZwZXJtaXQtcG9ydC1mb3J3YXJkaW5nAAAAAAAAAApwZXJtaXQtcHR5AAAAAAAAAA5wZXJtaXQtdXNlci1yYwAAAAAAAAAAAAABFwAAAAdzc2gtcnNhAAAAAwEAAQAAAQEA0c6EklYvC9B041qEGWDNuot6G4tTVm9LCQC0vA+v2n25ru9CINV68IljmXBORXBwfG6PdLhg0SEabZUbsNX5WrIVbGovcghKS6GRsqI5+Quhm+o8eG042JE/hBoYdZ19TcMEyPOGzHsx0U/BSN9ZJWVCxqN51iI6qyhz9f6jlX2LQBFEvXlhxgF3owBEf8UCZt/UvbZdmeeyKNQElPmiVLIJEAPCueECp7a2mjCiP3zqjDvSeeGk4CelB/1qZZ4V2n7fvbHZjAB5JJs4KXs5o8KgvQnqgQMxiLFZ4PATt4+mxEzh4JymppbqJOo2rYwOA3TAIEWWtYRV/ZKJ0AyhhQAAAQ8AAAAHc3NoLXJzYQAAAQC2lL+6JYTGOdz1zNnck6onrFcVpO2onCVAKP8HdLoCeH0/upIugaCocPKuzoURYEfiHQotviNeprE/2CyAroJ5VBdqWftEeHn3FFvBCQ1gwRQ7oci4C5n72t0vjWWE6WBylS0RqpJjr6EQ8a1vuwIqAQrEJPp2yNLjRH2WD7eicBh5f43VKOMr73DtyTh4xoF0C2sNBROudt58npTaYqRHQgoI25V/aCmuYBgM3wdAGcoEZGoSerMfhID7GcWkvemq2hF8mQsspG3zgnyQXk+ahagmefzxutDnr3KdrZ637La0/XwABvBZ9L4l5RiEilVI1Shl96F2qbBW2YZ64pUQ test@cloudflare.com

sshserver/testdata/id_rsa.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDrQqd6jzuo+yH7MStYWXu/H3dzt/JPFtn2VQ2pp+BfL8Es4Jz+dBIxyixVIuxCtseWFjN+ZkSFOK5ua5hSsdyGayfWaXUcio/xPloOC01supidLgfmg7hKPX5ceVC1TufvMdeBxr1Kcxa3UWGTHWa+R6I4qzR6ruFXZO4hnF+Zth8hW8kbCsvS5bNtIQTWC/GBNdkfEQKEhDKs2+KWOCLi7YwP9wiLoox5hSdpMvUXE70bgYgWLLu54dHA5k8gUeBib5Wu1ftdp9IXT///MKsLrfPNa3/oDTqEV6k7PZVe+btJ1yRpHuoxcBvuSKmxzT50qT2CBTsWqnFljlxV5mj1GVFWFFcYPLuYAED8UzC+/6pgKAneeVlt0ptqI8tC7xJxOUn2wwjTr9mIRjThuZ5nQBIznaBoMaOsVtue4c1/ldX+it4RSG0KuvIWegjGYqAfOcaNnnmrvsW1MeesAoWYoU/XWzcwshf2jhGC2uI62Xgp6eQh9S6aC2vBj9QaM5o5BQ6tUfVqbnjCRSJUQXRB3cAPwoKX+X/FPn63pw6yln0YiHBrY0GZGM3/BUan5itvTxF7eTE8Jf1uzmIt76Qc3S+4WvtEOj55SiJI+FaXoVSrHZb01mp6SYSAqfegqt9sZowfVUuRopaXNDjEnM/SJp1BshcavIsRgBLJ16Zm2w== test@cloudflare.com

sshserver/testdata/other_ca

@@ -0,0 +1,27 @@
+-----BEGIN OPENSSH PRIVATE KEY-----
+b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABFwAAAAdzc2gtcn
+NhAAAAAwEAAQAAAQEAzBO7TXxbpk7sGQm/Wa29N/NFe5uuoEQGC5hxfihmcvVgeKeNKiSS
+snxzCE1Y6SmNMoE4aQs92wtcn48GmxRwZSXbCqLq2CJrHfe9B2k3aPkJZpQkFMshcJGo7p
+G0Vlo7dWAbYf99/YKddf290uLK7vxw9ty0pM1hXSXHNShv1b+bTQm/COMZ5jNsncjc1yBH
+KGkFVHee9Dh4Z0xLlHipIyyNXXzI0RFYuHSNJz9GD310XQLIIroptr7+/7g6+sPPGsNlI+
+95OScba1/PQ2b/qy+KyIwNIMSd9ziJy5xnO7Vo3LrqQrza1Pkn2i29PljUcbc/F0hhXNIq
+ITdNWwVqsQAAA8iKllTIipZUyAAAAAdzc2gtcnNhAAABAQDME7tNfFumTuwZCb9Zrb0380
+V7m66gRAYLmHF+KGZy9WB4p40qJJKyfHMITVjpKY0ygThpCz3bC1yfjwabFHBlJdsKourY
+Imsd970HaTdo+QlmlCQUyyFwkajukbRWWjt1YBth/339gp11/b3S4sru/HD23LSkzWFdJc
+c1KG/Vv5tNCb8I4xnmM2ydyNzXIEcoaQVUd570OHhnTEuUeKkjLI1dfMjREVi4dI0nP0YP
+fXRdAsgiuim2vv7/uDr6w88aw2Uj73k5JxtrX89DZv+rL4rIjA0gxJ33OInLnGc7tWjcuu
+pCvNrU+SfaLb0+WNRxtz8XSGFc0iohN01bBWqxAAAAAwEAAQAAAQAKEtNFEOVpQS4QUlXa
+tGPJtj1wy4+EI7d0rRK1GoNsG0amzgZ+1Q1UuCXpe//uinmIy64gKUjlXhs1WRcHYqvlok
+e8r6wN/Szybr8q9Xuht+FJ6fgZ+qjs6JPBKvoO5SdYNOVFIhpzABaLs3nCRiWkRFvDI8Pa
++rRap7m8mwFiOJtmdiIZYFxzw6xXwTsGCrWPKgTv3FKGZzXnCB9i7jC2vwT1MDYbcnzEH4
+Ba4dxI8bp6WWEX0biRIXj3jCtLb5gisNTSxdZs254Syh75HEXunSh2YO+yVSWQtZj19ewW
+6Rb1Z3x5rVfXcgSkg7gZd9EpbckIIg6+MFSH3wdGW6atAAAAgQDFXiMuNd4ZYwdyhjlM5n
+nFqQDXGgnwyNdiIqAapoqTdF5aZwNnbTU0fCFaDMLCQAHgntcgCEsW9A4HzDzYhOABKElv
+j973vXWF165wFiZwuKSfroq/6JH6CiIcjiqpszbnqSOzy1hq913RWILS6e9yMjxRv8PUjm
+E+IkcnfcFUwAAAAIEA+jwI3ICe8PGEIezV2tvQFeQy2Z2wGslu1yvqfTYEztSmtygns3wn
+ZBb+cBXCnpqUCtznG7hZhq7I4m1I47BYznULwwFiBTVtBASG5wNP7zeVKTVZ4SKprze+Fe
+I/nUZDJ5Q26um7eDbhvZ/n95GY+fucMVHoSBfX1wE16XBfp88AAACBANDHcgC4qP2oyOw/
++p9HineMQd/ppG3fePe07jyZXLHLf0rByFveFgRAQ1m77O7FtP3fFKy3Y9nNy18LGq35ZK
+Blsz2B23bO8NuffgAhchDG7KzKFXCo+AraIj5znp/znK5zIkaiiSOQaYywJ36EooYVpRtj
+ep5ap6bBFDZ2e+V/AAAAEW1pa2VAQzAyWTUwVEdKR0g4AQ==
+-----END OPENSSH PRIVATE KEY-----

sshserver/testdata/other_ca.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDME7tNfFumTuwZCb9Zrb0380V7m66gRAYLmHF+KGZy9WB4p40qJJKyfHMITVjpKY0ygThpCz3bC1yfjwabFHBlJdsKourYImsd970HaTdo+QlmlCQUyyFwkajukbRWWjt1YBth/339gp11/b3S4sru/HD23LSkzWFdJcc1KG/Vv5tNCb8I4xnmM2ydyNzXIEcoaQVUd570OHhnTEuUeKkjLI1dfMjREVi4dI0nP0YPfXRdAsgiuim2vv7/uDr6w88aw2Uj73k5JxtrX89DZv+rL4rIjA0gxJ33OInLnGc7tWjcuupCvNrU+SfaLb0+WNRxtz8XSGFc0iohN01bBWqx mike@C02Y50TGJGH8

streamhandler/request.go

@@ -0,0 +1,64 @@
+package streamhandler
+
+import (
+	"fmt"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+
+	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/pkg/errors"
+)
+
+const (
+	lbProbeUserAgentPrefix = "Mozilla/5.0 (compatible; Cloudflare-Traffic-Manager/1.0; +https://www.cloudflare.com/traffic-manager/;"
+)
+
+func FindCfRayHeader(h1 *http.Request) string {
+	return h1.Header.Get("Cf-Ray")
+}
+
+func IsLBProbeRequest(req *http.Request) bool {
+	return strings.HasPrefix(req.UserAgent(), lbProbeUserAgentPrefix)
+}
+
+func createRequest(stream *h2mux.MuxedStream, url *url.URL) (*http.Request, error) {
+	req, err := http.NewRequest(http.MethodGet, url.String(), h2mux.MuxedStreamReader{MuxedStream: stream})
+	if err != nil {
+		return nil, errors.Wrap(err, "unexpected error from http.NewRequest")
+	}
+	err = H2RequestHeadersToH1Request(stream.Headers, req)
+	if err != nil {
+		return nil, errors.Wrap(err, "invalid request received")
+	}
+	return req, nil
+}
+
+func H2RequestHeadersToH1Request(h2 []h2mux.Header, h1 *http.Request) error {
+	for _, header := range h2 {
+		switch header.Name {
+		case ":method":
+			h1.Method = header.Value
+		case ":scheme":
+		case ":authority":
+			// Otherwise the host header will be based on the origin URL
+			h1.Host = header.Value
+		case ":path":
+			// We can't just set `URL.Path`, because there's no way to ask the library to *not* escape it,
+			// causing https://github.com/cloudflare/cloudflared/issues/124.
+			// The only way to bypass the URL escape seems to currently be `URL.Opaque`.
+			// See https://github.com/golang/go/issues/5777
+			h1.URL.Opaque = fmt.Sprintf("//%v%v", h1.URL.Host, header.Value)
+		case "content-length":
+			contentLength, err := strconv.ParseInt(header.Value, 10, 64)
+			if err != nil {
+				return fmt.Errorf("unparseable content length")
+			}
+			h1.ContentLength = contentLength
+		default:
+			h1.Header.Add(http.CanonicalHeaderKey(header.Name), header.Value)
+		}
+	}
+	return nil
+}

streamhandler/request_test.go

@@ -0,0 +1,192 @@
+package streamhandler
+
+import (
+	"fmt"
+	"net/http"
+	"testing"
+
+	"github.com/cloudflare/cloudflared/h2mux"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestH2RequestHeadersToH1Request_RegularHeaders(t *testing.T) {
+	request, err := http.NewRequest(http.MethodGet, "http://example.com", nil)
+	assert.NoError(t, err)
+
+	headersConversionErr := H2RequestHeadersToH1Request(
+		[]h2mux.Header{
+			h2mux.Header{
+				Name:  "Mock header 1",
+				Value: "Mock value 1",
+			},
+			h2mux.Header{
+				Name:  "Mock header 2",
+				Value: "Mock value 2",
+			},
+		},
+		request,
+	)
+
+	assert.Equal(t, http.Header{
+		"Mock header 1": []string{"Mock value 1"},
+		"Mock header 2": []string{"Mock value 2"},
+	}, request.Header)
+
+	assert.NoError(t, headersConversionErr)
+}
+
+func TestH2RequestHeadersToH1Request_NoHeaders(t *testing.T) {
+	request, err := http.NewRequest(http.MethodGet, "http://example.com", nil)
+	assert.NoError(t, err)
+
+	headersConversionErr := H2RequestHeadersToH1Request(
+		[]h2mux.Header{},
+		request,
+	)
+
+	assert.Equal(t, http.Header{}, request.Header)
+
+	assert.NoError(t, headersConversionErr)
+}
+
+func TestH2RequestHeadersToH1Request_InvalidHostPath(t *testing.T) {
+	request, err := http.NewRequest(http.MethodGet, "http://example.com", nil)
+	assert.NoError(t, err)
+
+	headersConversionErr := H2RequestHeadersToH1Request(
+		[]h2mux.Header{
+			h2mux.Header{
+				Name:  ":path",
+				Value: "//bad_path/",
+			},
+			h2mux.Header{
+				Name:  "Mock header",
+				Value: "Mock value",
+			},
+		},
+		request,
+	)
+
+	assert.Equal(t, http.Header{
+		"Mock header": []string{"Mock value"},
+	}, request.Header)
+
+	assert.Equal(t, "http://example.com//bad_path/", request.URL.String())
+
+	assert.NoError(t, headersConversionErr)
+}
+
+func TestH2RequestHeadersToH1Request_HostPathWithQuery(t *testing.T) {
+	request, err := http.NewRequest(http.MethodGet, "http://example.com/", nil)
+	assert.NoError(t, err)
+
+	headersConversionErr := H2RequestHeadersToH1Request(
+		[]h2mux.Header{
+			h2mux.Header{
+				Name:  ":path",
+				Value: "/?query=mock%20value",
+			},
+			h2mux.Header{
+				Name:  "Mock header",
+				Value: "Mock value",
+			},
+		},
+		request,
+	)
+
+	assert.Equal(t, http.Header{
+		"Mock header": []string{"Mock value"},
+	}, request.Header)
+
+	assert.Equal(t, "http://example.com/?query=mock%20value", request.URL.String())
+
+	assert.NoError(t, headersConversionErr)
+}
+
+func TestH2RequestHeadersToH1Request_HostPathWithURLEncoding(t *testing.T) {
+	request, err := http.NewRequest(http.MethodGet, "http://example.com/", nil)
+	assert.NoError(t, err)
+
+	headersConversionErr := H2RequestHeadersToH1Request(
+		[]h2mux.Header{
+			h2mux.Header{
+				Name:  ":path",
+				Value: "/mock%20path",
+			},
+			h2mux.Header{
+				Name:  "Mock header",
+				Value: "Mock value",
+			},
+		},
+		request,
+	)
+
+	assert.Equal(t, http.Header{
+		"Mock header": []string{"Mock value"},
+	}, request.Header)
+
+	assert.Equal(t, "http://example.com/mock%20path", request.URL.String())
+
+	assert.NoError(t, headersConversionErr)
+}
+
+func TestH2RequestHeadersToH1Request_WeirdURLs(t *testing.T) {
+	expectedPaths := []string{
+		"",
+		"/",
+		"//",
+		"/%20",
+		"/ ",
+		"/ a ",
+		"/a%20b",
+		"/foo/bar;param?query#frag",
+		"/a␠b",
+		"/a-umlaut-ä",
+		"/a-umlaut-%C3%A4",
+		"/a-umlaut-%c3%a4",
+		"/a#b#c",
+		"/a#b␠c",
+		"/a#b%20c",
+		"/a#b c",
+		"/\\",
+		"/a\\",
+		"/a\\b",
+		"/a,b.c.",
+		"/.",
+		"/a`",
+		"/a[0]",
+		"/?a[0]=5 &b[]=",
+		"/?a=%22b%20%22",
+	}
+
+	for index, expectedPath := range expectedPaths {
+		requestURL := "https://example.com"
+		expectedURL := fmt.Sprintf("https://example.com%v", expectedPath)
+
+		request, err := http.NewRequest(http.MethodGet, requestURL, nil)
+		assert.NoError(t, err)
+
+		headersConversionErr := H2RequestHeadersToH1Request(
+			[]h2mux.Header{
+				h2mux.Header{
+					Name:  ":path",
+					Value: expectedPath,
+				},
+				h2mux.Header{
+					Name:  "Mock header",
+					Value: "Mock value",
+				},
+			},
+			request,
+		)
+
+		assert.Equal(t, http.Header{
+			"Mock header": []string{"Mock value"},
+		}, request.Header)
+
+		assert.Equal(t, expectedURL, request.URL.String(), fmt.Sprintf("Failed URL index: %v", index))
+
+		assert.NoError(t, headersConversionErr)
+	}
+}

streamhandler/stream_handler.go

@@ -0,0 +1,190 @@
+package streamhandler
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"strconv"
+
+	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/tunnelhostnamemapper"
+	"github.com/cloudflare/cloudflared/tunnelrpc"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"zombiezen.com/go/capnproto2/rpc"
+)
+
+const (
+	statusPseudoHeader = ":status"
+)
+
+type httpErrorStatus struct {
+	status string
+	text   []byte
+}
+
+var (
+	statusBadRequest = newHTTPErrorStatus(http.StatusBadRequest)
+	statusNotFound   = newHTTPErrorStatus(http.StatusNotFound)
+	statusBadGateway = newHTTPErrorStatus(http.StatusBadGateway)
+)
+
+func newHTTPErrorStatus(status int) *httpErrorStatus {
+	return &httpErrorStatus{
+		status: strconv.Itoa(status),
+		text:   []byte(http.StatusText(status)),
+	}
+}
+
+// StreamHandler handles new stream opened by the edge. The streams can be used to proxy requests or make RPC.
+type StreamHandler struct {
+	// newConfigChan is a send-only channel to notify Supervisor of a new ClientConfig
+	newConfigChan chan<- *pogs.ClientConfig
+	// useConfigResultChan is a receive-only channel for Supervisor to communicate the result of applying a new ClientConfig
+	useConfigResultChan <-chan *pogs.UseConfigurationResult
+	// originMapper maps tunnel hostname to origin service
+	tunnelHostnameMapper *tunnelhostnamemapper.TunnelHostnameMapper
+	logger               *logrus.Entry
+}
+
+// NewStreamHandler creates a new StreamHandler
+func NewStreamHandler(newConfigChan chan<- *pogs.ClientConfig,
+	useConfigResultChan <-chan *pogs.UseConfigurationResult,
+	logger *logrus.Logger,
+) *StreamHandler {
+	return &StreamHandler{
+		newConfigChan:        newConfigChan,
+		useConfigResultChan:  useConfigResultChan,
+		tunnelHostnameMapper: tunnelhostnamemapper.NewTunnelHostnameMapper(),
+		logger:               logger.WithField("subsystem", "streamHandler"),
+	}
+}
+
+// UseConfiguration implements ClientService
+func (s *StreamHandler) UseConfiguration(ctx context.Context, config *pogs.ClientConfig) (*pogs.UseConfigurationResult, error) {
+	select {
+	case <-ctx.Done():
+		err := fmt.Errorf("Timeout while sending new config to Supervisor")
+		s.logger.Error(err)
+		return nil, err
+	case s.newConfigChan <- config:
+	}
+	select {
+	case <-ctx.Done():
+		err := fmt.Errorf("Timeout applying new configuration")
+		s.logger.Error(err)
+		return nil, err
+	case result := <-s.useConfigResultChan:
+		return result, nil
+	}
+}
+
+// UpdateConfig replaces current originmapper mapping with mappings from newConfig
+func (s *StreamHandler) UpdateConfig(newConfig []*pogs.ReverseProxyConfig) (failedConfigs []*pogs.FailedConfig) {
+
+	// Delete old configs that aren't in the `newConfig`
+	toRemove := s.tunnelHostnameMapper.ToRemove(newConfig)
+	for _, hostnameToRemove := range toRemove {
+		s.tunnelHostnameMapper.Delete(hostnameToRemove)
+	}
+
+	// Add new configs that weren't in the old mapper
+	toAdd := s.tunnelHostnameMapper.ToAdd(newConfig)
+	for _, tunnelConfig := range toAdd {
+		tunnelHostname := tunnelConfig.TunnelHostname
+		originSerice, err := tunnelConfig.OriginConfig.Service()
+		if err != nil {
+			s.logger.WithField("tunnelHostname", tunnelHostname).WithError(err).Error("Invalid origin service config")
+			failedConfigs = append(failedConfigs, &pogs.FailedConfig{
+				Config: tunnelConfig,
+				Reason: tunnelConfig.FailReason(err),
+			})
+			continue
+		}
+		s.tunnelHostnameMapper.Add(tunnelConfig.TunnelHostname, originSerice)
+		s.logger.WithField("tunnelHostname", tunnelHostname).Infof("New origin service config: %v", originSerice.Summary())
+	}
+	return
+}
+
+// ServeStream implements MuxedStreamHandler interface
+func (s *StreamHandler) ServeStream(stream *h2mux.MuxedStream) error {
+	if stream.IsRPCStream() {
+		return s.serveRPC(stream)
+	}
+	if err := s.serveRequest(stream); err != nil {
+		s.logger.Error(err)
+		return err
+	}
+	return nil
+}
+
+func (s *StreamHandler) serveRPC(stream *h2mux.MuxedStream) error {
+	stream.WriteHeaders([]h2mux.Header{{Name: ":status", Value: "200"}})
+	main := pogs.ClientService_ServerToClient(s)
+	rpcLogger := s.logger.WithField("subsystem", "clientserver-rpc")
+	rpcConn := rpc.NewConn(
+		tunnelrpc.NewTransportLogger(rpcLogger, rpc.StreamTransport(stream)),
+		rpc.MainInterface(main.Client),
+		tunnelrpc.ConnLog(s.logger.WithField("subsystem", "clientserver-rpc-transport")),
+	)
+	return rpcConn.Wait()
+}
+
+func (s *StreamHandler) serveRequest(stream *h2mux.MuxedStream) error {
+	tunnelHostname := stream.TunnelHostname()
+	if !tunnelHostname.IsSet() {
+		s.writeErrorStatus(stream, statusBadRequest)
+		return fmt.Errorf("stream doesn't have tunnelHostname")
+	}
+
+	originService, ok := s.tunnelHostnameMapper.Get(tunnelHostname)
+	if !ok {
+		s.writeErrorStatus(stream, statusNotFound)
+		return fmt.Errorf("cannot map tunnel hostname %s to origin", tunnelHostname)
+	}
+
+	req, err := createRequest(stream, originService.URL())
+	if err != nil {
+		s.writeErrorStatus(stream, statusBadRequest)
+		return errors.Wrap(err, "cannot create request")
+	}
+
+	logger := s.requestLogger(req, tunnelHostname)
+	logger.Debugf("Request Headers %+v", req.Header)
+
+	resp, err := originService.Proxy(stream, req)
+	if err != nil {
+		s.writeErrorStatus(stream, statusBadGateway)
+		return errors.Wrap(err, "cannot proxy request")
+	}
+
+	logger.WithField("status", resp.Status).Debugf("Response Headers %+v", resp.Header)
+	return nil
+}
+
+func (s *StreamHandler) requestLogger(req *http.Request, tunnelHostname h2mux.TunnelHostname) *logrus.Entry {
+	cfRay := FindCfRayHeader(req)
+	lbProbe := IsLBProbeRequest(req)
+	logger := s.logger.WithField("tunnelHostname", tunnelHostname)
+	if cfRay != "" {
+		logger = logger.WithField("CF-RAY", cfRay)
+		logger.Debugf("%s %s %s", req.Method, req.URL, req.Proto)
+	} else if lbProbe {
+		logger.Debugf("Load Balancer health check %s %s %s", req.Method, req.URL, req.Proto)
+	} else {
+		logger.Warnf("Requests %v does not have CF-RAY header. Please open a support ticket with Cloudflare.", req)
+	}
+	return logger
+}
+
+func (s *StreamHandler) writeErrorStatus(stream *h2mux.MuxedStream, status *httpErrorStatus) {
+	stream.WriteHeaders([]h2mux.Header{
+		{
+			Name:  statusPseudoHeader,
+			Value: status.status,
+		},
+	})
+	stream.Write(status.text)
+}

streamhandler/stream_handler_test.go

@@ -0,0 +1,223 @@
+package streamhandler
+
+import (
+	"context"
+	"io"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+	"golang.org/x/sync/errgroup"
+)
+
+const (
+	testOpenStreamTimeout = time.Millisecond * 5000
+	testHandshakeTimeout  = time.Millisecond * 1000
+)
+
+var (
+	testTunnelHostname = h2mux.TunnelHostname("123.cftunnel.com")
+	baseHeaders        = []h2mux.Header{
+		{Name: ":method", Value: "GET"},
+		{Name: ":scheme", Value: "http"},
+		{Name: ":authority", Value: "example.com"},
+		{Name: ":path", Value: "/"},
+	}
+	tunnelHostnameHeader = h2mux.Header{
+		Name:  h2mux.CloudflaredProxyTunnelHostnameHeader,
+		Value: testTunnelHostname.String(),
+	}
+)
+
+func TestServeRequest(t *testing.T) {
+	configChan := make(chan *pogs.ClientConfig)
+	useConfigResultChan := make(chan *pogs.UseConfigurationResult)
+	streamHandler := NewStreamHandler(configChan, useConfigResultChan, logrus.New())
+
+	message := []byte("Hello cloudflared")
+	httpServer := httptest.NewServer(&mockHTTPHandler{message})
+
+	reverseProxyConfigs := []*pogs.ReverseProxyConfig{
+		{
+			TunnelHostname: testTunnelHostname,
+			OriginConfig: &pogs.HTTPOriginConfig{
+				URLString: httpServer.URL,
+			},
+		},
+	}
+	streamHandler.UpdateConfig(reverseProxyConfigs)
+
+	muxPair := NewDefaultMuxerPair(t, streamHandler)
+	muxPair.Serve(t)
+
+	ctx, cancel := context.WithTimeout(context.Background(), testOpenStreamTimeout)
+	defer cancel()
+
+	headers := append(baseHeaders, tunnelHostnameHeader)
+	stream, err := muxPair.EdgeMux.OpenStream(ctx, headers, nil)
+	assert.NoError(t, err)
+	assertStatusHeader(t, http.StatusOK, stream.Headers)
+	assertRespBody(t, message, stream)
+}
+
+func TestServeBadRequest(t *testing.T) {
+	configChan := make(chan *pogs.ClientConfig)
+	useConfigResultChan := make(chan *pogs.UseConfigurationResult)
+	streamHandler := NewStreamHandler(configChan, useConfigResultChan, logrus.New())
+
+	muxPair := NewDefaultMuxerPair(t, streamHandler)
+	muxPair.Serve(t)
+
+	ctx, cancel := context.WithTimeout(context.Background(), testOpenStreamTimeout)
+	defer cancel()
+
+	// No tunnel hostname header, expect to get 400 Bad Request
+	stream, err := muxPair.EdgeMux.OpenStream(ctx, baseHeaders, nil)
+	assert.NoError(t, err)
+	assertStatusHeader(t, http.StatusBadRequest, stream.Headers)
+	assertRespBody(t, statusBadRequest.text, stream)
+
+	// No mapping for the tunnel hostname, expect to get 404 Not Found
+	headers := append(baseHeaders, tunnelHostnameHeader)
+	stream, err = muxPair.EdgeMux.OpenStream(ctx, headers, nil)
+	assert.NoError(t, err)
+	assertStatusHeader(t, http.StatusNotFound, stream.Headers)
+	assertRespBody(t, statusNotFound.text, stream)
+
+	// Nothing listening on empty url, so proxy would fail. Expect to get 502 Bad Gateway
+	reverseProxyConfigs := []*pogs.ReverseProxyConfig{
+		{
+			TunnelHostname: testTunnelHostname,
+			OriginConfig: &pogs.HTTPOriginConfig{
+				URLString: "",
+			},
+		},
+	}
+	streamHandler.UpdateConfig(reverseProxyConfigs)
+	stream, err = muxPair.EdgeMux.OpenStream(ctx, headers, nil)
+	assert.NoError(t, err)
+	assertStatusHeader(t, http.StatusBadGateway, stream.Headers)
+	assertRespBody(t, statusBadGateway.text, stream)
+
+	// Invalid content-length, wouldn't not be able to create a request. Expect to get 400 Bad Request
+	headers = append(headers, h2mux.Header{
+		Name:  "content-length",
+		Value: "x",
+	})
+	stream, err = muxPair.EdgeMux.OpenStream(ctx, headers, nil)
+	assert.NoError(t, err)
+	assertStatusHeader(t, http.StatusBadRequest, stream.Headers)
+	assertRespBody(t, statusBadRequest.text, stream)
+}
+
+func assertStatusHeader(t *testing.T, expectedStatus int, headers []h2mux.Header) {
+	assert.Equal(t, statusPseudoHeader, headers[0].Name)
+	assert.Equal(t, strconv.Itoa(expectedStatus), headers[0].Value)
+}
+
+func assertRespBody(t *testing.T, expectedRespBody []byte, stream *h2mux.MuxedStream) {
+	respBody := make([]byte, len(expectedRespBody))
+	_, err := stream.Read(respBody)
+	assert.NoError(t, err)
+	assert.Equal(t, expectedRespBody, respBody)
+}
+
+type DefaultMuxerPair struct {
+	OriginMuxConfig h2mux.MuxerConfig
+	OriginMux       *h2mux.Muxer
+	OriginConn      net.Conn
+	EdgeMuxConfig   h2mux.MuxerConfig
+	EdgeMux         *h2mux.Muxer
+	EdgeConn        net.Conn
+	doneC           chan struct{}
+}
+
+func NewDefaultMuxerPair(t assert.TestingT, h h2mux.MuxedStreamHandler) *DefaultMuxerPair {
+	origin, edge := net.Pipe()
+	p := &DefaultMuxerPair{
+		OriginMuxConfig: h2mux.MuxerConfig{
+			Timeout:                 testHandshakeTimeout,
+			Handler:                 h,
+			IsClient:                true,
+			Name:                    "origin",
+			Logger:                  logrus.NewEntry(logrus.New()),
+			DefaultWindowSize:       (1 << 8) - 1,
+			MaxWindowSize:           (1 << 15) - 1,
+			StreamWriteBufferMaxLen: 1024,
+		},
+		OriginConn: origin,
+		EdgeMuxConfig: h2mux.MuxerConfig{
+			Timeout:                 testHandshakeTimeout,
+			IsClient:                false,
+			Name:                    "edge",
+			Logger:                  logrus.NewEntry(logrus.New()),
+			DefaultWindowSize:       (1 << 8) - 1,
+			MaxWindowSize:           (1 << 15) - 1,
+			StreamWriteBufferMaxLen: 1024,
+		},
+		EdgeConn: edge,
+		doneC:    make(chan struct{}),
+	}
+	assert.NoError(t, p.Handshake())
+	return p
+}
+
+func (p *DefaultMuxerPair) Handshake() error {
+	ctx, cancel := context.WithTimeout(context.Background(), testHandshakeTimeout)
+	defer cancel()
+	errGroup, _ := errgroup.WithContext(ctx)
+	errGroup.Go(func() (err error) {
+		p.EdgeMux, err = h2mux.Handshake(p.EdgeConn, p.EdgeConn, p.EdgeMuxConfig)
+		return errors.Wrap(err, "edge handshake failure")
+	})
+	errGroup.Go(func() (err error) {
+		p.OriginMux, err = h2mux.Handshake(p.OriginConn, p.OriginConn, p.OriginMuxConfig)
+		return errors.Wrap(err, "origin handshake failure")
+	})
+
+	return errGroup.Wait()
+}
+
+func (p *DefaultMuxerPair) Serve(t assert.TestingT) {
+	ctx := context.Background()
+	var wg sync.WaitGroup
+	wg.Add(2)
+	go func() {
+		err := p.EdgeMux.Serve(ctx)
+		if err != nil && err != io.EOF && err != io.ErrClosedPipe {
+			t.Errorf("error in edge muxer Serve(): %s", err)
+		}
+		p.OriginMux.Shutdown()
+		wg.Done()
+	}()
+	go func() {
+		err := p.OriginMux.Serve(ctx)
+		if err != nil && err != io.EOF && err != io.ErrClosedPipe {
+			t.Errorf("error in origin muxer Serve(): %s", err)
+		}
+		p.EdgeMux.Shutdown()
+		wg.Done()
+	}()
+	go func() {
+		// notify when both muxes have stopped serving
+		wg.Wait()
+		close(p.doneC)
+	}()
+}
+
+type mockHTTPHandler struct {
+	message []byte
+}
+
+func (mth *mockHTTPHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	w.Write(mth.message)
+}

supervisor/supervisor.go

@@ -0,0 +1,179 @@
+package supervisor
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"os"
+	"os/signal"
+	"sync"
+	"syscall"
+
+	"golang.org/x/sync/errgroup"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/updater"
+	"github.com/cloudflare/cloudflared/connection"
+	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/streamhandler"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/sirupsen/logrus"
+)
+
+type Supervisor struct {
+	connManager         *connection.EdgeManager
+	streamHandler       *streamhandler.StreamHandler
+	autoupdater         *updater.AutoUpdater
+	supportAutoupdate   bool
+	newConfigChan       <-chan *pogs.ClientConfig
+	useConfigResultChan chan<- *pogs.UseConfigurationResult
+	state               *state
+	logger              *logrus.Entry
+}
+
+func NewSupervisor(
+	defaultClientConfig *pogs.ClientConfig,
+	userCredential []byte,
+	tlsConfig *tls.Config,
+	serviceDiscoverer connection.EdgeServiceDiscoverer,
+	cloudflaredConfig *connection.CloudflaredConfig,
+	autoupdater *updater.AutoUpdater,
+	supportAutoupdate bool,
+	logger *logrus.Logger,
+) (*Supervisor, error) {
+	newConfigChan := make(chan *pogs.ClientConfig)
+	useConfigResultChan := make(chan *pogs.UseConfigurationResult)
+	streamHandler := streamhandler.NewStreamHandler(newConfigChan, useConfigResultChan, logger)
+	invalidConfigs := streamHandler.UpdateConfig(defaultClientConfig.ReverseProxyConfigs)
+
+	if len(invalidConfigs) > 0 {
+		for _, invalidConfig := range invalidConfigs {
+			logger.Errorf("Tunnel %+v is invalid, reason: %s", invalidConfig.Config, invalidConfig.Reason)
+		}
+		return nil, fmt.Errorf("At least 1 Tunnel config is invalid")
+	}
+
+	tunnelHostnames := make([]h2mux.TunnelHostname, len(defaultClientConfig.ReverseProxyConfigs))
+	for i, reverseProxyConfig := range defaultClientConfig.ReverseProxyConfigs {
+		tunnelHostnames[i] = reverseProxyConfig.TunnelHostname
+	}
+	defaultEdgeMgrConfigurable := &connection.EdgeManagerConfigurable{
+		TunnelHostnames:      tunnelHostnames,
+		EdgeConnectionConfig: defaultClientConfig.EdgeConnectionConfig,
+	}
+	return &Supervisor{
+		connManager: connection.NewEdgeManager(streamHandler, defaultEdgeMgrConfigurable, userCredential, tlsConfig,
+			serviceDiscoverer, cloudflaredConfig, logger),
+		streamHandler:       streamHandler,
+		autoupdater:         autoupdater,
+		supportAutoupdate:   supportAutoupdate,
+		newConfigChan:       newConfigChan,
+		useConfigResultChan: useConfigResultChan,
+		state:               newState(defaultClientConfig),
+		logger:              logger.WithField("subsystem", "supervisor"),
+	}, nil
+}
+
+func (s *Supervisor) Run(ctx context.Context) error {
+	errGroup, groupCtx := errgroup.WithContext(ctx)
+
+	errGroup.Go(func() error {
+		return s.connManager.Run(groupCtx)
+	})
+
+	errGroup.Go(func() error {
+		return s.listenToNewConfig(groupCtx)
+	})
+
+	errGroup.Go(func() error {
+		return s.listenToShutdownSignal(groupCtx)
+	})
+
+	if s.supportAutoupdate {
+		errGroup.Go(func() error {
+			return s.autoupdater.Run(groupCtx)
+		})
+	}
+
+	err := errGroup.Wait()
+	s.logger.Warnf("Supervisor terminated, reason: %v", err)
+	return err
+}
+
+func (s *Supervisor) listenToShutdownSignal(serveCtx context.Context) error {
+	signals := make(chan os.Signal, 10)
+	signal.Notify(signals, syscall.SIGTERM, syscall.SIGINT)
+	defer signal.Stop(signals)
+
+	select {
+	case <-serveCtx.Done():
+		return serveCtx.Err()
+	case sig := <-signals:
+		return fmt.Errorf("received %v signal", sig)
+	}
+}
+
+func (s *Supervisor) listenToNewConfig(ctx context.Context) error {
+	for {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case newConfig := <-s.newConfigChan:
+			s.useConfigResultChan <- s.notifySubsystemsNewConfig(newConfig)
+		}
+	}
+}
+
+func (s *Supervisor) notifySubsystemsNewConfig(newConfig *pogs.ClientConfig) *pogs.UseConfigurationResult {
+	s.logger.Infof("Received configuration %v", newConfig.Version)
+	if s.state.hasAppliedVersion(newConfig.Version) {
+		s.logger.Infof("%v has been applied", newConfig.Version)
+		return &pogs.UseConfigurationResult{
+			Success: true,
+		}
+	}
+
+	s.state.updateConfig(newConfig)
+	var tunnelHostnames []h2mux.TunnelHostname
+	for _, tunnelConfig := range newConfig.ReverseProxyConfigs {
+		tunnelHostnames = append(tunnelHostnames, tunnelConfig.TunnelHostname)
+	}
+	// Update connManager configurable
+	s.connManager.UpdateConfigurable(&connection.EdgeManagerConfigurable{
+		TunnelHostnames:      tunnelHostnames,
+		EdgeConnectionConfig: newConfig.EdgeConnectionConfig,
+	})
+	// Update streamHandler tunnelHostnameMapper mapping
+	failedConfigs := s.streamHandler.UpdateConfig(newConfig.ReverseProxyConfigs)
+
+	if s.supportAutoupdate {
+		s.autoupdater.Update(newConfig.SupervisorConfig.AutoUpdateFrequency)
+	}
+
+	return &pogs.UseConfigurationResult{
+		Success:       len(failedConfigs) == 0,
+		FailedConfigs: failedConfigs,
+	}
+}
+
+type state struct {
+	sync.RWMutex
+	currentConfig *pogs.ClientConfig
+}
+
+func newState(currentConfig *pogs.ClientConfig) *state {
+	return &state{
+		currentConfig: currentConfig,
+	}
+}
+
+func (s *state) hasAppliedVersion(incomingVersion pogs.Version) bool {
+	s.RLock()
+	defer s.RUnlock()
+	return s.currentConfig.Version.IsNewerOrEqual(incomingVersion)
+}
+
+func (s *state) updateConfig(newConfig *pogs.ClientConfig) {
+	s.Lock()
+	defer s.Unlock()
+	s.currentConfig = newConfig
+}

tlsconfig/certreloader.go

@@ -2,10 +2,23 @@ package tlsconfig
 
 import (
 	"crypto/tls"
+	"crypto/x509"
 	"fmt"
+	"io/ioutil"
+	"runtime"
 	"sync"
 
 	"github.com/getsentry/raven-go"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"gopkg.in/urfave/cli.v2"
+)
+
+const (
+	OriginCAPoolFlag = "origin-ca-pool"
+	CaCertFlag       = "cacert"
+
+	edgeTLSServerName = "cftunnel.com"
 )
 
 // CertReloader can load and reload a TLS certificate from a particular filepath.
@@ -51,3 +64,135 @@ func (cr *CertReloader) LoadCert() error {
 	cr.certificate = &cert
 	return nil
 }
+
+func LoadOriginCA(c *cli.Context, logger *logrus.Logger) (*x509.CertPool, error) {
+	var originCustomCAPool []byte
+
+	originCAPoolFilename := c.String(OriginCAPoolFlag)
+	if originCAPoolFilename != "" {
+		var err error
+		originCustomCAPool, err = ioutil.ReadFile(originCAPoolFilename)
+		if err != nil {
+			return nil, errors.Wrap(err, fmt.Sprintf("unable to read the file %s for --%s", originCAPoolFilename, OriginCAPoolFlag))
+		}
+	}
+
+	originCertPool, err := loadOriginCertPool(originCustomCAPool, logger)
+	if err != nil {
+		return nil, errors.Wrap(err, "error loading the certificate pool")
+	}
+
+	// Windows users should be notified that they can use the flag
+	if runtime.GOOS == "windows" && originCAPoolFilename == "" {
+		logger.Infof("cloudflared does not support loading the system root certificate pool on Windows. Please use the --%s to specify it", OriginCAPoolFlag)
+	}
+
+	return originCertPool, nil
+}
+
+func LoadCustomOriginCA(originCAFilename string) (*x509.CertPool, error) {
+	// First, obtain the system certificate pool
+	certPool, err := x509.SystemCertPool()
+	if err != nil {
+		certPool = x509.NewCertPool()
+	}
+
+	// Next, append the Cloudflare CAs into the system pool
+	cfRootCA, err := GetCloudflareRootCA()
+	if err != nil {
+		return nil, errors.Wrap(err, "could not append Cloudflare Root CAs to cloudflared certificate pool")
+	}
+	for _, cert := range cfRootCA {
+		certPool.AddCert(cert)
+	}
+
+	if originCAFilename == "" {
+		return certPool, nil
+	}
+
+	customOriginCA, err := ioutil.ReadFile(originCAFilename)
+	if err != nil {
+		return nil, errors.Wrap(err, fmt.Sprintf("unable to read the file %s", originCAFilename))
+	}
+
+	if !certPool.AppendCertsFromPEM(customOriginCA) {
+		return nil, fmt.Errorf("error appending custom CA to cert pool")
+	}
+	return certPool, nil
+}
+
+func CreateTunnelConfig(c *cli.Context) (*tls.Config, error) {
+	var rootCAs []string
+	if c.String(CaCertFlag) != "" {
+		rootCAs = append(rootCAs, c.String(CaCertFlag))
+	}
+
+	userConfig := &TLSParameters{RootCAs: rootCAs, ServerName: edgeTLSServerName}
+	tlsConfig, err := GetConfig(userConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	if tlsConfig.RootCAs == nil {
+		rootCAPool := x509.NewCertPool()
+		cfRootCA, err := GetCloudflareRootCA()
+		if err != nil {
+			return nil, errors.Wrap(err, "could not append Cloudflare Root CAs to cloudflared certificate pool")
+		}
+		for _, cert := range cfRootCA {
+			rootCAPool.AddCert(cert)
+		}
+		tlsConfig.RootCAs = rootCAPool
+	}
+
+	if tlsConfig.ServerName == "" && !tlsConfig.InsecureSkipVerify {
+		return nil, fmt.Errorf("either ServerName or InsecureSkipVerify must be specified in the tls.Config")
+	}
+	return tlsConfig, nil
+}
+
+func loadOriginCertPool(originCAPoolPEM []byte, logger *logrus.Logger) (*x509.CertPool, error) {
+	// Get the global pool
+	certPool, err := loadGlobalCertPool(logger)
+	if err != nil {
+		return nil, err
+	}
+
+	// Then, add any custom origin CA pool the user may have passed
+	if originCAPoolPEM != nil {
+		if !certPool.AppendCertsFromPEM(originCAPoolPEM) {
+			logger.Warn("could not append the provided origin CA to the cloudflared certificate pool")
+		}
+	}
+
+	return certPool, nil
+}
+
+func loadGlobalCertPool(logger *logrus.Logger) (*x509.CertPool, error) {
+	// First, obtain the system certificate pool
+	certPool, err := x509.SystemCertPool()
+	if err != nil {
+		if runtime.GOOS != "windows" { // See https://github.com/golang/go/issues/16736
+			logger.WithError(err).Warn("error obtaining the system certificates")
+		}
+		certPool = x509.NewCertPool()
+	}
+
+	// Next, append the Cloudflare CAs into the system pool
+	cfRootCA, err := GetCloudflareRootCA()
+	if err != nil {
+		return nil, errors.Wrap(err, "could not append Cloudflare Root CAs to cloudflared certificate pool")
+	}
+	for _, cert := range cfRootCA {
+		certPool.AddCert(cert)
+	}
+
+	// Finally, add the Hello certificate into the pool (since it's self-signed)
+	helloCert, err := GetHelloCertificateX509()
+	if err != nil {
+		return nil, errors.Wrap(err, "could not append Hello server certificate to cloudflared certificate pool")
+	}
+	certPool.AddCert(helloCert)
+
+	return certPool, nil
+}

tlsconfig/tlsconfig_test.go

@@ -1,5 +1,3 @@
-// TODO: Remove the above build tag and include this test when we start compiling with Golang 1.10.0+
-
 package tlsconfig
 
 import (

tunneldns/https_upstream.go

@@ -82,7 +82,7 @@ func (u *UpstreamHTTPS) exchangeWireformat(msg []byte) ([]byte, error) {
 		return nil, errors.Wrap(err, "failed to create an HTTPS request")
 	}
 
-	req.Header.Add("Content-Type", "application/dns-udpwireformat")
+	req.Header.Add("Content-Type", "application/dns-message")
 	req.Host = u.endpoint.Hostname()
 
 	resp, err := u.client.Do(req)

tunnelhostnamemapper/tunnelhostnamemapper.go

@@ -0,0 +1,93 @@
+package tunnelhostnamemapper
+
+import (
+	"sync"
+
+	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/originservice"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+)
+
+// TunnelHostnameMapper maps TunnelHostname to an OriginService
+type TunnelHostnameMapper struct {
+	sync.RWMutex
+	tunnelHostnameToOrigin map[h2mux.TunnelHostname]originservice.OriginService
+}
+
+func NewTunnelHostnameMapper() *TunnelHostnameMapper {
+	return &TunnelHostnameMapper{
+		tunnelHostnameToOrigin: make(map[h2mux.TunnelHostname]originservice.OriginService),
+	}
+}
+
+// Get an OriginService given a TunnelHostname
+func (om *TunnelHostnameMapper) Get(key h2mux.TunnelHostname) (originservice.OriginService, bool) {
+	om.RLock()
+	defer om.RUnlock()
+	originService, ok := om.tunnelHostnameToOrigin[key]
+	return originService, ok
+}
+
+// Add a mapping. If there is already an OriginService with this key, shutdown the old origin service and replace it
+// with the new one
+func (om *TunnelHostnameMapper) Add(key h2mux.TunnelHostname, os originservice.OriginService) {
+	om.Lock()
+	defer om.Unlock()
+	if oldOS, ok := om.tunnelHostnameToOrigin[key]; ok {
+		oldOS.Shutdown()
+	}
+	om.tunnelHostnameToOrigin[key] = os
+}
+
+// Delete a mapping, and shutdown its OriginService
+func (om *TunnelHostnameMapper) Delete(key h2mux.TunnelHostname) (keyFound bool) {
+	om.Lock()
+	defer om.Unlock()
+	if os, ok := om.tunnelHostnameToOrigin[key]; ok {
+		os.Shutdown()
+		delete(om.tunnelHostnameToOrigin, key)
+		return true
+	}
+	return false
+}
+
+// ToRemove finds all keys that should be removed from the TunnelHostnameMapper.
+func (om *TunnelHostnameMapper) ToRemove(newConfigs []*pogs.ReverseProxyConfig) (toRemove []h2mux.TunnelHostname) {
+	om.Lock()
+	defer om.Unlock()
+
+	// Convert into a set, for O(1) lookups instead of O(n)
+	newConfigSet := toSet(newConfigs)
+
+	// If a config in `om` isn't in `newConfigs`, it must be removed.
+	for hostname := range om.tunnelHostnameToOrigin {
+		if _, ok := newConfigSet[hostname]; !ok {
+			toRemove = append(toRemove, hostname)
+		}
+	}
+
+	return
+}
+
+// ToAdd filters the given configs, keeping those that should be added to the TunnelHostnameMapper.
+func (om *TunnelHostnameMapper) ToAdd(newConfigs []*pogs.ReverseProxyConfig) (toAdd []*pogs.ReverseProxyConfig) {
+	om.Lock()
+	defer om.Unlock()
+
+	// If a config in `newConfigs` isn't in `om`, it must be added.
+	for _, config := range newConfigs {
+		if _, ok := om.tunnelHostnameToOrigin[config.TunnelHostname]; !ok {
+			toAdd = append(toAdd, config)
+		}
+	}
+
+	return
+}
+
+func toSet(configs []*pogs.ReverseProxyConfig) map[h2mux.TunnelHostname]*pogs.ReverseProxyConfig {
+	m := make(map[h2mux.TunnelHostname]*pogs.ReverseProxyConfig)
+	for _, config := range configs {
+		m[config.TunnelHostname] = config
+	}
+	return m
+}

tunnelhostnamemapper/tunnelhostnamemapper_test.go

@@ -0,0 +1,212 @@
+package tunnelhostnamemapper
+
+import (
+	"fmt"
+	"net/http"
+	"net/url"
+	"reflect"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/originservice"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/stretchr/testify/assert"
+)
+
+const (
+	routines = 1000
+)
+
+func TestTunnelHostnameMapperConcurrentAccess(t *testing.T) {
+	thm := NewTunnelHostnameMapper()
+
+	concurrentOps(t, func(i int) {
+		// om is empty
+		os, ok := thm.Get(tunnelHostname(i))
+		assert.False(t, ok)
+		assert.Nil(t, os)
+	})
+
+	firstURL, err := url.Parse("https://127.0.0.1:8080")
+	assert.NoError(t, err)
+	httpOS := originservice.NewHTTPService(http.DefaultTransport, firstURL, false)
+	concurrentOps(t, func(i int) {
+		thm.Add(tunnelHostname(i), httpOS)
+	})
+
+	concurrentOps(t, func(i int) {
+		os, ok := thm.Get(tunnelHostname(i))
+		assert.True(t, ok)
+		assert.Equal(t, httpOS, os)
+	})
+
+	secondURL, err := url.Parse("https://127.0.0.1:8080")
+	assert.NoError(t, err)
+	secondHTTPOS := originservice.NewHTTPService(http.DefaultTransport, secondURL, true)
+	concurrentOps(t, func(i int) {
+		// Add should httpOS with secondHTTPOS
+		thm.Add(tunnelHostname(i), secondHTTPOS)
+	})
+
+	concurrentOps(t, func(i int) {
+		os, ok := thm.Get(tunnelHostname(i))
+		assert.True(t, ok)
+		assert.Equal(t, secondHTTPOS, os)
+	})
+}
+
+func concurrentOps(t *testing.T, f func(i int)) {
+	var wg sync.WaitGroup
+	wg.Add(routines)
+	for i := 0; i < routines; i++ {
+		go func(i int) {
+			f(i)
+			wg.Done()
+		}(i)
+	}
+	wg.Wait()
+}
+
+func tunnelHostname(i int) h2mux.TunnelHostname {
+	return h2mux.TunnelHostname(fmt.Sprintf("%d.cftunnel.com", i))
+}
+
+func Test_toSet(t *testing.T) {
+
+	type args struct {
+		configs []*pogs.ReverseProxyConfig
+	}
+	tests := []struct {
+		name string
+		args args
+		want map[h2mux.TunnelHostname]*pogs.ReverseProxyConfig
+	}{
+		{
+			name: "empty slice should yield empty map",
+			args: args{},
+			want: map[h2mux.TunnelHostname]*pogs.ReverseProxyConfig{},
+		},
+		{
+			name: "multiple elements",
+			args: args{[]*pogs.ReverseProxyConfig{sampleConfig1(), sampleConfig2()}},
+			want: map[h2mux.TunnelHostname]*pogs.ReverseProxyConfig{
+				"mock.example.com":  sampleConfig1(),
+				"mock2.example.com": sampleConfig2(),
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := toSet(tt.args.configs); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("toSet() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestTunnelHostnameMapper_ToAdd(t *testing.T) {
+	type fields struct {
+		tunnelHostnameToOrigin map[h2mux.TunnelHostname]originservice.OriginService
+	}
+	type args struct {
+		newConfigs []*pogs.ReverseProxyConfig
+	}
+	tests := []struct {
+		name      string
+		fields    fields
+		args      args
+		wantToAdd []*pogs.ReverseProxyConfig
+	}{
+		{
+			name: "Mapper={}, NewConfig={}, toAdd={}",
+		},
+		{
+			name:      "Mapper={}, NewConfig={x}, toAdd={x}",
+			args:      args{newConfigs: []*pogs.ReverseProxyConfig{sampleConfig1()}},
+			wantToAdd: []*pogs.ReverseProxyConfig{sampleConfig1()},
+		},
+		{
+			name:      "Mapper={x}, NewConfig={x,y}, toAdd={y}",
+			args:      args{newConfigs: []*pogs.ReverseProxyConfig{sampleConfig2()}},
+			wantToAdd: []*pogs.ReverseProxyConfig{sampleConfig2()},
+			fields: fields{tunnelHostnameToOrigin: map[h2mux.TunnelHostname]originservice.OriginService{
+				h2mux.TunnelHostname(sampleConfig1().TunnelHostname): &originservice.HelloWorldService{},
+			}},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			om := &TunnelHostnameMapper{
+				tunnelHostnameToOrigin: tt.fields.tunnelHostnameToOrigin,
+			}
+			if gotToAdd := om.ToAdd(tt.args.newConfigs); !reflect.DeepEqual(gotToAdd, tt.wantToAdd) {
+				t.Errorf("TunnelHostnameMapper.ToAdd() = %v, want %v", gotToAdd, tt.wantToAdd)
+			}
+		})
+	}
+}
+
+func TestTunnelHostnameMapper_ToRemove(t *testing.T) {
+	type fields struct {
+		tunnelHostnameToOrigin map[h2mux.TunnelHostname]originservice.OriginService
+	}
+	type args struct {
+		newConfigs []*pogs.ReverseProxyConfig
+	}
+	tests := []struct {
+		name         string
+		fields       fields
+		args         args
+		wantToRemove []h2mux.TunnelHostname
+	}{
+		{
+			name: "Mapper={}, NewConfig={}, toRemove={}",
+		},
+		{
+			name:         "Mapper={x}, NewConfig={}, toRemove={x}",
+			wantToRemove: []h2mux.TunnelHostname{sampleConfig1().TunnelHostname},
+			fields: fields{tunnelHostnameToOrigin: map[h2mux.TunnelHostname]originservice.OriginService{
+				h2mux.TunnelHostname(sampleConfig1().TunnelHostname): &originservice.HelloWorldService{},
+			}},
+		},
+		{
+			name: "Mapper={x}, NewConfig={x}, toRemove={}",
+			args: args{newConfigs: []*pogs.ReverseProxyConfig{sampleConfig1()}},
+			fields: fields{tunnelHostnameToOrigin: map[h2mux.TunnelHostname]originservice.OriginService{
+				h2mux.TunnelHostname(sampleConfig1().TunnelHostname): &originservice.HelloWorldService{},
+			}},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			om := &TunnelHostnameMapper{
+				tunnelHostnameToOrigin: tt.fields.tunnelHostnameToOrigin,
+			}
+			if gotToRemove := om.ToRemove(tt.args.newConfigs); !reflect.DeepEqual(gotToRemove, tt.wantToRemove) {
+				t.Errorf("TunnelHostnameMapper.ToRemove() = %v, want %v", gotToRemove, tt.wantToRemove)
+			}
+		})
+	}
+}
+
+func sampleConfig1() *pogs.ReverseProxyConfig {
+	return &pogs.ReverseProxyConfig{
+		TunnelHostname:     "mock.example.com",
+		OriginConfig:       &pogs.HelloWorldOriginConfig{},
+		Retries:            18,
+		ConnectionTimeout:  5 * time.Second,
+		CompressionQuality: 3,
+	}
+}
+
+func sampleConfig2() *pogs.ReverseProxyConfig {
+	return &pogs.ReverseProxyConfig{
+		TunnelHostname:     "mock2.example.com",
+		OriginConfig:       &pogs.HelloWorldOriginConfig{},
+		Retries:            18,
+		ConnectionTimeout:  5 * time.Second,
+		CompressionQuality: 3,
+	}
+}

tunnelrpc/pogs/config.go

@@ -2,108 +2,341 @@ package pogs
 
 import (
 	"context"
+	"crypto/tls"
+	"crypto/x509"
 	"fmt"
+	"net"
+	"net/http"
+	"net/url"
 	"time"
 
+	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/originservice"
+	"github.com/cloudflare/cloudflared/tlsconfig"
 	"github.com/cloudflare/cloudflared/tunnelrpc"
+
+	"github.com/pkg/errors"
 	capnp "zombiezen.com/go/capnproto2"
 	"zombiezen.com/go/capnproto2/pogs"
 	"zombiezen.com/go/capnproto2/rpc"
+	"zombiezen.com/go/capnproto2/server"
 )
 
 ///
 /// Structs
 ///
 
-type CloudflaredConfig struct {
-	Timestamp              time.Time
+// ClientConfig is a collection of FallibleConfig that determines how cloudflared should function
+type ClientConfig struct {
+	Version              Version
+	SupervisorConfig     *SupervisorConfig
+	EdgeConnectionConfig *EdgeConnectionConfig
+	DoHProxyConfigs      []*DoHProxyConfig `capnp:"dohProxyConfigs"`
+	ReverseProxyConfigs  []*ReverseProxyConfig
+}
+
+func (c *ClientConfig) MarshalBytes() ([]byte, error) {
+	msg, firstSeg, err := capnp.NewMessage(capnp.SingleSegment(nil))
+	if err != nil {
+		return nil, err
+	}
+	capnpEntity, err := tunnelrpc.NewRootClientConfig(firstSeg)
+	if err != nil {
+		return nil, err
+	}
+	err = MarshalClientConfig(capnpEntity, c)
+	if err != nil {
+		return nil, err
+	}
+	return msg.Marshal()
+}
+
+func UnmarshalClientConfigFromBytes(clientConfigBytes []byte) (*ClientConfig, error) {
+	msg, err := capnp.Unmarshal(clientConfigBytes)
+	if err != nil {
+		return nil, err
+	}
+	capnpClientConfig, err := tunnelrpc.ReadRootClientConfig(msg)
+	if err != nil {
+		return nil, err
+	}
+	pogsClientConfig, err := UnmarshalClientConfig(capnpClientConfig)
+	if err != nil {
+		return nil, err
+	}
+	return pogsClientConfig, nil
+}
+
+// Version type models the version of a ClientConfig
+type Version uint64
+
+func InitVersion() Version {
+	return Version(0)
+}
+
+func (v Version) IsNewerOrEqual(comparedVersion Version) bool {
+	return v >= comparedVersion
+}
+
+func (v Version) String() string {
+	return fmt.Sprintf("Version: %d", v)
+}
+
+// FallibleConfig is an interface implemented by configs that cloudflared might not be able to apply
+//go-sumtype:decl FallibleConfig
+type FallibleConfig interface {
+	FailReason(err error) string
+	isFallibleConfig()
+}
+
+// SupervisorConfig specifies config of components managed by Supervisor other than ConnectionManager
+type SupervisorConfig struct {
 	AutoUpdateFrequency    time.Duration
 	MetricsUpdateFrequency time.Duration
-	HeartbeatInterval      time.Duration
-	MaxFailedHeartbeats    uint64
 	GracePeriod            time.Duration
-	DoHProxyConfigs        []*DoHProxyConfig
-	ReverseProxyConfigs    []*ReverseProxyConfig
 }
 
-type UseConfigurationResult struct {
-	Success      bool
-	ErrorMessage string
+// FailReason impelents FallibleConfig interface for SupervisorConfig
+func (sc *SupervisorConfig) FailReason(err error) string {
+	return fmt.Sprintf("Cannot apply SupervisorConfig, err: %v", err)
 }
 
+func (_ *SupervisorConfig) isFallibleConfig() {}
+
+// EdgeConnectionConfig specifies what parameters and how may connections should ConnectionManager establish with edge
+type EdgeConnectionConfig struct {
+	NumHAConnections    uint8
+	HeartbeatInterval   time.Duration
+	Timeout             time.Duration
+	MaxFailedHeartbeats uint64
+	UserCredentialPath  string
+}
+
+// FailReason impelents FallibleConfig interface for EdgeConnectionConfig
+func (cmc *EdgeConnectionConfig) FailReason(err error) string {
+	return fmt.Sprintf("Cannot apply EdgeConnectionConfig, err: %v", err)
+}
+
+func (_ *EdgeConnectionConfig) isFallibleConfig() {}
+
+// DoHProxyConfig is configuration for DNS over HTTPS service
 type DoHProxyConfig struct {
 	ListenHost string
 	ListenPort uint16
 	Upstreams  []string
 }
 
+// FailReason impelents FallibleConfig interface for DoHProxyConfig
+func (dpc *DoHProxyConfig) FailReason(err error) string {
+	return fmt.Sprintf("Cannot apply DoHProxyConfig, err: %v", err)
+}
+
+func (_ *DoHProxyConfig) isFallibleConfig() {}
+
+// ReverseProxyConfig how and for what hostnames can this cloudflared proxy
 type ReverseProxyConfig struct {
-	TunnelID           string
-	Origin             OriginConfig
+	TunnelHostname     h2mux.TunnelHostname
+	OriginConfig       OriginConfig
 	Retries            uint64
 	ConnectionTimeout  time.Duration
-	ChunkedEncoding    bool
 	CompressionQuality uint64
 }
 
+func NewReverseProxyConfig(
+	tunnelHostname string,
+	originConfig OriginConfig,
+	retries uint64,
+	connectionTimeout time.Duration,
+	compressionQuality uint64,
+) (*ReverseProxyConfig, error) {
+	if originConfig == nil {
+		return nil, fmt.Errorf("NewReverseProxyConfig: originConfigUnmarshaler was null")
+	}
+	return &ReverseProxyConfig{
+		TunnelHostname:     h2mux.TunnelHostname(tunnelHostname),
+		OriginConfig:       originConfig,
+		Retries:            retries,
+		ConnectionTimeout:  connectionTimeout,
+		CompressionQuality: compressionQuality,
+	}, nil
+}
+
+// FailReason impelents FallibleConfig interface for ReverseProxyConfig
+func (rpc *ReverseProxyConfig) FailReason(err error) string {
+	return fmt.Sprintf("Cannot apply ReverseProxyConfig, err: %v", err)
+}
+
+func (_ *ReverseProxyConfig) isFallibleConfig() {}
+
 //go-sumtype:decl OriginConfig
 type OriginConfig interface {
+	// Service returns a OriginService used to proxy to the origin
+	Service() (originservice.OriginService, error)
+	// go-sumtype requires at least one unexported method, otherwise it will complain that interface is not sealed
 	isOriginConfig()
 }
 
 type HTTPOriginConfig struct {
-	URL                   string        `capnp:"url"`
-	TCPKeepAlive          time.Duration `capnp:"tcpKeepAlive"`
-	DialDualStack         bool
-	TLSHandshakeTimeout   time.Duration `capnp:"tlsHandshakeTimeout"`
-	TLSVerify             bool          `capnp:"tlsVerify"`
-	OriginCAPool          string
-	OriginServerName      string
-	MaxIdleConnections    uint64
-	IdleConnectionTimeout time.Duration
+	URLString              string        `capnp:"urlString"`
+	TCPKeepAlive           time.Duration `capnp:"tcpKeepAlive"`
+	DialDualStack          bool
+	TLSHandshakeTimeout    time.Duration `capnp:"tlsHandshakeTimeout"`
+	TLSVerify              bool          `capnp:"tlsVerify"`
+	OriginCAPool           string
+	OriginServerName       string
+	MaxIdleConnections     uint64
+	IdleConnectionTimeout  time.Duration
+	ProxyConnectionTimeout time.Duration
+	ExpectContinueTimeout  time.Duration
+	ChunkedEncoding        bool
 }
 
-func (_ *HTTPOriginConfig) isOriginConfig() {}
+func (hc *HTTPOriginConfig) Service() (originservice.OriginService, error) {
+	rootCAs, err := tlsconfig.LoadCustomOriginCA(hc.OriginCAPool)
+	if err != nil {
+		return nil, err
+	}
 
-type UnixSocketOriginConfig struct {
-	Path string
+	dialContext := (&net.Dialer{
+		Timeout:   hc.ProxyConnectionTimeout,
+		KeepAlive: hc.TCPKeepAlive,
+		DualStack: hc.DialDualStack,
+	}).DialContext
+	transport := &http.Transport{
+		Proxy:       http.ProxyFromEnvironment,
+		DialContext: dialContext,
+		TLSClientConfig: &tls.Config{
+			RootCAs:            rootCAs,
+			ServerName:         hc.OriginServerName,
+			InsecureSkipVerify: hc.TLSVerify,
+		},
+		TLSHandshakeTimeout:   hc.TLSHandshakeTimeout,
+		MaxIdleConns:          int(hc.MaxIdleConnections),
+		IdleConnTimeout:       hc.IdleConnectionTimeout,
+		ExpectContinueTimeout: hc.ExpectContinueTimeout,
+	}
+	url, err := url.Parse(hc.URLString)
+	if err != nil {
+		return nil, errors.Wrapf(err, "%s is not a valid URL", hc.URLString)
+	}
+	if url.Scheme == "unix" {
+		transport.DialContext = func(ctx context.Context, _, _ string) (net.Conn, error) {
+			return dialContext(ctx, "unix", url.Host)
+		}
+	}
+	return originservice.NewHTTPService(transport, url, hc.ChunkedEncoding), nil
 }
 
-func (_ *UnixSocketOriginConfig) isOriginConfig() {}
+func (*HTTPOriginConfig) isOriginConfig() {}
 
 type WebSocketOriginConfig struct {
-	URL string `capnp:"url"`
+	URLString        string `capnp:"urlString"`
+	TLSVerify        bool   `capnp:"tlsVerify"`
+	OriginCAPool     string
+	OriginServerName string
 }
 
-func (_ *WebSocketOriginConfig) isOriginConfig() {}
+func (wsc *WebSocketOriginConfig) Service() (originservice.OriginService, error) {
+	rootCAs, err := tlsconfig.LoadCustomOriginCA(wsc.OriginCAPool)
+	if err != nil {
+		return nil, err
+	}
+	tlsConfig := &tls.Config{
+		RootCAs:            rootCAs,
+		ServerName:         wsc.OriginServerName,
+		InsecureSkipVerify: wsc.TLSVerify,
+	}
+
+	url, err := url.Parse(wsc.URLString)
+	if err != nil {
+		return nil, errors.Wrapf(err, "%s is not a valid URL", wsc.URLString)
+	}
+	return originservice.NewWebSocketService(tlsConfig, url)
+}
+
+func (*WebSocketOriginConfig) isOriginConfig() {}
 
 type HelloWorldOriginConfig struct{}
 
-func (_ *HelloWorldOriginConfig) isOriginConfig() {}
-
-///
-/// Boilerplate to convert between these structs and the primitive structs generated by capnp-go
-///
-
-func MarshalCloudflaredConfig(s tunnelrpc.CloudflaredConfig, p *CloudflaredConfig) error {
-	s.SetTimestamp(p.Timestamp.UnixNano())
-	s.SetAutoUpdateFrequency(p.AutoUpdateFrequency.Nanoseconds())
-	s.SetMetricsUpdateFrequency(p.MetricsUpdateFrequency.Nanoseconds())
-	s.SetHeartbeatInterval(p.HeartbeatInterval.Nanoseconds())
-	s.SetMaxFailedHeartbeats(p.MaxFailedHeartbeats)
-	s.SetGracePeriod(p.GracePeriod.Nanoseconds())
-	err := marshalDoHProxyConfigs(s, p.DoHProxyConfigs)
+func (*HelloWorldOriginConfig) Service() (originservice.OriginService, error) {
+	helloCert, err := tlsconfig.GetHelloCertificateX509()
 	if err != nil {
-		return err
+		return nil, errors.Wrap(err, "Cannot get Hello World server certificate")
 	}
-	err = marshalReverseProxyConfigs(s, p.ReverseProxyConfigs)
+	rootCAs := x509.NewCertPool()
+	rootCAs.AddCert(helloCert)
+	transport := &http.Transport{
+		Proxy: http.ProxyFromEnvironment,
+		DialContext: (&net.Dialer{
+			Timeout:   30 * time.Second,
+			KeepAlive: 30 * time.Second,
+			DualStack: true,
+		}).DialContext,
+		TLSClientConfig: &tls.Config{
+			RootCAs: rootCAs,
+		},
+		MaxIdleConns:          100,
+		IdleConnTimeout:       90 * time.Second,
+		TLSHandshakeTimeout:   10 * time.Second,
+		ExpectContinueTimeout: 1 * time.Second,
+	}
+	return originservice.NewHelloWorldService(transport)
+}
+
+func (*HelloWorldOriginConfig) isOriginConfig() {}
+
+/*
+ * Boilerplate to convert between these structs and the primitive structs
+ * generated by capnp-go.
+ * Mnemonics for variable names in this section:
+ *   - `p` is for POGS (plain old Go struct)
+ *   - `s` (and `ss`) is for "capnp.Struct", which is the fundamental type
+ *     underlying the capnp-go data structures.
+ */
+
+func MarshalClientConfig(s tunnelrpc.ClientConfig, p *ClientConfig) error {
+	s.SetVersion(uint64(p.Version))
+
+	supervisorConfig, err := s.NewSupervisorConfig()
 	if err != nil {
-		return err
+		return errors.Wrap(err, "failed to get SupervisorConfig")
+	}
+	if err = MarshalSupervisorConfig(supervisorConfig, p.SupervisorConfig); err != nil {
+		return errors.Wrap(err, "MarshalSupervisorConfig error")
+	}
+
+	edgeConnectionConfig, err := s.NewEdgeConnectionConfig()
+	if err != nil {
+		return errors.Wrap(err, "failed to get EdgeConnectionConfig")
+	}
+	if err := MarshalEdgeConnectionConfig(edgeConnectionConfig, p.EdgeConnectionConfig); err != nil {
+		return errors.Wrap(err, "MarshalEdgeConnectionConfig error")
+	}
+
+	if err := marshalDoHProxyConfigs(s, p.DoHProxyConfigs); err != nil {
+		return errors.Wrap(err, "marshalDoHProxyConfigs error")
+	}
+	if err := marshalReverseProxyConfigs(s, p.ReverseProxyConfigs); err != nil {
+		return errors.Wrap(err, "marshalReverseProxyConfigs error")
 	}
 	return nil
 }
 
-func marshalDoHProxyConfigs(s tunnelrpc.CloudflaredConfig, dohProxyConfigs []*DoHProxyConfig) error {
+func MarshalSupervisorConfig(s tunnelrpc.SupervisorConfig, p *SupervisorConfig) error {
+	if err := pogs.Insert(tunnelrpc.SupervisorConfig_TypeID, s.Struct, p); err != nil {
+		return errors.Wrap(err, "failed to insert SupervisorConfig")
+	}
+	return nil
+}
+
+func MarshalEdgeConnectionConfig(s tunnelrpc.EdgeConnectionConfig, p *EdgeConnectionConfig) error {
+	if err := pogs.Insert(tunnelrpc.EdgeConnectionConfig_TypeID, s.Struct, p); err != nil {
+		return errors.Wrap(err, "failed to insert EdgeConnectionConfig")
+	}
+	return nil
+}
+
+func marshalDoHProxyConfigs(s tunnelrpc.ClientConfig, dohProxyConfigs []*DoHProxyConfig) error {
 	capnpList, err := s.NewDohProxyConfigs(int32(len(dohProxyConfigs)))
 	if err != nil {
 		return err
@@ -117,7 +350,7 @@ func marshalDoHProxyConfigs(s tunnelrpc.CloudflaredConfig, dohProxyConfigs []*Do
 	return nil
 }
 
-func marshalReverseProxyConfigs(s tunnelrpc.CloudflaredConfig, reverseProxyConfigs []*ReverseProxyConfig) error {
+func marshalReverseProxyConfigs(s tunnelrpc.ClientConfig, reverseProxyConfigs []*ReverseProxyConfig) error {
 	capnpList, err := s.NewReverseProxyConfigs(int32(len(reverseProxyConfigs)))
 	if err != nil {
 		return err
@@ -131,28 +364,54 @@ func marshalReverseProxyConfigs(s tunnelrpc.CloudflaredConfig, reverseProxyConfi
 	return nil
 }
 
-func UnmarshalCloudflaredConfig(s tunnelrpc.CloudflaredConfig) (*CloudflaredConfig, error) {
-	p := new(CloudflaredConfig)
-	p.Timestamp = time.Unix(0, s.Timestamp()).UTC()
-	p.AutoUpdateFrequency = time.Duration(s.AutoUpdateFrequency())
-	p.MetricsUpdateFrequency = time.Duration(s.MetricsUpdateFrequency())
-	p.HeartbeatInterval = time.Duration(s.HeartbeatInterval())
-	p.MaxFailedHeartbeats = s.MaxFailedHeartbeats()
-	p.GracePeriod = time.Duration(s.GracePeriod())
-	dohProxyConfigs, err := unmarshalDoHProxyConfigs(s)
+func UnmarshalClientConfig(s tunnelrpc.ClientConfig) (*ClientConfig, error) {
+	p := new(ClientConfig)
+	p.Version = Version(s.Version())
+
+	supervisorConfig, err := s.SupervisorConfig()
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrap(err, "failed to get SupervisorConfig")
 	}
-	p.DoHProxyConfigs = dohProxyConfigs
-	reverseProxyConfigs, err := unmarshalReverseProxyConfigs(s)
+	p.SupervisorConfig, err = UnmarshalSupervisorConfig(supervisorConfig)
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrap(err, "UnmarshalSupervisorConfig error")
 	}
-	p.ReverseProxyConfigs = reverseProxyConfigs
+
+	edgeConnectionConfig, err := s.EdgeConnectionConfig()
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get ConnectionManagerConfig")
+	}
+	p.EdgeConnectionConfig, err = UnmarshalEdgeConnectionConfig(edgeConnectionConfig)
+	if err != nil {
+		return nil, errors.Wrap(err, "UnmarshalConnectionManagerConfig error")
+	}
+
+	p.DoHProxyConfigs, err = unmarshalDoHProxyConfigs(s)
+	if err != nil {
+		return nil, errors.Wrap(err, "unmarshalDoHProxyConfigs error")
+	}
+
+	p.ReverseProxyConfigs, err = unmarshalReverseProxyConfigs(s)
+	if err != nil {
+		return nil, errors.Wrap(err, "unmarshalReverseProxyConfigs error")
+	}
+
+	return p, nil
+}
+
+func UnmarshalSupervisorConfig(s tunnelrpc.SupervisorConfig) (*SupervisorConfig, error) {
+	p := new(SupervisorConfig)
+	err := pogs.Extract(p, tunnelrpc.SupervisorConfig_TypeID, s.Struct)
 	return p, err
 }
 
-func unmarshalDoHProxyConfigs(s tunnelrpc.CloudflaredConfig) ([]*DoHProxyConfig, error) {
+func UnmarshalEdgeConnectionConfig(s tunnelrpc.EdgeConnectionConfig) (*EdgeConnectionConfig, error) {
+	p := new(EdgeConnectionConfig)
+	err := pogs.Extract(p, tunnelrpc.EdgeConnectionConfig_TypeID, s.Struct)
+	return p, err
+}
+
+func unmarshalDoHProxyConfigs(s tunnelrpc.ClientConfig) ([]*DoHProxyConfig, error) {
 	var result []*DoHProxyConfig
 	marshalledDoHProxyConfigs, err := s.DohProxyConfigs()
 	if err != nil {
@@ -169,7 +428,7 @@ func unmarshalDoHProxyConfigs(s tunnelrpc.CloudflaredConfig) ([]*DoHProxyConfig,
 	return result, nil
 }
 
-func unmarshalReverseProxyConfigs(s tunnelrpc.CloudflaredConfig) ([]*ReverseProxyConfig, error) {
+func unmarshalReverseProxyConfigs(s tunnelrpc.ClientConfig) ([]*ReverseProxyConfig, error) {
 	var result []*ReverseProxyConfig
 	marshalledReverseProxyConfigs, err := s.ReverseProxyConfigs()
 	if err != nil {
@@ -187,13 +446,38 @@ func unmarshalReverseProxyConfigs(s tunnelrpc.CloudflaredConfig) ([]*ReverseProx
 }
 
 func MarshalUseConfigurationResult(s tunnelrpc.UseConfigurationResult, p *UseConfigurationResult) error {
-	return pogs.Insert(tunnelrpc.UseConfigurationResult_TypeID, s.Struct, p)
+	capnpList, err := s.NewFailedConfigs(int32(len(p.FailedConfigs)))
+	if err != nil {
+		return errors.Wrap(err, "Cannot create new FailedConfigs")
+	}
+	for i, unmarshalledFailedConfig := range p.FailedConfigs {
+		err := MarshalFailedConfig(capnpList.At(i), unmarshalledFailedConfig)
+		if err != nil {
+			return errors.Wrapf(err, "Cannot MarshalFailedConfig at index %d", i)
+		}
+	}
+	s.SetSuccess(p.Success)
+	return nil
 }
 
 func UnmarshalUseConfigurationResult(s tunnelrpc.UseConfigurationResult) (*UseConfigurationResult, error) {
 	p := new(UseConfigurationResult)
-	err := pogs.Extract(p, tunnelrpc.UseConfigurationResult_TypeID, s.Struct)
-	return p, err
+	var failedConfigs []*FailedConfig
+	marshalledFailedConfigs, err := s.FailedConfigs()
+	if err != nil {
+		return nil, errors.Wrap(err, "Cannot get FailedConfigs")
+	}
+	for i := 0; i < marshalledFailedConfigs.Len(); i++ {
+		ss := marshalledFailedConfigs.At(i)
+		failedConfig, err := UnmarshalFailedConfig(ss)
+		if err != nil {
+			return nil, errors.Wrapf(err, "Cannot UnmarshalFailedConfig at index %d", i)
+		}
+		failedConfigs = append(failedConfigs, failedConfig)
+	}
+	p.FailedConfigs = failedConfigs
+	p.Success = s.Success()
+	return p, nil
 }
 
 func MarshalDoHProxyConfig(s tunnelrpc.DoHProxyConfig, p *DoHProxyConfig) error {
@@ -207,52 +491,51 @@ func UnmarshalDoHProxyConfig(s tunnelrpc.DoHProxyConfig) (*DoHProxyConfig, error
 }
 
 func MarshalReverseProxyConfig(s tunnelrpc.ReverseProxyConfig, p *ReverseProxyConfig) error {
-	s.SetTunnelID(p.TunnelID)
-	switch config := p.Origin.(type) {
+	s.SetTunnelHostname(p.TunnelHostname.String())
+	switch config := p.OriginConfig.(type) {
 	case *HTTPOriginConfig:
-		ss, err := s.Origin().NewHttp()
+		ss, err := s.OriginConfig().NewHttp()
 		if err != nil {
 			return err
 		}
-		MarshalHTTPOriginConfig(ss, config)
-	case *UnixSocketOriginConfig:
-		ss, err := s.Origin().NewSocket()
-		if err != nil {
+		if err := MarshalHTTPOriginConfig(ss, config); err != nil {
 			return err
 		}
-		MarshalUnixSocketOriginConfig(ss, config)
 	case *WebSocketOriginConfig:
-		ss, err := s.Origin().NewWebsocket()
+		ss, err := s.OriginConfig().NewWebsocket()
 		if err != nil {
 			return err
 		}
-		MarshalWebSocketOriginConfig(ss, config)
+		if err := MarshalWebSocketOriginConfig(ss, config); err != nil {
+			return err
+		}
 	case *HelloWorldOriginConfig:
-		ss, err := s.Origin().NewHelloWorld()
+		ss, err := s.OriginConfig().NewHelloWorld()
 		if err != nil {
 			return err
 		}
-		MarshalHelloWorldOriginConfig(ss, config)
+		if err := MarshalHelloWorldOriginConfig(ss, config); err != nil {
+			return err
+		}
 	default:
 		return fmt.Errorf("Unknown type for config: %T", config)
 	}
 	s.SetRetries(p.Retries)
 	s.SetConnectionTimeout(p.ConnectionTimeout.Nanoseconds())
-	s.SetChunkedEncoding(p.ChunkedEncoding)
 	s.SetCompressionQuality(p.CompressionQuality)
 	return nil
 }
 
 func UnmarshalReverseProxyConfig(s tunnelrpc.ReverseProxyConfig) (*ReverseProxyConfig, error) {
 	p := new(ReverseProxyConfig)
-	tunnelID, err := s.TunnelID()
+	tunnelHostname, err := s.TunnelHostname()
 	if err != nil {
 		return nil, err
 	}
-	p.TunnelID = tunnelID
-	switch s.Origin().Which() {
-	case tunnelrpc.ReverseProxyConfig_origin_Which_http:
-		ss, err := s.Origin().Http()
+	p.TunnelHostname = h2mux.TunnelHostname(tunnelHostname)
+	switch s.OriginConfig().Which() {
+	case tunnelrpc.ReverseProxyConfig_originConfig_Which_http:
+		ss, err := s.OriginConfig().Http()
 		if err != nil {
 			return nil, err
 		}
@@ -260,19 +543,9 @@ func UnmarshalReverseProxyConfig(s tunnelrpc.ReverseProxyConfig) (*ReverseProxyC
 		if err != nil {
 			return nil, err
 		}
-		p.Origin = config
-	case tunnelrpc.ReverseProxyConfig_origin_Which_socket:
-		ss, err := s.Origin().Socket()
-		if err != nil {
-			return nil, err
-		}
-		config, err := UnmarshalUnixSocketOriginConfig(ss)
-		if err != nil {
-			return nil, err
-		}
-		p.Origin = config
-	case tunnelrpc.ReverseProxyConfig_origin_Which_websocket:
-		ss, err := s.Origin().Websocket()
+		p.OriginConfig = config
+	case tunnelrpc.ReverseProxyConfig_originConfig_Which_websocket:
+		ss, err := s.OriginConfig().Websocket()
 		if err != nil {
 			return nil, err
 		}
@@ -280,9 +553,9 @@ func UnmarshalReverseProxyConfig(s tunnelrpc.ReverseProxyConfig) (*ReverseProxyC
 		if err != nil {
 			return nil, err
 		}
-		p.Origin = config
-	case tunnelrpc.ReverseProxyConfig_origin_Which_helloWorld:
-		ss, err := s.Origin().HelloWorld()
+		p.OriginConfig = config
+	case tunnelrpc.ReverseProxyConfig_originConfig_Which_helloWorld:
+		ss, err := s.OriginConfig().HelloWorld()
 		if err != nil {
 			return nil, err
 		}
@@ -290,11 +563,10 @@ func UnmarshalReverseProxyConfig(s tunnelrpc.ReverseProxyConfig) (*ReverseProxyC
 		if err != nil {
 			return nil, err
 		}
-		p.Origin = config
+		p.OriginConfig = config
 	}
 	p.Retries = s.Retries()
 	p.ConnectionTimeout = time.Duration(s.ConnectionTimeout())
-	p.ChunkedEncoding = s.ChunkedEncoding()
 	p.CompressionQuality = s.CompressionQuality()
 	return p, nil
 }
@@ -309,16 +581,6 @@ func UnmarshalHTTPOriginConfig(s tunnelrpc.HTTPOriginConfig) (*HTTPOriginConfig,
 	return p, err
 }
 
-func MarshalUnixSocketOriginConfig(s tunnelrpc.UnixSocketOriginConfig, p *UnixSocketOriginConfig) error {
-	return pogs.Insert(tunnelrpc.UnixSocketOriginConfig_TypeID, s.Struct, p)
-}
-
-func UnmarshalUnixSocketOriginConfig(s tunnelrpc.UnixSocketOriginConfig) (*UnixSocketOriginConfig, error) {
-	p := new(UnixSocketOriginConfig)
-	err := pogs.Extract(p, tunnelrpc.UnixSocketOriginConfig_TypeID, s.Struct)
-	return p, err
-}
-
 func MarshalWebSocketOriginConfig(s tunnelrpc.WebSocketOriginConfig, p *WebSocketOriginConfig) error {
 	return pogs.Insert(tunnelrpc.WebSocketOriginConfig_TypeID, s.Struct, p)
 }
@@ -339,31 +601,30 @@ func UnmarshalHelloWorldOriginConfig(s tunnelrpc.HelloWorldOriginConfig) (*Hello
 	return p, err
 }
 
-type CloudflaredServer interface {
-	UseConfiguration(ctx context.Context, config *CloudflaredConfig) (*CloudflaredConfig, error)
-	GetConfiguration(ctx context.Context) (*CloudflaredConfig, error)
+type ClientService interface {
+	UseConfiguration(ctx context.Context, config *ClientConfig) (*UseConfigurationResult, error)
 }
 
-type CloudflaredServer_PogsClient struct {
+type ClientService_PogsClient struct {
 	Client capnp.Client
 	Conn   *rpc.Conn
 }
 
-func (c *CloudflaredServer_PogsClient) Close() error {
+func (c *ClientService_PogsClient) Close() error {
 	return c.Conn.Close()
 }
 
-func (c *CloudflaredServer_PogsClient) UseConfiguration(
+func (c *ClientService_PogsClient) UseConfiguration(
 	ctx context.Context,
-	config *CloudflaredConfig,
+	config *ClientConfig,
 ) (*UseConfigurationResult, error) {
-	client := tunnelrpc.CloudflaredServer{Client: c.Client}
-	promise := client.UseConfiguration(ctx, func(p tunnelrpc.CloudflaredServer_useConfiguration_Params) error {
-		cloudflaredConfig, err := p.NewCloudflaredConfig()
+	client := tunnelrpc.ClientService{Client: c.Client}
+	promise := client.UseConfiguration(ctx, func(p tunnelrpc.ClientService_useConfiguration_Params) error {
+		clientServiceConfig, err := p.NewClientServiceConfig()
 		if err != nil {
 			return err
 		}
-		return MarshalCloudflaredConfig(cloudflaredConfig, config)
+		return MarshalClientConfig(clientServiceConfig, config)
 	})
 	retval, err := promise.Result().Struct()
 	if err != nil {
@@ -371,3 +632,141 @@ func (c *CloudflaredServer_PogsClient) UseConfiguration(
 	}
 	return UnmarshalUseConfigurationResult(retval)
 }
+
+func ClientService_ServerToClient(s ClientService) tunnelrpc.ClientService {
+	return tunnelrpc.ClientService_ServerToClient(ClientService_PogsImpl{s})
+}
+
+type ClientService_PogsImpl struct {
+	impl ClientService
+}
+
+func (i ClientService_PogsImpl) UseConfiguration(p tunnelrpc.ClientService_useConfiguration) error {
+	config, err := p.Params.ClientServiceConfig()
+	if err != nil {
+		return errors.Wrap(err, "Cannot get CloudflaredConfig parameter")
+	}
+	pogsConfig, err := UnmarshalClientConfig(config)
+	if err != nil {
+		return errors.Wrap(err, "Cannot unmarshal tunnelrpc.CloudflaredConfig to *CloudflaredConfig")
+	}
+	server.Ack(p.Options)
+	userConfigResult, err := i.impl.UseConfiguration(p.Ctx, pogsConfig)
+	if err != nil {
+		return err
+	}
+	result, err := p.Results.NewResult()
+	if err != nil {
+		return err
+	}
+	return MarshalUseConfigurationResult(result, userConfigResult)
+}
+
+type UseConfigurationResult struct {
+	Success       bool
+	FailedConfigs []*FailedConfig
+}
+
+type FailedConfig struct {
+	Config FallibleConfig
+	Reason string
+}
+
+func MarshalFailedConfig(s tunnelrpc.FailedConfig, p *FailedConfig) error {
+	switch config := p.Config.(type) {
+	case *SupervisorConfig:
+		ss, err := s.Config().NewSupervisor()
+		if err != nil {
+			return err
+		}
+		err = MarshalSupervisorConfig(ss, config)
+		if err != nil {
+			return err
+		}
+	case *EdgeConnectionConfig:
+		ss, err := s.Config().NewEdgeConnection()
+		if err != nil {
+			return err
+		}
+		err = MarshalEdgeConnectionConfig(ss, config)
+		if err != nil {
+			return err
+		}
+	case *DoHProxyConfig:
+		ss, err := s.Config().NewDoh()
+		if err != nil {
+			return err
+		}
+		err = MarshalDoHProxyConfig(ss, config)
+		if err != nil {
+			return err
+		}
+	case *ReverseProxyConfig:
+		ss, err := s.Config().NewReverseProxy()
+		if err != nil {
+			return err
+		}
+		err = MarshalReverseProxyConfig(ss, config)
+		if err != nil {
+			return err
+		}
+	default:
+		return fmt.Errorf("Unknown type for Config: %T", config)
+	}
+	s.SetReason(p.Reason)
+	return nil
+}
+
+func UnmarshalFailedConfig(s tunnelrpc.FailedConfig) (*FailedConfig, error) {
+	p := new(FailedConfig)
+	switch s.Config().Which() {
+	case tunnelrpc.FailedConfig_config_Which_supervisor:
+		ss, err := s.Config().Supervisor()
+		if err != nil {
+			return nil, errors.Wrap(err, "Cannot get SupervisorConfig from Config")
+		}
+		config, err := UnmarshalSupervisorConfig(ss)
+		if err != nil {
+			return nil, errors.Wrap(err, "Cannot UnmarshalSupervisorConfig")
+		}
+		p.Config = config
+	case tunnelrpc.FailedConfig_config_Which_edgeConnection:
+		ss, err := s.Config().EdgeConnection()
+		if err != nil {
+			return nil, errors.Wrap(err, "Cannot get ConnectionManager from Config")
+		}
+		config, err := UnmarshalEdgeConnectionConfig(ss)
+		if err != nil {
+			return nil, errors.Wrap(err, "Cannot UnmarshalConnectionManagerConfig")
+		}
+		p.Config = config
+	case tunnelrpc.FailedConfig_config_Which_doh:
+		ss, err := s.Config().Doh()
+		if err != nil {
+			return nil, errors.Wrap(err, "Cannot get Doh from Config")
+		}
+		config, err := UnmarshalDoHProxyConfig(ss)
+		if err != nil {
+			return nil, errors.Wrap(err, "Cannot UnmarshalDoHProxyConfig")
+		}
+		p.Config = config
+	case tunnelrpc.FailedConfig_config_Which_reverseProxy:
+		ss, err := s.Config().ReverseProxy()
+		if err != nil {
+			return nil, errors.Wrap(err, "Cannot get ReverseProxy from Config")
+		}
+		config, err := UnmarshalReverseProxyConfig(ss)
+		if err != nil {
+			return nil, errors.Wrap(err, "Cannot UnmarshalReverseProxyConfig")
+		}
+		p.Config = config
+	default:
+		return nil, fmt.Errorf("Unknown type for FailedConfig: %v", s.Config().Which())
+	}
+	reason, err := s.Reason()
+	if err != nil {
+		return nil, errors.Wrap(err, "Cannot get Reason")
+	}
+	p.Reason = reason
+	return p, nil
+}

tunnelrpc/pogs/config_test.go

@@ -1,6 +1,8 @@
 package pogs
 
 import (
+	"fmt"
+	"reflect"
 	"testing"
 	"time"
 
@@ -10,44 +12,35 @@ import (
 	capnp "zombiezen.com/go/capnproto2"
 )
 
-func TestCloudflaredConfig(t *testing.T) {
-	addDoHProxyConfigs := func(c *CloudflaredConfig) {
-		c.DoHProxyConfigs = []*DoHProxyConfig{
-			sampleDoHProxyConfig(),
-		}
-	}
-	addReverseProxyConfigs := func(c *CloudflaredConfig) {
-		c.ReverseProxyConfigs = []*ReverseProxyConfig{
-			sampleReverseProxyConfig(),
-			sampleReverseProxyConfig(func(c *ReverseProxyConfig) {
-				c.Origin = sampleHTTPOriginConfig()
-			}),
-			sampleReverseProxyConfig(func(c *ReverseProxyConfig) {
-				c.Origin = sampleUnixSocketOriginConfig()
-			}),
-			sampleReverseProxyConfig(func(c *ReverseProxyConfig) {
-				c.Origin = sampleWebSocketOriginConfig()
-			}),
-		}
-	}
+// Assert *HTTPOriginConfig implements OriginConfig
+var _ OriginConfig = (*HTTPOriginConfig)(nil)
 
-	testCases := []*CloudflaredConfig{
-		sampleCloudflaredConfig(),
-		sampleCloudflaredConfig(addDoHProxyConfigs),
-		sampleCloudflaredConfig(addReverseProxyConfigs),
-		sampleCloudflaredConfig(addDoHProxyConfigs, addReverseProxyConfigs),
-	}
-	for i, testCase := range testCases {
+// Assert *WebSocketOriginConfig implements OriginConfig
+var _ OriginConfig = (*WebSocketOriginConfig)(nil)
+
+// Assert *HelloWorldOriginConfig implements OriginConfig
+var _ OriginConfig = (*HelloWorldOriginConfig)(nil)
+
+func TestVersion(t *testing.T) {
+	firstVersion := InitVersion()
+	secondVersion := Version(1)
+	assert.False(t, firstVersion.IsNewerOrEqual(secondVersion))
+	assert.True(t, secondVersion.IsNewerOrEqual(firstVersion))
+	assert.True(t, secondVersion.IsNewerOrEqual(secondVersion))
+}
+
+func TestClientConfigCapnp(t *testing.T) {
+	for i, testCase := range ClientConfigTestCases() {
 		_, seg, err := capnp.NewMessage(capnp.SingleSegment(nil))
-		capnpEntity, err := tunnelrpc.NewCloudflaredConfig(seg)
+		capnpEntity, err := tunnelrpc.NewClientConfig(seg)
 		if !assert.NoError(t, err) {
 			t.Fatal("Couldn't initialize a new message")
 		}
-		err = MarshalCloudflaredConfig(capnpEntity, testCase)
+		err = MarshalClientConfig(capnpEntity, testCase)
 		if !assert.NoError(t, err, "testCase index %v failed to marshal", i) {
 			continue
 		}
-		result, err := UnmarshalCloudflaredConfig(capnpEntity)
+		result, err := UnmarshalClientConfig(capnpEntity)
 		if !assert.NoError(t, err, "testCase index %v failed to unmarshal", i) {
 			continue
 		}
@@ -55,14 +48,69 @@ func TestCloudflaredConfig(t *testing.T) {
 	}
 }
 
+func ClientConfigTestCases() []*ClientConfig {
+
+	addDoHProxyConfigs := func(c *ClientConfig) {
+		c.DoHProxyConfigs = []*DoHProxyConfig{
+			sampleDoHProxyConfig(),
+		}
+	}
+	addReverseProxyConfigs := func(c *ClientConfig) {
+		c.ReverseProxyConfigs = []*ReverseProxyConfig{
+			sampleReverseProxyConfig(),
+			sampleReverseProxyConfig(func(c *ReverseProxyConfig) {
+			}),
+			sampleReverseProxyConfig(func(c *ReverseProxyConfig) {
+				c.OriginConfig = sampleHTTPOriginConfig()
+			}),
+			sampleReverseProxyConfig(func(c *ReverseProxyConfig) {
+				c.OriginConfig = sampleHTTPOriginConfigUnixPath()
+			}),
+			sampleReverseProxyConfig(func(c *ReverseProxyConfig) {
+				c.OriginConfig = sampleWebSocketOriginConfig()
+			}),
+		}
+	}
+
+	testCases := []*ClientConfig{
+		sampleClientConfig(),
+		sampleClientConfig(addDoHProxyConfigs),
+		sampleClientConfig(addReverseProxyConfigs),
+		sampleClientConfig(addDoHProxyConfigs, addReverseProxyConfigs),
+	}
+
+	return testCases
+}
+
+func TestClientConfig(t *testing.T) {
+	for _, testCase := range ClientConfigTestCases() {
+		b, err := testCase.MarshalBytes()
+		assert.NoError(t, err)
+
+		clientConfig, err := UnmarshalClientConfigFromBytes(b)
+		assert.NoError(t, err)
+
+		assert.Equal(t, testCase, clientConfig)
+	}
+}
+
 func TestUseConfigurationResult(t *testing.T) {
 	testCases := []*UseConfigurationResult{
 		&UseConfigurationResult{
 			Success: true,
 		},
 		&UseConfigurationResult{
-			Success:      false,
-			ErrorMessage: "the quick brown fox jumped over the lazy dogs",
+			Success: false,
+			FailedConfigs: []*FailedConfig{
+				{
+					Config: sampleReverseProxyConfig(),
+					Reason: "Invalid certificate",
+				},
+				{
+					Config: sampleDoHProxyConfig(),
+					Reason: "Cannot listen on port 53",
+				},
+			},
 		},
 	}
 	for i, testCase := range testCases {
@@ -112,13 +160,13 @@ func TestReverseProxyConfig(t *testing.T) {
 	testCases := []*ReverseProxyConfig{
 		sampleReverseProxyConfig(),
 		sampleReverseProxyConfig(func(c *ReverseProxyConfig) {
-			c.Origin = sampleHTTPOriginConfig()
+			c.OriginConfig = sampleHTTPOriginConfig()
 		}),
 		sampleReverseProxyConfig(func(c *ReverseProxyConfig) {
-			c.Origin = sampleUnixSocketOriginConfig()
+			c.OriginConfig = sampleHTTPOriginConfigUnixPath()
 		}),
 		sampleReverseProxyConfig(func(c *ReverseProxyConfig) {
-			c.Origin = sampleWebSocketOriginConfig()
+			c.OriginConfig = sampleWebSocketOriginConfig()
 		}),
 	}
 	for i, testCase := range testCases {
@@ -161,28 +209,6 @@ func TestHTTPOriginConfig(t *testing.T) {
 	}
 }
 
-func TestUnixSocketOriginConfig(t *testing.T) {
-	testCases := []*UnixSocketOriginConfig{
-		sampleUnixSocketOriginConfig(),
-	}
-	for i, testCase := range testCases {
-		_, seg, err := capnp.NewMessage(capnp.SingleSegment(nil))
-		capnpEntity, err := tunnelrpc.NewUnixSocketOriginConfig(seg)
-		if !assert.NoError(t, err) {
-			t.Fatal("Couldn't initialize a new message")
-		}
-		err = MarshalUnixSocketOriginConfig(capnpEntity, testCase)
-		if !assert.NoError(t, err, "testCase index %v failed to marshal", i) {
-			continue
-		}
-		result, err := UnmarshalUnixSocketOriginConfig(capnpEntity)
-		if !assert.NoError(t, err, "testCase index %v failed to unmarshal", i) {
-			continue
-		}
-		assert.Equal(t, testCase, result, "testCase index %v didn't preserve struct through marshalling and unmarshalling", i)
-	}
-}
-
 func TestWebSocketOriginConfig(t *testing.T) {
 	testCases := []*WebSocketOriginConfig{
 		sampleWebSocketOriginConfig(),
@@ -205,48 +231,98 @@ func TestWebSocketOriginConfig(t *testing.T) {
 	}
 }
 
+func TestOriginConfigInvalidURL(t *testing.T) {
+	invalidConfigs := []OriginConfig{
+		&HTTPOriginConfig{
+			// this url doesn't have a scheme
+			URLString: "127.0.0.1:36192",
+		},
+		&WebSocketOriginConfig{
+			URLString: "127.0.0.1:36192",
+		},
+	}
+
+	for _, config := range invalidConfigs {
+		service, err := config.Service()
+		assert.Error(t, err)
+		assert.Nil(t, service)
+	}
+}
+
 //////////////////////////////////////////////////////////////////////////////
 // Functions to generate sample data for ease of testing
+//
+// There's one "sample" function per struct type. Each goes like this:
+//   1. Initialize an instance of the relevant struct.
+//   2. Ensure the instance has no zero-valued fields. (This catches the
+//      error-case where a field was added, but we forgot to add code to
+//      marshal/unmarshal this field in CapnProto.)
+//   3. Apply one or more "override" functions (which accept a
+//      pointer-to-struct, so they can mutate the instance).
 
-func sampleCloudflaredConfig(overrides ...func(*CloudflaredConfig)) *CloudflaredConfig {
-	// strip the location and monotonic clock reading so that assert.Equals()
-	// will work correctly
-	now := time.Now().UTC().Round(0)
-	sample := &CloudflaredConfig{
-		Timestamp:              now,
-		AutoUpdateFrequency:    21 * time.Hour,
-		MetricsUpdateFrequency: 11 * time.Minute,
-		HeartbeatInterval:      5 * time.Second,
-		MaxFailedHeartbeats:    9001,
-		GracePeriod:            31 * time.Second,
+// sampleClientConfig initializes a new ClientConfig literal,
+// applies any number of overrides to it, and returns it.
+func sampleClientConfig(overrides ...func(*ClientConfig)) *ClientConfig {
+	sample := &ClientConfig{
+		Version:              Version(1337),
+		SupervisorConfig:     sampleSupervisorConfig(),
+		EdgeConnectionConfig: sampleEdgeConnectionConfig(),
 	}
+	sample.ensureNoZeroFields()
 	for _, f := range overrides {
 		f(sample)
 	}
 	return sample
 }
 
+func sampleSupervisorConfig() *SupervisorConfig {
+	sample := &SupervisorConfig{
+		AutoUpdateFrequency:    21 * time.Hour,
+		MetricsUpdateFrequency: 11 * time.Minute,
+		GracePeriod:            31 * time.Second,
+	}
+	sample.ensureNoZeroFields()
+	return sample
+}
+
+func sampleEdgeConnectionConfig() *EdgeConnectionConfig {
+	sample := &EdgeConnectionConfig{
+		NumHAConnections:    49,
+		HeartbeatInterval:   5 * time.Second,
+		Timeout:             9 * time.Second,
+		MaxFailedHeartbeats: 9001,
+		UserCredentialPath:  "/Users/example/.cloudflared/cert.pem",
+	}
+	sample.ensureNoZeroFields()
+	return sample
+}
+
+// sampleDoHProxyConfig initializes a new DoHProxyConfig struct,
+// applies any number of overrides to it, and returns it.
 func sampleDoHProxyConfig(overrides ...func(*DoHProxyConfig)) *DoHProxyConfig {
 	sample := &DoHProxyConfig{
 		ListenHost: "127.0.0.1",
 		ListenPort: 53,
-		Upstreams:  []string{"https://1.example.com", "https://2.example.com"},
+		Upstreams:  []string{"1.1.1.1", "1.0.0.1"},
 	}
+	sample.ensureNoZeroFields()
 	for _, f := range overrides {
 		f(sample)
 	}
 	return sample
 }
 
+// sampleReverseProxyConfig initializes a new ReverseProxyConfig struct,
+// applies any number of overrides to it, and returns it.
 func sampleReverseProxyConfig(overrides ...func(*ReverseProxyConfig)) *ReverseProxyConfig {
 	sample := &ReverseProxyConfig{
-		TunnelID:           "hijk",
-		Origin:             &HelloWorldOriginConfig{},
+		TunnelHostname:     "mock-non-lb-tunnel.example.com",
+		OriginConfig:       &HelloWorldOriginConfig{},
 		Retries:            18,
 		ConnectionTimeout:  5 * time.Second,
-		ChunkedEncoding:    false,
-		CompressionQuality: 4,
+		CompressionQuality: 3,
 	}
+	sample.ensureNoZeroFields()
 	for _, f := range overrides {
 		f(sample)
 	}
@@ -255,26 +331,42 @@ func sampleReverseProxyConfig(overrides ...func(*ReverseProxyConfig)) *ReversePr
 
 func sampleHTTPOriginConfig(overrides ...func(*HTTPOriginConfig)) *HTTPOriginConfig {
 	sample := &HTTPOriginConfig{
-		URL:                   "https://example.com",
-		TCPKeepAlive:          7 * time.Second,
-		DialDualStack:         true,
-		TLSHandshakeTimeout:   11 * time.Second,
-		TLSVerify:             true,
-		OriginCAPool:          "/etc/cert.pem",
-		OriginServerName:      "secure.example.com",
-		MaxIdleConnections:    19,
-		IdleConnectionTimeout: 17 * time.Second,
+		URLString:              "https.example.com",
+		TCPKeepAlive:           7 * time.Second,
+		DialDualStack:          true,
+		TLSHandshakeTimeout:    11 * time.Second,
+		TLSVerify:              true,
+		OriginCAPool:           "/etc/cert.pem",
+		OriginServerName:       "secure.example.com",
+		MaxIdleConnections:     19,
+		IdleConnectionTimeout:  17 * time.Second,
+		ProxyConnectionTimeout: 15 * time.Second,
+		ExpectContinueTimeout:  21 * time.Second,
+		ChunkedEncoding:        true,
 	}
+	sample.ensureNoZeroFields()
 	for _, f := range overrides {
 		f(sample)
 	}
 	return sample
 }
 
-func sampleUnixSocketOriginConfig(overrides ...func(*UnixSocketOriginConfig)) *UnixSocketOriginConfig {
-	sample := &UnixSocketOriginConfig{
-		Path: "/var/lib/file.sock",
+func sampleHTTPOriginConfigUnixPath(overrides ...func(*HTTPOriginConfig)) *HTTPOriginConfig {
+	sample := &HTTPOriginConfig{
+		URLString:              "unix:/var/lib/file.sock",
+		TCPKeepAlive:           7 * time.Second,
+		DialDualStack:          true,
+		TLSHandshakeTimeout:    11 * time.Second,
+		TLSVerify:              true,
+		OriginCAPool:           "/etc/cert.pem",
+		OriginServerName:       "secure.example.com",
+		MaxIdleConnections:     19,
+		IdleConnectionTimeout:  17 * time.Second,
+		ProxyConnectionTimeout: 15 * time.Second,
+		ExpectContinueTimeout:  21 * time.Second,
+		ChunkedEncoding:        true,
 	}
+	sample.ensureNoZeroFields()
 	for _, f := range overrides {
 		f(sample)
 	}
@@ -283,10 +375,79 @@ func sampleUnixSocketOriginConfig(overrides ...func(*UnixSocketOriginConfig)) *U
 
 func sampleWebSocketOriginConfig(overrides ...func(*WebSocketOriginConfig)) *WebSocketOriginConfig {
 	sample := &WebSocketOriginConfig{
-		URL: "ssh://example.com",
+		URLString:        "ssh://example.com",
+		TLSVerify:        true,
+		OriginCAPool:     "/etc/cert.pem",
+		OriginServerName: "secure.example.com",
 	}
+	sample.ensureNoZeroFields()
 	for _, f := range overrides {
 		f(sample)
 	}
 	return sample
 }
+
+func (c *ClientConfig) ensureNoZeroFields() {
+	ensureNoZeroFieldsInSample(reflect.ValueOf(c), []string{"DoHProxyConfigs", "ReverseProxyConfigs"})
+}
+
+func (c *SupervisorConfig) ensureNoZeroFields() {
+	ensureNoZeroFieldsInSample(reflect.ValueOf(c), []string{})
+}
+
+func (c *EdgeConnectionConfig) ensureNoZeroFields() {
+	ensureNoZeroFieldsInSample(reflect.ValueOf(c), []string{})
+}
+
+func (c *DoHProxyConfig) ensureNoZeroFields() {
+	ensureNoZeroFieldsInSample(reflect.ValueOf(c), []string{})
+}
+
+func (c *ReverseProxyConfig) ensureNoZeroFields() {
+	ensureNoZeroFieldsInSample(reflect.ValueOf(c), []string{})
+}
+
+func (c *HTTPOriginConfig) ensureNoZeroFields() {
+	ensureNoZeroFieldsInSample(reflect.ValueOf(c), []string{})
+}
+
+func (c *WebSocketOriginConfig) ensureNoZeroFields() {
+	ensureNoZeroFieldsInSample(reflect.ValueOf(c), []string{})
+}
+
+// ensureNoZeroFieldsInSample checks that all fields in the sample struct,
+// except those listed in `allowedZeroFieldNames`, are initialized to nonzero
+// values. Note that the value has to be a pointer for reflection to work
+// correctly:
+//     e := &ExampleStruct{ ... }
+//     ensureNoZeroFieldsInSample(reflect.ValueOf(e), []string{})
+//
+// Context:
+// Our tests work by taking a sample struct and marshalling/unmarshalling it.
+// This makes them easy to write, but introduces some risk: if we don't
+// include a field in the sample value, it won't be covered under tests.
+// This check reduces that risk by requiring fields to be either initialized
+// or explicitly uninitialized.
+func ensureNoZeroFieldsInSample(ptrToSampleValue reflect.Value, allowedZeroFieldNames []string) {
+	sampleValue := ptrToSampleValue.Elem()
+	structType := ptrToSampleValue.Type().Elem()
+
+	allowedZeroFieldSet := make(map[string]bool)
+	for _, name := range allowedZeroFieldNames {
+		if _, ok := structType.FieldByName(name); !ok {
+			panic(fmt.Sprintf("struct %v has no field %v", structType.Name(), name))
+		}
+		allowedZeroFieldSet[name] = true
+	}
+
+	for i := 0; i < structType.NumField(); i++ {
+		if allowedZeroFieldSet[structType.Field(i).Name] {
+			continue
+		}
+
+		zeroValue := reflect.Zero(structType.Field(i).Type)
+		if reflect.DeepEqual(zeroValue.Interface(), sampleValue.Field(i).Interface()) {
+			panic(fmt.Sprintf("In the sample value for struct %v, field %v was not initialized", structType.Name(), structType.Field(i).Name))
+		}
+	}
+}

tunnelrpc/pogs/tunnelrpc.go

@@ -2,10 +2,12 @@ package pogs
 
 import (
 	"context"
+	"fmt"
 	"time"
 
 	"github.com/cloudflare/cloudflared/tunnelrpc"
 	"github.com/google/uuid"
+	"github.com/pkg/errors"
 
 	log "github.com/sirupsen/logrus"
 	capnp "zombiezen.com/go/capnproto2"
@@ -73,27 +75,129 @@ func UnmarshalRegistrationOptions(s tunnelrpc.RegistrationOptions) (*Registratio
 	return p, err
 }
 
-type ConnectResult struct {
-	Err        *ConnectError
-	ServerInfo ServerInfo
+// ConnectResult models the result of Connect RPC, implemented by ConnectError and ConnectSuccess.
+type ConnectResult interface {
+	ConnectError() *ConnectError
+	ConnectedTo() string
+	ClientConfig() *ClientConfig
+	Marshal(s tunnelrpc.ConnectResult) error
 }
 
-func MarshalConnectResult(s tunnelrpc.ConnectResult, p *ConnectResult) error {
-	return pogs.Insert(tunnelrpc.ConnectResult_TypeID, s.Struct, p)
+func MarshalConnectResult(s tunnelrpc.ConnectResult, p ConnectResult) error {
+	return p.Marshal(s)
 }
 
-func UnmarshalConnectResult(s tunnelrpc.ConnectResult) (*ConnectResult, error) {
-	p := new(ConnectResult)
-	err := pogs.Extract(p, tunnelrpc.ConnectResult_TypeID, s.Struct)
-	return p, err
+func UnmarshalConnectResult(s tunnelrpc.ConnectResult) (ConnectResult, error) {
+	switch s.Result().Which() {
+	case tunnelrpc.ConnectResult_result_Which_err:
+		capnpConnectError, err := s.Result().Err()
+		if err != nil {
+			return nil, err
+		}
+		return UnmarshalConnectError(capnpConnectError)
+	case tunnelrpc.ConnectResult_result_Which_success:
+		capnpConnectSuccess, err := s.Result().Success()
+		if err != nil {
+			return nil, err
+		}
+		return UnmarshalConnectSuccess(capnpConnectSuccess)
+	default:
+		return nil, fmt.Errorf("Unmarshal %v not implemented yet", s.Result().Which().String())
+	}
 }
 
+// ConnectSuccess is the concrete returned type when Connect RPC succeed
+type ConnectSuccess struct {
+	ServerLocationName string
+	Config             *ClientConfig
+}
+
+func (*ConnectSuccess) ConnectError() *ConnectError {
+	return nil
+}
+
+func (cs *ConnectSuccess) ConnectedTo() string {
+	return cs.ServerLocationName
+}
+
+func (cs *ConnectSuccess) ClientConfig() *ClientConfig {
+	return cs.Config
+}
+
+func (cs *ConnectSuccess) Marshal(s tunnelrpc.ConnectResult) error {
+	capnpConnectSuccess, err := s.Result().NewSuccess()
+	if err != nil {
+		return err
+	}
+
+	err = capnpConnectSuccess.SetServerLocationName(cs.ServerLocationName)
+	if err != nil {
+		return errors.Wrap(err, "failed to set ConnectSuccess.ServerLocationName")
+	}
+
+	if cs.Config != nil {
+		capnpClientConfig, err := capnpConnectSuccess.NewClientConfig()
+		if err != nil {
+			return errors.Wrap(err, "failed to initialize ConnectSuccess.ClientConfig")
+		}
+		if err := MarshalClientConfig(capnpClientConfig, cs.Config); err != nil {
+			return errors.Wrap(err, "failed to marshal ClientConfig")
+		}
+	}
+
+	return nil
+}
+
+func UnmarshalConnectSuccess(s tunnelrpc.ConnectSuccess) (*ConnectSuccess, error) {
+	p := new(ConnectSuccess)
+
+	serverLocationName, err := s.ServerLocationName()
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get tunnelrpc.ConnectSuccess.ServerLocationName")
+	}
+	p.ServerLocationName = serverLocationName
+
+	if s.HasClientConfig() {
+		capnpClientConfig, err := s.ClientConfig()
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to get tunnelrpc.ConnectSuccess.ClientConfig")
+		}
+		p.Config, err = UnmarshalClientConfig(capnpClientConfig)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to get unmarshal ClientConfig")
+		}
+	}
+
+	return p, nil
+}
+
+// ConnectError is the concrete returned type when Connect RPC encounters some error
 type ConnectError struct {
 	Cause       string
 	RetryAfter  time.Duration
 	ShouldRetry bool
 }
 
+func (ce *ConnectError) ConnectError() *ConnectError {
+	return ce
+}
+
+func (*ConnectError) ConnectedTo() string {
+	return ""
+}
+
+func (*ConnectError) ClientConfig() *ClientConfig {
+	return nil
+}
+
+func (ce *ConnectError) Marshal(s tunnelrpc.ConnectResult) error {
+	capnpConnectError, err := s.Result().NewErr()
+	if err != nil {
+		return err
+	}
+	return MarshalConnectError(capnpConnectError, ce)
+}
+
 func MarshalConnectError(s tunnelrpc.ConnectError, p *ConnectError) error {
 	return pogs.Insert(tunnelrpc.ConnectError_TypeID, s.Struct, p)
 }
@@ -133,46 +237,88 @@ type ConnectParameters struct {
 	NumPreviousAttempts uint8
 	Tags                []Tag
 	CloudflaredVersion  string
-}
-
-// CapnpConnectParameters is ConnectParameters represented in Cap'n Proto build-in types
-type CapnpConnectParameters struct {
-	OriginCert          []byte
-	CloudflaredID       []byte
-	NumPreviousAttempts uint8
-	Tags                []Tag
-	CloudflaredVersion  string
+	IntentLabel         string
 }
 
 func MarshalConnectParameters(s tunnelrpc.CapnpConnectParameters, p *ConnectParameters) error {
+	if err := s.SetOriginCert(p.OriginCert); err != nil {
+		return err
+	}
 	cloudflaredIDBytes, err := p.CloudflaredID.MarshalBinary()
 	if err != nil {
 		return err
 	}
-	capnpConnectParameters := &CapnpConnectParameters{
-		OriginCert:          p.OriginCert,
-		CloudflaredID:       cloudflaredIDBytes,
-		NumPreviousAttempts: p.NumPreviousAttempts,
-		CloudflaredVersion:  p.CloudflaredVersion,
+	if err := s.SetCloudflaredID(cloudflaredIDBytes); err != nil {
+		return err
 	}
-	return pogs.Insert(tunnelrpc.CapnpConnectParameters_TypeID, s.Struct, capnpConnectParameters)
+	s.SetNumPreviousAttempts(p.NumPreviousAttempts)
+	if len(p.Tags) > 0 {
+		tagsCapnpList, err := s.NewTags(int32(len(p.Tags)))
+		if err != nil {
+			return err
+		}
+		for i, tag := range p.Tags {
+			tagCapnp := tagsCapnpList.At(i)
+			if err := tagCapnp.SetName(tag.Name); err != nil {
+				return err
+			}
+			if err := tagCapnp.SetValue(tag.Value); err != nil {
+				return err
+			}
+		}
+	}
+	if err := s.SetCloudflaredVersion(p.CloudflaredVersion); err != nil {
+		return err
+	}
+	return s.SetIntentLabel(p.IntentLabel)
 }
 
 func UnmarshalConnectParameters(s tunnelrpc.CapnpConnectParameters) (*ConnectParameters, error) {
-	p := new(CapnpConnectParameters)
-	err := pogs.Extract(p, tunnelrpc.CapnpConnectParameters_TypeID, s.Struct)
+	originCert, err := s.OriginCert()
 	if err != nil {
 		return nil, err
 	}
-	cloudflaredID, err := uuid.FromBytes(p.CloudflaredID)
+
+	cloudflaredIDBytes, err := s.CloudflaredID()
 	if err != nil {
 		return nil, err
 	}
+	cloudflaredID, err := uuid.FromBytes(cloudflaredIDBytes)
+	if err != nil {
+		return nil, err
+	}
+
+	tagsCapnpList, err := s.Tags()
+	if err != nil {
+		return nil, err
+	}
+	var tags []Tag
+	for i := 0; i < tagsCapnpList.Len(); i++ {
+		tagCapnp := tagsCapnpList.At(i)
+		name, err := tagCapnp.Name()
+		if err != nil {
+			return nil, err
+		}
+		value, err := tagCapnp.Value()
+		if err != nil {
+			return nil, err
+		}
+		tags = append(tags, Tag{Name: name, Value: value})
+	}
+
+	cloudflaredVersion, err := s.CloudflaredVersion()
+	if err != nil {
+		return nil, err
+	}
+
+	intentLabel, err := s.IntentLabel()
 	return &ConnectParameters{
-		OriginCert:          p.OriginCert,
+		OriginCert:          originCert,
 		CloudflaredID:       cloudflaredID,
-		NumPreviousAttempts: p.NumPreviousAttempts,
-		CloudflaredVersion:  p.CloudflaredVersion,
+		NumPreviousAttempts: s.NumPreviousAttempts(),
+		Tags:                tags,
+		CloudflaredVersion:  cloudflaredVersion,
+		IntentLabel:         intentLabel,
 	}, nil
 }
 
@@ -180,7 +326,7 @@ type TunnelServer interface {
 	RegisterTunnel(ctx context.Context, originCert []byte, hostname string, options *RegistrationOptions) (*TunnelRegistration, error)
 	GetServerInfo(ctx context.Context) (*ServerInfo, error)
 	UnregisterTunnel(ctx context.Context, gracePeriodNanoSec int64) error
-	Connect(ctx context.Context, paramaters *ConnectParameters) (*ConnectResult, error)
+	Connect(ctx context.Context, parameters *ConnectParameters) (ConnectResult, error)
 }
 
 func TunnelServer_ServerToClient(s TunnelServer) tunnelrpc.TunnelServer {
@@ -241,11 +387,11 @@ func (i TunnelServer_PogsImpl) UnregisterTunnel(p tunnelrpc.TunnelServer_unregis
 }
 
 func (i TunnelServer_PogsImpl) Connect(p tunnelrpc.TunnelServer_connect) error {
-	paramaters, err := p.Params.Parameters()
+	parameters, err := p.Params.Parameters()
 	if err != nil {
 		return err
 	}
-	pogsParameters, err := UnmarshalConnectParameters(paramaters)
+	pogsParameters, err := UnmarshalConnectParameters(parameters)
 	if err != nil {
 		return err
 	}
@@ -258,7 +404,7 @@ func (i TunnelServer_PogsImpl) Connect(p tunnelrpc.TunnelServer_connect) error {
 	if err != nil {
 		return err
 	}
-	return MarshalConnectResult(result, connectResult)
+	return connectResult.Marshal(result)
 }
 
 type TunnelServer_PogsClient struct {
@@ -322,7 +468,7 @@ func (c TunnelServer_PogsClient) UnregisterTunnel(ctx context.Context, gracePeri
 
 func (c TunnelServer_PogsClient) Connect(ctx context.Context,
 	parameters *ConnectParameters,
-) (*ConnectResult, error) {
+) (ConnectResult, error) {
 	client := tunnelrpc.TunnelServer{Client: c.Client}
 	promise := client.Connect(ctx, func(p tunnelrpc.TunnelServer_connect_Params) error {
 		connectParameters, err := p.NewParameters()

tunnelrpc/pogs/tunnelrpc_test.go

@@ -0,0 +1,103 @@
+package pogs
+
+import (
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/cloudflare/cloudflared/tunnelrpc"
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/assert"
+	capnp "zombiezen.com/go/capnproto2"
+)
+
+func TestConnectResult(t *testing.T) {
+	testCases := []ConnectResult{
+		&ConnectError{
+			Cause:       "it broke",
+			ShouldRetry: false,
+			RetryAfter:  2 * time.Second,
+		},
+		&ConnectSuccess{
+			ServerLocationName: "SFO",
+			Config:             sampleClientConfig(),
+		},
+		&ConnectSuccess{
+			ServerLocationName: "",
+			Config:             nil,
+		},
+	}
+	for i, testCase := range testCases {
+		_, seg, err := capnp.NewMessage(capnp.SingleSegment(nil))
+		capnpEntity, err := tunnelrpc.NewConnectResult(seg)
+		if !assert.NoError(t, err) {
+			t.Fatal("Couldn't initialize a new message")
+		}
+		err = MarshalConnectResult(capnpEntity, testCase)
+		if !assert.NoError(t, err, "testCase #%v failed to marshal", i) {
+			continue
+		}
+		result, err := UnmarshalConnectResult(capnpEntity)
+		if !assert.NoError(t, err, "testCase #%v failed to unmarshal", i) {
+			continue
+		}
+		assert.Equal(t, testCase, result, "testCase index %v didn't preserve struct through marshalling and unmarshalling", i)
+	}
+}
+
+func TestConnectParameters(t *testing.T) {
+	testCases := []*ConnectParameters{
+		sampleConnectParameters(),
+		sampleConnectParameters(func(c *ConnectParameters) {
+			c.IntentLabel = "my_intent"
+		}),
+		sampleConnectParameters(func(c *ConnectParameters) {
+			c.Tags = nil
+		}),
+	}
+	for i, testCase := range testCases {
+		_, seg, err := capnp.NewMessage(capnp.SingleSegment(nil))
+		capnpEntity, err := tunnelrpc.NewCapnpConnectParameters(seg)
+		if !assert.NoError(t, err) {
+			t.Fatal("Couldn't initialize a new message")
+		}
+		err = MarshalConnectParameters(capnpEntity, testCase)
+		if !assert.NoError(t, err, "testCase index %v failed to marshal", i) {
+			continue
+		}
+		result, err := UnmarshalConnectParameters(capnpEntity)
+		if !assert.NoError(t, err, "testCase index %v failed to unmarshal", i) {
+			continue
+		}
+		assert.Equal(t, testCase, result, "testCase index %v didn't preserve struct through marshalling and unmarshalling", i)
+	}
+}
+
+func sampleConnectParameters(overrides ...func(*ConnectParameters)) *ConnectParameters {
+	cloudflaredID, err := uuid.Parse("ED7BA470-8E54-465E-825C-99712043E01C")
+	if err != nil {
+		panic(err)
+	}
+	sample := &ConnectParameters{
+		OriginCert:          []byte("my-origin-cert"),
+		CloudflaredID:       cloudflaredID,
+		NumPreviousAttempts: 19,
+		Tags: []Tag{
+			Tag{
+				Name:  "provision-method",
+				Value: "new",
+			},
+		},
+		CloudflaredVersion: "7.0",
+		IntentLabel:        "my_intent",
+	}
+	sample.ensureNoZeroFields()
+	for _, f := range overrides {
+		f(sample)
+	}
+	return sample
+}
+
+func (c *ConnectParameters) ensureNoZeroFields() {
+	ensureNoZeroFieldsInSample(reflect.ValueOf(c), []string{})
+}

tunnelrpc/tunnelrpc.capnp

@@ -57,12 +57,15 @@ struct CapnpConnectParameters {
     tags @3 :List(Tag);
     # release version of cloudflared
     cloudflaredVersion @4 :Text;
+    # which intent this cloudflared instance should get its behaviour from
+    intentLabel @5 :Text;
 }
 
 struct ConnectResult {
-    err @0 :ConnectError;
-    # Information about the server this connection is established with
-    serverInfo @1 :ServerInfo;
+    result :union {
+        err @0 :ConnectError;
+        success @1 :ConnectSuccess;
+    }
 }
 
 struct ConnectError {
@@ -72,83 +75,110 @@ struct ConnectError {
     shouldRetry @2 :Bool;
 }
 
-struct CloudflaredConfig {
-    # Timestamp (in ns) of this configuration. Any configuration supplied to
-    # useConfiguration() with an older timestamp should be ignored.
-    timestamp @0 :Int64;
+struct ConnectSuccess {
+    # Information about the server this connection is established with
+    serverLocationName @0 :Text;
+    # How this cloudflared instance should be configured. This can be null if there isn't an intent for this origin yet
+    clientConfig @1 :ClientConfig;
+}
+
+struct ClientConfig {
+    # Version of this configuration. This value is opaque, but is guaranteed
+    # to monotonically increase in value. Any configuration supplied to
+    # useConfiguration() with a smaller `version` should be ignored.
+    version @0 :UInt64;
+    # supervisorConfig  is configuration for supervisor, the component that manages connection manager,
+    # autoupdater and metrics server
+    supervisorConfig @1 :SupervisorConfig;
+    # edgeConnectionConfig is configuration for connection manager, the componenet that manages connections with the edge
+    edgeConnectionConfig @2 :EdgeConnectionConfig;
+    # Configuration for cloudflared to run as a DNS-over-HTTPS proxy.
+    # cloudflared CLI option: `proxy-dns`
+    dohProxyConfigs @3 :List(DoHProxyConfig);
+    # Configuration for cloudflared to run as an HTTP reverse proxy.
+    reverseProxyConfigs @4 :List(ReverseProxyConfig);
+}
+
+struct SupervisorConfig {
     # Frequency (in ns) to check Equinox for updates.
     # Zero means auto-update is disabled.
     # cloudflared CLI option: `autoupdate-freq`
-    autoUpdateFrequency @1 :Int64;
+    autoUpdateFrequency @0 :Int64;
     # Frequency (in ns) to update connection-based metrics.
     # cloudflared CLI option: `metrics-update-freq`
-    metricsUpdateFrequency @2 :Int64;
-    # interval (in ns) between heartbeats with the Cloudflare edge
-    # cloudflared CLI option: `heartbeat-interval`
-    heartbeatInterval @3 :Int64;
-    # Minimum number of unacked heartbeats for cloudflared to send before
-    # closing the connection to the edge.
-    # cloudflared CLI option: `heartbeat-count`
-    maxFailedHeartbeats @4 :UInt64;
+    metricsUpdateFrequency @1 :Int64;
     # Time (in ns) to continue serving requests after cloudflared receives its
     # first SIGINT/SIGTERM. A second SIGINT/SIGTERM will force cloudflared to
     # shutdown immediately. For example, this field can be used to gracefully
     # transition traffic to another cloudflared instance.
     # cloudflared CLI option: `grace-period`
-    gracePeriod @5 :Int64;
-    # Configuration for cloudflared to run as a DNS-over-HTTPS proxy.
-    # cloudflared CLI option: `proxy-dns`
-    dohProxyConfigs @6 :List(DoHProxyConfig);
-    # Configuration for cloudflared to run as an HTTP reverse proxy.
-    reverseProxyConfigs @7 :List(ReverseProxyConfig);
+    gracePeriod @2 :Int64;
+}
+
+struct EdgeConnectionConfig {
+    # cloudflared CLI option: `ha-connections`
+    numHAConnections @0 :UInt8;
+    # Interval (in ns) between heartbeats with the Cloudflare edge
+    # cloudflared CLI option: `heartbeat-interval`
+    heartbeatInterval @1 :Int64;
+    # Maximum wait time to connect with the edge.
+    timeout  @2 :Int64;
+    # Number of unacked heartbeats for cloudflared to send before
+    # closing the connection to the edge.
+    # cloudflared CLI option: `heartbeat-count`
+    maxFailedHeartbeats @3 :UInt64;
+    # Absolute path of the file containing certificate and token to connect with the edge
+    userCredentialPath @4 :Text;
 }
 
 struct ReverseProxyConfig {
-    tunnelID @0 :Text;
-    origin :union {
+    tunnelHostname @0 :Text;
+    originConfig :union {
         http @1 :HTTPOriginConfig;
-        socket @2 :UnixSocketOriginConfig;
-        websocket @3 :WebSocketOriginConfig;
-        helloWorld @4 :HelloWorldOriginConfig;
+        websocket @2 :WebSocketOriginConfig;
+        helloWorld @3 :HelloWorldOriginConfig;
     }
     # Maximum number of retries for connection/protocol errors.
     # cloudflared CLI option: `retries`
-    retries @5 :UInt64;
+    retries @4 :UInt64;
     # maximum time (in ns) for cloudflared to wait to establish a connection
     # to the origin. Zero means no timeout.
     # cloudflared CLI option: `proxy-connect-timeout`
-    connectionTimeout @6 :Int64;
-    # Whether cloudflared should allow chunked transfer encoding to the
-    # origin. (This should be disabled for WSGI origins, for example.)
-    # negation of cloudflared CLI option: `no-chunked-encoding`
-    chunkedEncoding @7 :Bool;
+    connectionTimeout @5 :Int64;
     # (beta) Use cross-stream compression instead of HTTP compression.
     # 0=off, 1=low, 2=medium, 3=high.
     # For more context see the mapping here: https://github.com/cloudflare/cloudflared/blob/2019.3.2/h2mux/h2_dictionaries.go#L62
     # cloudflared CLI option: `compression-quality`
-    compressionQuality @8 :UInt64;
+    compressionQuality @6 :UInt64;
 }
 
-struct UnixSocketOriginConfig {
-    # path to the socket file.
-    # cloudflared will send data to this socket via a Unix socket connection.
-    # cloudflared CLI option: `unix-socket`
-    path @0 :Text;
-}
-
-#
 struct WebSocketOriginConfig {
     # URI of the origin service.
     # cloudflared will start a websocket server that forwards data to this URI
     # cloudflared CLI option: `url`
     # cloudflared logic: https://github.com/cloudflare/cloudflared/blob/2019.3.2/cmd/cloudflared/tunnel/cmd.go#L304
-    url @0 :Text;
+    urlString @0 :Text;
+    # Whether cloudflared should verify TLS connections to the origin.
+    # negation of cloudflared CLI option: `no-tls-verify`
+    tlsVerify @1 :Bool;
+    # originCAPool specifies the root CA that cloudflared should use when
+    # verifying TLS connections to the origin.
+    #   - if tlsVerify is false, originCAPool will be ignored.
+    #   - if tlsVerify is true and originCAPool is empty, the system CA pool
+    #     will be loaded if possible.
+    #   - if tlsVerify is true and originCAPool is non-empty, cloudflared will
+    #     treat it as the filepath to the root CA.
+    # cloudflared CLI option: `origin-ca-pool`
+    originCAPool @2 :Text;
+    # Hostname to use when verifying TLS connections to the origin.
+    # cloudflared CLI option: `origin-server-name`
+    originServerName @3 :Text;
 }
 
 struct HTTPOriginConfig {
     # HTTP(S) URL of the origin service.
     # cloudflared CLI option: `url`
-    url @0 :Text;
+    urlString @0 :Text;
     # the TCP keep-alive period (in ns) for an active network connection.
     # Zero means keep-alives are not enabled.
     # cloudflared CLI option: `proxy-tcp-keepalive`
@@ -184,6 +214,17 @@ struct HTTPOriginConfig {
     # idle before closing itself. Zero means no timeout.
     # cloudflared CLI option: `proxy-keepalive-timeout`
     idleConnectionTimeout @8 :Int64;
+    # maximum amount of time a dial will wait for a connect to complete.
+    proxyConnectionTimeout @9 :Int64;
+    # The amount of time to wait for origin's first response headers after fully
+    # writing the request headers if the request has an "Expect: 100-continue" header.
+    # Zero means no timeout and causes the body to be sent immediately, without
+    # waiting for the server to approve.
+    expectContinueTimeout @10 :Int64;
+    # Whether cloudflared should allow chunked transfer encoding to the
+    # origin. (This should be disabled for WSGI origins, for example.)
+    # negation of cloudflared CLI option: `no-chunked-encoding`
+	chunkedEncoding @11 :Bool;
 }
 
 # configuration for cloudflared to provide a DNS over HTTPS proxy server
@@ -220,7 +261,17 @@ struct ServerInfo {
 
 struct UseConfigurationResult {
     success @0 :Bool;
-    errorMessage @1 :Text;
+    failedConfigs @1 :List(FailedConfig);
+}
+
+struct FailedConfig {
+    config :union {
+        supervisor @0 :SupervisorConfig;
+        edgeConnection @1 :EdgeConnectionConfig;
+        doh @2 :DoHProxyConfig;
+        reverseProxy @3 :ReverseProxyConfig;
+    }
+	reason @4 :Text;
 }
 
 interface TunnelServer {
@@ -230,6 +281,6 @@ interface TunnelServer {
     connect @3 (parameters :CapnpConnectParameters) -> (result :ConnectResult);
 }
 
-interface CloudflaredServer {
-    useConfiguration @0 (cloudflaredConfig :CloudflaredConfig) -> (result :UseConfigurationResult);
+interface ClientService {
+    useConfiguration @0 (clientServiceConfig :ClientConfig) -> (result :UseConfigurationResult);
 }

validation/validation_test.go

@@ -180,33 +180,6 @@ func TestValidateHTTPService_HTTP2HTTP(t *testing.T) {
 		}
 		panic("Shouldn't reach here")
 	})))
-
-	// Integration-style test with a mock server
-	server, client, err := createMockServerAndClient(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		assert.Equal(t, hostname, r.Host)
-		w.WriteHeader(200)
-	}))
-	assert.NoError(t, err)
-	defer server.Close()
-	assert.Nil(t, ValidateHTTPService(originURL, hostname, client.Transport))
-
-	// this will fail if the client follows the 302
-	redirectServer, redirectClient, err := createMockServerAndClient(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if r.URL.Path == "/followedRedirect" {
-			t.Fatal("shouldn't have followed the 302")
-		}
-		if r.Method == "CONNECT" {
-			assert.Equal(t, "127.0.0.1:443", r.Host)
-		} else {
-			assert.Equal(t, hostname, r.Host)
-		}
-		w.Header().Set("Location", "/followedRedirect")
-		w.WriteHeader(302)
-	}))
-	assert.NoError(t, err)
-	defer redirectServer.Close()
-	assert.Nil(t, ValidateHTTPService(originURL, hostname, redirectClient.Transport))
-
 }
 
 // Happy path 2: originURL is HTTPS, and HTTPS connections work
@@ -235,36 +208,6 @@ func TestValidateHTTPService_HTTPS2HTTPS(t *testing.T) {
 		}
 		panic("Shouldn't reach here")
 	})))
-
-	// Integration-style test with a mock server
-	server, client, err := createSecureMockServerAndClient(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if r.Method == "CONNECT" {
-			assert.Equal(t, "127.0.0.1:443", r.Host)
-		} else {
-			assert.Equal(t, hostname, r.Host)
-		}
-		w.WriteHeader(200)
-	}))
-	assert.NoError(t, err)
-	defer server.Close()
-	assert.Nil(t, ValidateHTTPService(originURL, hostname, client.Transport))
-
-	// this will fail if the client follows the 302
-	redirectServer, redirectClient, err := createSecureMockServerAndClient(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if r.URL.Path == "/followedRedirect" {
-			t.Fatal("shouldn't have followed the 302")
-		}
-		if r.Method == "CONNECT" {
-			assert.Equal(t, "127.0.0.1:443", r.Host)
-		} else {
-			assert.Equal(t, hostname, r.Host)
-		}
-		w.Header().Set("Location", "/followedRedirect")
-		w.WriteHeader(302)
-	}))
-	assert.NoError(t, err)
-	defer redirectServer.Close()
-	assert.Nil(t, ValidateHTTPService(originURL, hostname, redirectClient.Transport))
 }
 
 // Error path 1: originURL is HTTPS, but HTTP connections work
@@ -293,37 +236,6 @@ func TestValidateHTTPService_HTTPS2HTTP(t *testing.T) {
 		}
 		panic("Shouldn't reach here")
 	})))
-
-	// Integration-style test with a mock server
-	server, client, err := createMockServerAndClient(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if r.Method == "CONNECT" {
-			assert.Equal(t, "127.0.0.1:1234", r.Host)
-		} else {
-			assert.Equal(t, hostname, r.Host)
-		}
-		w.WriteHeader(200)
-	}))
-	assert.NoError(t, err)
-	defer server.Close()
-	assert.Error(t, ValidateHTTPService(originURL, hostname, client.Transport))
-
-	// this will fail if the client follows the 302
-	redirectServer, redirectClient, err := createMockServerAndClient(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if r.URL.Path == "/followedRedirect" {
-			t.Fatal("shouldn't have followed the 302")
-		}
-		if r.Method == "CONNECT" {
-			assert.Equal(t, "127.0.0.1:1234", r.Host)
-		} else {
-			assert.Equal(t, hostname, r.Host)
-		}
-		w.Header().Set("Location", "/followedRedirect")
-		w.WriteHeader(302)
-	}))
-	assert.NoError(t, err)
-	defer redirectServer.Close()
-	assert.Error(t, ValidateHTTPService(originURL, hostname, redirectClient.Transport))
-
 }
 
 // Error path 2: originURL is HTTP, but HTTPS connections work
@@ -352,21 +264,11 @@ func TestValidateHTTPService_HTTP2HTTPS(t *testing.T) {
 		}
 		panic("Shouldn't reach here")
 	})))
+}
 
-	// Integration-style test with a mock server
-	server, client, err := createSecureMockServerAndClient(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if r.Method == "CONNECT" {
-			assert.Equal(t, "127.0.0.1:1234", r.Host)
-		} else {
-			assert.Equal(t, hostname, r.Host)
-		}
-		w.WriteHeader(200)
-	}))
-	assert.NoError(t, err)
-	defer server.Close()
-	assert.Error(t, ValidateHTTPService(originURL, hostname, client.Transport))
-
-	// this will fail if the client follows the 302
+// Ensure the client does not follow 302 responses
+func TestValidateHTTPService_NoFollowRedirects(t *testing.T) {
+	hostname := "example.com"
 	redirectServer, redirectClient, err := createSecureMockServerAndClient(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if r.URL.Path == "/followedRedirect" {
 			t.Fatal("shouldn't have followed the 302")
@@ -381,12 +283,12 @@ func TestValidateHTTPService_HTTP2HTTPS(t *testing.T) {
 	}))
 	assert.NoError(t, err)
 	defer redirectServer.Close()
-	assert.Error(t, ValidateHTTPService(originURL, hostname, redirectClient.Transport))
+	assert.NoError(t, ValidateHTTPService(redirectServer.URL, hostname, redirectClient.Transport))
 }
 
-// error path 3: origin URL is nonresponsive
+// Ensure validation times out when origin URL is nonresponsive
 func TestValidateHTTPService_NonResponsiveOrigin(t *testing.T) {
-	originURL := "https://127.0.0.1/"
+	originURL := "http://127.0.0.1/"
 	hostname := "example.com"
 	oldValidationTimeout := validationTimeout
 	defer func() {
@@ -394,7 +296,10 @@ func TestValidateHTTPService_NonResponsiveOrigin(t *testing.T) {
 	}()
 	validationTimeout = 500 * time.Millisecond
 
-	server, client, err := createSecureMockServerAndClient(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	// Use createMockServerAndClient, not createSecureMockServerAndClient.
+	// The latter will bail with HTTP 400 immediately on an http:// request,
+	// which defeats the purpose of a 'nonresponsive origin' test.
+	server, client, err := createMockServerAndClient(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if r.Method == "CONNECT" {
 			assert.Equal(t, "127.0.0.1:443", r.Host)
 		} else {
@@ -409,6 +314,7 @@ func TestValidateHTTPService_NonResponsiveOrigin(t *testing.T) {
 	defer server.Close()
 
 	err = ValidateHTTPService(originURL, hostname, client.Transport)
+	fmt.Println(err)
 	if err, ok := err.(net.Error); assert.True(t, ok) {
 		assert.True(t, err.Timeout())
 	}

vendor/github.com/anmitsu/go-shlex/.gitignore

@@ -0,0 +1 @@
+shlex.test

vendor/github.com/anmitsu/go-shlex/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) anmitsu <anmitsu.s@gmail.com>
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

vendor/github.com/anmitsu/go-shlex/README.md

@@ -0,0 +1,38 @@
+# go-shlex
+
+go-shlex is a library to make a lexical analyzer like Unix shell for
+Go.
+
+## Install
+
+    go get -u "github.com/anmitsu/go-shlex"
+
+## Usage
+
+```go
+package main
+
+import (
+    "fmt"
+    "log"
+
+    "github.com/anmitsu/go-shlex"
+)
+
+func main() {
+    cmd := `cp -Rdp "file name" 'file name2' dir\ name`
+    words, err := shlex.Split(cmd, true)
+    if err != nil {
+        log.Fatal(err)
+    }
+
+    for _, w := range words {
+        fmt.Println(w)
+    }
+}
+```
+
+## Documentation
+
+http://godoc.org/github.com/anmitsu/go-shlex
+

vendor/github.com/anmitsu/go-shlex/shlex.go

@@ -0,0 +1,193 @@
+// Package shlex provides a simple lexical analysis like Unix shell.
+package shlex
+
+import (
+	"bufio"
+	"errors"
+	"io"
+	"strings"
+	"unicode"
+)
+
+var (
+	ErrNoClosing = errors.New("No closing quotation")
+	ErrNoEscaped = errors.New("No escaped character")
+)
+
+// Tokenizer is the interface that classifies a token according to
+// words, whitespaces, quotations, escapes and escaped quotations.
+type Tokenizer interface {
+	IsWord(rune) bool
+	IsWhitespace(rune) bool
+	IsQuote(rune) bool
+	IsEscape(rune) bool
+	IsEscapedQuote(rune) bool
+}
+
+// DefaultTokenizer implements a simple tokenizer like Unix shell.
+type DefaultTokenizer struct{}
+
+func (t *DefaultTokenizer) IsWord(r rune) bool {
+	return r == '_' || unicode.IsLetter(r) || unicode.IsNumber(r)
+}
+func (t *DefaultTokenizer) IsQuote(r rune) bool {
+	switch r {
+	case '\'', '"':
+		return true
+	default:
+		return false
+	}
+}
+func (t *DefaultTokenizer) IsWhitespace(r rune) bool {
+	return unicode.IsSpace(r)
+}
+func (t *DefaultTokenizer) IsEscape(r rune) bool {
+	return r == '\\'
+}
+func (t *DefaultTokenizer) IsEscapedQuote(r rune) bool {
+	return r == '"'
+}
+
+// Lexer represents a lexical analyzer.
+type Lexer struct {
+	reader          *bufio.Reader
+	tokenizer       Tokenizer
+	posix           bool
+	whitespacesplit bool
+}
+
+// NewLexer creates a new Lexer reading from io.Reader.  This Lexer
+// has a DefaultTokenizer according to posix and whitespacesplit
+// rules.
+func NewLexer(r io.Reader, posix, whitespacesplit bool) *Lexer {
+	return &Lexer{
+		reader:          bufio.NewReader(r),
+		tokenizer:       &DefaultTokenizer{},
+		posix:           posix,
+		whitespacesplit: whitespacesplit,
+	}
+}
+
+// NewLexerString creates a new Lexer reading from a string.  This
+// Lexer has a DefaultTokenizer according to posix and whitespacesplit
+// rules.
+func NewLexerString(s string, posix, whitespacesplit bool) *Lexer {
+	return NewLexer(strings.NewReader(s), posix, whitespacesplit)
+}
+
+// Split splits a string according to posix or non-posix rules.
+func Split(s string, posix bool) ([]string, error) {
+	return NewLexerString(s, posix, true).Split()
+}
+
+// SetTokenizer sets a Tokenizer.
+func (l *Lexer) SetTokenizer(t Tokenizer) {
+	l.tokenizer = t
+}
+
+func (l *Lexer) Split() ([]string, error) {
+	result := make([]string, 0)
+	for {
+		token, err := l.readToken()
+		if token != "" {
+			result = append(result, token)
+		}
+
+		if err == io.EOF {
+			break
+		} else if err != nil {
+			return result, err
+		}
+	}
+	return result, nil
+}
+
+func (l *Lexer) readToken() (string, error) {
+	t := l.tokenizer
+	token := ""
+	quoted := false
+	state := ' '
+	escapedstate := ' '
+scanning:
+	for {
+		next, _, err := l.reader.ReadRune()
+		if err != nil {
+			if t.IsQuote(state) {
+				return token, ErrNoClosing
+			} else if t.IsEscape(state) {
+				return token, ErrNoEscaped
+			}
+			return token, err
+		}
+
+		switch {
+		case t.IsWhitespace(state):
+			switch {
+			case t.IsWhitespace(next):
+				break scanning
+			case l.posix && t.IsEscape(next):
+				escapedstate = 'a'
+				state = next
+			case t.IsWord(next):
+				token += string(next)
+				state = 'a'
+			case t.IsQuote(next):
+				if !l.posix {
+					token += string(next)
+				}
+				state = next
+			default:
+				token = string(next)
+				if l.whitespacesplit {
+					state = 'a'
+				} else if token != "" || (l.posix && quoted) {
+					break scanning
+				}
+			}
+		case t.IsQuote(state):
+			quoted = true
+			switch {
+			case next == state:
+				if !l.posix {
+					token += string(next)
+					break scanning
+				} else {
+					state = 'a'
+				}
+			case l.posix && t.IsEscape(next) && t.IsEscapedQuote(state):
+				escapedstate = state
+				state = next
+			default:
+				token += string(next)
+			}
+		case t.IsEscape(state):
+			if t.IsQuote(escapedstate) && next != state && next != escapedstate {
+				token += string(state)
+			}
+			token += string(next)
+			state = escapedstate
+		case t.IsWord(state):
+			switch {
+			case t.IsWhitespace(next):
+				if token != "" || (l.posix && quoted) {
+					break scanning
+				}
+			case l.posix && t.IsQuote(next):
+				state = next
+			case l.posix && t.IsEscape(next):
+				escapedstate = 'a'
+				state = next
+			case t.IsWord(next) || t.IsQuote(next):
+				token += string(next)
+			default:
+				if l.whitespacesplit {
+					token += string(next)
+				} else if token != "" {
+					l.reader.UnreadRune()
+					break scanning
+				}
+			}
+		}
+	}
+	return token, nil
+}

vendor/github.com/aws/aws-sdk-go/LICENSE.txt

@@ -1,4 +1,5 @@
-Apache License
+
+                                 Apache License
                            Version 2.0, January 2004
                         http://www.apache.org/licenses/
 
@@ -178,7 +179,7 @@ Apache License
    APPENDIX: How to apply the Apache License to your work.
 
       To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "{}"
+      boilerplate notice, with the fields enclosed by brackets "[]"
       replaced with your own identifying information. (Don't include
       the brackets!)  The text should be enclosed in the appropriate
       comment syntax for the file format. We also recommend that a
@@ -186,7 +187,7 @@ Apache License
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright {yyyy} {name of copyright owner}
+   Copyright [yyyy] [name of copyright owner]
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
@@ -199,4 +200,3 @@ Apache License
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
-