Release 2019.11.0

This drops the default size auf the h2mux write buffer from 512 MB to 1 MB.
This massively reduces memory usage, since each stream has its own buffer.

.gitignore

@@ -9,5 +9,8 @@ guide/public
 \#*\#
 cscope.*
 cloudflared
+cloudflared.exe
 !cmd/cloudflared/
-.DS_Store
+.DS_Store
+*-session.log
+ssh_server_tests/.env

Gopkg.lock

@@ -1,709 +0,0 @@
-# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
-
-
-[[projects]]
-  digest = "1:9f3b30d9f8e0d7040f729b82dcbc8f0dead820a133b3147ce355fc451f32d761"
-  name = "github.com/BurntSushi/toml"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "3012a1dbe2e4bd1391d42b32f0577cb7bbc7f005"
-  version = "v0.3.1"
-
-[[projects]]
-  branch = "master"
-  digest = "1:1a200e7e73293b75eb8e5c93d023b5472663432da0b663e1532624fcfede9ca8"
-  name = "github.com/anmitsu/go-shlex"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "648efa622239a2f6ff949fed78ee37b48d499ba4"
-
-[[projects]]
-  digest = "1:4a296b8f487dab7ea37c4558d6e84be8103eb6bd297202e870266ff99ebd769d"
-  name = "github.com/aws/aws-sdk-go"
-  packages = [
-    "aws",
-    "aws/awserr",
-    "aws/awsutil",
-    "aws/client",
-    "aws/client/metadata",
-    "aws/corehandlers",
-    "aws/credentials",
-    "aws/credentials/ec2rolecreds",
-    "aws/credentials/endpointcreds",
-    "aws/credentials/processcreds",
-    "aws/credentials/stscreds",
-    "aws/csm",
-    "aws/defaults",
-    "aws/ec2metadata",
-    "aws/endpoints",
-    "aws/request",
-    "aws/session",
-    "aws/signer/v4",
-    "internal/ini",
-    "internal/s3err",
-    "internal/sdkio",
-    "internal/sdkrand",
-    "internal/sdkuri",
-    "internal/shareddefaults",
-    "private/protocol",
-    "private/protocol/eventstream",
-    "private/protocol/eventstream/eventstreamapi",
-    "private/protocol/json/jsonutil",
-    "private/protocol/query",
-    "private/protocol/query/queryutil",
-    "private/protocol/rest",
-    "private/protocol/restxml",
-    "private/protocol/xml/xmlutil",
-    "service/s3",
-    "service/sts",
-    "service/sts/stsiface",
-  ]
-  pruneopts = "UT"
-  revision = "2eb9a651f22769d792f974e8a8efe16afea3370f"
-  version = "v1.23.9"
-
-[[projects]]
-  digest = "1:d6afaeed1502aa28e80a4ed0981d570ad91b2579193404256ce672ed0a609e0d"
-  name = "github.com/beorn7/perks"
-  packages = ["quantile"]
-  pruneopts = "UT"
-  revision = "4b2b341e8d7715fae06375aa633dbb6e91b3fb46"
-  version = "v1.0.0"
-
-[[projects]]
-  digest = "1:fed1f537c2f1269fe475a8556c393fe466641682d73ef8fd0491cd3aa1e47bad"
-  name = "github.com/certifi/gocertifi"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "deb3ae2ef2610fde3330947281941c562861188b"
-  version = "2018.01.18"
-
-[[projects]]
-  digest = "1:e5003c19d396d8b3cf1324ea0bf49b00f13e9466d0297d1268b641f1c617c3a2"
-  name = "github.com/cloudflare/brotli-go"
-  packages = ["."]
-  pruneopts = "T"
-  revision = "18c9f6c67e3dfc12e0ddaca748d2887f97a7ac28"
-
-[[projects]]
-  digest = "1:6dbb2bbc7e6333e691c4d82fd86485f0695a35902fbb9b2df5f72e22ab0040f3"
-  name = "github.com/cloudflare/golibs"
-  packages = ["lrucache"]
-  pruneopts = "UT"
-  revision = "333127dbecfcc23a8db7d9a4f52785d23aff44a1"
-
-[[projects]]
-  digest = "1:3f9506ee991cdee1f05bf0cd3e34b5cd922dc00d6a950fb4beb4e07ab1c4d3d1"
-  name = "github.com/coredns/coredns"
-  packages = [
-    "core/dnsserver",
-    "coremain",
-    "pb",
-    "plugin",
-    "plugin/cache",
-    "plugin/cache/freq",
-    "plugin/etcd/msg",
-    "plugin/metrics",
-    "plugin/metrics/vars",
-    "plugin/pkg/cache",
-    "plugin/pkg/dnstest",
-    "plugin/pkg/dnsutil",
-    "plugin/pkg/doh",
-    "plugin/pkg/edns",
-    "plugin/pkg/fuzz",
-    "plugin/pkg/log",
-    "plugin/pkg/nonwriter",
-    "plugin/pkg/rcode",
-    "plugin/pkg/response",
-    "plugin/pkg/trace",
-    "plugin/pkg/uniq",
-    "plugin/pkg/watch",
-    "plugin/test",
-    "request",
-  ]
-  pruneopts = "UT"
-  revision = "2e322f6e8a54f18c6aef9c25a7c432c291a3d9f7"
-  version = "v1.2.0"
-
-[[projects]]
-  digest = "1:6f70106e7bc1c803e8a0a4519e09c12d154771acfa2559206e97b033bbd1dd38"
-  name = "github.com/coreos/go-oidc"
-  packages = ["jose"]
-  pruneopts = "UT"
-  revision = "a93f71fdfe73d2c0f5413c0565eea0af6523a6df"
-
-[[projects]]
-  digest = "1:1da3a221f0bc090792d3a2a080ff09008427c0e0f0533a4ed6abd8994421da73"
-  name = "github.com/coreos/go-systemd"
-  packages = ["daemon"]
-  pruneopts = "UT"
-  revision = "95778dfbb74eb7e4dbaf43bf7d71809650ef8076"
-  version = "v19"
-
-[[projects]]
-  digest = "1:385dff5bf83bf1c9d40ef2b0f2e697f0d1973fc9e24ae9401cf41698612bbebc"
-  name = "github.com/creack/pty"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "2769f65a3a94eb8f876f44a0459d24ae7ad2e488"
-  version = "v1.1.7"
-
-[[projects]]
-  digest = "1:ffe9824d294da03b391f44e1ae8281281b4afc1bdaa9588c9097785e3af10cec"
-  name = "github.com/davecgh/go-spew"
-  packages = ["spew"]
-  pruneopts = "UT"
-  revision = "8991bc29aa16c548c550c7ff78260e27b9ab7c73"
-  version = "v1.1.1"
-
-[[projects]]
-  branch = "master"
-  digest = "1:c013ffc6e15f9f898078f9d38441c68b228aa7b899659452170250ccb27f5f1e"
-  name = "github.com/elgs/gosqljson"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "027aa4915315a0b2825c0f025cea347829b974fa"
-
-[[projects]]
-  digest = "1:d4268b2a09b1f736633577c4ac93f2a5356c73742fff5344e2451aeec60a7ad0"
-  name = "github.com/equinox-io/equinox"
-  packages = [
-    ".",
-    "internal/go-update",
-    "internal/go-update/internal/binarydist",
-    "internal/go-update/internal/osext",
-    "internal/osext",
-    "proto",
-  ]
-  pruneopts = "UT"
-  revision = "5205c98a6c11dc72747ce12fff6cd620a99fde05"
-  version = "v1.2.0"
-
-[[projects]]
-  digest = "1:433763f10d88dba9b533a7ea2fe9f5ee11e57e00306eb97a1f6090fd978e8fa1"
-  name = "github.com/facebookgo/grace"
-  packages = ["gracenet"]
-  pruneopts = "UT"
-  revision = "75cf19382434e82df4dd84953f566b8ad23d6e9e"
-
-[[projects]]
-  branch = "master"
-  digest = "1:50a46ab1d5edbbdd55125b4d37f1bf503d0807c26461f9ad7b358d6006641d09"
-  name = "github.com/flynn/go-shlex"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "3f9db97f856818214da2e1057f8ad84803971cff"
-
-[[projects]]
-  digest = "1:d4623fc7bf7e281d9107367cc4a9e76ed3e86b1eec1a4e30630c870bef1fedd0"
-  name = "github.com/getsentry/raven-go"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "ed7bcb39ff10f39ab08e317ce16df282845852fa"
-
-[[projects]]
-  digest = "1:0a01355005024757ae2a1f62e8fe68a30b4f5fde7a08ac88b3685e7999ac354b"
-  name = "github.com/gliderlabs/ssh"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "5b6cc7030f17095c0cf23bb063b0bfe824fe5f8b"
-  version = "v0.2.2"
-
-[[projects]]
-  branch = "master"
-  digest = "1:3e6afc3ed8a72949aa735c00fddc23427dc9384ccfd51cf0d91a412e668da632"
-  name = "github.com/golang-collections/collections"
-  packages = ["queue"]
-  pruneopts = "UT"
-  revision = "604e922904d35e97f98a774db7881f049cd8d970"
-
-[[projects]]
-  digest = "1:239c4c7fd2159585454003d9be7207167970194216193a8a210b8d29576f19c9"
-  name = "github.com/golang/protobuf"
-  packages = [
-    "proto",
-    "ptypes",
-    "ptypes/any",
-    "ptypes/duration",
-    "ptypes/timestamp",
-  ]
-  pruneopts = "UT"
-  revision = "b5d812f8a3706043e23a9cd5babf2e5423744d30"
-  version = "v1.3.1"
-
-[[projects]]
-  digest = "1:582b704bebaa06b48c29b0cec224a6058a09c86883aaddabde889cd1a5f73e1b"
-  name = "github.com/google/uuid"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "0cd6bf5da1e1c83f8b45653022c74f71af0538a4"
-  version = "v1.1.1"
-
-[[projects]]
-  digest = "1:d5f97fc268267ec1b61c3453058c738246fc3e746f14b1ae25161513b7367b0c"
-  name = "github.com/gorilla/mux"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "c5c6c98bc25355028a63748a498942a6398ccd22"
-  version = "v1.7.1"
-
-[[projects]]
-  digest = "1:43dd08a10854b2056e615d1b1d22ac94559d822e1f8b6fcc92c1a1057e85188e"
-  name = "github.com/gorilla/websocket"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "ea4d1f681babbce9545c9c5f3d5194a789c89f5b"
-  version = "v1.2.0"
-
-[[projects]]
-  branch = "master"
-  digest = "1:1a1206efd03a54d336dce7bb8719e74f2f8932f661cb9f57d5813a1d99c083d8"
-  name = "github.com/grpc-ecosystem/grpc-opentracing"
-  packages = ["go/otgrpc"]
-  pruneopts = "UT"
-  revision = "8e809c8a86450a29b90dcc9efbf062d0fe6d9746"
-
-[[projects]]
-  digest = "1:bb81097a5b62634f3e9fec1014657855610c82d19b9a40c17612e32651e35dca"
-  name = "github.com/jmespath/go-jmespath"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "c2b33e84"
-
-[[projects]]
-  digest = "1:6c41d4f998a03b6604227ccad36edaed6126c397e5d78709ef4814a1145a6757"
-  name = "github.com/jmoiron/sqlx"
-  packages = [
-    ".",
-    "reflectx",
-  ]
-  pruneopts = "UT"
-  revision = "d161d7a76b5661016ad0b085869f77fd410f3e6a"
-  version = "v1.2.0"
-
-[[projects]]
-  digest = "1:31e761d97c76151dde79e9d28964a812c46efc5baee4085b86f68f0c654450de"
-  name = "github.com/konsorten/go-windows-terminal-sequences"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "f55edac94c9bbba5d6182a4be46d86a2c9b5b50e"
-  version = "v1.0.2"
-
-[[projects]]
-  digest = "1:bc1c0be40c67b6b4aee09d7508d5a2a52c1c116b1fa43806dad2b0d6b4d4003b"
-  name = "github.com/lib/pq"
-  packages = [
-    ".",
-    "oid",
-    "scram",
-  ]
-  pruneopts = "UT"
-  revision = "51e2106eed1cea199c802d2a49e91e2491b02056"
-  version = "v1.1.0"
-
-[[projects]]
-  digest = "1:2fa7b0155cd54479a755c629de26f888a918e13f8857a2c442205d825368e084"
-  name = "github.com/mattn/go-colorable"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "3a70a971f94a22f2fa562ffcc7a0eb45f5daf045"
-  version = "v0.1.1"
-
-[[projects]]
-  digest = "1:e150b5fafbd7607e2d638e4e5cf43aa4100124e5593385147b0a74e2733d8b0d"
-  name = "github.com/mattn/go-isatty"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "c2a7a6ca930a4cd0bc33a3f298eb71960732a3a7"
-  version = "v0.0.7"
-
-[[projects]]
-  digest = "1:ff5ebae34cfbf047d505ee150de27e60570e8c394b3b8fdbb720ff6ac71985fc"
-  name = "github.com/matttproud/golang_protobuf_extensions"
-  packages = ["pbutil"]
-  pruneopts = "UT"
-  revision = "c12348ce28de40eed0136aa2b644d0ee0650e56c"
-  version = "v1.0.1"
-
-[[projects]]
-  digest = "1:75fa16a231ef40da3e462d651c20b9df20bde0777bdc1ac0982242c79057ee71"
-  name = "github.com/mholt/caddy"
-  packages = [
-    ".",
-    "caddyfile",
-    "telemetry",
-  ]
-  pruneopts = "UT"
-  revision = "d3b731e9255b72d4571a5aac125634cf1b6031dc"
-
-[[projects]]
-  digest = "1:2b4b4b2e5544c2a11a486c1b631357aa2ddf766e50c1b2483cf809da2c511234"
-  name = "github.com/miekg/dns"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "73601d4aed9d844322611759d7f3619110b7c88e"
-  version = "v1.1.8"
-
-[[projects]]
-  digest = "1:5d231480e1c64a726869bc4142d270184c419749d34f167646baa21008eb0a79"
-  name = "github.com/mitchellh/go-homedir"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "af06845cf3004701891bf4fdb884bfe4920b3727"
-  version = "v1.1.0"
-
-[[projects]]
-  digest = "1:53bc4cd4914cd7cd52139990d5170d6dc99067ae31c56530621b18b35fc30318"
-  name = "github.com/mitchellh/mapstructure"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "3536a929edddb9a5b34bd6861dc4a9647cb459fe"
-  version = "v1.1.2"
-
-[[projects]]
-  digest = "1:11e62d6050198055e6cd87ed57e5d8c669e84f839c16e16f192374d913d1a70d"
-  name = "github.com/opentracing/opentracing-go"
-  packages = [
-    ".",
-    "ext",
-    "log",
-  ]
-  pruneopts = "UT"
-  revision = "659c90643e714681897ec2521c60567dd21da733"
-  version = "v1.1.0"
-
-[[projects]]
-  digest = "1:40e195917a951a8bf867cd05de2a46aaf1806c50cf92eebf4c16f78cd196f747"
-  name = "github.com/pkg/errors"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "645ef00459ed84a119197bfb8d8205042c6df63d"
-  version = "v0.8.0"
-
-[[projects]]
-  digest = "1:0028cb19b2e4c3112225cd871870f2d9cf49b9b4276531f03438a88e94be86fe"
-  name = "github.com/pmezard/go-difflib"
-  packages = ["difflib"]
-  pruneopts = "UT"
-  revision = "792786c7400a136282c1664665ae0a8db921c6c2"
-  version = "v1.0.0"
-
-[[projects]]
-  digest = "1:c968b29db5d68ec97de404b6d058d5937fa015a141b3b4f7a0d87d5f8226f04c"
-  name = "github.com/prometheus/client_golang"
-  packages = [
-    "prometheus",
-    "prometheus/promhttp",
-  ]
-  pruneopts = "UT"
-  revision = "967789050ba94deca04a5e84cce8ad472ce313c1"
-  version = "v0.9.0-pre1"
-
-[[projects]]
-  branch = "master"
-  digest = "1:2d5cd61daa5565187e1d96bae64dbbc6080dacf741448e9629c64fd93203b0d4"
-  name = "github.com/prometheus/client_model"
-  packages = ["go"]
-  pruneopts = "UT"
-  revision = "fd36f4220a901265f90734c3183c5f0c91daa0b8"
-
-[[projects]]
-  digest = "1:35cf6bdf68db765988baa9c4f10cc5d7dda1126a54bd62e252dbcd0b1fc8da90"
-  name = "github.com/prometheus/common"
-  packages = [
-    "expfmt",
-    "internal/bitbucket.org/ww/goautoneg",
-    "model",
-  ]
-  pruneopts = "UT"
-  revision = "a82f4c12f983cc2649298185f296632953e50d3e"
-  version = "v0.3.0"
-
-[[projects]]
-  branch = "master"
-  digest = "1:49b09905e781d7775c086604cc00083e1832d0783f1f421b79f42657c457d029"
-  name = "github.com/prometheus/procfs"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "8368d24ba045f26503eb745b624d930cbe214c79"
-
-[[projects]]
-  digest = "1:1a23fdd843129ef761ffe7651bc5fe7c5b09fbe933e92783ab06cc11c37b7b37"
-  name = "github.com/rifflock/lfshook"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "b9218ef580f59a2e72dad1aa33d660150445d05a"
-  version = "v2.4"
-
-[[projects]]
-  digest = "1:31538f8d774e8bf2fb9380d2d2a82d69e206a7d0603946586926f1819c546c26"
-  name = "github.com/sirupsen/logrus"
-  packages = [
-    ".",
-    "hooks/test",
-  ]
-  pruneopts = "UT"
-  revision = "839c75faf7f98a33d445d181f3018b5c3409a45e"
-  version = "v1.4.2"
-
-[[projects]]
-  digest = "1:7e8d267900c7fa7f35129a2a37596e38ed0f11ca746d6d9ba727980ee138f9f6"
-  name = "github.com/stretchr/testify"
-  packages = [
-    "assert",
-    "require",
-  ]
-  pruneopts = "UT"
-  revision = "12b6f73e6084dad08a7c6e575284b177ecafbc71"
-  version = "v1.2.1"
-
-[[projects]]
-  branch = "master"
-  digest = "1:a84d5ec8b40a827962ea250f2cf03434138ccae9d83fcac12fb49b70c70b80cc"
-  name = "golang.org/x/crypto"
-  packages = [
-    "curve25519",
-    "ed25519",
-    "ed25519/internal/edwards25519",
-    "internal/chacha20",
-    "internal/subtle",
-    "nacl/box",
-    "nacl/secretbox",
-    "poly1305",
-    "salsa20/salsa",
-    "ssh",
-    "ssh/terminal",
-  ]
-  pruneopts = "UT"
-  revision = "f416ebab96af27ca70b6e5c23d6a0747530da626"
-
-[[projects]]
-  branch = "master"
-  digest = "1:52d140f7ab52e491cc1cbc93e6637aa5e9a7f3beae7545d675b02e52ca9d7290"
-  name = "golang.org/x/net"
-  packages = [
-    "bpf",
-    "context",
-    "http/httpguts",
-    "http2",
-    "http2/hpack",
-    "idna",
-    "internal/iana",
-    "internal/socket",
-    "internal/timeseries",
-    "ipv4",
-    "ipv6",
-    "trace",
-    "websocket",
-  ]
-  pruneopts = "UT"
-  revision = "1da14a5a36f220ea3f03470682b737b1dfd5de22"
-
-[[projects]]
-  digest = "1:39ebcc2b11457b703ae9ee2e8cca0f68df21969c6102cb3b705f76cca0ea0239"
-  name = "golang.org/x/sync"
-  packages = ["errgroup"]
-  pruneopts = "UT"
-  revision = "1d60e4601c6fd243af51cc01ddf169918a5407ca"
-
-[[projects]]
-  branch = "master"
-  digest = "1:77751d02e939d7078faedaeec10c09af575a09c528d84d18f2cb45a84bd1889a"
-  name = "golang.org/x/sys"
-  packages = [
-    "cpu",
-    "unix",
-    "windows",
-    "windows/registry",
-    "windows/svc",
-    "windows/svc/eventlog",
-    "windows/svc/mgr",
-  ]
-  pruneopts = "UT"
-  revision = "12500544f89f9420afe9529ba8940bf72d294972"
-
-[[projects]]
-  digest = "1:a2ab62866c75542dd18d2b069fec854577a20211d7c0ea6ae746072a1dccdd18"
-  name = "golang.org/x/text"
-  packages = [
-    "collate",
-    "collate/build",
-    "internal/colltab",
-    "internal/gen",
-    "internal/tag",
-    "internal/triegen",
-    "internal/ucd",
-    "language",
-    "secure/bidirule",
-    "transform",
-    "unicode/bidi",
-    "unicode/cldr",
-    "unicode/norm",
-    "unicode/rangetable",
-  ]
-  pruneopts = "UT"
-  revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0"
-  version = "v0.3.0"
-
-[[projects]]
-  branch = "master"
-  digest = "1:c3076e7defee87de1236f1814beb588f40a75544c60121e6eb38b3b3721783e2"
-  name = "google.golang.org/genproto"
-  packages = ["googleapis/rpc/status"]
-  pruneopts = "UT"
-  revision = "d1146b9035b912113a38af3b138eb2af567b2c67"
-
-[[projects]]
-  digest = "1:31d87f39886fb38a2b6c097ff3b9f985d6960772170d64a68246f7790e955746"
-  name = "google.golang.org/grpc"
-  packages = [
-    ".",
-    "balancer",
-    "balancer/base",
-    "balancer/roundrobin",
-    "binarylog/grpc_binarylog_v1",
-    "codes",
-    "connectivity",
-    "credentials",
-    "credentials/internal",
-    "encoding",
-    "encoding/proto",
-    "grpclog",
-    "internal",
-    "internal/backoff",
-    "internal/balancerload",
-    "internal/binarylog",
-    "internal/channelz",
-    "internal/envconfig",
-    "internal/grpcrand",
-    "internal/grpcsync",
-    "internal/syscall",
-    "internal/transport",
-    "keepalive",
-    "metadata",
-    "naming",
-    "peer",
-    "resolver",
-    "resolver/dns",
-    "resolver/passthrough",
-    "stats",
-    "status",
-    "tap",
-  ]
-  pruneopts = "UT"
-  revision = "236199dd5f8031d698fb64091194aecd1c3895b2"
-  version = "v1.20.0"
-
-[[projects]]
-  branch = "altsrc-parse-durations"
-  digest = "1:0370b1bceda03dbfade3abbde639a43f1113bab711ec760452e5c0dcc0c14787"
-  name = "gopkg.in/urfave/cli.v2"
-  packages = [
-    ".",
-    "altsrc",
-  ]
-  pruneopts = "UT"
-  revision = "d604b6ffeee878fbf084fd2761466b6649989cee"
-  source = "https://github.com/cbranch/cli"
-
-[[projects]]
-  digest = "1:4d2e5a73dc1500038e504a8d78b986630e3626dc027bc030ba5c75da257cdb96"
-  name = "gopkg.in/yaml.v2"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "51d6538a90f86fe93ac480b35f37b2be17fef232"
-  version = "v2.2.2"
-
-[[projects]]
-  digest = "1:8ffc3ddc31414c0a71220957bb723b16510d7fcb5b3880dc0da4cf6d39c31642"
-  name = "zombiezen.com/go/capnproto2"
-  packages = [
-    ".",
-    "encoding/text",
-    "internal/fulfiller",
-    "internal/nodemap",
-    "internal/packed",
-    "internal/queue",
-    "internal/schema",
-    "internal/strquote",
-    "pogs",
-    "rpc",
-    "rpc/internal/refcount",
-    "schemas",
-    "server",
-    "std/capnp/rpc",
-  ]
-  pruneopts = "UT"
-  revision = "7cfd211c19c7f5783c695f3654efa46f0df259c3"
-  source = "https://github.com/zombiezen/go-capnproto2"
-  version = "v2.17.1"
-
-[solve-meta]
-  analyzer-name = "dep"
-  analyzer-version = 1
-  input-imports = [
-    "github.com/DATA-DOG/go-sqlmock",
-    "github.com/aws/aws-sdk-go/aws",
-    "github.com/aws/aws-sdk-go/aws/credentials",
-    "github.com/aws/aws-sdk-go/aws/session",
-    "github.com/aws/aws-sdk-go/service/s3",
-    "github.com/cloudflare/brotli-go",
-    "github.com/cloudflare/golibs/lrucache",
-    "github.com/coredns/coredns/core/dnsserver",
-    "github.com/coredns/coredns/plugin",
-    "github.com/coredns/coredns/plugin/cache",
-    "github.com/coredns/coredns/plugin/metrics/vars",
-    "github.com/coredns/coredns/plugin/pkg/dnstest",
-    "github.com/coredns/coredns/plugin/pkg/rcode",
-    "github.com/coredns/coredns/request",
-    "github.com/coreos/go-oidc/jose",
-    "github.com/coreos/go-systemd/daemon",
-    "github.com/creack/pty",
-    "github.com/elgs/gosqljson",
-    "github.com/equinox-io/equinox",
-    "github.com/facebookgo/grace/gracenet",
-    "github.com/getsentry/raven-go",
-    "github.com/gliderlabs/ssh",
-    "github.com/golang-collections/collections/queue",
-    "github.com/google/uuid",
-    "github.com/gorilla/mux",
-    "github.com/gorilla/websocket",
-    "github.com/lib/pq",
-    "github.com/mattn/go-colorable",
-    "github.com/miekg/dns",
-    "github.com/mitchellh/go-homedir",
-    "github.com/mitchellh/mapstructure",
-    "github.com/pkg/errors",
-    "github.com/prometheus/client_golang/prometheus",
-    "github.com/prometheus/client_golang/prometheus/promhttp",
-    "github.com/rifflock/lfshook",
-    "github.com/sirupsen/logrus",
-    "github.com/sirupsen/logrus/hooks/test",
-    "github.com/stretchr/testify/assert",
-    "github.com/stretchr/testify/require",
-    "golang.org/x/crypto/nacl/box",
-    "golang.org/x/crypto/ssh",
-    "golang.org/x/crypto/ssh/terminal",
-    "golang.org/x/net/context",
-    "golang.org/x/net/http2",
-    "golang.org/x/net/http2/hpack",
-    "golang.org/x/net/idna",
-    "golang.org/x/net/trace",
-    "golang.org/x/net/websocket",
-    "golang.org/x/sync/errgroup",
-    "golang.org/x/sys/windows",
-    "golang.org/x/sys/windows/svc",
-    "golang.org/x/sys/windows/svc/eventlog",
-    "golang.org/x/sys/windows/svc/mgr",
-    "gopkg.in/urfave/cli.v2",
-    "gopkg.in/urfave/cli.v2/altsrc",
-    "zombiezen.com/go/capnproto2",
-    "zombiezen.com/go/capnproto2/encoding/text",
-    "zombiezen.com/go/capnproto2/pogs",
-    "zombiezen.com/go/capnproto2/rpc",
-    "zombiezen.com/go/capnproto2/schemas",
-    "zombiezen.com/go/capnproto2/server",
-    "zombiezen.com/go/capnproto2/std/capnp/rpc",
-  ]
-  solver-name = "gps-cdcl"
-  solver-version = 1

Gopkg.toml

@@ -1,103 +0,0 @@
-[prune]
-  go-tests = true
-  unused-packages = true
-
-  [[prune.project]]
-    name = "github.com/cloudflare/brotli-go"
-    unused-packages = false
-
-
-
-[[constraint]]
-  name = "github.com/facebookgo/grace"
-  revision = "75cf19382434e82df4dd84953f566b8ad23d6e9e"
-
-[[constraint]]
-  name = "github.com/getsentry/raven-go"
-  revision = "ed7bcb39ff10f39ab08e317ce16df282845852fa"
-
-[[constraint]]
-  name = "github.com/pkg/errors"
-  version = "=0.8.0"
-
-[[constraint]]
-  name = "github.com/prometheus/client_golang"
-  version = "=0.9.0-pre1"
-
-[[constraint]]
-  name = "github.com/sirupsen/logrus"
-  version = "=1.4.2"
-
-[[constraint]]
-  name = "github.com/stretchr/testify"
-  version = "=1.2.1"
-
-[[constraint]]
-  name = "golang.org/x/net"
-  branch = "master"  # master required by github.com/miekg/dns
-
-[[constraint]]
-  name = "golang.org/x/sync"
-  revision = "1d60e4601c6fd243af51cc01ddf169918a5407ca"
-
-[[constraint]]
-  name = "gopkg.in/urfave/cli.v2"
-  source = "https://github.com/cbranch/cli"
-  branch = "altsrc-parse-durations"
-
-[[constraint]]
-  name = "zombiezen.com/go/capnproto2"
-  source = "https://github.com/zombiezen/go-capnproto2"
-  version = "=2.17.1"
-
-[[constraint]]
-  name = "github.com/gorilla/websocket"
-  version = "=1.2.0"
-
-[[constraint]]
-  name = "github.com/coredns/coredns"
-  version = "=1.2.0"
-
-[[constraint]]
-  name = "github.com/miekg/dns"
-  version = "=1.1.8"
-
-[[constraint]]
-  name = "github.com/cloudflare/brotli-go"
-  revision = "18c9f6c67e3dfc12e0ddaca748d2887f97a7ac28"
-
-[[override]]
-  name = "github.com/mholt/caddy"
-  revision = "d3b731e9255b72d4571a5aac125634cf1b6031dc"
-
-[[constraint]]
-  name = "github.com/coreos/go-oidc"
-  revision = "a93f71fdfe73d2c0f5413c0565eea0af6523a6df"
-
-[[constraint]]
-  name = "golang.org/x/crypto"
-  branch = "master"  # master required by github.com/miekg/dns
-
-[[constraint]]
-  name = "github.com/cloudflare/golibs"
-  revision = "333127dbecfcc23a8db7d9a4f52785d23aff44a1"
-
-[[constraint]]
-  name = "github.com/google/uuid"
-  version = "=1.1.1"
-
-[[constraint]]
-  name = "github.com/mitchellh/mapstructure"
-  version = "1.1.2"
-
-[[constraint]]
-  name = "github.com/gliderlabs/ssh"
-  version = "0.2.2"
-
-[[constraint]]
-  name = "github.com/creack/pty"
-  version = "1.1.7"
-
-[[constraint]]
-  name = "github.com/aws/aws-sdk-go"
-  version = "1.23.9"

Makefile

@@ -29,7 +29,7 @@ clean:
 
 .PHONY: cloudflared
 cloudflared: tunnel-deps
-	go build -v $(VERSION_FLAGS) $(IMPORT_PATH)/cmd/cloudflared
+	go build -v -mod=vendor $(VERSION_FLAGS) $(IMPORT_PATH)/cmd/cloudflared
 
 .PHONY: container
 container:
@@ -37,7 +37,11 @@ container:
 
 .PHONY: test
 test: vet
-	go test -v -race $(VERSION_FLAGS) ./...
+	go test -v -mod=vendor -race $(VERSION_FLAGS) ./...
+
+.PHONY: test-ssh-server
+test-ssh-server:
+	docker-compose -f ssh_server_tests/docker-compose.yml up
 
 .PHONY: cloudflared-deb
 cloudflared-deb: cloudflared
@@ -78,6 +82,6 @@ tunnelrpc/tunnelrpc.capnp.go: tunnelrpc/tunnelrpc.capnp
 
 .PHONY: vet
 vet:
-	go vet ./...
+	go vet -mod=vendor ./...
 	which go-sumtype  # go get github.com/BurntSushi/go-sumtype
-	go-sumtype $$(go list ./...)
+	go-sumtype $$(go list -mod=vendor ./...)

RELEASE_NOTES

@@ -1,3 +1,66 @@
+2019.11.0
+- 2019-11-04 TUN-2502: Switch to go modules
+- 2019-11-04 TUN-2500: Don't send client registration errors to Sentry
+- 2019-11-04 TUN-2489: Delete stream from activestreammap when read and write are both closed
+- 2019-11-05 TUN-2505: Terminate stream on receipt of RST_STREAM; MuxedStream.CloseWrite() should terminate the MuxedStream.Write() loop
+- 2019-10-30 TUN-2451: Log inavlid path
+- 2019-10-22 TUN-2425: Enable cloudflared to serve multiple Hello World servers by having each of them create its own ServeMux
+- 2019-10-22 AUTH-2173: Prepends access login url with scheme if one doesnt exist
+- 2019-10-23 TUN-2460: Configure according to the ClientConfig recevied from a successful Connect
+- 2019-10-23 AUTH-2177: Reads and writes error streams
+
+2019.10.4
+- 2019-10-21 TUN-2450: Remove Brew publishing formula
+
+2019.10.3
+- 2019-10-18 Fix #129: Excessive memory usage streaming large files (#142)
+
+2019.10.2
+- 2019-10-17 AUTH-2167: Adds CLI option for host key directory
+
+2019.10.1
+- 2019-10-17 Adds variable to fix windows build
+
+2019.10.0
+- 2019-10-11 AUTH-2105: Dont require --destination arg
+- 2019-10-14 TUN-2344: log more details: http2.Framer.ErrorDetail() if available, connectionID
+- 2019-10-16 AUTH-2159: Moves shutdownC close into error handling AUTH-2161: Lowers size of preamble length AUTH-2160: Fixes url parsing logic
+- 2019-10-16 AUTH-2135: Adds support for IPv6 and tests
+- 2019-10-02 AUTH-2105: Adds support for local forwarding. Refactor auditlogger creation. AUTH-2088: Adds dynamic destination routing
+- 2019-10-09 AUTH-2114: Uses short lived cert auth for outgoing client connection
+- 2019-09-30 AUTH-2089: Revise ssh server to function as a proxy
+
+2019.9.2
+- 2019-09-26 TUN-2355: Roll back TUN-2276
+
+2019.9.1
+- 2019-09-23 TUN-2334: remove tlsConfig.ServerName special case
+- 2019-09-23 AUTH-2077: Quotes open browser command in windows
+- 2019-09-11 AUTH-2050: Adds time.sleep to temporarily avoid hitting tunnel muxer dealock issue
+- 2019-09-10 AUTH-2056: Writes stderr to its own stream for non-pty connections
+- 2019-09-16 TUN-2307: Capnp is the only serialization format used in tunnelpogs
+- 2019-09-18 TUN-2315: Replace Scope with IntentLabel
+- 2019-09-17 TUN-2309: Split ConnectResult into ConnectError and ConnectSuccess, each implementing its own capnp serialization logic
+- 2019-09-18 AUTH-2052: Adds tests for SSH server
+- 2019-09-18 AUTH-2067: Log commands correctly
+- 2019-09-19 AUTH-2055: Verifies token at edge on access login
+- 2019-09-04 TUN-2201: change SRV records used by cloudflared
+- 2019-09-06 TUN-2280: Revert "TUN-2260: add name/group to CapnpConnectParameters, remove Scope"
+- 2019-09-03 AUTH-1943 hooked up uploader to logger, added timestamp to session logs, add tests
+- 2019-09-04 AUTH-2036: Refactor user retrieval, shutdown after ssh server stops, add custom version string
+- 2019-09-06 AUTH-1942 added event log to ssh server
+- 2019-09-04 AUTH-2037: Adds support for ssh port forwarding
+- 2019-09-05 TUN-2276: Path encoding broken
+
+2019.9.0
+- 2019-09-05 TUN-2279: Revert path encoding fix
+- 2019-08-30 AUTH-2021 - check error for failing tests
+- 2019-08-29 AUTH-2030: Support both authorized_key and short lived cert authentication simultaniously without specifiying at start time
+- 2019-08-29 AUTH-2026: Adds support for non-pty sessions and inline command exec
+- 2019-08-26 AUTH-1943: Adds session logging
+- 2019-08-26 TUN-2162: Decomplect OpenStream to allow finer-grained timeouts
+- 2019-08-29 TUN-2260: add name/group to CapnpConnectParameters, remove Scope
+
 2019.8.4
 - 2019-08-30 Fix #111: Add support for specifying a specific HTTP Host: header on the origin. (#114)
 - 2019-08-22 TUN-2165: Add ClientConfig to tunnelrpc.ConnectResult

awsuploader/directory_upload_manager.go

@@ -69,7 +69,7 @@ func (m *DirectoryUploadManager) run() {
 // sweep the directory and kick off uploads
 func (m *DirectoryUploadManager) sweep() {
 	filepath.Walk(m.rootDirectory, func(path string, info os.FileInfo, err error) error {
-		if info.IsDir() {
+		if err != nil || info.IsDir() {
 			return nil
 		}
 		//30 days ago
@@ -77,7 +77,7 @@ func (m *DirectoryUploadManager) sweep() {
 		checkTime := time.Now().Add(-time.Duration(retentionTime))
 
 		//delete the file it is stale
-		if info.ModTime().After(checkTime) {
+		if info.ModTime().Before(checkTime) {
 			os.Remove(path)
 			return nil
 		}

carrier/carrier.go

@@ -114,7 +114,7 @@ func createWebsocketStream(options *StartOptions) (*cloudflaredWebsocket.Conn, e
 
 	wsConn, resp, err := cloudflaredWebsocket.ClientConnect(req, nil)
 	defer closeRespBody(resp)
-	if err != nil && isAccessResponse(resp) {
+	if err != nil && IsAccessResponse(resp) {
 		wsConn, err = createAccessAuthenticatedStream(options)
 		if err != nil {
 			return nil, err
@@ -126,10 +126,10 @@ func createWebsocketStream(options *StartOptions) (*cloudflaredWebsocket.Conn, e
 	return &cloudflaredWebsocket.Conn{Conn: wsConn}, nil
 }
 
-// isAccessResponse checks the http Response to see if the url location
+// IsAccessResponse checks the http Response to see if the url location
 // contains the Access structure.
-func isAccessResponse(resp *http.Response) bool {
-	if resp == nil || resp.StatusCode <= 300 {
+func IsAccessResponse(resp *http.Response) bool {
+	if resp == nil || resp.StatusCode != http.StatusFound {
 		return false
 	}
 
@@ -156,7 +156,7 @@ func createAccessAuthenticatedStream(options *StartOptions) (*websocket.Conn, er
 		return wsConn, nil
 	}
 
-	if !isAccessResponse(resp) {
+	if !IsAccessResponse(resp) {
 		return nil, err
 	}
 
@@ -179,7 +179,7 @@ func createAccessAuthenticatedStream(options *StartOptions) (*websocket.Conn, er
 
 // createAccessWebSocketStream builds an Access request and makes a connection
 func createAccessWebSocketStream(options *StartOptions) (*websocket.Conn, *http.Response, error) {
-	req, err := buildAccessRequest(options)
+	req, err := BuildAccessRequest(options)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -187,7 +187,7 @@ func createAccessWebSocketStream(options *StartOptions) (*websocket.Conn, *http.
 }
 
 // buildAccessRequest builds an HTTP request with the Access token set
-func buildAccessRequest(options *StartOptions) (*http.Request, error) {
+func BuildAccessRequest(options *StartOptions) (*http.Request, error) {
 	req, err := http.NewRequest(http.MethodGet, options.OriginURL, nil)
 	if err != nil {
 		return nil, err

carrier/carrier_test.go

@@ -102,14 +102,14 @@ func TestIsAccessResponse(t *testing.T) {
 		ExpectedOut bool
 	}{
 		{"nil response", nil, false},
-		{"redirect with no location", &http.Response{StatusCode: http.StatusPermanentRedirect}, false},
+		{"redirect with no location", &http.Response{StatusCode: http.StatusFound}, false},
 		{"200 ok", &http.Response{StatusCode: http.StatusOK}, false},
-		{"redirect with location", &http.Response{StatusCode: http.StatusPermanentRedirect, Header: validLocationHeader}, true},
-		{"redirect with invalid location", &http.Response{StatusCode: http.StatusPermanentRedirect, Header: invalidLocationHeader}, false},
+		{"redirect with location", &http.Response{StatusCode: http.StatusFound, Header: validLocationHeader}, true},
+		{"redirect with invalid location", &http.Response{StatusCode: http.StatusFound, Header: invalidLocationHeader}, false},
 	}
 
 	for i, tc := range testCases {
-		if isAccessResponse(tc.In) != tc.ExpectedOut {
+		if IsAccessResponse(tc.In) != tc.ExpectedOut {
 			t.Fatalf("Failed case %d -- %s", i, tc.Description)
 		}
 	}

cfsetup.yaml

@@ -1,5 +1,5 @@
 pinned_go: &pinned_go go=1.12.7-1
-build_dir: &build_dir /cfsetup_build/src/github.com/cloudflare/cloudflared/
+build_dir: &build_dir /cfsetup_build
 default-flavor: stretch
 stretch: &stretch
   build:
@@ -8,7 +8,6 @@ stretch: &stretch
       - *pinned_go
       - build-essential
     post-cache:
-      - export GOPATH=/cfsetup_build/
       - export GOOS=linux
       - export GOARCH=amd64
       - make cloudflared
@@ -20,7 +19,6 @@ stretch: &stretch
       - fakeroot
       - rubygem-fpm
     post-cache:
-      - export GOPATH=/cfsetup_build/
       - export GOOS=linux
       - export GOARCH=amd64
       - make cloudflared-deb
@@ -30,7 +28,6 @@ stretch: &stretch
       - *pinned_go
       - build-essential
     post-cache:
-      - export GOPATH=/cfsetup_build/
       - export GOOS=linux
       - export GOARCH=amd64
       - make release
@@ -41,7 +38,6 @@ stretch: &stretch
       - crossbuild-essential-armhf
       - gcc-arm-linux-gnueabihf
     post-cache:
-      - export GOPATH=/cfsetup_build/
       - export GOOS=linux
       - export GOARCH=arm
       - export CC=arm-linux-gnueabihf-gcc
@@ -52,7 +48,6 @@ stretch: &stretch
       - *pinned_go
       - gcc-multilib
     post-cache:
-      - export GOPATH=/cfsetup_build/
       - export GOOS=linux
       - export GOARCH=386
       - make release
@@ -62,7 +57,6 @@ stretch: &stretch
       - *pinned_go
       - gcc-mingw-w64
     post-cache:
-      - export GOPATH=/cfsetup_build/
       - export GOOS=windows
       - export GOARCH=amd64
       - export CC=x86_64-w64-mingw32-gcc
@@ -73,7 +67,6 @@ stretch: &stretch
       - *pinned_go
       - gcc-mingw-w64
     post-cache:
-      - export GOPATH=/cfsetup_build/
       - export GOOS=windows
       - export GOARCH=386
       - export CC=i686-w64-mingw32-gcc-win32
@@ -84,12 +77,11 @@ stretch: &stretch
       - *pinned_go
       - build-essential
     post-cache:
-      - export GOPATH=/cfsetup_build/
       - export GOOS=linux
       - export GOARCH=amd64
-      - sudo chown -R $(whoami) /cfsetup_build/
-      - go get github.com/BurntSushi/go-sumtype
-      - export PATH="$GOPATH/bin:$PATH"
+      # cd to a non-module directory: https://github.com/golang/go/issues/24250
+      - (cd / && go get github.com/BurntSushi/go-sumtype)
+      - export PATH="$HOME/go/bin:$PATH"
       - make test
 
 jessie: *stretch

cmd/cloudflared/access/carrier.go

@@ -23,7 +23,7 @@ func ssh(c *cli.Context) error {
 	if err != nil || rawHostName == "" {
 		return cli.ShowCommandHelp(c, "ssh")
 	}
-	originURL := "https://" + hostname
+	originURL := ensureURLScheme(hostname)
 
 	// get the headers from the cmdline and add them
 	headers := buildRequestHeaders(c.StringSlice(sshHeaderFlag))
@@ -34,6 +34,11 @@ func ssh(c *cli.Context) error {
 		headers.Add("CF-Access-Client-Secret", c.String(sshTokenSecretFlag))
 	}
 
+	destination := c.String(sshDestinationFlag)
+	if destination != "" {
+		headers.Add("CF-Access-SSH-Destination", destination)
+	}
+
 	options := &carrier.StartOptions{
 		OriginURL: originURL,
 		Headers:   headers,

cmd/cloudflared/access/cmd.go

@@ -1,17 +1,20 @@
 package access
 
 import (
-	"errors"
 	"fmt"
+	"net/http"
 	"net/url"
 	"os"
 	"strings"
 	"text/template"
+	"time"
 
+	"github.com/cloudflare/cloudflared/carrier"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/shell"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/token"
 	"github.com/cloudflare/cloudflared/sshgen"
 	"github.com/cloudflare/cloudflared/validation"
+	"github.com/pkg/errors"
 	"golang.org/x/net/idna"
 
 	"github.com/cloudflare/cloudflared/log"
@@ -21,6 +24,7 @@ import (
 
 const (
 	sshHostnameFlag    = "hostname"
+	sshDestinationFlag = "destination"
 	sshURLFlag         = "url"
 	sshHeaderFlag      = "header"
 	sshTokenIDFlag     = "service-token-id"
@@ -124,6 +128,10 @@ func Commands() []*cli.Command {
 							Name:  sshHostnameFlag,
 							Usage: "specify the hostname of your application.",
 						},
+						&cli.StringFlag{
+							Name:  sshDestinationFlag,
+							Usage: "specify the destination address of your SSH server.",
+						},
 						&cli.StringFlag{
 							Name:  sshURLFlag,
 							Usage: "specify the host:port to forward data to Cloudflare edge.",
@@ -183,14 +191,20 @@ func login(c *cli.Context) error {
 	raven.SetDSN(sentryDSN)
 	logger := log.CreateLogger()
 	args := c.Args()
-	appURL, err := url.Parse(args.First())
+	rawURL := ensureURLScheme(args.First())
+	appURL, err := url.Parse(rawURL)
 	if args.Len() < 1 || err != nil {
 		logger.Errorf("Please provide the url of the Access application\n")
 		return err
 	}
-	token, err := token.FetchToken(appURL)
-	if err != nil {
-		logger.Errorf("Failed to fetch token: %s\n", err)
+	if err := verifyTokenAtEdge(appURL, c); err != nil {
+		logger.WithError(err).Error("Could not verify token")
+		return err
+	}
+
+	token, err := token.GetTokenIfExists(appURL)
+	if err != nil || token == "" {
+		fmt.Fprintln(os.Stderr, "Unable to find token for provided application.")
 		return err
 	}
 	fmt.Fprintf(os.Stdout, "Successfully fetched your token:\n\n%s\n\n", string(token))
@@ -198,6 +212,16 @@ func login(c *cli.Context) error {
 	return nil
 }
 
+// ensureURLScheme prepends a URL with https:// if it doesnt have a scheme. http:// URLs will not be converted.
+func ensureURLScheme(url string) string {
+	url = strings.Replace(strings.ToLower(url), "http://", "https://", 1)
+	if !strings.HasPrefix(url, "https://") {
+		url = fmt.Sprintf("https://%s", url)
+
+	}
+	return url
+}
+
 // curl provides a wrapper around curl, passing Access JWT along in request
 func curl(c *cli.Context) error {
 	raven.SetDSN(sentryDSN)
@@ -281,7 +305,7 @@ func sshGen(c *cli.Context) error {
 		return cli.ShowCommandHelp(c, "ssh-gen")
 	}
 
-	originURL, err := url.Parse("https://" + hostname)
+	originURL, err := url.Parse(ensureURLScheme(hostname))
 	if err != nil {
 		return err
 	}
@@ -372,3 +396,59 @@ func isFileThere(candidate string) bool {
 	}
 	return true
 }
+
+// verifyTokenAtEdge checks for a token on disk, or generates a new one.
+// Then makes a request to to the origin with the token to ensure it is valid.
+// Returns nil if token is valid.
+func verifyTokenAtEdge(appUrl *url.URL, c *cli.Context) error {
+	headers := buildRequestHeaders(c.StringSlice(sshHeaderFlag))
+	if c.IsSet(sshTokenIDFlag) {
+		headers.Add("CF-Access-Client-Id", c.String(sshTokenIDFlag))
+	}
+	if c.IsSet(sshTokenSecretFlag) {
+		headers.Add("CF-Access-Client-Secret", c.String(sshTokenSecretFlag))
+	}
+	options := &carrier.StartOptions{OriginURL: appUrl.String(), Headers: headers}
+
+	if valid, err := isTokenValid(options); err != nil {
+		return err
+	} else if valid {
+		return nil
+	}
+
+	if err := token.RemoveTokenIfExists(appUrl); err != nil {
+		return err
+	}
+
+	if valid, err := isTokenValid(options); err != nil {
+		return err
+	} else if !valid {
+		return errors.New("failed to verify token")
+	}
+
+	return nil
+}
+
+// isTokenValid makes a request to the origin and returns true if the response was not a 302.
+func isTokenValid(options *carrier.StartOptions) (bool, error) {
+	req, err := carrier.BuildAccessRequest(options)
+	if err != nil {
+		return false, errors.Wrap(err, "Could not create access request")
+	}
+
+	// Do not follow redirects
+	client := &http.Client{
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			return http.ErrUseLastResponse
+		},
+		Timeout: time.Second * 5,
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		return false, err
+	}
+	defer resp.Body.Close()
+
+	// A redirect to login means the token was invalid.
+	return !carrier.IsAccessResponse(resp), nil
+}

cmd/cloudflared/access/cmd_test.go

@@ -0,0 +1,25 @@
+package access
+
+import "testing"
+
+func Test_ensureURLScheme(t *testing.T) {
+	type args struct {
+		url string
+	}
+	tests := []struct {
+		name string
+		args args
+		want string
+	}{
+		{"no scheme", args{"localhost:123"}, "https://localhost:123"},
+		{"http scheme", args{"http://test"}, "https://test"},
+		{"https scheme", args{"https://test"}, "https://test"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := ensureURLScheme(tt.args.url); got != tt.want {
+				t.Errorf("ensureURLScheme() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

cmd/cloudflared/shell/launch_browser_darwin.go

@@ -0,0 +1,11 @@
+//+build darwin
+
+package shell
+
+import (
+	"os/exec"
+)
+
+func getBrowserCmd(url string) *exec.Cmd {
+	return exec.Command("open", url)
+}

cmd/cloudflared/shell/launch_browser_other.go

@@ -0,0 +1,11 @@
+//+build !windows,!darwin,!linux,!netbsd,!freebsd,!openbsd
+
+package shell
+
+import (
+	"os/exec"
+)
+
+func getBrowserCmd(url string) *exec.Cmd {
+	return nil
+}

cmd/cloudflared/shell/launch_browser_unix.go

@@ -0,0 +1,11 @@
+//+build linux freebsd openbsd netbsd
+
+package shell
+
+import (
+	"os/exec"
+)
+
+func getBrowserCmd(url string) *exec.Cmd {
+	return exec.Command("xdg-open", url)
+}

cmd/cloudflared/shell/launch_browser_windows.go

@@ -0,0 +1,18 @@
+//+build windows
+
+package shell
+
+import (
+	"fmt"
+	"os/exec"
+	"syscall"
+)
+
+func getBrowserCmd(url string) *exec.Cmd {
+	cmd := exec.Command("cmd")
+	// CmdLine is only defined when compiling for windows.
+	// Empty string is the cmd proc "Title". Needs to be included because the start command will interpret the first
+	// quoted string as that field and we want to quote the URL.
+	cmd.SysProcAttr = &syscall.SysProcAttr{CmdLine: fmt.Sprintf(`/c start "" "%s"`, url)}
+	return cmd
+}

cmd/cloudflared/shell/shell.go

@@ -4,25 +4,11 @@ import (
 	"io"
 	"os"
 	"os/exec"
-	"runtime"
 )
 
 // OpenBrowser opens the specified URL in the default browser of the user
 func OpenBrowser(url string) error {
-	var cmd string
-	var args []string
-
-	switch runtime.GOOS {
-	case "windows":
-		cmd = "cmd"
-		args = []string{"/c", "start"}
-	case "darwin":
-		cmd = "open"
-	default: // "linux", "freebsd", "openbsd", "netbsd"
-		cmd = "xdg-open"
-	}
-	args = append(args, url)
-	return exec.Command(cmd, args...).Start()
+	return getBrowserCmd(url).Start()
 }
 
 // Run will kick off a shell task and pipe the results to the respective std pipes

cmd/cloudflared/transfer/transfer.go

@@ -18,7 +18,7 @@ import (
 )
 
 const (
-	baseStoreURL  = "https://login.cloudflarewarp.com/"
+	baseStoreURL  = "https://login.argotunnel.com/"
 	clientTimeout = time.Second * 60
 )
 
@@ -45,7 +45,7 @@ func Run(transferURL *url.URL, resourceName, key, value, path string, shouldEncr
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "Please open the following URL and log in with your Cloudflare account:\n\n%s\n\nLeave cloudflared running to download the %s automatically.\n", requestURL, resourceName)
 	} else {
-		fmt.Fprintf(os.Stderr, "A browser window should have opened at the following URL:\n\n%s\n\nIf the browser failed to open, open it yourself and visit the URL above.\n", requestURL)
+		fmt.Fprintf(os.Stderr, "A browser window should have opened at the following URL:\n\n%s\n\nIf the browser failed to open, please visit the URL above directly in your browser.\n", requestURL)
 	}
 
 	var resourceData []byte

cmd/cloudflared/tunnel/cmd.go

@@ -23,6 +23,7 @@ import (
 	"github.com/cloudflare/cloudflared/metrics"
 	"github.com/cloudflare/cloudflared/origin"
 	"github.com/cloudflare/cloudflared/signal"
+	"github.com/cloudflare/cloudflared/sshlog"
 	"github.com/cloudflare/cloudflared/sshserver"
 	"github.com/cloudflare/cloudflared/supervisor"
 	"github.com/cloudflare/cloudflared/tlsconfig"
@@ -44,14 +45,11 @@ import (
 const (
 	sentryDSN = "https://56a9c9fa5c364ab28f34b14f35ea0f1b:3e8827f6f9f740738eb11138f7bebb68@sentry.io/189878"
 
-	sshLogFileDirectory = "/var/log/cloudflared/"
+	sshLogFileDirectory = "/usr/local/var/log/cloudflared/"
 
 	// sshPortFlag is the port on localhost the cloudflared ssh server will run on
 	sshPortFlag = "local-ssh-port"
 
-	// shortLivedCertFlag enables short lived cert authentication
-	shortLivedCertFlag = "short-lived-certs"
-
 	// sshIdleTimeoutFlag defines the duration a SSH session can remain idle before being closed
 	sshIdleTimeoutFlag = "ssh-idle-timeout"
 
@@ -75,6 +73,11 @@ const (
 
 	// s3URLFlag is the S3 URL of SSH log uploader (e.g. don't use AWS s3 and use google storage bucket instead)
 	s3URLFlag = "s3-url-host"
+
+	// hostKeyPath is the path of the dir to save SSH host keys too
+	hostKeyPath = "host-key-path"
+
+	noIntentMsg = "The --intent argument is required. Cloudflared looks up an Intent to determine what configuration to use (i.e. which tunnels to start). If you don't have any Intents yet, you can use a placeholder Intent Label for now. Then, when you make an Intent with that label, cloudflared will get notified and open the tunnels you specified in that Intent."
 )
 
 var (
@@ -365,42 +368,53 @@ func StartServer(c *cli.Context, version string, shutdownC, graceShutdownC chan
 
 	if c.IsSet("ssh-server") {
 		if runtime.GOOS != "darwin" && runtime.GOOS != "linux" {
-			logger.Errorf("--ssh-server is not supported on %s", runtime.GOOS)
-			return errors.New(fmt.Sprintf("--ssh-server is not supported on %s", runtime.GOOS))
+			msg := fmt.Sprintf("--ssh-server is not supported on %s", runtime.GOOS)
+			logger.Error(msg)
+			return errors.New(msg)
 
 		}
 
 		logger.Infof("ssh-server set")
 
+		logManager := sshlog.NewEmptyManager()
 		if c.IsSet(bucketNameFlag) && c.IsSet(regionNameFlag) && c.IsSet(accessKeyIDFlag) && c.IsSet(secretIDFlag) {
 			uploader, err := awsuploader.NewFileUploader(c.String(bucketNameFlag), c.String(regionNameFlag),
 				c.String(accessKeyIDFlag), c.String(secretIDFlag), c.String(sessionTokenIDFlag), c.String(s3URLFlag))
 			if err != nil {
-				logger.WithError(err).Error("Cannot create uploader for SSH Server")
-				return errors.Wrap(err, "Cannot create uploader for SSH Server")
+				msg := "Cannot create uploader for SSH Server"
+				logger.WithError(err).Error(msg)
+				return errors.Wrap(err, msg)
 			}
 
-			os.Mkdir(sshLogFileDirectory, 0600)
+			if err := os.MkdirAll(sshLogFileDirectory, 0700); err != nil {
+				msg := fmt.Sprintf("Cannot create SSH log file directory %s", sshLogFileDirectory)
+				logger.WithError(err).Errorf(msg)
+				return errors.Wrap(err, msg)
+			}
+
+			logManager = sshlog.New(sshLogFileDirectory)
 
 			uploadManager := awsuploader.NewDirectoryUploadManager(logger, uploader, sshLogFileDirectory, 30*time.Minute, shutdownC)
 			uploadManager.Start()
 		}
 
-		sshServerAddress := "127.0.0.1:" + c.String(sshPortFlag)
-		server, err := sshserver.New(logger, sshServerAddress, shutdownC, c.Bool(shortLivedCertFlag), c.Duration(sshIdleTimeoutFlag), c.Duration(sshMaxTimeoutFlag))
-
+		localServerAddress := "127.0.0.1:" + c.String(sshPortFlag)
+		server, err := sshserver.New(logManager, logger, version, localServerAddress, c.String("hostname"), c.Path(hostKeyPath), shutdownC, c.Duration(sshIdleTimeoutFlag), c.Duration(sshMaxTimeoutFlag))
 		if err != nil {
-			logger.WithError(err).Error("Cannot create new SSH Server")
-			return errors.Wrap(err, "Cannot create new SSH Server")
+			msg := "Cannot create new SSH Server"
+			logger.WithError(err).Error(msg)
+			return errors.Wrap(err, msg)
 		}
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
 			if err = server.Start(); err != nil && err != ssh.ErrServerClosed {
 				logger.WithError(err).Error("SSH server error")
+				// TODO: remove when declarative tunnels are implemented.
+				close(shutdownC)
 			}
 		}()
-		c.Set("url", "ssh://"+sshServerAddress)
+		c.Set("url", "ssh://"+localServerAddress)
 	}
 
 	if host := hostnameFromURI(c.String("url")); host != "" {
@@ -508,21 +522,16 @@ func startDeclarativeTunnel(ctx context.Context,
 		return err
 	}
 
-	var scope pogs.Scope
-	if c.IsSet("group") == c.IsSet("system-name") {
-		err = fmt.Errorf("exactly one of --group or --system-name must be specified")
-		logger.WithError(err).Error("unable to determine scope")
-		return err
-	} else if c.IsSet("group") {
-		scope = pogs.NewGroup(c.String("group"))
-	} else {
-		scope = pogs.NewSystemName(c.String("system-name"))
+	intentLabel := c.String("intent")
+	if intentLabel == "" {
+		logger.Error("--intent was empty")
+		return fmt.Errorf(noIntentMsg)
 	}
 
 	cloudflaredConfig := &connection.CloudflaredConfig{
 		BuildInfo:     buildInfo,
 		CloudflaredID: cloudflaredID,
-		Scope:         scope,
+		IntentLabel:   intentLabel,
 		Tags:          tags,
 	}
 
@@ -946,15 +955,9 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			Hidden:  true,
 		}),
 		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:    "system-name",
-			Usage:   "Unique identifier for this cloudflared instance. It can be configured individually in the Declarative Tunnel UI. Mutually exclusive with `--group`.",
-			EnvVars: []string{"TUNNEL_SYSTEM_NAME"},
-			Hidden:  true,
-		}),
-		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:    "group",
-			Usage:   "Name of a group of cloudflared instances, of which this instance should be an identical copy. They can be configured collectively in the Declarative Tunnel UI. Mutually exclusive with `--system-name`.",
-			EnvVars: []string{"TUNNEL_GROUP"},
+			Name:    "intent",
+			Usage:   "The label of an Intent from which `cloudflared` should gets its tunnels from. Intents can be created in the Origin Registry UI.",
+			EnvVars: []string{"TUNNEL_INTENT"},
 			Hidden:  true,
 		}),
 		altsrc.NewDurationFlag(&cli.DurationFlag{
@@ -967,16 +970,10 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 		altsrc.NewStringFlag(&cli.StringFlag{
 			Name:    sshPortFlag,
 			Usage:   "Localhost port that cloudflared SSH server will run on",
-			Value:   "22",
+			Value:   "2222",
 			EnvVars: []string{"LOCAL_SSH_PORT"},
 			Hidden:  true,
 		}),
-		altsrc.NewBoolFlag(&cli.BoolFlag{
-			Name:    shortLivedCertFlag,
-			Usage:   "Enable short lived cert authentication for SSH server",
-			EnvVars: []string{"SHORT_LIVED_CERTS"},
-			Hidden:  true,
-		}),
 		altsrc.NewDurationFlag(&cli.DurationFlag{
 			Name:    sshIdleTimeoutFlag,
 			Usage:   "Connection timeout after no activity",
@@ -1025,5 +1022,12 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			EnvVars: []string{"S3_URL"},
 			Hidden:  true,
 		}),
+		altsrc.NewPathFlag(&cli.PathFlag{
+			Name:    hostKeyPath,
+			Usage:   "Absolute path of directory to save SSH host keys in",
+			EnvVars: []string{"HOST_KEY_PATH"},
+			Hidden:  true,
+
+		}),
 	}
-}
+}

cmd/cloudflared/tunnel/login.go

@@ -15,7 +15,7 @@ import (
 
 const (
 	baseLoginURL     = "https://dash.cloudflare.com/argotunnel"
-	callbackStoreURL = "https://login.cloudflarewarp.com/"
+	callbackStoreURL = "https://login.argotunnel.com/"
 )
 
 func login(c *cli.Context) error {

connection/connection.go

@@ -50,7 +50,7 @@ func (c *Connection) Serve(ctx context.Context) error {
 }
 
 // Connect is used to establish connections with cloudflare's edge network
-func (c *Connection) Connect(ctx context.Context, parameters *tunnelpogs.ConnectParameters, logger *logrus.Entry) (*pogs.ConnectResult, error) {
+func (c *Connection) Connect(ctx context.Context, parameters *tunnelpogs.ConnectParameters, logger *logrus.Entry) (pogs.ConnectResult, error) {
 	openStreamCtx, cancel := context.WithTimeout(ctx, openStreamTimeout)
 	defer cancel()
 

connection/discovery.go

@@ -13,13 +13,13 @@ import (
 )
 
 const (
-	// Used to discover HA Warp servers
-	srvService = "warp"
+	// Used to discover HA origintunneld servers
+	srvService = "origintunneld"
 	srvProto   = "tcp"
-	srvName    = "cloudflarewarp.com"
+	srvName    = "argotunnel.com"
 
 	// Used to fallback to DoT when we can't use the default resolver to
-	// discover HA Warp servers (GitHub issue #75).
+	// discover HA origintunneld servers (GitHub issue #75).
 	dotServerName = "cloudflare-dns.com"
 	dotServerAddr = "1.1.1.1:853"
 	dotTimeout    = time.Duration(15 * time.Second)
@@ -30,8 +30,8 @@ const (
 
 var friendlyDNSErrorLines = []string{
 	`Please try the following things to diagnose this issue:`,
-	`  1. ensure that cloudflarewarp.com is returning "warp" service records.`,
-	`     Run your system's equivalent of: dig srv _warp._tcp.cloudflarewarp.com`,
+	`  1. ensure that argotunnel.com is returning "origintunneld" service records.`,
+	`     Run your system's equivalent of: dig srv _origintunneld._tcp.argotunnel.com`,
 	`  2. ensure that your DNS resolver is not returning compressed SRV records.`,
 	`     See GitHub issue https://github.com/golang/go/issues/27546`,
 	`     For example, you could use Cloudflare's 1.1.1.1 as your resolver:`,
@@ -102,7 +102,7 @@ func EdgeDiscovery(logger *logrus.Entry) ([]*net.TCPAddr, error) {
 		// Try to fall back to DoT from Cloudflare directly.
 		//
 		// Note: Instead of DoT, we could also have used DoH. Either of these:
-		//     - directly via the JSON API (https://1.1.1.1/dns-query?ct=application/dns-json&name=_warp._tcp.cloudflarewarp.com&type=srv)
+		//     - directly via the JSON API (https://1.1.1.1/dns-query?ct=application/dns-json&name=_origintunneld._tcp.argotunnel.com&type=srv)
 		//     - indirectly via `tunneldns.NewUpstreamHTTPS()`
 		// But both of these cases miss out on a key feature from the stdlib:
 		//     "The returned records are sorted by priority and randomized by weight within a priority."
@@ -119,7 +119,7 @@ func EdgeDiscovery(logger *logrus.Entry) ([]*net.TCPAddr, error) {
 			for _, s := range friendlyDNSErrorLines {
 				logger.Errorln(s)
 			}
-			return nil, errors.Wrap(err, "Could not lookup srv records on _warp._tcp.cloudflarewarp.com")
+			return nil, errors.Wrap(err, "Could not lookup srv records on _origintunneld._tcp.argotunnel.com")
 		}
 		// Accept the fallback results and keep going
 		addrs = fallbackAddrs

connection/manager.go

@@ -10,21 +10,24 @@ import (
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
 	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/streamhandler"
 	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+
 	"github.com/google/uuid"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
 
 const (
-	quickStartLink = "https://developers.cloudflare.com/argo-tunnel/quickstart/"
-	faqLink        = "https://developers.cloudflare.com/argo-tunnel/faq/"
+	quickStartLink    = "https://developers.cloudflare.com/argo-tunnel/quickstart/"
+	faqLink           = "https://developers.cloudflare.com/argo-tunnel/faq/"
+	defaultRetryAfter = time.Second * 5
 )
 
 // EdgeManager manages connections with the edge
 type EdgeManager struct {
 	// streamHandler handles stream opened by the edge
-	streamHandler h2mux.MuxedStreamHandler
+	streamHandler *streamhandler.StreamHandler
 	// TLSConfig is the TLS configuration to connect with edge
 	tlsConfig *tls.Config
 	// cloudflaredConfig is the cloudflared configuration that is determined when the process first starts
@@ -47,11 +50,11 @@ type CloudflaredConfig struct {
 	CloudflaredID uuid.UUID
 	Tags          []pogs.Tag
 	BuildInfo     *buildinfo.BuildInfo
-	Scope         pogs.Scope
+	IntentLabel   string
 }
 
 func NewEdgeManager(
-	streamHandler h2mux.MuxedStreamHandler,
+	streamHandler *streamhandler.StreamHandler,
 	edgeConnMgrConfigurable *EdgeManagerConfigurable,
 	userCredential []byte,
 	tlsConfig *tls.Config,
@@ -87,8 +90,12 @@ func (em *EdgeManager) Run(ctx context.Context) error {
 		// Create/delete connection one at a time, so we don't need to adjust for connections that are being created/deleted
 		// in shouldCreateConnection or shouldReduceConnection calculation
 		if em.state.shouldCreateConnection(em.serviceDiscoverer.AvailableAddrs()) {
-			if err := em.newConnection(ctx); err != nil {
-				em.logger.WithError(err).Error("cannot create new connection")
+			if connErr := em.newConnection(ctx); connErr != nil {
+				if !connErr.ShouldRetry {
+					em.logger.WithError(connErr).Error(em.noRetryMessage())
+					return connErr
+				}
+				em.logger.WithError(connErr).Error("cannot create new connection")
 			}
 		} else if em.state.shouldReduceConnection() {
 			if err := em.closeConnection(ctx); err != nil {
@@ -103,11 +110,11 @@ func (em *EdgeManager) UpdateConfigurable(newConfigurable *EdgeManagerConfigurab
 	em.state.updateConfigurable(newConfigurable)
 }
 
-func (em *EdgeManager) newConnection(ctx context.Context) error {
+func (em *EdgeManager) newConnection(ctx context.Context) *pogs.ConnectError {
 	edgeIP := em.serviceDiscoverer.Addr()
 	edgeConn, err := em.dialEdge(ctx, edgeIP)
 	if err != nil {
-		return errors.Wrap(err, "dial edge error")
+		return retryConnection(fmt.Sprintf("dial edge error: %v", err))
 	}
 	configurable := em.state.getConfigurable()
 	// Establish a muxed connection with the edge
@@ -121,12 +128,12 @@ func (em *EdgeManager) newConnection(ctx context.Context) error {
 		Logger:            em.logger.WithField("subsystem", "muxer"),
 	})
 	if err != nil {
-		return errors.Wrap(err, "couldn't perform handshake with edge")
+		retryConnection(fmt.Sprintf("couldn't perform handshake with edge: %v", err))
 	}
 
 	h2muxConn, err := newConnection(muxer, edgeIP)
 	if err != nil {
-		return errors.Wrap(err, "couldn't create h2mux connection")
+		return retryConnection(fmt.Sprintf("couldn't create h2mux connection: %v", err))
 	}
 
 	go em.serveConn(ctx, h2muxConn)
@@ -136,23 +143,22 @@ func (em *EdgeManager) newConnection(ctx context.Context) error {
 		CloudflaredVersion:  em.cloudflaredConfig.BuildInfo.CloudflaredVersion,
 		NumPreviousAttempts: 0,
 		OriginCert:          em.state.getUserCredential(),
-		Scope:               em.cloudflaredConfig.Scope,
+		IntentLabel:         em.cloudflaredConfig.IntentLabel,
 		Tags:                em.cloudflaredConfig.Tags,
 	}, em.logger)
 	if err != nil {
 		h2muxConn.Shutdown()
-		return errors.Wrap(err, "couldn't connect to edge")
+		return retryConnection(fmt.Sprintf("couldn't connect to edge: %v", err))
 	}
 
-	if connErr := connResult.Err; connErr != nil {
-		if !connErr.ShouldRetry {
-			return errors.Wrap(connErr, em.noRetryMessage())
-		}
-		return errors.Wrapf(connErr, "edge responded with RetryAfter=%v", connErr.RetryAfter)
+	if connErr := connResult.ConnectError(); connErr != nil {
+		return connErr
 	}
 
 	em.state.newConnection(h2muxConn)
-	em.logger.Infof("connected to %s", connResult.ServerInfo.LocationName)
+	em.logger.Infof("connected to %s", connResult.ConnectedTo())
+
+	em.streamHandler.UseConfiguration(ctx, connResult.ClientConfig())
 	return nil
 }
 
@@ -282,3 +288,11 @@ func (ems *edgeManagerState) getUserCredential() []byte {
 	defer ems.RUnlock()
 	return ems.userCredential
 }
+
+func retryConnection(cause string) *pogs.ConnectError {
+	return &pogs.ConnectError{
+		Cause:       cause,
+		RetryAfter:  defaultRetryAfter,
+		ShouldRetry: true,
+	}
+}

connection/manager_test.go

@@ -8,7 +8,9 @@ import (
 	"github.com/stretchr/testify/assert"
 
 	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/streamhandler"
 	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+
 	"github.com/google/uuid"
 	"github.com/sirupsen/logrus"
 )
@@ -42,16 +44,12 @@ var (
 	}
 )
 
-type mockStreamHandler struct {
-}
-
-func (msh *mockStreamHandler) ServeStream(*h2mux.MuxedStream) error {
-	return nil
-}
-
 func mockEdgeManager() *EdgeManager {
+	newConfigChan := make(chan<- *pogs.ClientConfig)
+	useConfigResultChan := make(<-chan *pogs.UseConfigurationResult)
+	logger := logrus.New()
 	return NewEdgeManager(
-		&mockStreamHandler{},
+		streamhandler.NewStreamHandler(newConfigChan, useConfigResultChan, logger),
 		configurable,
 		[]byte{},
 		nil,

go.mod

@@ -0,0 +1,61 @@
+module github.com/cloudflare/cloudflared
+
+go 1.12
+
+require (
+	github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 // indirect
+	github.com/aws/aws-sdk-go v1.25.8
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261 // indirect
+	github.com/cloudflare/brotli-go v0.0.0-20191101163834-d34379f7ff93
+	github.com/cloudflare/golibs v0.0.0-20170913112048-333127dbecfc
+	github.com/coredns/coredns v1.2.0
+	github.com/coreos/go-oidc v0.0.0-20171002155002-a93f71fdfe73
+	github.com/coreos/go-systemd v0.0.0-20190620071333-e64a0ec8b42a
+	github.com/elgs/gosqljson v0.0.0-20160403005647-027aa4915315
+	github.com/equinox-io/equinox v1.2.0
+	github.com/facebookgo/grace v0.0.0-20180706040059-75cf19382434
+	github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 // indirect
+	github.com/getsentry/raven-go v0.0.0-20180517221441-ed7bcb39ff10
+	github.com/gliderlabs/ssh v0.0.0-20191009160644-63518b5243e0
+	github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3
+	github.com/google/uuid v1.1.1
+	github.com/gorilla/mux v1.7.3
+	github.com/gorilla/websocket v1.2.0
+	github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 // indirect
+	github.com/json-iterator/go v1.1.6 // indirect
+	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
+	github.com/lib/pq v1.2.0
+	github.com/mattn/go-colorable v0.1.4
+	github.com/mattn/go-isatty v0.0.10 // indirect
+	github.com/mholt/caddy v0.0.0-20180807230124-d3b731e9255b // indirect
+	github.com/miekg/dns v1.1.8
+	github.com/mitchellh/go-homedir v1.1.0
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.1 // indirect
+	github.com/opentracing/opentracing-go v1.1.0 // indirect
+	github.com/pkg/errors v0.8.1
+	github.com/prometheus/client_golang v1.0.0
+	github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 // indirect
+	github.com/prometheus/common v0.7.0 // indirect
+	github.com/prometheus/procfs v0.0.5 // indirect
+	github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5
+	github.com/sirupsen/logrus v1.4.2
+	github.com/stretchr/testify v1.3.0
+	golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc
+	golang.org/x/net v0.0.0-20191007182048-72f939374954
+	golang.org/x/sync v0.0.0-20190423024810-112230192c58
+	golang.org/x/sys v0.0.0-20191008105621-543471e840be
+	golang.org/x/text v0.3.2 // indirect
+	google.golang.org/appengine v1.4.0 // indirect
+	google.golang.org/genproto v0.0.0-20191007204434-a023cd5227bd // indirect
+	google.golang.org/grpc v1.24.0 // indirect
+	gopkg.in/urfave/cli.v2 v2.0.0-20180128181224-d604b6ffeee8
+	gopkg.in/yaml.v2 v2.2.4 // indirect
+	zombiezen.com/go/capnproto2 v0.0.0-20180616160808-7cfd211c19c7
+)
+
+// ../../go/pkg/mod/github.com/coredns/coredns@v1.2.0/plugin/metrics/metrics.go:40:49: too many arguments in call to prometheus.NewProcessCollector
+// have (int, string)
+// want (prometheus.ProcessCollectorOpts)
+replace github.com/prometheus/client_golang => github.com/prometheus/client_golang v0.9.0-pre1

go.sum

@@ -0,0 +1,198 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA=
+github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
+github.com/aws/aws-sdk-go v1.25.8 h1:n7I+HUUXjun2CsX7JK+1hpRIkZrlKhd3nayeb+Xmavs=
+github.com/aws/aws-sdk-go v1.25.8/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
+github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261 h1:6/yVvBsKeAw05IUj4AzvrxaCnDjN4nUqKjW9+w5wixg=
+github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cloudflare/brotli-go v0.0.0-20180507233613-18c9f6c67e3d h1:4ZpmBY49TvTEw8yB2UtvS/kne36synubxO2EY1y9HGc=
+github.com/cloudflare/brotli-go v0.0.0-20180507233613-18c9f6c67e3d/go.mod h1:C7j4+vDRAawlVO14WU7MAP4Z7+K7lFxRcuXm6bezBDk=
+github.com/cloudflare/brotli-go v0.0.0-20191101163834-d34379f7ff93 h1:QrGfkZDnMxcWHaYDdB7CmqS9i26OAnUj/xcus/abYkY=
+github.com/cloudflare/brotli-go v0.0.0-20191101163834-d34379f7ff93/go.mod h1:QiTe66jFdP7cUKMCCf/WrvDyYdtdmdZfVcdoLbzaKVY=
+github.com/cloudflare/golibs v0.0.0-20170913112048-333127dbecfc h1:Dvk3ySBsOm5EviLx6VCyILnafPcQinXGP5jbTdHUJgE=
+github.com/cloudflare/golibs v0.0.0-20170913112048-333127dbecfc/go.mod h1:HlgKKR8V5a1wroIDDIz3/A+T+9Janfq+7n1P5sEFdi0=
+github.com/coredns/coredns v1.2.0 h1:YEI38K2BJYzL/SxO2tZFD727T/C68DqVWkBQjT0sWPU=
+github.com/coredns/coredns v1.2.0/go.mod h1:zASH/MVDgR6XZTbxvOnsZfffS+31vg6Ackf/wo1+AM0=
+github.com/coreos/go-oidc v0.0.0-20171002155002-a93f71fdfe73 h1:7CNPV0LWRCa1FNmqg700pbXhzvmoaXKyfxWRkjRym7Q=
+github.com/coreos/go-oidc v0.0.0-20171002155002-a93f71fdfe73/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
+github.com/coreos/go-systemd v0.0.0-20190620071333-e64a0ec8b42a h1:W8b4lQ4tFF21aspRGoBuCNV6V2fFJBF+pm1J6OY8Lys=
+github.com/coreos/go-systemd v0.0.0-20190620071333-e64a0ec8b42a/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/elgs/gosqljson v0.0.0-20160403005647-027aa4915315 h1:0NXTHkloWMqOB1KHv1pZ/LAXsq9mhH0VsqyBu0MjB1Q=
+github.com/elgs/gosqljson v0.0.0-20160403005647-027aa4915315/go.mod h1:QhwWWjjii22wonb8HPfn6AkBRA95VpJ+xVqyhnCSbbU=
+github.com/equinox-io/equinox v1.2.0 h1:bBS7Ou+Y7Jwgmy8TWSYxEh85WctuFn7FPlgbUzX4DBA=
+github.com/equinox-io/equinox v1.2.0/go.mod h1:6s3HJB0PYUNgs0mxmI8fHdfVl3TQ25ieA/PVfr+eyVo=
+github.com/facebookgo/grace v0.0.0-20180706040059-75cf19382434 h1:mOp33BLbcbJ8fvTAmZacbBiOASfxN+MLcLxymZCIrGE=
+github.com/facebookgo/grace v0.0.0-20180706040059-75cf19382434/go.mod h1:KigFdumBXUPSwzLDbeuzyt0elrL7+CP7TKuhrhT4bcU=
+github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ=
+github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
+github.com/getsentry/raven-go v0.0.0-20180517221441-ed7bcb39ff10 h1:YO10pIIBftO/kkTFdWhctH96grJ7qiy7bMdiZcIvPKs=
+github.com/getsentry/raven-go v0.0.0-20180517221441-ed7bcb39ff10/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ=
+github.com/gliderlabs/ssh v0.0.0-20191009160644-63518b5243e0 h1:gF8ngtda767ddth2SH0YSAhswhz6qUkvyI9EZFYCWJA=
+github.com/gliderlabs/ssh v0.0.0-20191009160644-63518b5243e0/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
+github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
+github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3 h1:zN2lZNZRflqFyxVaTIU61KNKQ9C0055u9CAfpmqUvo4=
+github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3/go.mod h1:nPpo7qLxd6XL3hWJG/O60sR8ZKfMCiIoNap5GvD12KU=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw=
+github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
+github.com/gorilla/websocket v1.2.0 h1:VJtLvh6VQym50czpZzx07z/kw9EgAxI3x1ZB8taTMQQ=
+github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
+github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 h1:MJG/KsmcqMwFAkh8mTnAwhyKoB+sTAnY4CACC110tbU=
+github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645/go.mod h1:6iZfnjpejD4L/4DwD7NryNaJyCQdzwWwH2MWhCA90Kw=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
+github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
+github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
+github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
+github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0=
+github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA=
+github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.10 h1:qxFzApOv4WsAL965uUPIsXzAKCZxN2p9UqdhFS4ZW10=
+github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84=
+github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
+github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/mholt/caddy v0.0.0-20180807230124-d3b731e9255b h1:/BbY4n99iMazlr2igipph+hj0MwlZIWpcsP8Iy+na+s=
+github.com/mholt/caddy v0.0.0-20180807230124-d3b731e9255b/go.mod h1:Wb1PlT4DAYSqOEd03MsqkdkXnTxA8v9pKjdpxbqM1kY=
+github.com/miekg/dns v1.1.8 h1:1QYRAKU3lN5cRfLCkPU08hwvLJFhvjP6MqNMmQz6ZVI=
+github.com/miekg/dns v1.1.8/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
+github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU=
+github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_golang v0.9.0-pre1 h1:AWTOhsOI9qxeirTuA0A4By/1Es1+y9EcCGY6bBZ2fhM=
+github.com/prometheus/client_golang v0.9.0-pre1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM=
+github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
+github.com/prometheus/client_golang v1.2.1 h1:JnMpQc6ppsNgw9QPAGF6Dod479itz7lvlsMzzNayLOI=
+github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/common v0.1.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.3.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.5.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc=
+github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc=
+github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLyCY=
+github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA=
+github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.0.5 h1:3+auTFlqw+ZaQYJARz6ArODtkaIwtvBTx3N2NehQlL8=
+github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
+github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5 h1:mZHayPoR0lNmnHyvtYjDeq0zlVHn9K/ZXoy17ylucdo=
+github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5/go.mod h1:GEXHk5HgEKCvEIIrSpFI3ozzG5xOKA2DVlEX/gGnewM=
+github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc h1:c0o/qxkaO2LF5t6fQrT4b5hzyggAkLLlCUjqfRxd8Q4=
+golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191007182048-72f939374954 h1:JGZucVF/L/TotR719NbujzadOZ2AgnYlqphQGHDCKaU=
+golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191008105621-543471e840be h1:QAcqgptGM8IQBC9K/RC4o+O9YmqEm0diQn9QmZw/0mU=
+golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20191007204434-a023cd5227bd h1:84VQPzup3IpKLxuIAZjHMhVjJ8fZ4/i3yUnj3k6fUdw=
+google.golang.org/genproto v0.0.0-20191007204434-a023cd5227bd/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.24.0 h1:vb/1TCsVn3DcJlQ0Gs1yB1pKI6Do2/QNwxdKqmc/b0s=
+google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/urfave/cli.v2 v2.0.0-20180128181224-d604b6ffeee8 h1:/pLAskKF+d5SawboKd8GB8ew4ClHDbt2c3K9EBFeRGU=
+gopkg.in/urfave/cli.v2 v2.0.0-20180128181224-d604b6ffeee8/go.mod h1:cKXr3E0k4aosgycml1b5z33BVV6hai1Kh7uDgFOkbcs=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+zombiezen.com/go/capnproto2 v0.0.0-20180616160808-7cfd211c19c7 h1:CZoOFlTPbKfAShKYrMuUfYbnXexFT1rYRUX1SPnrdE4=
+zombiezen.com/go/capnproto2 v0.0.0-20180616160808-7cfd211c19c7/go.mod h1:TMGa8HWGJkXiq4nHe9Zu/JgRF5oUtg4XizFC+Vexbec=

h2mux/error.go

@@ -20,11 +20,12 @@ var (
 	ErrUnknownStream       = MuxerProtocolError{"2002 unknown stream", http2.ErrCodeProtocol}
 	ErrInvalidStream       = MuxerProtocolError{"2003 invalid stream", http2.ErrCodeProtocol}
 
-	ErrStreamHeadersSent      = MuxerApplicationError{"3000 headers already sent"}
-	ErrConnectionClosed       = MuxerApplicationError{"3001 connection closed"}
-	ErrConnectionDropped      = MuxerApplicationError{"3002 connection dropped"}
-	ErrOpenStreamTimeout      = MuxerApplicationError{"3003 open stream timeout"}
-	ErrResponseHeadersTimeout = MuxerApplicationError{"3004 timeout waiting for initial response headers"}
+	ErrStreamHeadersSent               = MuxerApplicationError{"3000 headers already sent"}
+	ErrStreamRequestConnectionClosed   = MuxerApplicationError{"3001 connection closed while opening stream"}
+	ErrConnectionDropped               = MuxerApplicationError{"3002 connection dropped"}
+	ErrStreamRequestTimeout            = MuxerApplicationError{"3003 open stream timeout"}
+	ErrResponseHeadersTimeout          = MuxerApplicationError{"3004 timeout waiting for initial response headers"}
+	ErrResponseHeadersConnectionClosed = MuxerApplicationError{"3005 connection closed while waiting for initial response headers"}
 
 	ErrClosedStream = MuxerStreamError{"4000 stream closed", http2.ErrCodeStreamClosed}
 )

h2mux/h2mux.go

@@ -1,7 +1,6 @@
 package h2mux
 
 import (
-	"bytes"
 	"context"
 	"io"
 	"strings"
@@ -20,7 +19,7 @@ const (
 	maxWindowSize            uint32        = (1 << 31) - 1 // 2^31-1 = 2147483647, max window size in http2 spec
 	defaultTimeout           time.Duration = 5 * time.Second
 	defaultRetries           uint64        = 5
-	defaultWriteBufferMaxLen int           = 1024 * 1024 * 512 // 500mb
+	defaultWriteBufferMaxLen int           = 1024 * 1024 // 1mb
 
 	SettingMuxerMagic http2.SettingID = 0x42db
 	MuxerMagicOrigin  uint32          = 0xa2e43c8b
@@ -388,72 +387,58 @@ func isConnectionClosedError(err error) bool {
 // OpenStream opens a new data stream with the given headers.
 // Called by proxy server and tunnel
 func (m *Muxer) OpenStream(ctx context.Context, headers []Header, body io.Reader) (*MuxedStream, error) {
-	stream := &MuxedStream{
-		responseHeadersReceived: make(chan struct{}),
-		readBuffer:              NewSharedBuffer(),
-		writeBuffer:             &bytes.Buffer{},
-		writeBufferMaxLen:       m.config.StreamWriteBufferMaxLen,
-		writeBufferHasSpace:     make(chan struct{}, 1),
-		receiveWindow:           m.config.DefaultWindowSize,
-		receiveWindowCurrentMax: m.config.DefaultWindowSize,
-		receiveWindowMax:        m.config.MaxWindowSize,
-		sendWindow:              m.config.DefaultWindowSize,
-		readyList:               m.readyList,
-		writeHeaders:            headers,
-		dictionaries:            m.muxReader.dictionaries,
+	stream := m.NewStream(headers)
+	if err := m.MakeMuxedStreamRequest(ctx, MuxedStreamRequest{stream, body}); err != nil {
+		return nil, err
 	}
-
-	select {
-	// Will be received by mux writer
-	case <-ctx.Done():
-		return nil, ErrOpenStreamTimeout
-	case <-m.abortChan:
-		return nil, ErrConnectionClosed
-	case m.newStreamChan <- MuxedStreamRequest{stream: stream, body: body}:
-	}
-
-	select {
-	case <-ctx.Done():
-		return nil, ErrResponseHeadersTimeout
-	case <-m.abortChan:
-		return nil, ErrConnectionClosed
-	case <-stream.responseHeadersReceived:
-		return stream, nil
+	if err := m.AwaitResponseHeaders(ctx, stream); err != nil {
+		return nil, err
 	}
+	return stream, nil
 }
 
 func (m *Muxer) OpenRPCStream(ctx context.Context) (*MuxedStream, error) {
-	stream := &MuxedStream{
-		responseHeadersReceived: make(chan struct{}),
-		readBuffer:              NewSharedBuffer(),
-		writeBuffer:             &bytes.Buffer{},
-		writeBufferMaxLen:       m.config.StreamWriteBufferMaxLen,
-		writeBufferHasSpace:     make(chan struct{}, 1),
-		receiveWindow:           m.config.DefaultWindowSize,
-		receiveWindowCurrentMax: m.config.DefaultWindowSize,
-		receiveWindowMax:        m.config.MaxWindowSize,
-		sendWindow:              m.config.DefaultWindowSize,
-		readyList:               m.readyList,
-		writeHeaders:            RPCHeaders(),
-		dictionaries:            m.muxReader.dictionaries,
+	stream := m.NewStream(RPCHeaders())
+	if err := m.MakeMuxedStreamRequest(ctx, MuxedStreamRequest{stream: stream, body: nil}); err != nil {
+		return nil, err
 	}
+	if err := m.AwaitResponseHeaders(ctx, stream); err != nil {
+		return nil, err
+	}
+	return stream, nil
+}
 
+func (m *Muxer) NewStream(headers []Header) *MuxedStream {
+	return NewStream(m.config, headers, m.readyList, m.muxReader.dictionaries)
+}
+
+func (m *Muxer) MakeMuxedStreamRequest(ctx context.Context, request MuxedStreamRequest) error {
 	select {
+	case <-ctx.Done():
+		return ErrStreamRequestTimeout
+	case <-m.abortChan:
+		return ErrStreamRequestConnectionClosed
 	// Will be received by mux writer
-	case <-ctx.Done():
-		return nil, ErrOpenStreamTimeout
-	case <-m.abortChan:
-		return nil, ErrConnectionClosed
-	case m.newStreamChan <- MuxedStreamRequest{stream: stream, body: nil}:
+	case m.newStreamChan <- request:
+		return nil
 	}
+}
 
+func (m *Muxer) CloseStreamRead(stream *MuxedStream) {
+	stream.CloseRead()
+	if stream.WriteClosed() {
+		m.streams.Delete(stream.streamID)
+	}
+}
+
+func (m *Muxer) AwaitResponseHeaders(ctx context.Context, stream *MuxedStream) error {
 	select {
 	case <-ctx.Done():
-		return nil, ErrResponseHeadersTimeout
+		return ErrResponseHeadersTimeout
 	case <-m.abortChan:
-		return nil, ErrConnectionClosed
+		return ErrResponseHeadersConnectionClosed
 	case <-stream.responseHeadersReceived:
-		return stream, nil
+		return nil
 	}
 }
 

h2mux/h2mux_test.go

@@ -584,8 +584,8 @@ func TestOpenAfterDisconnect(t *testing.T) {
 			[]Header{{Name: "test-header", Value: "headerValue"}},
 			nil,
 		)
-		if err != ErrConnectionClosed {
-			t.Fatalf("unexpected error in OpenStream: %s", err)
+		if err != ErrStreamRequestConnectionClosed && err != ErrResponseHeadersConnectionClosed {
+			t.Fatalf("case %v: unexpected error in OpenStream: %v", i, err)
 		}
 	}
 }

h2mux/muxedstream.go

@@ -88,6 +88,23 @@ func (th TunnelHostname) IsSet() bool {
 	return th != ""
 }
 
+func NewStream(config MuxerConfig, writeHeaders []Header, readyList *ReadyList, dictionaries h2Dictionaries) *MuxedStream {
+	return &MuxedStream{
+		responseHeadersReceived: make(chan struct{}),
+		readBuffer:              NewSharedBuffer(),
+		writeBuffer:             &bytes.Buffer{},
+		writeBufferMaxLen:       config.StreamWriteBufferMaxLen,
+		writeBufferHasSpace:     make(chan struct{}, 1),
+		receiveWindow:           config.DefaultWindowSize,
+		receiveWindowCurrentMax: config.DefaultWindowSize,
+		receiveWindowMax:        config.MaxWindowSize,
+		sendWindow:              config.DefaultWindowSize,
+		readyList:               readyList,
+		writeHeaders:            writeHeaders,
+		dictionaries:            dictionaries,
+	}
+}
+
 func (s *MuxedStream) Read(p []byte) (n int, err error) {
 	var readBuffer ReadWriteClosedCloser
 	if s.dictionaries.read != nil {
@@ -120,9 +137,10 @@ func (s *MuxedStream) Write(p []byte) (int, error) {
 		// If the buffer is full, block till there is more room.
 		// Use a loop to recheck the buffer size after the lock is reacquired.
 		for s.writeBufferMaxLen <= s.writeBuffer.Len() {
-			s.writeLock.Unlock()
-			<-s.writeBufferHasSpace
-			s.writeLock.Lock()
+			s.awaitWriteBufferHasSpace()
+			if s.writeEOF {
+				return totalWritten, io.EOF
+			}
 		}
 		amountToWrite := len(p) - totalWritten
 		spaceAvailable := s.writeBufferMaxLen - s.writeBuffer.Len()
@@ -171,10 +189,19 @@ func (s *MuxedStream) CloseWrite() error {
 	if c, ok := s.writeBuffer.(io.Closer); ok {
 		c.Close()
 	}
+	// Allow MuxedStream::Write() to terminate its loop with err=io.EOF, if needed
+	s.notifyWriteBufferHasSpace()
+	// We need to send something over the wire, even if it's an END_STREAM with no data
 	s.writeNotify()
 	return nil
 }
 
+func (s *MuxedStream) WriteClosed() bool {
+	s.writeLock.Lock()
+	defer s.writeLock.Unlock()
+	return s.writeEOF
+}
+
 func (s *MuxedStream) WriteHeaders(headers []Header) error {
 	s.writeLock.Lock()
 	defer s.writeLock.Unlock()
@@ -215,6 +242,23 @@ func (s *MuxedStream) TunnelHostname() TunnelHostname {
 	return s.tunnelHostname
 }
 
+// Block until a value is sent on writeBufferHasSpace.
+// Must be called while holding writeLock
+func (s *MuxedStream) awaitWriteBufferHasSpace() {
+	s.writeLock.Unlock()
+	<-s.writeBufferHasSpace
+	s.writeLock.Lock()
+}
+
+// Send a value on writeBufferHasSpace without blocking.
+// Must be called while holding writeLock
+func (s *MuxedStream) notifyWriteBufferHasSpace() {
+	select {
+	case s.writeBufferHasSpace <- struct{}{}:
+	default:
+	}
+}
+
 func (s *MuxedStream) getReceiveWindow() uint32 {
 	s.writeLock.Lock()
 	defer s.writeLock.Unlock()
@@ -334,17 +378,13 @@ func (s *MuxedStream) getChunk() *streamChunk {
 		sendData:     !s.sentEOF,
 		eof:          s.writeEOF && uint32(s.writeBuffer.Len()) <= s.sendWindow,
 	}
-
 	// Copy at most s.sendWindow bytes, adjust the sendWindow accordingly
 	writeLen, _ := io.CopyN(&chunk.buffer, s.writeBuffer, int64(s.sendWindow))
 	s.sendWindow -= uint32(writeLen)
 
-	// Non-blocking channel send. This will allow MuxedStream::Write() to continue, if needed
+	// Allow MuxedStream::Write() to continue, if needed
 	if s.writeBuffer.Len() < s.writeBufferMaxLen {
-		select {
-		case s.writeBufferHasSpace <- struct{}{}:
-		default:
-		}
+		s.notifyWriteBufferHasSpace()
 	}
 
 	// When we write the chunk, we'll write the WINDOW_UPDATE frame if needed

h2mux/muxreader.go

@@ -87,23 +87,28 @@ func (r *MuxReader) run(parentLogger *log.Entry) error {
 	for {
 		frame, err := r.f.ReadFrame()
 		if err != nil {
+			errLogger := logger.WithError(err)
+			if errorDetail := r.f.ErrorDetail(); errorDetail != nil {
+				errLogger = errLogger.WithField("errorDetail", errorDetail)
+			}
 			switch e := err.(type) {
 			case http2.StreamError:
-				logger.WithError(err).Warn("stream error")
+				errLogger.Warn("stream error")
 				r.streamError(e.StreamID, e.Code)
 			case http2.ConnectionError:
-				logger.WithError(err).Warn("connection error")
+				errLogger.Warn("connection error")
 				return r.connectionError(err)
 			default:
 				if isConnectionClosedError(err) {
 					if r.streams.Len() == 0 {
+						// don't log the error here -- that would just be extra noise
 						logger.Debug("shutting down")
 						return nil
 					}
-					logger.Warn("connection closed unexpectedly")
+					errLogger.Warn("connection closed unexpectedly")
 					return err
 				} else {
-					logger.WithError(err).Warn("frame read error")
+					errLogger.Warn("frame read error")
 					return r.connectionError(err)
 				}
 			}
@@ -120,6 +125,9 @@ func (r *MuxReader) run(parentLogger *log.Entry) error {
 			if streamID == 0 {
 				return ErrInvalidStream
 			}
+			if stream, ok := r.streams.Get(streamID); ok {
+				stream.Close()
+			}
 			r.streams.Delete(streamID)
 		case *http2.PingFrame:
 			r.receivePingData(f)

hello/hello.go

@@ -103,15 +103,16 @@ func StartHelloWorldServer(logger *logrus.Logger, listener net.Listener, shutdow
 		WriteBufferSize: 1024,
 	}
 
-	httpServer := &http.Server{Addr: listener.Addr().String(), Handler: nil}
+	muxer := http.NewServeMux()
+	muxer.HandleFunc("/uptime", uptimeHandler(time.Now()))
+	muxer.HandleFunc("/ws", websocketHandler(logger, upgrader))
+	muxer.HandleFunc("/", rootHandler(serverName))
+	httpServer := &http.Server{Addr: listener.Addr().String(), Handler: muxer}
 	go func() {
 		<-shutdownC
 		httpServer.Close()
 	}()
 
-	http.HandleFunc("/uptime", uptimeHandler(time.Now()))
-	http.HandleFunc("/ws", websocketHandler(logger, upgrader))
-	http.HandleFunc("/", rootHandler(serverName))
 	err := httpServer.Serve(listener)
 	return err
 }

origin/tunnel.go

@@ -23,7 +23,6 @@ import (
 	"github.com/cloudflare/cloudflared/validation"
 	"github.com/cloudflare/cloudflared/websocket"
 
-	raven "github.com/getsentry/raven-go"
 	"github.com/google/uuid"
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
@@ -152,7 +151,7 @@ func ServeTunnelLoop(ctx context.Context,
 	connectedSignal *signal.Signal,
 	u uuid.UUID,
 ) error {
-	logger := config.Logger
+	connectionLogger := config.Logger.WithField("connectionID", connectionID)
 	config.Metrics.incrementHaConnections()
 	defer config.Metrics.decrementHaConnections()
 	backoff := BackoffHandler{MaxRetries: config.Retries}
@@ -165,10 +164,18 @@ func ServeTunnelLoop(ctx context.Context,
 	// Ensure the above goroutine will terminate if we return without connecting
 	defer connectedFuse.Fuse(false)
 	for {
-		err, recoverable := ServeTunnel(ctx, config, addr, connectionID, connectedFuse, &backoff, u)
+		err, recoverable := ServeTunnel(
+			ctx,
+			config,
+			connectionLogger,
+			addr, connectionID,
+			connectedFuse,
+			&backoff,
+			u,
+		)
 		if recoverable {
 			if duration, ok := backoff.GetBackoffDuration(ctx); ok {
-				logger.Infof("Retrying in %s seconds", duration)
+				connectionLogger.Infof("Retrying in %s seconds", duration)
 				backoff.Backoff(ctx)
 				continue
 			}
@@ -180,6 +187,7 @@ func ServeTunnelLoop(ctx context.Context,
 func ServeTunnel(
 	ctx context.Context,
 	config *TunnelConfig,
+	logger *log.Entry,
 	addr *net.TCPAddr,
 	connectionID uint8,
 	connectedFuse *h2mux.BooleanFuse,
@@ -199,7 +207,6 @@ func ServeTunnel(
 	}()
 
 	connectionTag := uint8ToString(connectionID)
-	logger := config.Logger.WithField("connectionID", connectionTag)
 
 	// additional tags to send other than hostname which is set in cloudflared main package
 	tags := make(map[string]string)
@@ -208,7 +215,7 @@ func ServeTunnel(
 	// Returns error from parsing the origin URL or handshake errors
 	handler, originLocalIP, err := NewTunnelHandler(ctx, config, addr.String(), connectionID)
 	if err != nil {
-		errLog := config.Logger.WithError(err)
+		errLog := logger.WithError(err)
 		switch err.(type) {
 		case dialError:
 			errLog.Error("Unable to dial edge")
@@ -224,7 +231,7 @@ func ServeTunnel(
 	errGroup, serveCtx := errgroup.WithContext(ctx)
 
 	errGroup.Go(func() error {
-		err := RegisterTunnel(serveCtx, handler.muxer, config, connectionID, originLocalIP, u)
+		err := RegisterTunnel(serveCtx, handler.muxer, config, logger, connectionID, originLocalIP, u)
 		if err == nil {
 			connectedFuse.Fuse(true)
 			backoff.SetGracePeriod()
@@ -260,6 +267,8 @@ func ServeTunnel(
 
 	err = errGroup.Wait()
 	if err != nil {
+		_ = newClientRegisterTunnelError(err, config.Metrics.regFail)
+
 		switch castedErr := err.(type) {
 		case dupConnRegisterTunnelError:
 			logger.Info("Already connected to this server, selecting a different one")
@@ -274,14 +283,12 @@ func ServeTunnel(
 			return castedErr.cause, !castedErr.permanent
 		case clientRegisterTunnelError:
 			logger.WithError(castedErr.cause).Error("Register tunnel error on client side")
-			raven.CaptureError(castedErr.cause, tags)
 			return err, true
 		case muxerShutdownError:
 			logger.Infof("Muxer shutdown")
 			return err, true
 		default:
 			logger.WithError(err).Error("Serve tunnel error")
-			raven.CaptureError(err, tags)
 			return err, true
 		}
 	}
@@ -302,6 +309,7 @@ func RegisterTunnel(
 	ctx context.Context,
 	muxer *h2mux.Muxer,
 	config *TunnelConfig,
+	logger *log.Entry,
 	connectionID uint8,
 	originLocalIP string,
 	uuid uuid.UUID,
@@ -333,13 +341,13 @@ func RegisterTunnel(
 		config.Hostname,
 		config.RegistrationOptions(connectionID, originLocalIP, uuid),
 	)
-	LogServerInfo(serverInfoPromise.Result(), connectionID, config.Metrics, config.Logger)
+	LogServerInfo(serverInfoPromise.Result(), connectionID, config.Metrics, logger)
 	if err != nil {
 		// RegisterTunnel RPC failure
 		return newClientRegisterTunnelError(err, config.Metrics.regFail)
 	}
 	for _, logLine := range registration.LogLines {
-		config.Logger.Info(logLine)
+		logger.Info(logLine)
 	}
 
 	if regErr := processRegisterTunnelError(registration.Err, registration.PermanentFailure, config.Metrics); regErr != nil {
@@ -348,24 +356,24 @@ func RegisterTunnel(
 
 	if registration.TunnelID != "" {
 		config.Metrics.tunnelsHA.AddTunnelID(connectionID, registration.TunnelID)
-		config.Logger.Infof("Each HA connection's tunnel IDs: %v", config.Metrics.tunnelsHA.String())
+		logger.Infof("Each HA connection's tunnel IDs: %v", config.Metrics.tunnelsHA.String())
 	}
 
 	// Print out the user's trial zone URL in a nice box (if they requested and got one)
 	if isTrialTunnel := config.Hostname == ""; isTrialTunnel {
 		if url, err := url.Parse(registration.Url); err == nil {
 			for _, line := range asciiBox(trialZoneMsg(url.String()), 2) {
-				config.Logger.Infoln(line)
+				logger.Infoln(line)
 			}
 		} else {
-			config.Logger.Errorln("Failed to connect tunnel, please try again.")
+			logger.Errorln("Failed to connect tunnel, please try again.")
 			return fmt.Errorf("empty URL in response from Cloudflare edge")
 		}
 	}
 
 	config.Metrics.userHostnamesCounts.WithLabelValues(registration.Url).Inc()
 
-	config.Logger.Infof("Route propagating, it may take up to 1 minute for your new route to become functional")
+	logger.Infof("Route propagating, it may take up to 1 minute for your new route to become functional")
 	return nil
 }
 
@@ -421,7 +429,7 @@ func LogServerInfo(
 	promise tunnelrpc.ServerInfo_Promise,
 	connectionID uint8,
 	metrics *TunnelMetrics,
-	logger *log.Logger,
+	logger *log.Entry,
 ) {
 	serverInfoMessage, err := promise.Struct()
 	if err != nil {

publish-homebrew-formula.sh

@@ -1,31 +0,0 @@
-#!/bin/bash
-
-FILENAME=$1
-VERSION=$2
-TAP_ROOT=$3
-URL="https://developers.cloudflare.com/argo-tunnel/dl/cloudflared-${VERSION}-darwin-amd64.tgz"
-SHA256=$(sha256sum -b "${FILENAME}" | cut -b1-64)
-
-cd "${TAP_ROOT}" || exit 1
-git checkout -f master
-git reset --hard origin/master
-
-tee cloudflared.rb <<EOF
-class Cloudflared < Formula
-  desc 'Argo Tunnel'
-  homepage 'https://developers.cloudflare.com/argo-tunnel/'
-  url '${URL}'
-  sha256 '${SHA256}'
-  version '${VERSION}'
-  def install
-    bin.install 'cloudflared'
-  end
-end
-EOF
-git add cloudflared.rb
-
-git config user.name "cloudflare-warp-bot"
-git config user.email "warp-bot@cloudflare.com"
-git commit -m "Release Argo Tunnel ${VERSION}"
-git version
-GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=../github_known_hosts" git push -v origin master

ssh_server_tests/Dockerfile

@@ -0,0 +1,14 @@
+FROM python:3-buster
+
+RUN wget https://bin.equinox.io/c/VdrWdbjqyF/cloudflared-stable-linux-amd64.deb \
+  && dpkg -i cloudflared-stable-linux-amd64.deb
+
+RUN pip install pexpect
+
+COPY tests.py .
+COPY ssh /root/.ssh
+RUN chmod 600 /root/.ssh/id_rsa
+
+ARG SSH_HOSTNAME
+RUN bash -c 'sed -i "s/{{hostname}}/${SSH_HOSTNAME}/g" /root/.ssh/authorized_keys_config'
+RUN bash -c 'sed -i "s/{{hostname}}/${SSH_HOSTNAME}/g" /root/.ssh/short_lived_cert_config'

ssh_server_tests/README.md

@@ -0,0 +1,23 @@
+# Cloudflared SSH server smoke tests
+
+Runs several tests in a docker container against a server that is started out of band of these tests.
+Cloudflared token also needs to be retrieved out of band.
+SSH server hostname and user need to be configured in a docker environment file
+
+
+## Running tests
+
+* Build cloudflared:
+make cloudflared
+
+* Start server:
+sudo ./cloudflared tunnel --hostname HOSTNAME --ssh-server
+
+* Fetch token: 
+./cloudflared access login HOSTNAME
+
+* Create docker env file:
+echo "SSH_HOSTNAME=HOSTNAME\nSSH_USER=USERNAME\n" > ssh_server_tests/.env
+
+* Run tests:
+make test-ssh-server

ssh_server_tests/docker-compose.yml

@@ -0,0 +1,18 @@
+version: "3.1"
+
+services:
+  ssh_test:
+    build:
+      context: .
+      args:
+        - SSH_HOSTNAME=${SSH_HOSTNAME}
+    volumes:
+      - "~/.cloudflared/:/root/.cloudflared"
+    env_file:
+     - .env
+    environment:
+      - AUTHORIZED_KEYS_SSH_CONFIG=/root/.ssh/authorized_keys_config
+      - SHORT_LIVED_CERT_SSH_CONFIG=/root/.ssh/short_lived_cert_config
+      - REMOTE_SCP_FILENAME=scp_test.txt
+      - ROOT_ONLY_TEST_FILE_PATH=~/permission_test.txt
+    entrypoint: "python tests.py"

ssh_server_tests/ssh/authorized_keys_config

@@ -0,0 +1,5 @@
+Host *
+    AddressFamily inet
+
+Host {{hostname}}
+  ProxyCommand /usr/local/bin/cloudflared access ssh --hostname %h

ssh_server_tests/ssh/id_rsa

@@ -0,0 +1,49 @@
+-----BEGIN OPENSSH PRIVATE KEY-----
+b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAACFwAAAAdzc2gtcn
+NhAAAAAwEAAQAAAgEAvi26NDQ8cYTTztqPe9ZgF5HR/rIo5FoDgL5NbbZKW6h0txP9Fd8s
+id9Bgmo+aGkeM327tPVVMQ6UFmdRksOCIDWQDjNLF8b6S+Fu95tvMKSbGreRoR32OvgZKV
+I6KmOsF4z4GIv9naPplZswtKEUhSSI+/gPdAs9wfwalqZ77e82QJ727bYMeC3lzuoT+KBI
+dYufJ4OQhLtpHrqhB5sn7s6+oCv/u85GSln5SIC18Hi2t9lW4tgb5tH8P0kEDDWGfPS5ok
+qGi4kFTvwBXOCS2r4dhi5hRkpP7PqG4np0OCfvK5IRRJ27fCnj0loc+puZJAxnPMbuJr64
+vwxRx78PM/V0PDUsl0P6aR/vbe0XmF9FGqbWf2Tar1p4r6C9/bMzcDz8seYT8hzLIHP3+R
+l1hdlsTLm+1EzhaExKId+tjXegKGG4nU24h6qHEnRxLQDMwEsdkfj4E1pVypZJXVyNj99D
+o84vi0EUnu7R4HmQb/C+Pu7qMDtLT3Zk7O5Mg4LQ+cTz9V0noYEAyG46nAB4U/nJzBnV1J
++aAdpioHmUAYhLYlQ9Kiy7LCJi92g9Wqa4wxMKxBbO5ZeH++p2p2lUi/oQNqx/2dLYFmy0
+wxvJHbZIhAaFbOeCvHg1ucIAQznli2jOr2qoB+yKRRPAp/3NXnZg1v7ce2CkwiAD52wjtC
+kAAAdILMJUeyzCVHsAAAAHc3NoLXJzYQAAAgEAvi26NDQ8cYTTztqPe9ZgF5HR/rIo5FoD
+gL5NbbZKW6h0txP9Fd8sid9Bgmo+aGkeM327tPVVMQ6UFmdRksOCIDWQDjNLF8b6S+Fu95
+tvMKSbGreRoR32OvgZKVI6KmOsF4z4GIv9naPplZswtKEUhSSI+/gPdAs9wfwalqZ77e82
+QJ727bYMeC3lzuoT+KBIdYufJ4OQhLtpHrqhB5sn7s6+oCv/u85GSln5SIC18Hi2t9lW4t
+gb5tH8P0kEDDWGfPS5okqGi4kFTvwBXOCS2r4dhi5hRkpP7PqG4np0OCfvK5IRRJ27fCnj
+0loc+puZJAxnPMbuJr64vwxRx78PM/V0PDUsl0P6aR/vbe0XmF9FGqbWf2Tar1p4r6C9/b
+MzcDz8seYT8hzLIHP3+Rl1hdlsTLm+1EzhaExKId+tjXegKGG4nU24h6qHEnRxLQDMwEsd
+kfj4E1pVypZJXVyNj99Do84vi0EUnu7R4HmQb/C+Pu7qMDtLT3Zk7O5Mg4LQ+cTz9V0noY
+EAyG46nAB4U/nJzBnV1J+aAdpioHmUAYhLYlQ9Kiy7LCJi92g9Wqa4wxMKxBbO5ZeH++p2
+p2lUi/oQNqx/2dLYFmy0wxvJHbZIhAaFbOeCvHg1ucIAQznli2jOr2qoB+yKRRPAp/3NXn
+Zg1v7ce2CkwiAD52wjtCkAAAADAQABAAACAQCbnVsyAFQ9J00Rg/HIiUATyTQlzq57O9SF
+8jH1RiZOHedzLx32WaleH5rBFiJ+2RTnWUjQ57aP77fpJR2wk93UcT+w/vPBPwXsNUjRvx
+Qan3ZzRCYbyiKDWiNslmYV7X0RwD36CAK8jTVDP7t48h2SXLTiSLaMY+5i3uD6yLu7k/O2
+qNyw4jgN1rCmwQ8acD0aQec3NAZ7NcbsaBX/3Uutsup0scwOZtlJWZoLY5Z8cKpCgcsAz4
+j1NHnNZvey7dFgSffj/ktdvf7kBH0w/GnuJ4aNF0Jte70u0kiw5TZYBQVFh74tgUu6a6SJ
+qUbxIYUL5EJNjxGsDn+phHEemw3aMv0CwZG6Tqaionlna7bLsl9Bg1HTGclczVWx8uqC+M
+6agLmkhYCHG0rVj8h5smjXAQXtmvIDVYDOlJZZoF9VAOCj6QfmJUH1NAGpCs1HDHbeOxGA
+OLCh4d3F4rScPqhGdtSt4W13VFIvXn2Qqoz9ufepZsee1SZqpcerxywx2wN9ZAzu+X8lTN
+i+TA2B3vWpqqucOEsp4JwDN+VMKZqKUGUDWcm/eHSaG6wq0q734LUlgM85TjaIg8QsNtWV
+giB1nWwsYIuH4rsFNFGEwURYdGBcw6idH0GZ7I4RaIB5F9oOza1d601E0APHYrtnx9yOiK
+nOtJ+5ZmVZovaDRfu1aQAAAQBU/EFaNUzoVhO04pS2L6BlByt963bOIsSJhdlEzek5AAli
+eaf1S/PD6xWCc0IGY+GZE0HPbhsKYanjqOpWldcA2T7fzf4oz4vFBfUkPYo/MLSlLCYsDd
+IH3wBkCssnfR5EkzNgxnOvq646Nl64BMvxwSIXGPktdq9ZALxViwricSRzCFURnh5vLHWU
+wBzSgAA0UlZ9E64GtAv066+AoZCp83GhTLRC4o0naE2e/K4op4BCFHLrZ8eXmDRK3NJj80
+Vkn+uhrk+SHmbjIhmS57Pv9p8TWyRvemph/nMUuZGKBUu2X+JQxggck0KigIrXjsmciCsM
+BIM3mYDDfjYbyVhTAAABAQDkV8O1bWUsAIqk7RU+iDZojN5kaO+zUvj1TafX8QX1sY6pu4
+Z2cfSEka1532BaehM95bQm7BCPw4cYg56XidmCQTZ9WaWqxVrOo48EKXUtZMZx6nKFOKlq
+MT2XTMnGT9n7kFCfEjSVkAjuJ9ZTFLOaoXAaVRnxeHQwOKaup5KKP9GSzNIw328U+96s3V
+WKHeT4pMjHBccgW/qX/tRRidZw5in5uBC9Ew5y3UACFTkNOnhUwVfyUNbBZJ2W36msQ3KD
+AN7nOrQHqhd3NFyCEy2ovIAKVBacr/VEX6EsRUshIehJzz8EY9f3kXL7WT2QDoz2giPeBJ
+HJdEpXt43UpszjAAABAQDVNpqNdHUlCs9XnbIvc6ZRrNh79wt65YFfvh/QEuA33KnA6Ri6
+EgnV5IdUWXS/UFaYcm2udydrBpVIVifSYl3sioHBylpri23BEy38PKwVXvghUtfpN6dWGn
+NZUG25fQPtIzqi+lo953ZjIj+Adi17AeVv4P4NiLrZeM9lXfWf2pEPOecxXs1IwAf9IiDQ
+WepAwRLsu42eEnHA+DSJPZUkSbISfM5X345k0g6EVATX/yLL3CsqClPzPtsqjh6rbEfFg3
+2OfIMcWV77gOlGWGQ+bUHc8kV6xJqV9QVacLWzfLvIqHF0wQMf8WLOVHEzkfiq4VjwhVqr
+/+FFvljm5nSDAAAAEW1pa2VAQzAyWTUwVEdKR0g4AQ==
+-----END OPENSSH PRIVATE KEY-----

ssh_server_tests/ssh/id_rsa.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC+Lbo0NDxxhNPO2o971mAXkdH+sijkWgOAvk1ttkpbqHS3E/0V3yyJ30GCaj5oaR4zfbu09VUxDpQWZ1GSw4IgNZAOM0sXxvpL4W73m28wpJsat5GhHfY6+BkpUjoqY6wXjPgYi/2do+mVmzC0oRSFJIj7+A90Cz3B/BqWpnvt7zZAnvbttgx4LeXO6hP4oEh1i58ng5CEu2keuqEHmyfuzr6gK/+7zkZKWflIgLXweLa32Vbi2Bvm0fw/SQQMNYZ89LmiSoaLiQVO/AFc4JLavh2GLmFGSk/s+obienQ4J+8rkhFEnbt8KePSWhz6m5kkDGc8xu4mvri/DFHHvw8z9XQ8NSyXQ/ppH+9t7ReYX0UaptZ/ZNqvWnivoL39szNwPPyx5hPyHMsgc/f5GXWF2WxMub7UTOFoTEoh362Nd6AoYbidTbiHqocSdHEtAMzASx2R+PgTWlXKlkldXI2P30Ojzi+LQRSe7tHgeZBv8L4+7uowO0tPdmTs7kyDgtD5xPP1XSehgQDIbjqcAHhT+cnMGdXUn5oB2mKgeZQBiEtiVD0qLLssImL3aD1aprjDEwrEFs7ll4f76nanaVSL+hA2rH/Z0tgWbLTDG8kdtkiEBoVs54K8eDW5wgBDOeWLaM6vaqgH7IpFE8Cn/c1edmDW/tx7YKTCIAPnbCO0KQ== mike@C02Y50TGJGH8

ssh_server_tests/ssh/short_lived_cert_config

@@ -0,0 +1,11 @@
+Host *
+    AddressFamily inet
+
+Host {{hostname}}
+  ProxyCommand bash -c '/usr/local/bin/cloudflared access ssh-gen --hostname %h; ssh -F /root/.ssh/short_lived_cert_config -tt %r@cfpipe-{{hostname}} >&2 <&1'
+
+Host cfpipe-{{hostname}}
+  HostName {{hostname}}
+  ProxyCommand /usr/local/bin/cloudflared access ssh --hostname %h
+  IdentityFile ~/.cloudflared/{{hostname}}-cf_key
+  CertificateFile ~/.cloudflared/{{hostname}}-cf_key-cert.pub

ssh_server_tests/tests.py

@@ -0,0 +1,195 @@
+"""
+Cloudflared Integration tests
+"""
+
+import unittest
+import subprocess
+import os
+import tempfile
+from contextlib import contextmanager
+
+from pexpect import pxssh
+
+
+class TestSSHBase(unittest.TestCase):
+    """
+    SSH test base class containing constants and helper funcs
+    """
+
+    HOSTNAME = os.environ["SSH_HOSTNAME"]
+    SSH_USER = os.environ["SSH_USER"]
+    SSH_TARGET = f"{SSH_USER}@{HOSTNAME}"
+    AUTHORIZED_KEYS_SSH_CONFIG = os.environ["AUTHORIZED_KEYS_SSH_CONFIG"]
+    SHORT_LIVED_CERT_SSH_CONFIG = os.environ["SHORT_LIVED_CERT_SSH_CONFIG"]
+    SSH_OPTIONS = {"StrictHostKeyChecking": "no"}
+
+    @classmethod
+    def get_ssh_command(cls, pty=True):
+        """
+        Return ssh command arg list. If pty is true, a PTY is forced for the session.
+        """
+        cmd = [
+            "ssh",
+            "-o",
+            "StrictHostKeyChecking=no",
+            "-F",
+            cls.AUTHORIZED_KEYS_SSH_CONFIG,
+            cls.SSH_TARGET,
+        ]
+        if not pty:
+            cmd += ["-T"]
+        else:
+            cmd += ["-tt"]
+
+        return cmd
+
+    @classmethod
+    @contextmanager
+    def ssh_session_manager(cls, *args, **kwargs):
+        """
+        Context manager for interacting with a pxssh session.
+        Disables pty echo on the remote server and ensures session is terminated afterward.
+        """
+        session = pxssh.pxssh(options=cls.SSH_OPTIONS)
+
+        session.login(
+            cls.HOSTNAME,
+            username=cls.SSH_USER,
+            original_prompt=r"[#@$]",
+            ssh_config=kwargs.get("ssh_config", cls.AUTHORIZED_KEYS_SSH_CONFIG),
+            ssh_tunnels=kwargs.get("ssh_tunnels", {}),
+        )
+        try:
+            session.sendline("stty -echo")
+            session.prompt()
+            yield session
+        finally:
+            session.logout()
+
+    @staticmethod
+    def get_command_output(session, cmd):
+        """
+        Executes command on remote ssh server and waits for prompt.
+        Returns command output
+        """
+        session.sendline(cmd)
+        session.prompt()
+        return session.before.decode().strip()
+
+    def exec_command(self, cmd, shell=False):
+        """
+        Executes command locally. Raises Assertion error for non-zero return code.
+        Returns stdout and stderr
+        """
+        proc = subprocess.Popen(
+            cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=shell
+        )
+        raw_out, raw_err = proc.communicate()
+
+        out = raw_out.decode()
+        err = raw_err.decode()
+        self.assertEqual(proc.returncode, 0, msg=f"stdout: {out} stderr: {err}")
+        return out.strip(), err.strip()
+
+
+class TestSSHCommandExec(TestSSHBase):
+    """
+    Tests inline ssh command exec
+    """
+
+    # Name of file to be downloaded over SCP on remote server.
+    REMOTE_SCP_FILENAME = os.environ["REMOTE_SCP_FILENAME"]
+
+    @classmethod
+    def get_scp_base_command(cls):
+        return [
+            "scp",
+            "-o",
+            "StrictHostKeyChecking=no",
+            "-v",
+            "-F",
+            cls.AUTHORIZED_KEYS_SSH_CONFIG,
+        ]
+
+    @unittest.skip(
+        "This creates files on the remote. Should be skipped until server is dockerized."
+    )
+    def test_verbose_scp_sink_mode(self):
+        with tempfile.NamedTemporaryFile() as fl:
+            self.exec_command(
+                self.get_scp_base_command() + [fl.name, f"{self.SSH_TARGET}:"]
+            )
+
+    def test_verbose_scp_source_mode(self):
+        with tempfile.TemporaryDirectory() as tmpdirname:
+            self.exec_command(
+                self.get_scp_base_command()
+                + [f"{self.SSH_TARGET}:{self.REMOTE_SCP_FILENAME}", tmpdirname]
+            )
+            local_filename = os.path.join(tmpdirname, self.REMOTE_SCP_FILENAME)
+
+            self.assertTrue(os.path.exists(local_filename))
+            self.assertTrue(os.path.getsize(local_filename) > 0)
+
+    def test_pty_command(self):
+        base_cmd = self.get_ssh_command()
+
+        out, _ = self.exec_command(base_cmd + ["whoami"])
+        self.assertEqual(out.strip().lower(), self.SSH_USER.lower())
+
+        out, _ = self.exec_command(base_cmd + ["tty"])
+        self.assertNotEqual(out, "not a tty")
+
+    def test_non_pty_command(self):
+        base_cmd = self.get_ssh_command(pty=False)
+
+        out, _ = self.exec_command(base_cmd + ["whoami"])
+        self.assertEqual(out.strip().lower(), self.SSH_USER.lower())
+
+        out, _ = self.exec_command(base_cmd + ["tty"])
+        self.assertEqual(out, "not a tty")
+
+
+class TestSSHShell(TestSSHBase):
+    """
+    Tests interactive SSH shell
+    """
+
+    # File path to a file on the remote server with root only read privileges.
+    ROOT_ONLY_TEST_FILE_PATH = os.environ["ROOT_ONLY_TEST_FILE_PATH"]
+
+    def test_ssh_pty(self):
+        with self.ssh_session_manager() as session:
+
+            # Test shell launched as correct user
+            username = self.get_command_output(session, "whoami")
+            self.assertEqual(username.lower(), self.SSH_USER.lower())
+
+            # Test USER env variable set
+            user_var = self.get_command_output(session, "echo $USER")
+            self.assertEqual(user_var.lower(), self.SSH_USER.lower())
+
+            # Test HOME env variable set to true user home.
+            home_env = self.get_command_output(session, "echo $HOME")
+            pwd = self.get_command_output(session, "pwd")
+            self.assertEqual(pwd, home_env)
+
+            # Test shell launched in correct user home dir.
+            self.assertIn(username, pwd)
+
+            # Ensure shell launched with correct user's permissions and privs.
+            # Cant read root owned 0700 files.
+            output = self.get_command_output(
+                session, f"cat {self.ROOT_ONLY_TEST_FILE_PATH}"
+            )
+            self.assertIn("Permission denied", output)
+
+    def test_short_lived_cert_auth(self):
+        with self.ssh_session_manager(
+            ssh_config=self.SHORT_LIVED_CERT_SSH_CONFIG
+        ) as session:
+            username = self.get_command_output(session, "whoami")
+            self.assertEqual(username.lower(), self.SSH_USER.lower())
+
+
+unittest.main()

sshgen/sshgen.go

@@ -8,7 +8,6 @@ import (
 	"crypto/x509"
 	"encoding/json"
 	"encoding/pem"
-	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -20,6 +19,7 @@ import (
 	cfpath "github.com/cloudflare/cloudflared/cmd/cloudflared/path"
 	"github.com/coreos/go-oidc/jose"
 	homedir "github.com/mitchellh/go-homedir"
+	"github.com/pkg/errors"
 	gossh "golang.org/x/crypto/ssh"
 )
 
@@ -73,48 +73,54 @@ func GenerateShortLivedCertificate(appURL *url.URL, token string) error {
 // handleCertificateGeneration takes a JWT and uses it build a signPayload
 // to send to the Sign endpoint with the public key from the keypair it generated
 func handleCertificateGeneration(token, fullName string) (string, error) {
+	pub, err := generateKeyPair(fullName)
+	if err != nil {
+		return "", err
+	}
+
+	return SignCert(token, string(pub))
+}
+
+func SignCert(token, pubKey string) (string, error) {
 	if token == "" {
 		return "", errors.New("invalid token")
 	}
 
 	jwt, err := jose.ParseJWT(token)
 	if err != nil {
-		return "", err
+		return "", errors.Wrap(err, "failed to parse JWT")
 	}
 
 	claims, err := jwt.Claims()
 	if err != nil {
-		return "", err
+		return "", errors.Wrap(err, "failed to retrieve JWT claims")
 	}
 
 	issuer, _, err := claims.StringClaim("iss")
 	if err != nil {
-		return "", err
-	}
-
-	pub, err := generateKeyPair(fullName)
-	if err != nil {
-		return "", err
+		return "", errors.Wrap(err, "failed to retrieve JWT iss")
 	}
 
 	buf, err := json.Marshal(&signPayload{
-		PublicKey: string(pub),
+		PublicKey: pubKey,
 		JWT:       token,
 		Issuer:    issuer,
 	})
 	if err != nil {
-		return "", err
+		return "", errors.Wrap(err, "failed to marshal signPayload")
 	}
-
 	var res *http.Response
 	if mockRequest != nil {
 		res, err = mockRequest(issuer+signEndpoint, "application/json", bytes.NewBuffer(buf))
 	} else {
-		res, err = http.Post(issuer+signEndpoint, "application/json", bytes.NewBuffer(buf))
+		client := http.Client{
+			Timeout: 10 * time.Second,
+		}
+		res, err = client.Post(issuer+signEndpoint, "application/json", bytes.NewBuffer(buf))
 	}
 
 	if err != nil {
-		return "", err
+		return "", errors.Wrap(err, "failed to send request")
 	}
 	defer res.Body.Close()
 
@@ -130,9 +136,9 @@ func handleCertificateGeneration(token, fullName string) (string, error) {
 
 	var signRes signResponse
 	if err := decoder.Decode(&signRes); err != nil {
-		return "", err
+		return "", errors.Wrap(err, "failed to decode HTTP response")
 	}
-	return signRes.Certificate, err
+	return signRes.Certificate, nil
 }
 
 // generateKeyPair creates a EC keypair (P256) and stores them in the homedir.

sshlog/empty_manager.go

@@ -0,0 +1,37 @@
+package sshlog
+
+import (
+	"io"
+
+	"github.com/sirupsen/logrus"
+)
+
+//empty manager implements the Manager but does nothing (for testing and to disable logging unless the logs are set)
+type emptyManager struct {
+}
+
+type emptyWriteCloser struct {
+}
+
+// NewEmptyManager creates a new instance of a log empty log manager that does nothing
+func NewEmptyManager() Manager {
+	return &emptyManager{}
+}
+
+func (m *emptyManager) NewLogger(name string, logger *logrus.Logger) (io.WriteCloser, error) {
+	return &emptyWriteCloser{}, nil
+}
+
+func (m *emptyManager) NewSessionLogger(name string, logger *logrus.Logger) (io.WriteCloser, error) {
+	return &emptyWriteCloser{}, nil
+}
+
+// emptyWriteCloser
+
+func (w *emptyWriteCloser) Write(p []byte) (n int, err error) {
+	return len(p), nil
+}
+
+func (w *emptyWriteCloser) Close() error {
+	return nil
+}

sshlog/go.capnp

@@ -0,0 +1,15 @@
+# Generate go.capnp.out with:
+# capnp compile -o- go.capnp > go.capnp.out
+# Must run inside this directory to preserve paths.
+
+@0xd12a1c51fedd6c88;
+
+annotation package(file) :Text;
+annotation import(file) :Text;
+annotation doc(struct, field, enum) :Text;
+annotation tag(enumerant) :Text;
+annotation notag(enumerant) :Void;
+annotation customtype(field) :Text;
+annotation name(struct, field, union, enum, enumerant, interface, method, param, annotation, const, group) :Text;
+
+$package("capnp");

sshlog/logger.go

@@ -0,0 +1,167 @@
+package sshlog
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sync"
+	"time"
+
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	logTimeFormat        = "2006-01-02T15-04-05.000"
+	megabyte             = 1024 * 1024
+	defaultFileSizeLimit = 100 * megabyte
+)
+
+// Logger will buffer and write events to disk
+type Logger struct {
+	sync.Mutex
+	filename      string
+	file          *os.File
+	writeBuffer   *bufio.Writer
+	logger        *logrus.Logger
+	flushInterval time.Duration
+	maxFileSize   int64
+	done          chan struct{}
+	once          sync.Once
+}
+
+// NewLogger creates a Logger instance. A buffer is created that needs to be
+// drained and closed when the caller is finished, so instances should call
+// Close when finished with this Logger instance. Writes will be flushed to disk
+// every second (fsync). filename is the name of the logfile to be created. The
+// logger variable is a logrus that will log all i/o, filesystem error etc, that
+// that shouldn't end execution of the logger, but are useful to report to the
+// caller.
+func NewLogger(filename string, logger *logrus.Logger, flushInterval time.Duration, maxFileSize int64) (*Logger, error) {
+	if logger == nil {
+		return nil, errors.New("logger can't be nil")
+	}
+	f, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(0600))
+	if err != nil {
+		return nil, err
+	}
+	l := &Logger{filename: filename,
+		file:          f,
+		writeBuffer:   bufio.NewWriter(f),
+		logger:        logger,
+		flushInterval: flushInterval,
+		maxFileSize:   maxFileSize,
+		done:          make(chan struct{}),
+	}
+
+	go l.writer()
+	return l, nil
+}
+
+// Writes to a log buffer. Implements the io.Writer interface.
+func (l *Logger) Write(p []byte) (n int, err error) {
+	l.Lock()
+	defer l.Unlock()
+	return l.writeBuffer.Write(p)
+}
+
+// Close drains anything left in the buffer and cleans up any resources still
+// in use.
+func (l *Logger) Close() error {
+	l.once.Do(func() {
+		close(l.done)
+	})
+	if err := l.write(); err != nil {
+		return err
+	}
+	return l.file.Close()
+}
+
+// writer is the run loop that handles draining the write buffer and syncing
+// data to disk.
+func (l *Logger) writer() {
+	ticker := time.NewTicker(l.flushInterval)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ticker.C:
+			if err := l.write(); err != nil {
+				l.logger.Errorln(err)
+			}
+		case <-l.done:
+			return
+		}
+	}
+}
+
+// write does the actual system write calls to disk and does a rotation if the
+// file size limit has been reached. Since the rotation happens at the end,
+// the rotation is a soft limit (aka the file can be bigger than the max limit
+// because of the final buffer flush)
+func (l *Logger) write() error {
+	l.Lock()
+	defer l.Unlock()
+
+	if l.writeBuffer.Buffered() <= 0 {
+		return nil
+	}
+
+	if err := l.writeBuffer.Flush(); err != nil {
+		return err
+	}
+
+	if err := l.file.Sync(); err != nil {
+		return err
+	}
+
+	if l.shouldRotate() {
+		return l.rotate()
+	}
+	return nil
+}
+
+// shouldRotate checks to see if the current file should be rotated to a new
+// logfile.
+func (l *Logger) shouldRotate() bool {
+	info, err := l.file.Stat()
+	if err != nil {
+		return false
+	}
+
+	return info.Size() >= l.maxFileSize
+}
+
+// rotate creates a new logfile with the existing filename and renames the
+// existing file with a current timestamp.
+func (l *Logger) rotate() error {
+	if err := l.file.Close(); err != nil {
+		return err
+	}
+
+	// move the existing file
+	newname := rotationName(l.filename)
+	if err := os.Rename(l.filename, newname); err != nil {
+		return fmt.Errorf("can't rename log file: %s", err)
+	}
+
+	f, err := os.OpenFile(l.filename, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(0600))
+	if err != nil {
+		return fmt.Errorf("failed to open new logfile %s", err)
+	}
+	l.file = f
+	l.writeBuffer = bufio.NewWriter(f)
+	return nil
+}
+
+// rotationName creates a new filename from the given name, inserting a timestamp
+// between the filename and the extension.
+func rotationName(name string) string {
+	dir := filepath.Dir(name)
+	filename := filepath.Base(name)
+	ext := filepath.Ext(filename)
+	prefix := filename[:len(filename)-len(ext)]
+	t := time.Now()
+	timestamp := t.Format(logTimeFormat)
+	return filepath.Join(dir, fmt.Sprintf("%s-%s%s", prefix, timestamp, ext))
+}

sshlog/logger_test.go

@@ -0,0 +1,89 @@
+package sshlog
+
+import (
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/sirupsen/logrus"
+)
+
+const logFileName = "test-logger.log"
+
+func createLogger(t *testing.T) *Logger {
+	os.Remove(logFileName)
+	l := logrus.New()
+	logger, err := NewLogger(logFileName, l, time.Millisecond, 1024)
+	if err != nil {
+		t.Fatal("couldn't create the logger!", err)
+	}
+	return logger
+}
+// AUTH-2115 TODO: fix this test
+//func TestWrite(t *testing.T) {
+//	testStr := "hi"
+//	logger := createLogger(t)
+//	defer func() {
+//		logger.Close()
+//		os.Remove(logFileName)
+//	}()
+//
+//	logger.Write([]byte(testStr))
+//	time.Sleep(2 * time.Millisecond)
+//	data, err := ioutil.ReadFile(logFileName)
+//	if err != nil {
+//		t.Fatal("couldn't read the log file!", err)
+//	}
+//	checkStr := string(data)
+//	if checkStr != testStr {
+//		t.Fatal("file data doesn't match!")
+//	}
+//}
+
+func TestFilenameRotation(t *testing.T) {
+	newName := rotationName("dir/bob/acoolloggername.log")
+
+	dir := filepath.Dir(newName)
+	if dir != "dir/bob" {
+		t.Fatal("rotation name doesn't respect the directory filepath:", newName)
+	}
+
+	filename := filepath.Base(newName)
+	if !strings.HasPrefix(filename, "acoolloggername") {
+		t.Fatal("rotation filename is wrong:", filename)
+	}
+
+	ext := filepath.Ext(newName)
+	if ext != ".log" {
+		t.Fatal("rotation file extension is wrong:", ext)
+	}
+}
+
+func TestRotation(t *testing.T) {
+	logger := createLogger(t)
+
+	for i := 0; i < 2000; i++ {
+		logger.Write([]byte("a string for testing rotation\n"))
+	}
+	logger.Close()
+
+	count := 0
+	filepath.Walk(".", func(path string, info os.FileInfo, err error) error {
+		if err != nil || info.IsDir() {
+			return nil
+		}
+		if strings.HasPrefix(info.Name(), "test-logger") {
+			log.Println("deleting: ", path)
+			os.Remove(path)
+			count++
+		}
+		return nil
+	})
+	if count < 2 {
+		t.Fatal("rotation didn't roll files:", count)
+	}
+
+}

sshlog/manager.go

@@ -0,0 +1,34 @@
+package sshlog
+
+import (
+	"io"
+	"path/filepath"
+	"time"
+
+	"github.com/sirupsen/logrus"
+)
+
+// Manager be managing logs bruh
+type Manager interface {
+	NewLogger(string, *logrus.Logger) (io.WriteCloser, error)
+	NewSessionLogger(string, *logrus.Logger) (io.WriteCloser, error)
+}
+
+type manager struct {
+	baseDirectory string
+}
+
+// New creates a new instance of a log manager
+func New(baseDirectory string) Manager {
+	return &manager{
+		baseDirectory: baseDirectory,
+	}
+}
+
+func (m *manager) NewLogger(name string, logger *logrus.Logger) (io.WriteCloser, error) {
+	return NewLogger(filepath.Join(m.baseDirectory, name), logger, time.Second, defaultFileSizeLimit)
+}
+
+func (m *manager) NewSessionLogger(name string, logger *logrus.Logger) (io.WriteCloser, error) {
+	return NewSessionLogger(filepath.Join(m.baseDirectory, name), logger, time.Second, defaultFileSizeLimit)
+}

sshlog/session_log.capnp

@@ -0,0 +1,9 @@
+using Go = import "go.capnp";
+@0x8f43375162194466;
+$Go.package("sshlog");
+$Go.import("github.com/cloudflare/cloudflared/sshlog");
+
+struct SessionLog {
+    timestamp @0 :Text;
+    content @1 :Data;
+}

sshlog/session_log.capnp.go

@@ -0,0 +1,110 @@
+// Code generated by capnpc-go. DO NOT EDIT.
+
+package sshlog
+
+import (
+	capnp "zombiezen.com/go/capnproto2"
+	text "zombiezen.com/go/capnproto2/encoding/text"
+	schemas "zombiezen.com/go/capnproto2/schemas"
+)
+
+type SessionLog struct{ capnp.Struct }
+
+// SessionLog_TypeID is the unique identifier for the type SessionLog.
+const SessionLog_TypeID = 0xa13a07c504a5ab64
+
+func NewSessionLog(s *capnp.Segment) (SessionLog, error) {
+	st, err := capnp.NewStruct(s, capnp.ObjectSize{DataSize: 0, PointerCount: 2})
+	return SessionLog{st}, err
+}
+
+func NewRootSessionLog(s *capnp.Segment) (SessionLog, error) {
+	st, err := capnp.NewRootStruct(s, capnp.ObjectSize{DataSize: 0, PointerCount: 2})
+	return SessionLog{st}, err
+}
+
+func ReadRootSessionLog(msg *capnp.Message) (SessionLog, error) {
+	root, err := msg.RootPtr()
+	return SessionLog{root.Struct()}, err
+}
+
+func (s SessionLog) String() string {
+	str, _ := text.Marshal(0xa13a07c504a5ab64, s.Struct)
+	return str
+}
+
+func (s SessionLog) Timestamp() (string, error) {
+	p, err := s.Struct.Ptr(0)
+	return p.Text(), err
+}
+
+func (s SessionLog) HasTimestamp() bool {
+	p, err := s.Struct.Ptr(0)
+	return p.IsValid() || err != nil
+}
+
+func (s SessionLog) TimestampBytes() ([]byte, error) {
+	p, err := s.Struct.Ptr(0)
+	return p.TextBytes(), err
+}
+
+func (s SessionLog) SetTimestamp(v string) error {
+	return s.Struct.SetText(0, v)
+}
+
+func (s SessionLog) Content() ([]byte, error) {
+	p, err := s.Struct.Ptr(1)
+	return []byte(p.Data()), err
+}
+
+func (s SessionLog) HasContent() bool {
+	p, err := s.Struct.Ptr(1)
+	return p.IsValid() || err != nil
+}
+
+func (s SessionLog) SetContent(v []byte) error {
+	return s.Struct.SetData(1, v)
+}
+
+// SessionLog_List is a list of SessionLog.
+type SessionLog_List struct{ capnp.List }
+
+// NewSessionLog creates a new list of SessionLog.
+func NewSessionLog_List(s *capnp.Segment, sz int32) (SessionLog_List, error) {
+	l, err := capnp.NewCompositeList(s, capnp.ObjectSize{DataSize: 0, PointerCount: 2}, sz)
+	return SessionLog_List{l}, err
+}
+
+func (s SessionLog_List) At(i int) SessionLog { return SessionLog{s.List.Struct(i)} }
+
+func (s SessionLog_List) Set(i int, v SessionLog) error { return s.List.SetStruct(i, v.Struct) }
+
+func (s SessionLog_List) String() string {
+	str, _ := text.MarshalList(0xa13a07c504a5ab64, s.List)
+	return str
+}
+
+// SessionLog_Promise is a wrapper for a SessionLog promised by a client call.
+type SessionLog_Promise struct{ *capnp.Pipeline }
+
+func (p SessionLog_Promise) Struct() (SessionLog, error) {
+	s, err := p.Pipeline.Struct()
+	return SessionLog{s}, err
+}
+
+const schema_8f43375162194466 = "x\xda\x120q`\x12d\x8dg`\x08dae\xfb" +
+	"\x9f\xb2z)\xcbQv\xab\x85\x0c\x82B\x8c\xff\xd3\\" +
+	"$\x93\x02\xcd\x9d\xfb\x19X\x99\xd8\x19\x18\x04E_\x09" +
+	"*\x82h\xd9r\x06\xc6\xff\xc5\xa9\xc5\xc5\x99\xf9y\xf1" +
+	"L9\xf9\xe9z\xc9\x89\x05y\x05V\xc1`!\xfe<" +
+	"\x9f\xfc\xf4\x00F\xc6@\x0ef\x16\x06\x06\x16F\x06\x06" +
+	"A\xcd \x06\x86@\x0df\xc6@\x13&FAFF" +
+	"\x11F\x90\xa0\xa1\x13\x03C\xa0\x0e3c\xa0\x05\x13\xe3" +
+	"\xff\x92\xcc\xdc\xd4\xe2\x92\xc4\\\x06\xc6\x02F\x1e\x06&" +
+	"F\x1e\x06\xc6\xfa\xe4\xfc\xbc\x92\xd4\xbc\x12F^\x06&" +
+	"F^\x06F@\x00\x00\x00\xff\xff\xdaK$\x1a"
+
+func init() {
+	schemas.Register(schema_8f43375162194466,
+		0xa13a07c504a5ab64)
+}

sshlog/session_logger.go

@@ -0,0 +1,71 @@
+package sshlog
+
+import (
+	"time"
+
+	"github.com/sirupsen/logrus"
+	capnp "zombiezen.com/go/capnproto2"
+	"zombiezen.com/go/capnproto2/pogs"
+)
+
+// SessionLogger will buffer and write events to disk using capnp proto for session replay
+type SessionLogger struct {
+	logger  *Logger
+	encoder *capnp.Encoder
+}
+
+type sessionLogData struct {
+	Timestamp string // The UTC timestamp of when the log occurred
+	Content   []byte // The shell output
+}
+
+// NewSessionLogger creates a new session logger by encapsulating a Logger object and writing capnp encoded messages to it
+func NewSessionLogger(filename string, logger *logrus.Logger, flushInterval time.Duration, maxFileSize int64) (*SessionLogger, error) {
+	l, err := NewLogger(filename, logger, flushInterval, maxFileSize)
+	if err != nil {
+		return nil, err
+	}
+	sessionLogger := &SessionLogger{
+		logger:  l,
+		encoder: capnp.NewEncoder(l),
+	}
+	return sessionLogger, nil
+}
+
+// Writes to a log buffer. Implements the io.Writer interface.
+func (l *SessionLogger) Write(p []byte) (n int, err error) {
+	return l.writeSessionLog(&sessionLogData{
+		Timestamp: time.Now().UTC().Format(time.RFC3339),
+		Content:   p,
+	})
+}
+
+// Close drains anything left in the buffer and cleans up any resources still
+// in use.
+func (l *SessionLogger) Close() error {
+	return l.logger.Close()
+}
+
+func (l *SessionLogger) writeSessionLog(p *sessionLogData) (int, error) {
+	msg, seg, err := capnp.NewMessage(capnp.SingleSegment(nil))
+	if err != nil {
+		return 0, err
+	}
+	log, err := NewRootSessionLog(seg)
+	if err != nil {
+		return 0, err
+	}
+	log.SetTimestamp(p.Timestamp)
+	log.SetContent(p.Content)
+
+	if err := l.encoder.Encode(msg); err != nil {
+		return 0, err
+	}
+	return len(p.Content), nil
+}
+
+func unmarshalSessionLog(s SessionLog) (*sessionLogData, error) {
+	p := new(sessionLogData)
+	err := pogs.Extract(p, SessionLog_TypeID, s.Struct)
+	return p, err
+}

sshlog/session_logger_test.go

@@ -0,0 +1,69 @@
+package sshlog
+
+import (
+	"os"
+	"testing"
+	"time"
+
+	"github.com/sirupsen/logrus"
+	capnp "zombiezen.com/go/capnproto2"
+)
+
+const sessionLogFileName = "test-session-logger.log"
+
+func createSessionLogger(t *testing.T) *SessionLogger {
+	os.Remove(sessionLogFileName)
+	l := logrus.New()
+	logger, err := NewSessionLogger(sessionLogFileName, l, time.Millisecond, 1024)
+	if err != nil {
+		t.Fatal("couldn't create the logger!", err)
+	}
+	return logger
+}
+
+func TestSessionLogWrite(t *testing.T) {
+	testStr := "hi"
+	logger := createSessionLogger(t)
+	defer func() {
+		logger.Close()
+		os.Remove(sessionLogFileName)
+	}()
+
+	logger.Write([]byte(testStr))
+	time.Sleep(2 * time.Millisecond)
+	f, err := os.Open(sessionLogFileName)
+	if err != nil {
+		t.Fatal("couldn't read the log file!", err)
+	}
+	defer f.Close()
+
+	msg, err := capnp.NewDecoder(f).Decode()
+	if err != nil {
+		t.Fatal("couldn't read the capnp msg file!", err)
+	}
+
+	sessionLog, err := ReadRootSessionLog(msg)
+	if err != nil {
+		t.Fatal("couldn't read the session log from the msg!", err)
+	}
+
+	timeStr, err := sessionLog.Timestamp()
+	if err != nil {
+		t.Fatal("couldn't read the Timestamp field!", err)
+	}
+
+	_, terr := time.Parse(time.RFC3339, timeStr)
+	if terr != nil {
+		t.Fatal("couldn't parse the Timestamp into the expected RFC3339 format", terr)
+	}
+
+	data, err := sessionLog.Content()
+	if err != nil {
+		t.Fatal("couldn't read the Content field!", err)
+	}
+
+	checkStr := string(data)
+	if checkStr != testStr {
+		t.Fatal("file data doesn't match!")
+	}
+}

sshserver/authentication.go

@@ -1,98 +0,0 @@
-//+build !windows
-
-package sshserver
-
-import (
-	"fmt"
-	"io/ioutil"
-	"os"
-	"path"
-
-	"github.com/gliderlabs/ssh"
-	"github.com/pkg/errors"
-	gossh "golang.org/x/crypto/ssh"
-)
-
-var (
-	systemConfigPath  = "/etc/cloudflared/"
-	authorizedKeysDir = ".cloudflared/authorized_keys"
-)
-
-func (s *SSHServer) authorizedKeyHandler(ctx ssh.Context, key ssh.PublicKey) bool {
-	sshUser, err := s.getUserFunc(ctx.User())
-	if err != nil {
-		s.logger.Debugf("Invalid user: %s", ctx.User())
-		return false
-	}
-
-	authorizedKeysPath := path.Join(sshUser.HomeDir, authorizedKeysDir)
-	if _, err := os.Stat(authorizedKeysPath); os.IsNotExist(err) {
-		s.logger.Debugf("authorized_keys file %s not found", authorizedKeysPath)
-		return false
-	}
-
-	authorizedKeysBytes, err := ioutil.ReadFile(authorizedKeysPath)
-	if err != nil {
-		s.logger.WithError(err).Errorf("Failed to load authorized_keys %s", authorizedKeysPath)
-		return false
-	}
-
-	for len(authorizedKeysBytes) > 0 {
-
-		// Skips invalid keys. Returns error if no valid keys remain.
-		pubKey, _, _, rest, err := ssh.ParseAuthorizedKey(authorizedKeysBytes)
-		authorizedKeysBytes = rest
-		if err != nil {
-			s.logger.Errorf("Invalid key(s) found in %s", authorizedKeysPath)
-			return false
-		}
-
-		if ssh.KeysEqual(pubKey, key) {
-			ctx.SetValue("sshUser", sshUser)
-			return true
-		}
-	}
-	s.logger.Debugf("Matching public key not found in %s", authorizedKeysPath)
-	return false
-}
-
-func (s *SSHServer) shortLivedCertHandler(ctx ssh.Context, key ssh.PublicKey) bool {
-	userCert, ok := key.(*gossh.Certificate)
-	if !ok {
-		s.logger.Debug("Received key is not an SSH certificate")
-		return false
-	}
-
-	if !ssh.KeysEqual(s.caCert, userCert.SignatureKey) {
-		s.logger.Debug("CA certificate does not match user certificate signer")
-		return false
-	}
-
-	checker := gossh.CertChecker{}
-	if err := checker.CheckCert(ctx.User(), userCert); err != nil {
-		s.logger.Debug(err)
-		return false
-	} else {
-		sshUser, err := s.getUserFunc(ctx.User())
-		if err != nil {
-			s.logger.Debugf("Invalid user: %s", ctx.User())
-			return false
-		}
-		ctx.SetValue("sshUser", sshUser)
-	}
-	return true
-}
-
-func getCACert() (ssh.PublicKey, error) {
-	caCertPath := path.Join(systemConfigPath, "ca.pub")
-	caCertBytes, err := ioutil.ReadFile(caCertPath)
-	if err != nil {
-		return nil, errors.Wrap(err, fmt.Sprintf("Failed to load CA certertificate %s", caCertPath))
-	}
-	caCert, _, _, _, err := ssh.ParseAuthorizedKey(caCertBytes)
-	if err != nil {
-		return nil, errors.Wrap(err, "Failed to parse CA Certificate")
-	}
-
-	return caCert, nil
-}

sshserver/authentication_test.go

@@ -1,192 +0,0 @@
-//+build !windows
-
-package sshserver
-
-import (
-	"context"
-	"io/ioutil"
-	"net"
-	"os"
-	"os/user"
-	"path"
-	"sync"
-	"testing"
-
-	"github.com/cloudflare/cloudflared/log"
-	"github.com/gliderlabs/ssh"
-	"github.com/sirupsen/logrus"
-	"github.com/sirupsen/logrus/hooks/test"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-const (
-	validPrincipal       = "testUser"
-	testDir              = "testdata"
-	testUserKeyFilename  = "id_rsa.pub"
-	testCAFilename       = "ca.pub"
-	testOtherCAFilename  = "other_ca.pub"
-	testUserCertFilename = "id_rsa-cert.pub"
-)
-
-var logger, hook = test.NewNullLogger()
-
-func TestMain(m *testing.M) {
-	authorizedKeysDir = testUserKeyFilename
-	logger.SetLevel(logrus.DebugLevel)
-	code := m.Run()
-	os.Exit(code)
-}
-
-func TestPublicKeyAuth_Success(t *testing.T) {
-	context, cancel := newMockContext(validPrincipal)
-	defer cancel()
-
-	sshServer := SSHServer{getUserFunc: getMockUser}
-
-	pubKey := getKey(t, testUserKeyFilename)
-	assert.True(t, sshServer.authorizedKeyHandler(context, pubKey))
-}
-
-func TestPublicKeyAuth_MissingKey(t *testing.T) {
-	context, cancel := newMockContext(validPrincipal)
-	defer cancel()
-
-	sshServer := SSHServer{logger: logger, getUserFunc: getMockUser}
-
-	pubKey := getKey(t, testOtherCAFilename)
-	assert.False(t, sshServer.authorizedKeyHandler(context, pubKey))
-	assert.Contains(t, hook.LastEntry().Message, "Matching public key not found in")
-}
-
-func TestPublicKeyAuth_InvalidUser(t *testing.T) {
-	context, cancel := newMockContext("notAUser")
-	defer cancel()
-
-	sshServer := SSHServer{logger: logger, getUserFunc: lookupUser}
-
-	pubKey := getKey(t, testUserKeyFilename)
-	assert.False(t, sshServer.authorizedKeyHandler(context, pubKey))
-	assert.Contains(t, hook.LastEntry().Message, "Invalid user")
-}
-
-func TestPublicKeyAuth_MissingFile(t *testing.T) {
-	currentUser, err := user.Current()
-	require.Nil(t, err)
-	context, cancel := newMockContext(currentUser.Username)
-	defer cancel()
-
-	sshServer := SSHServer{Server: ssh.Server{}, logger: logger, getUserFunc: lookupUser}
-
-	pubKey := getKey(t, testUserKeyFilename)
-	assert.False(t, sshServer.authorizedKeyHandler(context, pubKey))
-	assert.Contains(t, hook.LastEntry().Message, "not found")
-}
-
-func TestShortLivedCerts_Success(t *testing.T) {
-	context, cancel := newMockContext(validPrincipal)
-	defer cancel()
-
-	caCert := getKey(t, testCAFilename)
-	sshServer := SSHServer{logger: log.CreateLogger(), caCert: caCert, getUserFunc: getMockUser}
-
-	userCert := getKey(t, testUserCertFilename)
-	assert.True(t, sshServer.shortLivedCertHandler(context, userCert))
-}
-
-func TestShortLivedCerts_CAsDontMatch(t *testing.T) {
-	context, cancel := newMockContext(validPrincipal)
-	defer cancel()
-
-	caCert := getKey(t, testOtherCAFilename)
-	sshServer := SSHServer{logger: logger, caCert: caCert, getUserFunc: getMockUser}
-
-	userCert := getKey(t, testUserCertFilename)
-	assert.False(t, sshServer.shortLivedCertHandler(context, userCert))
-	assert.Equal(t, "CA certificate does not match user certificate signer", hook.LastEntry().Message)
-}
-
-func TestShortLivedCerts_UserDoesNotExist(t *testing.T) {
-	context, cancel := newMockContext(validPrincipal)
-	defer cancel()
-
-	caCert := getKey(t, testCAFilename)
-	sshServer := SSHServer{logger: logger, caCert: caCert, getUserFunc: lookupUser}
-
-	userCert := getKey(t, testUserCertFilename)
-	assert.False(t, sshServer.shortLivedCertHandler(context, userCert))
-	assert.Contains(t, hook.LastEntry().Message, "Invalid user")
-}
-
-func TestShortLivedCerts_InvalidPrincipal(t *testing.T) {
-	context, cancel := newMockContext("notAUser")
-	defer cancel()
-
-	caCert := getKey(t, testCAFilename)
-	sshServer := SSHServer{logger: logger, caCert: caCert, getUserFunc: lookupUser}
-
-	userCert := getKey(t, testUserCertFilename)
-	assert.False(t, sshServer.shortLivedCertHandler(context, userCert))
-	assert.Contains(t, hook.LastEntry().Message, "not in the set of valid principals for given certificate")
-}
-
-func getMockUser(_ string) (*User, error) {
-	return &User{
-		Username: validPrincipal,
-		HomeDir:  testDir,
-	}, nil
-
-}
-
-func getKey(t *testing.T, filename string) ssh.PublicKey {
-	path := path.Join(testDir, filename)
-	bytes, err := ioutil.ReadFile(path)
-	require.Nil(t, err)
-	pubKey, _, _, _, err := ssh.ParseAuthorizedKey(bytes)
-	require.Nil(t, err)
-	return pubKey
-}
-
-type mockSSHContext struct {
-	context.Context
-	*sync.Mutex
-}
-
-func newMockContext(user string) (*mockSSHContext, context.CancelFunc) {
-	innerCtx, cancel := context.WithCancel(context.Background())
-	mockCtx := &mockSSHContext{innerCtx, &sync.Mutex{}}
-	mockCtx.SetValue("user", user)
-	return mockCtx, cancel
-}
-
-func (ctx *mockSSHContext) SetValue(key, value interface{}) {
-	ctx.Context = context.WithValue(ctx.Context, key, value)
-}
-
-func (ctx *mockSSHContext) User() string {
-	return ctx.Value("user").(string)
-}
-
-func (ctx *mockSSHContext) SessionID() string {
-	return ""
-}
-
-func (ctx *mockSSHContext) ClientVersion() string {
-	return ""
-}
-
-func (ctx *mockSSHContext) ServerVersion() string {
-	return ""
-}
-
-func (ctx *mockSSHContext) RemoteAddr() net.Addr {
-	return nil
-}
-
-func (ctx *mockSSHContext) LocalAddr() net.Addr {
-	return nil
-}
-
-func (ctx *mockSSHContext) Permissions() *ssh.Permissions {
-	return nil
-}

sshserver/get_user.go

@@ -1,226 +0,0 @@
-// Taken from https://github.com/golang/go/blob/ad644d2e86bab85787879d41c2d2aebbd7c57db8/src/os/user/user.go
-// and modified to return login shell in User struct. cloudflared requires cgo for compilation because of this addition.
-
-// Copyright 2011 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build aix darwin dragonfly freebsd !android,linux netbsd openbsd solaris
-// +build cgo,!osusergo
-
-package sshserver
-
-import (
-	"fmt"
-	"strconv"
-	"strings"
-	"syscall"
-	"unsafe"
-)
-
-/*
-#cgo solaris CFLAGS: -D_POSIX_PTHREAD_SEMANTICS
-#include <unistd.h>
-#include <sys/types.h>
-#include <pwd.h>
-#include <grp.h>
-#include <stdlib.h>
-
-static int mygetpwuid_r(int uid, struct passwd *pwd,
-	char *buf, size_t buflen, struct passwd **result) {
-	return getpwuid_r(uid, pwd, buf, buflen, result);
-}
-
-static int mygetpwnam_r(const char *name, struct passwd *pwd,
-	char *buf, size_t buflen, struct passwd **result) {
-	return getpwnam_r(name, pwd, buf, buflen, result);
-}
-
-static int mygetgrgid_r(int gid, struct group *grp,
-	char *buf, size_t buflen, struct group **result) {
- return getgrgid_r(gid, grp, buf, buflen, result);
-}
-
-static int mygetgrnam_r(const char *name, struct group *grp,
-	char *buf, size_t buflen, struct group **result) {
- return getgrnam_r(name, grp, buf, buflen, result);
-}
-*/
-import "C"
-
-type UnknownUserIdError int
-
-func (e UnknownUserIdError) Error() string {
-	return "user: unknown userid " + strconv.Itoa(int(e))
-}
-
-// UnknownUserError is returned by Lookup when
-// a user cannot be found.
-type UnknownUserError string
-
-func (e UnknownUserError) Error() string {
-	return "user: unknown user " + string(e)
-}
-
-// UnknownGroupIdError is returned by LookupGroupId when
-// a group cannot be found.
-type UnknownGroupIdError string
-
-func (e UnknownGroupIdError) Error() string {
-	return "group: unknown groupid " + string(e)
-}
-
-// UnknownGroupError is returned by LookupGroup when
-// a group cannot be found.
-type UnknownGroupError string
-
-func (e UnknownGroupError) Error() string {
-	return "group: unknown group " + string(e)
-}
-
-type User struct {
-	// Uid is the user ID.
-	// On POSIX systems, this is a decimal number representing the uid.
-	// On Windows, this is a security identifier (SID) in a string format.
-	// On Plan 9, this is the contents of /dev/user.
-	Uid string
-	// Gid is the primary group ID.
-	// On POSIX systems, this is a decimal number representing the gid.
-	// On Windows, this is a SID in a string format.
-	// On Plan 9, this is the contents of /dev/user.
-	Gid string
-	// Username is the login name.
-	Username string
-	// Name is the user's real or display name.
-	// It might be blank.
-	// On POSIX systems, this is the first (or only) entry in the GECOS field
-	// list.
-	// On Windows, this is the user's display name.
-	// On Plan 9, this is the contents of /dev/user.
-	Name string
-	// HomeDir is the path to the user's home directory (if they have one).
-	HomeDir string
-
-	/****************** Begin added code ******************/
-	// Login shell
-	Shell string
-	/****************** End added code ******************/
-}
-
-func lookupUser(username string) (*User, error) {
-	var pwd C.struct_passwd
-	var result *C.struct_passwd
-	nameC := make([]byte, len(username)+1)
-	copy(nameC, username)
-
-	buf := alloc(userBuffer)
-	defer buf.free()
-
-	err := retryWithBuffer(buf, func() syscall.Errno {
-		// mygetpwnam_r is a wrapper around getpwnam_r to avoid
-		// passing a size_t to getpwnam_r, because for unknown
-		// reasons passing a size_t to getpwnam_r doesn't work on
-		// Solaris.
-		return syscall.Errno(C.mygetpwnam_r((*C.char)(unsafe.Pointer(&nameC[0])),
-			&pwd,
-			(*C.char)(buf.ptr),
-			C.size_t(buf.size),
-			&result))
-	})
-	if err != nil {
-		return nil, fmt.Errorf("user: lookup username %s: %v", username, err)
-	}
-	if result == nil {
-		return nil, UnknownUserError(username)
-	}
-	return buildUser(&pwd), err
-}
-
-func buildUser(pwd *C.struct_passwd) *User {
-	u := &User{
-		Uid:      strconv.FormatUint(uint64(pwd.pw_uid), 10),
-		Gid:      strconv.FormatUint(uint64(pwd.pw_gid), 10),
-		Username: C.GoString(pwd.pw_name),
-		Name:     C.GoString(pwd.pw_gecos),
-		HomeDir:  C.GoString(pwd.pw_dir),
-		/****************** Begin added code ******************/
-		Shell: C.GoString(pwd.pw_shell),
-		/****************** End added code ******************/
-	}
-	// The pw_gecos field isn't quite standardized. Some docs
-	// say: "It is expected to be a comma separated list of
-	// personal data where the first item is the full name of the
-	// user."
-	if i := strings.Index(u.Name, ","); i >= 0 {
-		u.Name = u.Name[:i]
-	}
-	return u
-}
-
-type bufferKind C.int
-
-const (
-	userBuffer = bufferKind(C._SC_GETPW_R_SIZE_MAX)
-)
-
-func (k bufferKind) initialSize() C.size_t {
-	sz := C.sysconf(C.int(k))
-	if sz == -1 {
-		// DragonFly and FreeBSD do not have _SC_GETPW_R_SIZE_MAX.
-		// Additionally, not all Linux systems have it, either. For
-		// example, the musl libc returns -1.
-		return 1024
-	}
-	if !isSizeReasonable(int64(sz)) {
-		// Truncate.  If this truly isn't enough, retryWithBuffer will error on the first run.
-		return maxBufferSize
-	}
-	return C.size_t(sz)
-}
-
-type memBuffer struct {
-	ptr  unsafe.Pointer
-	size C.size_t
-}
-
-func alloc(kind bufferKind) *memBuffer {
-	sz := kind.initialSize()
-	return &memBuffer{
-		ptr:  C.malloc(sz),
-		size: sz,
-	}
-}
-
-func (mb *memBuffer) resize(newSize C.size_t) {
-	mb.ptr = C.realloc(mb.ptr, newSize)
-	mb.size = newSize
-}
-
-func (mb *memBuffer) free() {
-	C.free(mb.ptr)
-}
-
-// retryWithBuffer repeatedly calls f(), increasing the size of the
-// buffer each time, until f succeeds, fails with a non-ERANGE error,
-// or the buffer exceeds a reasonable limit.
-func retryWithBuffer(buf *memBuffer, f func() syscall.Errno) error {
-	for {
-		errno := f()
-		if errno == 0 {
-			return nil
-		} else if errno != syscall.ERANGE {
-			return errno
-		}
-		newSize := buf.size * 2
-		if !isSizeReasonable(int64(newSize)) {
-			return fmt.Errorf("internal buffer exceeds %d bytes", maxBufferSize)
-		}
-		buf.resize(newSize)
-	}
-}
-
-const maxBufferSize = 1 << 20
-
-func isSizeReasonable(sz int64) bool {
-	return sz > 0 && sz <= maxBufferSize
-}

sshserver/host_keys.go

@@ -19,46 +19,44 @@ import (
 )
 
 const (
-	rsaFilename   = "ssh_host_rsa_key"
-	ecdsaFilename = "ssh_host_ecdsa_key"
+	rsaFilename       = "ssh_host_rsa_key"
+	ecdsaFilename     = "ssh_host_ecdsa_key"
 )
 
-func (s *SSHServer) configureHostKeys() error {
-	if _, err := os.Stat(systemConfigPath); os.IsNotExist(err) {
-		if err := os.MkdirAll(systemConfigPath, 0755); err != nil {
-			return errors.Wrap(err, fmt.Sprintf("Error creating %s directory", systemConfigPath))
+var defaultHostKeyDir = filepath.Join(".cloudflared", "host_keys")
+
+func (s *SSHProxy) configureHostKeys(hostKeyDir string) error {
+	if hostKeyDir == "" {
+		homeDir, err := os.UserHomeDir()
+		if err != nil {
+			return err
+		}
+		hostKeyDir = filepath.Join(homeDir, defaultHostKeyDir)
+	}
+
+	if _, err := os.Stat(hostKeyDir); os.IsNotExist(err) {
+		if err := os.MkdirAll(hostKeyDir, 0755); err != nil {
+			return errors.Wrap(err, fmt.Sprintf("Error creating %s directory", hostKeyDir))
 		}
 	}
 
-	if err := s.configureHostKey(s.ensureECDSAKeyExists); err != nil {
+	if err := s.configureECDSAKey(hostKeyDir); err != nil {
 		return err
 	}
 
-	if err := s.configureHostKey(s.ensureRSAKeyExists); err != nil {
+	if err := s.configureRSAKey(hostKeyDir); err != nil {
 		return err
 	}
 
 	return nil
 }
 
-func (s *SSHServer) configureHostKey(keyFunc func() (string, error)) error {
-	path, err := keyFunc()
-	if err != nil {
-		return err
-	}
-
-	if err := s.SetOption(ssh.HostKeyFile(path)); err != nil {
-		return errors.Wrap(err, "Could not set SSH host key")
-	}
-	return nil
-}
-
-func (s *SSHServer) ensureRSAKeyExists() (string, error) {
-	keyPath := filepath.Join(systemConfigPath, rsaFilename)
+func (s *SSHProxy) configureRSAKey(basePath string) error {
+	keyPath := filepath.Join(basePath, rsaFilename)
 	if _, err := os.Stat(keyPath); os.IsNotExist(err) {
 		key, err := rsa.GenerateKey(rand.Reader, 2048)
 		if err != nil {
-			return "", errors.Wrap(err, "Error generating RSA host key")
+			return errors.Wrap(err, "Error generating RSA host key")
 		}
 
 		privateKey := &pem.Block{
@@ -67,25 +65,28 @@ func (s *SSHServer) ensureRSAKeyExists() (string, error) {
 		}
 
 		if err = writePrivateKey(keyPath, privateKey); err != nil {
-			return "", err
+			return err
 		}
 
 		s.logger.Debug("Created new RSA SSH host key: ", keyPath)
 	}
-	return keyPath, nil
+	if err := s.SetOption(ssh.HostKeyFile(keyPath)); err != nil {
+		return errors.Wrap(err, "Could not set SSH RSA host key")
+	}
+	return nil
 }
 
-func (s *SSHServer) ensureECDSAKeyExists() (string, error) {
-	keyPath := filepath.Join(systemConfigPath, ecdsaFilename)
+func (s *SSHProxy) configureECDSAKey(basePath string) error {
+	keyPath := filepath.Join(basePath, ecdsaFilename)
 	if _, err := os.Stat(keyPath); os.IsNotExist(err) {
 		key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
 		if err != nil {
-			return "", errors.Wrap(err, "Error generating ECDSA host key")
+			return errors.Wrap(err, "Error generating ECDSA host key")
 		}
 
 		keyBytes, err := x509.MarshalECPrivateKey(key)
 		if err != nil {
-			return "", errors.Wrap(err, "Error marshalling ECDSA key")
+			return errors.Wrap(err, "Error marshalling ECDSA key")
 		}
 
 		privateKey := &pem.Block{
@@ -94,12 +95,15 @@ func (s *SSHServer) ensureECDSAKeyExists() (string, error) {
 		}
 
 		if err = writePrivateKey(keyPath, privateKey); err != nil {
-			return "", err
+			return err
 		}
 
 		s.logger.Debug("Created new ECDSA SSH host key: ", keyPath)
 	}
-	return keyPath, nil
+	if err := s.SetOption(ssh.HostKeyFile(keyPath)); err != nil {
+		return errors.Wrap(err, "Could not set SSH ECDSA host key")
+	}
+	return nil
 }
 
 func writePrivateKey(keyPath string, privateKey *pem.Block) error {

sshserver/preamble_test.go

@@ -0,0 +1,29 @@
+package sshserver
+
+import (
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"testing"
+)
+
+func TestHasPort(t *testing.T) {
+	type testCase struct {
+		input          string
+		expectedOutput string
+	}
+
+	tests := []testCase{
+		{"localhost", "localhost:22"},
+		{"other.addr:22", "other.addr:22"},
+		{"[2001:db8::1]:8080", "[2001:db8::1]:8080"},
+		{"[::1]", "[::1]:22"},
+		{"2001:0db8:3c4d:0015:0000:0000:1a2f:1234", "[2001:0db8:3c4d:0015:0000:0000:1a2f:1234]:22"},
+		{"::1", "[::1]:22"},
+	}
+
+	for _, test := range tests {
+		out, err := canonicalizeDest(test.input)
+		require.Nil(t, err)
+		assert.Equal(t, test.expectedOutput, out)
+	}
+}

sshserver/sshserver_unix.go

@@ -3,66 +3,110 @@
 package sshserver
 
 import (
-	"bufio"
-	"errors"
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"encoding/binary"
+	"encoding/json"
 	"fmt"
 	"io"
-	"os"
-	"os/exec"
-	"os/user"
-	"strconv"
-	"syscall"
+	"net"
+	"runtime"
+	"strings"
 	"time"
-	"unsafe"
 
-	"github.com/creack/pty"
+	"github.com/cloudflare/cloudflared/sshgen"
+	"github.com/cloudflare/cloudflared/sshlog"
 	"github.com/gliderlabs/ssh"
+	"github.com/google/uuid"
+	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
+	gossh "golang.org/x/crypto/ssh"
 )
 
-type SSHServer struct {
+const (
+	auditEventStart       = "session_start"
+	auditEventStop        = "session_stop"
+	auditEventExec        = "exec"
+	auditEventScp         = "scp"
+	auditEventResize      = "resize"
+	auditEventShell       = "shell"
+	sshContextSessionID   = "sessionID"
+	sshContextEventLogger = "eventLogger"
+	sshContextPreamble    = "sshPreamble"
+	sshContextSSHClient   = "sshClient"
+	SSHPreambleLength     = 2
+	defaultSSHPort        = "22"
+)
+
+type auditEvent struct {
+	Event       string `json:"event,omitempty"`
+	EventType   string `json:"event_type,omitempty"`
+	SessionID   string `json:"session_id,omitempty"`
+	User        string `json:"user,omitempty"`
+	Login       string `json:"login,omitempty"`
+	Datetime    string `json:"datetime,omitempty"`
+	Hostname    string `json:"hostname,omitempty"`
+	Destination string `json:"destination,omitempty"`
+}
+
+// sshConn wraps the incoming net.Conn and a cleanup function
+// This is done to allow the outgoing SSH client to be retrieved and closed when the conn itself is closed.
+type sshConn struct {
+	net.Conn
+	cleanupFunc func()
+}
+
+// close calls the cleanupFunc before closing the conn
+func (c sshConn) Close() error {
+	c.cleanupFunc()
+	return c.Conn.Close()
+}
+
+type SSHProxy struct {
 	ssh.Server
-	logger      *logrus.Logger
-	shutdownC   chan struct{}
-	caCert      ssh.PublicKey
-	getUserFunc func(string) (*User, error)
+	hostname   string
+	logger     *logrus.Logger
+	shutdownC  chan struct{}
+	caCert     ssh.PublicKey
+	logManager sshlog.Manager
 }
 
-func New(logger *logrus.Logger, address string, shutdownC chan struct{}, shortLivedCertAuth bool, idleTimeout, maxTimeout time.Duration) (*SSHServer, error) {
-	currentUser, err := user.Current()
-	if err != nil {
-		return nil, err
-	}
-	if currentUser.Uid != "0" {
-		return nil, errors.New("cloudflared SSH server needs to run as root")
+type SSHPreamble struct {
+	Destination string
+	JWT         string
+}
+
+// New creates a new SSHProxy and configures its host keys and authentication by the data provided
+func New(logManager sshlog.Manager, logger *logrus.Logger, version, localAddress, hostname, hostKeyDir string, shutdownC chan struct{}, idleTimeout, maxTimeout time.Duration) (*SSHProxy, error) {
+	sshProxy := SSHProxy{
+		hostname:   hostname,
+		logger:     logger,
+		shutdownC:  shutdownC,
+		logManager: logManager,
 	}
 
-	sshServer := SSHServer{
-		Server:      ssh.Server{Addr: address, MaxTimeout: maxTimeout, IdleTimeout: idleTimeout},
-		logger:      logger,
-		shutdownC:   shutdownC,
-		getUserFunc: lookupUser,
+	sshProxy.Server = ssh.Server{
+		Addr:             localAddress,
+		MaxTimeout:       maxTimeout,
+		IdleTimeout:      idleTimeout,
+		Version:          fmt.Sprintf("SSH-2.0-Cloudflare-Access_%s_%s", version, runtime.GOOS),
+		PublicKeyHandler: sshProxy.proxyAuthCallback,
+		ConnCallback:     sshProxy.connCallback,
+		ChannelHandlers: map[string]ssh.ChannelHandler{
+			"default": sshProxy.channelHandler,
+		},
 	}
 
-	if err := sshServer.configureHostKeys(); err != nil {
+	if err := sshProxy.configureHostKeys(hostKeyDir); err != nil {
 		return nil, err
 	}
 
-	if shortLivedCertAuth {
-		caCert, err := getCACert()
-		if err != nil {
-			return nil, err
-		}
-		sshServer.caCert = caCert
-		sshServer.PublicKeyHandler = sshServer.shortLivedCertHandler
-	} else {
-		sshServer.PublicKeyHandler = sshServer.authorizedKeyHandler
-	}
-
-	return &sshServer, nil
+	return &sshProxy, nil
 }
 
-func (s *SSHServer) Start() error {
+// Start the SSH proxy listener to start handling SSH connections from clients
+func (s *SSHProxy) Start() error {
 	s.logger.Infof("Starting SSH server at %s", s.Addr)
 
 	go func() {
@@ -72,120 +116,375 @@ func (s *SSHServer) Start() error {
 		}
 	}()
 
-	s.Handle(s.connectionHandler)
 	return s.ListenAndServe()
 }
 
-func (s *SSHServer) connectionHandler(session ssh.Session) {
-	// Get uid and gid of user attempting to login
-	sshUser, ok := session.Context().Value("sshUser").(*User)
-	if !ok || sshUser == nil {
-		s.logger.Error("Error retrieving credentials from session")
-		s.CloseSession(session)
-		return
+// proxyAuthCallback attempts to connect to ultimate SSH destination. If successful, it allows the incoming connection
+// to connect to the proxy and saves the outgoing SSH client to the context. Otherwise, no connection to the
+// the proxy is allowed.
+func (s *SSHProxy) proxyAuthCallback(ctx ssh.Context, key ssh.PublicKey) bool {
+	client, err := s.dialDestination(ctx)
+	if err != nil {
+		return false
 	}
+	ctx.SetValue(sshContextSSHClient, client)
+	return true
+}
 
-	// Spawn shell under user
-	cmd := exec.Command(sshUser.Shell)
+// connCallback reads the preamble sent from the proxy server and saves an audit event logger to the context.
+// If any errors occur, the connection is terminated by returning nil from the callback.
+func (s *SSHProxy) connCallback(ctx ssh.Context, conn net.Conn) net.Conn {
+	// AUTH-2050: This is a temporary workaround of a timing issue in the tunnel muxer to allow further testing.
+	// TODO: Remove this
+	time.Sleep(10 * time.Millisecond)
 
-	ptyReq, winCh, isPty := session.Pty()
-	if !isPty {
-		if _, err := io.WriteString(session, "No PTY requested.\n"); err != nil {
-			s.logger.WithError(err).Error("No PTY requested: Failed to write to SSH session")
+	preamble, err := s.readPreamble(conn)
+	if err != nil {
+		if netErr, ok := err.(net.Error); ok && netErr.Timeout() {
+			s.logger.Warn("Could not establish session. Client likely does not have --destination set and is using old-style ssh config")
+		} else if err != io.EOF {
+			s.logger.WithError(err).Error("failed to read SSH preamble")
 		}
-
-		s.CloseSession(session)
-		return
+		return nil
 	}
+	ctx.SetValue(sshContextPreamble, preamble)
 
-	uidInt, err := stringToUint32(sshUser.Uid)
+	logger, sessionID, err := s.auditLogger()
 	if err != nil {
-		s.logger.WithError(err).Error("Invalid user")
-		s.CloseSession(session)
-		return
+		s.logger.WithError(err).Error("failed to configure logger")
+		return nil
 	}
-	gidInt, err := stringToUint32(sshUser.Gid)
-	if err != nil {
-		s.logger.WithError(err).Error("Invalid user group")
-		s.CloseSession(session)
+	ctx.SetValue(sshContextEventLogger, logger)
+	ctx.SetValue(sshContextSessionID, sessionID)
+
+	// attempts to retrieve and close the outgoing ssh client when the incoming conn is closed.
+	// If no client exists, the conn is being closed before the PublicKeyCallback was called (where the client is created).
+	cleanupFunc := func() {
+		client, ok := ctx.Value(sshContextSSHClient).(*gossh.Client)
+		if ok && client != nil {
+			client.Close()
+		}
+	}
+
+	return sshConn{conn, cleanupFunc}
+}
+
+// channelHandler proxies incoming and outgoing SSH traffic back and forth over an SSH Channel
+func (s *SSHProxy) channelHandler(srv *ssh.Server, conn *gossh.ServerConn, newChan gossh.NewChannel, ctx ssh.Context) {
+	if newChan.ChannelType() != "session" && newChan.ChannelType() != "direct-tcpip" {
+		msg := fmt.Sprintf("channel type %s is not supported", newChan.ChannelType())
+		s.logger.Info(msg)
+		if err := newChan.Reject(gossh.UnknownChannelType, msg); err != nil {
+			s.logger.WithError(err).Error("Error rejecting SSH channel")
+		}
 		return
 	}
 
-	cmd.SysProcAttr = &syscall.SysProcAttr{Credential: &syscall.Credential{Uid: uidInt, Gid: gidInt}}
-	cmd.Env = append(cmd.Env, fmt.Sprintf("TERM=%s", ptyReq.Term))
-	cmd.Env = append(cmd.Env, fmt.Sprintf("USER=%s", sshUser.Username))
-	cmd.Env = append(cmd.Env, fmt.Sprintf("HOME=%s", sshUser.HomeDir))
-	cmd.Dir = sshUser.HomeDir
-	psuedoTTY, err := pty.Start(cmd)
+	localChan, localChanReqs, err := newChan.Accept()
 	if err != nil {
-		s.logger.WithError(err).Error("Failed to start pty session")
-		s.CloseSession(session)
-		close(s.shutdownC)
+		s.logger.WithError(err).Error("Failed to accept session channel")
+		return
+	}
+	defer localChan.Close()
+
+	// client will be closed when the sshConn is closed
+	client, ok := ctx.Value(sshContextSSHClient).(*gossh.Client)
+	if !ok {
+		s.logger.Error("Could not retrieve client from context")
 		return
 	}
 
-	// Handle terminal window size changes
+	remoteChan, remoteChanReqs, err := client.OpenChannel(newChan.ChannelType(), newChan.ExtraData())
+	if err != nil {
+		s.logger.WithError(err).Error("Failed to open remote channel")
+		return
+	}
+
+	defer remoteChan.Close()
+
+	// Proxy ssh traffic back and forth between client and destination
+	s.proxyChannel(localChan, remoteChan, localChanReqs, remoteChanReqs, conn, ctx)
+}
+
+// proxyChannel couples two SSH channels and proxies SSH traffic and channel requests back and forth.
+func (s *SSHProxy) proxyChannel(localChan, remoteChan gossh.Channel, localChanReqs, remoteChanReqs <-chan *gossh.Request, conn *gossh.ServerConn, ctx ssh.Context) {
+	done := make(chan struct{}, 2)
 	go func() {
-		for win := range winCh {
-			if errNo := setWinsize(psuedoTTY, win.Width, win.Height); errNo != 0 {
-				s.logger.WithError(err).Error("Failed to set pty window size: ", err.Error())
-				s.CloseSession(session)
-				close(s.shutdownC)
+		if _, err := io.Copy(localChan, remoteChan); err != nil {
+			s.logger.WithError(err).Error("remote to local copy error")
+		}
+		done <- struct{}{}
+	}()
+	go func() {
+		if _, err := io.Copy(remoteChan, localChan); err != nil {
+			s.logger.WithError(err).Error("local to remote copy error")
+		}
+		done <- struct{}{}
+	}()
+
+	// stderr streams are used non-pty sessions since they have distinct IO streams.
+	remoteStderr := remoteChan.Stderr()
+	localStderr := localChan.Stderr()
+	go func() {
+		if _, err := io.Copy(remoteStderr, localStderr); err != nil {
+			s.logger.WithError(err).Error("stderr local to remote copy error")
+		}
+	}()
+	go func() {
+		if _, err := io.Copy(localStderr, remoteStderr); err != nil {
+			s.logger.WithError(err).Error("stderr remote to local copy error")
+		}
+	}()
+
+	s.logAuditEvent(conn, "", auditEventStart, ctx)
+	defer s.logAuditEvent(conn, "", auditEventStop, ctx)
+
+	// Proxy channel requests
+	for {
+		select {
+		case req := <-localChanReqs:
+			if req == nil {
 				return
 			}
+			if err := s.forwardChannelRequest(remoteChan, req); err != nil {
+				s.logger.WithError(err).Error("Failed to forward request")
+				return
+			}
+
+			s.logChannelRequest(req, conn, ctx)
+
+		case req := <-remoteChanReqs:
+			if req == nil {
+				return
+			}
+			if err := s.forwardChannelRequest(localChan, req); err != nil {
+				s.logger.WithError(err).Error("Failed to forward request")
+				return
+			}
+		case <-done:
+			return
+		}
+	}
+}
+
+// readPreamble reads a preamble from the SSH connection before any SSH traffic is sent.
+// This preamble is a JSON encoded struct containing the users JWT and ultimate destination.
+// The first 4 bytes contain the length of the preamble which follows immediately.
+func (s *SSHProxy) readPreamble(conn net.Conn) (*SSHPreamble, error) {
+	// Set conn read deadline while reading preamble to prevent hangs if preamble wasnt sent.
+	if err := conn.SetReadDeadline(time.Now().Add(500 * time.Millisecond)); err != nil {
+		return nil, errors.Wrap(err, "failed to set conn deadline")
+	}
+	defer func() {
+		if err := conn.SetReadDeadline(time.Time{}); err != nil {
+			s.logger.WithError(err).Error("Failed to unset conn read deadline")
 		}
 	}()
 
-	// Write incoming commands to PTY
-	go func() {
-		if _, err := io.Copy(psuedoTTY, session); err != nil {
-			s.logger.WithError(err).Error("Failed to write incoming command to pty")
+	size := make([]byte, SSHPreambleLength)
+	if _, err := io.ReadFull(conn, size); err != nil {
+		return nil, err
+	}
+	payloadLength := binary.BigEndian.Uint16(size)
+	payload := make([]byte, payloadLength)
+	if _, err := io.ReadFull(conn, payload); err != nil {
+		return nil, err
+	}
+
+	var preamble SSHPreamble
+	err := json.Unmarshal(payload, &preamble)
+	if err != nil {
+		return nil, err
+	}
+
+	preamble.Destination, err = canonicalizeDest(preamble.Destination)
+	if err != nil {
+		return nil, err
+	}
+	return &preamble, nil
+}
+
+// canonicalizeDest adds a default port if one doesnt exist
+func canonicalizeDest(dest string) (string, error) {
+	_, _, err := net.SplitHostPort(dest)
+	// if host and port are split without error, a port exists.
+	if err != nil {
+		addrErr, ok := err.(*net.AddrError)
+		if !ok {
+			return "", err
 		}
-	}()
-	pr, pw := io.Pipe()
-	scanner := bufio.NewScanner(pr)
-	go func() {
-		for scanner.Scan() {
-			s.logger.Info(scanner.Text())
+		// If the port is missing, append it.
+		if addrErr.Err == "missing port in address" {
+			return fmt.Sprintf("%s:%s", dest, defaultSSHPort), nil
 		}
-	}()
 
-	// Write outgoing command output to both the command recorder, and remote user
-	mw := io.MultiWriter(pw, session)
-	if _, err := io.Copy(mw, psuedoTTY); err != nil {
-		s.logger.WithError(err).Error("Failed to write command output to user")
+		// If there are too many colons and address is IPv6, wrap in brackets and append port. Otherwise invalid address
+		ip := net.ParseIP(dest)
+		if addrErr.Err == "too many colons in address" && ip != nil && ip.To4() == nil {
+			return fmt.Sprintf("[%s]:%s", dest, defaultSSHPort), nil
+		}
+		return "", addrErr
 	}
 
-	if err := pw.Close(); err != nil {
-		s.logger.WithError(err).Error("Failed to close pipe writer")
-	}
-
-	if err := pr.Close(); err != nil {
-		s.logger.WithError(err).Error("Failed to close pipe reader")
-	}
-
-	// Wait for all resources associated with cmd to be released
-	// Returns error if shell exited with a non-zero status or received a signal
-	if err := cmd.Wait(); err != nil {
-		s.logger.WithError(err).Debug("Shell did not close correctly")
-	}
+	return dest, nil
 }
 
-func (s *SSHServer) CloseSession(session ssh.Session) {
-	if err := session.Exit(1); err != nil {
-		s.logger.WithError(err).Error("Failed to close SSH session")
+// dialDestination creates a new SSH client and dials the destination server
+func (s *SSHProxy) dialDestination(ctx ssh.Context) (*gossh.Client, error) {
+	preamble, ok := ctx.Value(sshContextPreamble).(*SSHPreamble)
+	if !ok {
+		msg := "failed to retrieve SSH preamble from context"
+		s.logger.Error(msg)
+		return nil, errors.New(msg)
+	}
+
+	signer, err := s.genSSHSigner(preamble.JWT)
+	if err != nil {
+		s.logger.WithError(err).Error("Failed to generate signed short lived cert")
+		return nil, err
+	}
+	s.logger.Debugf("Short lived certificate for %s connecting to %s:\n\n%s", ctx.User(), preamble.Destination, gossh.MarshalAuthorizedKey(signer.PublicKey()))
+
+	clientConfig := &gossh.ClientConfig{
+		User: ctx.User(),
+		// AUTH-2103 TODO: proper host key check
+		HostKeyCallback: gossh.InsecureIgnoreHostKey(),
+		Auth:            []gossh.AuthMethod{gossh.PublicKeys(signer)},
+		ClientVersion:   ctx.ServerVersion(),
+	}
+
+	client, err := gossh.Dial("tcp", preamble.Destination, clientConfig)
+	if err != nil {
+		s.logger.WithError(err).Info("Failed to connect to destination SSH server")
+		return nil, err
+	}
+	return client, nil
+}
+
+// Generates a key pair and sends public key to get signed by CA
+func (s *SSHProxy) genSSHSigner(jwt string) (gossh.Signer, error) {
+	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to generate ecdsa key pair")
+	}
+
+	pub, err := gossh.NewPublicKey(&key.PublicKey)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to convert ecdsa public key to SSH public key")
+	}
+
+	pubBytes := gossh.MarshalAuthorizedKey(pub)
+	signedCertBytes, err := sshgen.SignCert(jwt, string(pubBytes))
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to retrieve cert from SSHCAAPI")
+	}
+
+	signedPub, _, _, _, err := gossh.ParseAuthorizedKey([]byte(signedCertBytes))
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to parse SSH public key")
+	}
+
+	cert, ok := signedPub.(*gossh.Certificate)
+	if !ok {
+		return nil, errors.Wrap(err, "failed to assert public key as certificate")
+	}
+	signer, err := gossh.NewSignerFromKey(key)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to create signer")
+	}
+
+	certSigner, err := gossh.NewCertSigner(cert, signer)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to create cert signer")
+	}
+	return certSigner, nil
+}
+
+// forwardChannelRequest sends request req to SSH channel sshChan, waits for reply, and sends the reply back.
+func (s *SSHProxy) forwardChannelRequest(sshChan gossh.Channel, req *gossh.Request) error {
+	reply, err := sshChan.SendRequest(req.Type, req.WantReply, req.Payload)
+	if err != nil {
+		return errors.Wrap(err, "Failed to send request")
+	}
+	if err := req.Reply(reply, nil); err != nil {
+		return errors.Wrap(err, "Failed to reply to request")
+	}
+	return nil
+}
+
+// logChannelRequest creates an audit log for different types of channel requests
+func (s *SSHProxy) logChannelRequest(req *gossh.Request, conn *gossh.ServerConn, ctx ssh.Context) {
+	var eventType string
+	var event string
+	switch req.Type {
+	case "exec":
+		var payload struct{ Value string }
+		if err := gossh.Unmarshal(req.Payload, &payload); err != nil {
+			s.logger.WithError(err).Errorf("Failed to unmarshal channel request payload: %s:%s", req.Type, req.Payload)
+		}
+		event = payload.Value
+
+		eventType = auditEventExec
+		if strings.HasPrefix(string(req.Payload), "scp") {
+			eventType = auditEventScp
+		}
+	case "shell":
+		eventType = auditEventShell
+	case "window-change":
+		eventType = auditEventResize
+	default:
+		return
+	}
+	s.logAuditEvent(conn, event, eventType, ctx)
+}
+
+func (s *SSHProxy) auditLogger() (io.WriteCloser, string, error) {
+	sessionUUID, err := uuid.NewRandom()
+	if err != nil {
+		return nil, "", errors.Wrap(err, "failed to create sessionID")
+	}
+	sessionID := sessionUUID.String()
+
+	writer, err := s.logManager.NewLogger(fmt.Sprintf("%s-event.log", sessionID), s.logger)
+	if err != nil {
+		return nil, "", errors.Wrap(err, "failed to create logger")
+	}
+	return writer, sessionID, nil
+}
+
+func (s *SSHProxy) logAuditEvent(conn *gossh.ServerConn, event, eventType string, ctx ssh.Context) {
+	sessionID, sessionIDOk := ctx.Value(sshContextSessionID).(string)
+	writer, writerOk := ctx.Value(sshContextEventLogger).(io.WriteCloser)
+	if !writerOk || !sessionIDOk {
+		s.logger.Error("Failed to retrieve audit logger from context")
+		return
+	}
+
+	var destination string
+	preamble, ok := ctx.Value(sshContextPreamble).(*SSHPreamble)
+	if ok {
+		destination = preamble.Destination
+	} else {
+		s.logger.Error("Failed to retrieve SSH preamble from context")
+	}
+
+	ae := auditEvent{
+		Event:       event,
+		EventType:   eventType,
+		SessionID:   sessionID,
+		User:        conn.User(),
+		Login:       conn.User(),
+		Datetime:    time.Now().UTC().Format(time.RFC3339),
+		Hostname:    s.hostname,
+		Destination: destination,
+	}
+	data, err := json.Marshal(&ae)
+	if err != nil {
+		s.logger.WithError(err).Error("Failed to marshal audit event. malformed audit object")
+		return
+	}
+	line := string(data) + "\n"
+	if _, err := writer.Write([]byte(line)); err != nil {
+		s.logger.WithError(err).Error("Failed to write audit event.")
 	}
 }
-
-// Sets PTY window size for terminal
-func setWinsize(f *os.File, w, h int) syscall.Errno {
-	_, _, errNo := syscall.Syscall(syscall.SYS_IOCTL, f.Fd(), uintptr(syscall.TIOCSWINSZ),
-		uintptr(unsafe.Pointer(&struct{ h, w, x, y uint16 }{uint16(h), uint16(w), 0, 0})))
-	return errNo
-}
-
-func stringToUint32(str string) (uint32, error) {
-	uid, err := strconv.ParseUint(str, 10, 32)
-	return uint32(uid), err
-
-}

sshserver/sshserver_windows.go

@@ -5,13 +5,22 @@ package sshserver
 import (
 	"errors"
 
-	"github.com/sirupsen/logrus"
 	"time"
+
+	"github.com/cloudflare/cloudflared/sshlog"
+	"github.com/sirupsen/logrus"
 )
 
+const SSHPreambleLength = 2
+
 type SSHServer struct{}
 
-func New(_ *logrus.Logger, _ string, _ chan struct{}, _ bool, _, _ time.Duration) (*SSHServer, error) {
+type SSHPreamble struct {
+	Destination string
+	JWT         string
+}
+
+func New(_ sshlog.Manager, _ *logrus.Logger, _, _, _, _ string, _ chan struct{}, _, _ time.Duration) (*SSHServer, error) {
 	return nil, errors.New("cloudflared ssh server is not supported on windows")
 }
 

streamhandler/request.go

@@ -45,13 +45,16 @@ func H2RequestHeadersToH1Request(h2 []h2mux.Header, h1 *http.Request) error {
 			// Otherwise the host header will be based on the origin URL
 			h1.Host = header.Value
 		case ":path":
-			// Separate the path and the (optional) query from each other
-			chunks := strings.SplitN(header.Value, "?", 2)
-
-			h1.URL.Path = chunks[0]
-			if len(chunks) > 1 {
-				h1.URL.RawQuery = chunks[1]
+			u, err := url.Parse(header.Value)
+			if err != nil {
+				return fmt.Errorf("unparseable path")
 			}
+			resolved := h1.URL.ResolveReference(u)
+			// prevent escaping base URL
+			if !strings.HasPrefix(resolved.String(), h1.URL.String()) {
+				return fmt.Errorf("invalid path %s", header.Value)
+			}
+			h1.URL = resolved
 		case "content-length":
 			contentLength, err := strconv.ParseInt(header.Value, 10, 64)
 			if err != nil {

streamhandler/request_test.go

@@ -1,104 +0,0 @@
-package streamhandler
-
-import (
-	"net/http"
-	"testing"
-
-	"github.com/cloudflare/cloudflared/h2mux"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestH2RequestHeadersToH1Request_RegularHeaders(t *testing.T) {
-	request, err := http.NewRequest(http.MethodGet, "http://example.com", nil)
-	assert.NoError(t, err)
-
-	headersConversionErr := H2RequestHeadersToH1Request(
-		[]h2mux.Header{
-			h2mux.Header{
-				Name:  "Mock header 1",
-				Value: "Mock value 1",
-			},
-			h2mux.Header{
-				Name:  "Mock header 2",
-				Value: "Mock value 2",
-			},
-		},
-		request,
-	)
-
-	assert.Equal(t, http.Header{
-		"Mock header 1": []string{"Mock value 1"},
-		"Mock header 2": []string{"Mock value 2"},
-	}, request.Header)
-
-	assert.NoError(t, headersConversionErr)
-}
-
-func TestH2RequestHeadersToH1Request_NoHeaders(t *testing.T) {
-	request, err := http.NewRequest(http.MethodGet, "http://example.com", nil)
-	assert.NoError(t, err)
-
-	headersConversionErr := H2RequestHeadersToH1Request(
-		[]h2mux.Header{},
-		request,
-	)
-
-	assert.Equal(t, http.Header{}, request.Header)
-
-	assert.NoError(t, headersConversionErr)
-}
-
-func TestH2RequestHeadersToH1Request_InvalidHostPath(t *testing.T) {
-	request, err := http.NewRequest(http.MethodGet, "http://example.com", nil)
-	assert.NoError(t, err)
-
-	headersConversionErr := H2RequestHeadersToH1Request(
-		[]h2mux.Header{
-			h2mux.Header{
-				Name:  ":path",
-				Value: "//bad_path/",
-			},
-			h2mux.Header{
-				Name:  "Mock header",
-				Value: "Mock value",
-			},
-		},
-		request,
-	)
-
-	assert.Equal(t, http.Header{
-		"Mock header": []string{"Mock value"},
-	}, request.Header)
-
-	assert.Equal(t, "http://example.com//bad_path/", request.URL.String())
-
-	assert.NoError(t, headersConversionErr)
-}
-
-func TestH2RequestHeadersToH1Request_HostPathWithQuery(t *testing.T) {
-	request, err := http.NewRequest(http.MethodGet, "http://example.com/", nil)
-	assert.NoError(t, err)
-
-	headersConversionErr := H2RequestHeadersToH1Request(
-		[]h2mux.Header{
-			h2mux.Header{
-				Name:  ":path",
-				Value: "/?query",
-			},
-			h2mux.Header{
-				Name:  "Mock header",
-				Value: "Mock value",
-			},
-		},
-		request,
-	)
-
-	assert.Equal(t, http.Header{
-		"Mock header": []string{"Mock value"},
-	}, request.Header)
-
-	assert.Equal(t, "http://example.com/?query", request.URL.String())
-
-	assert.NoError(t, headersConversionErr)
-}

streamhandler/stream_handler.go

@@ -93,7 +93,7 @@ func (s *StreamHandler) UpdateConfig(newConfig []*pogs.ReverseProxyConfig) (fail
 	toAdd := s.tunnelHostnameMapper.ToAdd(newConfig)
 	for _, tunnelConfig := range toAdd {
 		tunnelHostname := tunnelConfig.TunnelHostname
-		originSerice, err := tunnelConfig.OriginConfigJSONHandler.OriginConfig.Service()
+		originSerice, err := tunnelConfig.OriginConfig.Service()
 		if err != nil {
 			s.logger.WithField("tunnelHostname", tunnelHostname).WithError(err).Error("Invalid origin service config")
 			failedConfigs = append(failedConfigs, &pogs.FailedConfig{

streamhandler/stream_handler_test.go

@@ -49,10 +49,8 @@ func TestServeRequest(t *testing.T) {
 	reverseProxyConfigs := []*pogs.ReverseProxyConfig{
 		{
 			TunnelHostname: testTunnelHostname,
-			OriginConfigJSONHandler: &pogs.OriginConfigJSONHandler{
-				OriginConfig: &pogs.HTTPOriginConfig{
-					URLString: httpServer.URL,
-				},
+			OriginConfig: &pogs.HTTPOriginConfig{
+				URLString: httpServer.URL,
 			},
 		},
 	}
@@ -99,10 +97,8 @@ func TestServeBadRequest(t *testing.T) {
 	reverseProxyConfigs := []*pogs.ReverseProxyConfig{
 		{
 			TunnelHostname: testTunnelHostname,
-			OriginConfigJSONHandler: &pogs.OriginConfigJSONHandler{
-				OriginConfig: &pogs.HTTPOriginConfig{
-					URLString: "",
-				},
+			OriginConfig: &pogs.HTTPOriginConfig{
+				URLString: "",
 			},
 		},
 	}

tlsconfig/certreloader.go

@@ -5,7 +5,6 @@ import (
 	"crypto/x509"
 	"fmt"
 	"io/ioutil"
-	"net"
 	"runtime"
 	"sync"
 
@@ -18,6 +17,8 @@ import (
 const (
 	OriginCAPoolFlag = "origin-ca-pool"
 	CaCertFlag       = "cacert"
+
+	edgeTLSServerName = "cftunnel.com"
 )
 
 // CertReloader can load and reload a TLS certificate from a particular filepath.
@@ -126,7 +127,7 @@ func CreateTunnelConfig(c *cli.Context) (*tls.Config, error) {
 		rootCAs = append(rootCAs, c.String(CaCertFlag))
 	}
 
-	userConfig := &TLSParameters{RootCAs: rootCAs}
+	userConfig := &TLSParameters{RootCAs: rootCAs, ServerName: edgeTLSServerName}
 	tlsConfig, err := GetConfig(userConfig)
 	if err != nil {
 		return nil, err
@@ -142,10 +143,6 @@ func CreateTunnelConfig(c *cli.Context) (*tls.Config, error) {
 			rootCAPool.AddCert(cert)
 		}
 		tlsConfig.RootCAs = rootCAPool
-		tlsConfig.ServerName = "cftunnel.com"
-	} else if edgeAddrs := c.StringSlice("edge"); len(edgeAddrs) > 0 {
-		// Set for development environments and for testing specific origintunneld instances
-		tlsConfig.ServerName, _, _ = net.SplitHostPort(edgeAddrs[0])
 	}
 
 	if tlsConfig.ServerName == "" && !tlsConfig.InsecureSkipVerify {

tlsconfig/tlsconfig_test.go

@@ -1,5 +1,3 @@
-// TODO: Remove the above build tag and include this test when we start compiling with Golang 1.10.0+
-
 package tlsconfig
 
 import (

tunnelhostnamemapper/tunnelhostnamemapper_test.go

@@ -193,20 +193,20 @@ func TestTunnelHostnameMapper_ToRemove(t *testing.T) {
 
 func sampleConfig1() *pogs.ReverseProxyConfig {
 	return &pogs.ReverseProxyConfig{
-		TunnelHostname:          "mock.example.com",
-		OriginConfigJSONHandler: &pogs.OriginConfigJSONHandler{OriginConfig: &pogs.HelloWorldOriginConfig{}},
-		Retries:                 18,
-		ConnectionTimeout:       5 * time.Second,
-		CompressionQuality:      3,
+		TunnelHostname:     "mock.example.com",
+		OriginConfig:       &pogs.HelloWorldOriginConfig{},
+		Retries:            18,
+		ConnectionTimeout:  5 * time.Second,
+		CompressionQuality: 3,
 	}
 }
 
 func sampleConfig2() *pogs.ReverseProxyConfig {
 	return &pogs.ReverseProxyConfig{
-		TunnelHostname:          "mock2.example.com",
-		OriginConfigJSONHandler: &pogs.OriginConfigJSONHandler{OriginConfig: &pogs.HelloWorldOriginConfig{}},
-		Retries:                 18,
-		ConnectionTimeout:       5 * time.Second,
-		CompressionQuality:      3,
+		TunnelHostname:     "mock2.example.com",
+		OriginConfig:       &pogs.HelloWorldOriginConfig{},
+		Retries:            18,
+		ConnectionTimeout:  5 * time.Second,
+		CompressionQuality: 3,
 	}
 }

tunnelrpc/pogs/config.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"crypto/tls"
 	"crypto/x509"
-	"encoding/json"
 	"fmt"
 	"net"
 	"net/http"
@@ -29,11 +28,43 @@ import (
 
 // ClientConfig is a collection of FallibleConfig that determines how cloudflared should function
 type ClientConfig struct {
-	Version              Version               `json:"version"`
-	SupervisorConfig     *SupervisorConfig     `json:"supervisor_config"`
-	EdgeConnectionConfig *EdgeConnectionConfig `json:"edge_connection_config"`
-	DoHProxyConfigs      []*DoHProxyConfig     `json:"doh_proxy_configs" capnp:"dohProxyConfigs"`
-	ReverseProxyConfigs  []*ReverseProxyConfig `json:"reverse_proxy_configs"`
+	Version              Version
+	SupervisorConfig     *SupervisorConfig
+	EdgeConnectionConfig *EdgeConnectionConfig
+	DoHProxyConfigs      []*DoHProxyConfig `capnp:"dohProxyConfigs"`
+	ReverseProxyConfigs  []*ReverseProxyConfig
+}
+
+func (c *ClientConfig) MarshalBytes() ([]byte, error) {
+	msg, firstSeg, err := capnp.NewMessage(capnp.SingleSegment(nil))
+	if err != nil {
+		return nil, err
+	}
+	capnpEntity, err := tunnelrpc.NewRootClientConfig(firstSeg)
+	if err != nil {
+		return nil, err
+	}
+	err = MarshalClientConfig(capnpEntity, c)
+	if err != nil {
+		return nil, err
+	}
+	return msg.Marshal()
+}
+
+func UnmarshalClientConfigFromBytes(clientConfigBytes []byte) (*ClientConfig, error) {
+	msg, err := capnp.Unmarshal(clientConfigBytes)
+	if err != nil {
+		return nil, err
+	}
+	capnpClientConfig, err := tunnelrpc.ReadRootClientConfig(msg)
+	if err != nil {
+		return nil, err
+	}
+	pogsClientConfig, err := UnmarshalClientConfig(capnpClientConfig)
+	if err != nil {
+		return nil, err
+	}
+	return pogsClientConfig, nil
 }
 
 // Version type models the version of a ClientConfig
@@ -52,16 +83,17 @@ func (v Version) String() string {
 }
 
 // FallibleConfig is an interface implemented by configs that cloudflared might not be able to apply
+//go-sumtype:decl FallibleConfig
 type FallibleConfig interface {
 	FailReason(err error) string
-	jsonType() string
+	isFallibleConfig()
 }
 
 // SupervisorConfig specifies config of components managed by Supervisor other than ConnectionManager
 type SupervisorConfig struct {
-	AutoUpdateFrequency    time.Duration `json:"auto_update_frequency"`
-	MetricsUpdateFrequency time.Duration `json:"metrics_update_frequency"`
-	GracePeriod            time.Duration `json:"grace_period"`
+	AutoUpdateFrequency    time.Duration
+	MetricsUpdateFrequency time.Duration
+	GracePeriod            time.Duration
 }
 
 // FailReason impelents FallibleConfig interface for SupervisorConfig
@@ -69,23 +101,15 @@ func (sc *SupervisorConfig) FailReason(err error) string {
 	return fmt.Sprintf("Cannot apply SupervisorConfig, err: %v", err)
 }
 
-func (sc *SupervisorConfig) MarshalJSON() ([]byte, error) {
-	marshaler := make(map[string]SupervisorConfig, 1)
-	marshaler[sc.jsonType()] = *sc
-	return json.Marshal(marshaler)
-}
-
-func (sc *SupervisorConfig) jsonType() string {
-	return "supervisor_config"
-}
+func (_ *SupervisorConfig) isFallibleConfig() {}
 
 // EdgeConnectionConfig specifies what parameters and how may connections should ConnectionManager establish with edge
 type EdgeConnectionConfig struct {
-	NumHAConnections    uint8         `json:"num_ha_connections"`
-	HeartbeatInterval   time.Duration `json:"heartbeat_interval"`
-	Timeout             time.Duration `json:"timeout"`
-	MaxFailedHeartbeats uint64        `json:"max_failed_heartbeats"`
-	UserCredentialPath  string        `json:"user_credential_path"`
+	NumHAConnections    uint8
+	HeartbeatInterval   time.Duration
+	Timeout             time.Duration
+	MaxFailedHeartbeats uint64
+	UserCredentialPath  string
 }
 
 // FailReason impelents FallibleConfig interface for EdgeConnectionConfig
@@ -93,21 +117,13 @@ func (cmc *EdgeConnectionConfig) FailReason(err error) string {
 	return fmt.Sprintf("Cannot apply EdgeConnectionConfig, err: %v", err)
 }
 
-func (cmc *EdgeConnectionConfig) MarshalJSON() ([]byte, error) {
-	marshaler := make(map[string]EdgeConnectionConfig, 1)
-	marshaler[cmc.jsonType()] = *cmc
-	return json.Marshal(marshaler)
-}
-
-func (cmc *EdgeConnectionConfig) jsonType() string {
-	return "edge_connection_config"
-}
+func (_ *EdgeConnectionConfig) isFallibleConfig() {}
 
 // DoHProxyConfig is configuration for DNS over HTTPS service
 type DoHProxyConfig struct {
-	ListenHost string   `json:"listen_host"`
-	ListenPort uint16   `json:"listen_port"`
-	Upstreams  []string `json:"upstreams"`
+	ListenHost string
+	ListenPort uint16
+	Upstreams  []string
 }
 
 // FailReason impelents FallibleConfig interface for DoHProxyConfig
@@ -115,23 +131,15 @@ func (dpc *DoHProxyConfig) FailReason(err error) string {
 	return fmt.Sprintf("Cannot apply DoHProxyConfig, err: %v", err)
 }
 
-func (dpc *DoHProxyConfig) MarshalJSON() ([]byte, error) {
-	marshaler := make(map[string]DoHProxyConfig, 1)
-	marshaler[dpc.jsonType()] = *dpc
-	return json.Marshal(marshaler)
-}
-
-func (dpc *DoHProxyConfig) jsonType() string {
-	return "doh_proxy_config"
-}
+func (_ *DoHProxyConfig) isFallibleConfig() {}
 
 // ReverseProxyConfig how and for what hostnames can this cloudflared proxy
 type ReverseProxyConfig struct {
-	TunnelHostname          h2mux.TunnelHostname     `json:"tunnel_hostname"`
-	OriginConfigJSONHandler *OriginConfigJSONHandler `json:"origin_config"`
-	Retries                 uint64                   `json:"retries"`
-	ConnectionTimeout       time.Duration            `json:"connection_timeout"`
-	CompressionQuality      uint64                   `json:"compression_quality"`
+	TunnelHostname     h2mux.TunnelHostname
+	OriginConfig       OriginConfig
+	Retries            uint64
+	ConnectionTimeout  time.Duration
+	CompressionQuality uint64
 }
 
 func NewReverseProxyConfig(
@@ -145,11 +153,11 @@ func NewReverseProxyConfig(
 		return nil, fmt.Errorf("NewReverseProxyConfig: originConfigUnmarshaler was null")
 	}
 	return &ReverseProxyConfig{
-		TunnelHostname:          h2mux.TunnelHostname(tunnelHostname),
-		OriginConfigJSONHandler: &OriginConfigJSONHandler{originConfig},
-		Retries:                 retries,
-		ConnectionTimeout:       connectionTimeout,
-		CompressionQuality:      compressionQuality,
+		TunnelHostname:     h2mux.TunnelHostname(tunnelHostname),
+		OriginConfig:       originConfig,
+		Retries:            retries,
+		ConnectionTimeout:  connectionTimeout,
+		CompressionQuality: compressionQuality,
 	}, nil
 }
 
@@ -158,58 +166,29 @@ func (rpc *ReverseProxyConfig) FailReason(err error) string {
 	return fmt.Sprintf("Cannot apply ReverseProxyConfig, err: %v", err)
 }
 
-func (rpc *ReverseProxyConfig) MarshalJSON() ([]byte, error) {
-	marshaler := make(map[string]ReverseProxyConfig, 1)
-	marshaler[rpc.jsonType()] = *rpc
-	return json.Marshal(marshaler)
-}
-
-func (rpc *ReverseProxyConfig) jsonType() string {
-	return "reverse_proxy_config"
-}
+func (_ *ReverseProxyConfig) isFallibleConfig() {}
 
 //go-sumtype:decl OriginConfig
 type OriginConfig interface {
 	// Service returns a OriginService used to proxy to the origin
 	Service() (originservice.OriginService, error)
 	// go-sumtype requires at least one unexported method, otherwise it will complain that interface is not sealed
-	jsonType() string
-}
-
-type originType int
-
-const (
-	httpType originType = iota
-	wsType
-	helloWorldType
-)
-
-func (ot originType) String() string {
-	switch ot {
-	case httpType:
-		return "Http"
-	case wsType:
-		return "WebSocket"
-	case helloWorldType:
-		return "HelloWorld"
-	default:
-		return "unknown"
-	}
+	isOriginConfig()
 }
 
 type HTTPOriginConfig struct {
-	URLString              string        `capnp:"urlString" json:"url_string" mapstructure:"url_string"`
-	TCPKeepAlive           time.Duration `capnp:"tcpKeepAlive" json:"tcp_keep_alive" mapstructure:"tcp_keep_alive"`
-	DialDualStack          bool          `json:"dial_dual_stack" mapstructure:"dial_dual_stack"`
-	TLSHandshakeTimeout    time.Duration `capnp:"tlsHandshakeTimeout" json:"tls_handshake_timeout" mapstructure:"tls_handshake_timeout"`
-	TLSVerify              bool          `capnp:"tlsVerify" json:"tls_verify" mapstructure:"tls_verify"`
-	OriginCAPool           string        `json:"origin_ca_pool" mapstructure:"origin_ca_pool"`
-	OriginServerName       string        `json:"origin_server_name" mapstructure:"origin_server_name"`
-	MaxIdleConnections     uint64        `json:"max_idle_connections" mapstructure:"max_idle_connections"`
-	IdleConnectionTimeout  time.Duration `json:"idle_connection_timeout" mapstructure:"idle_connection_timeout"`
-	ProxyConnectionTimeout time.Duration `json:"proxy_connection_timeout" mapstructure:"proxy_connection_timeout"`
-	ExpectContinueTimeout  time.Duration `json:"expect_continue_timeout" mapstructure:"expect_continue_timeout"`
-	ChunkedEncoding        bool          `json:"chunked_encoding" mapstructure:"chunked_encoding"`
+	URLString              string        `capnp:"urlString"`
+	TCPKeepAlive           time.Duration `capnp:"tcpKeepAlive"`
+	DialDualStack          bool
+	TLSHandshakeTimeout    time.Duration `capnp:"tlsHandshakeTimeout"`
+	TLSVerify              bool          `capnp:"tlsVerify"`
+	OriginCAPool           string
+	OriginServerName       string
+	MaxIdleConnections     uint64
+	IdleConnectionTimeout  time.Duration
+	ProxyConnectionTimeout time.Duration
+	ExpectContinueTimeout  time.Duration
+	ChunkedEncoding        bool
 }
 
 func (hc *HTTPOriginConfig) Service() (originservice.OriginService, error) {
@@ -248,15 +227,13 @@ func (hc *HTTPOriginConfig) Service() (originservice.OriginService, error) {
 	return originservice.NewHTTPService(transport, url, hc.ChunkedEncoding), nil
 }
 
-func (_ *HTTPOriginConfig) jsonType() string {
-	return httpType.String()
-}
+func (*HTTPOriginConfig) isOriginConfig() {}
 
 type WebSocketOriginConfig struct {
-	URLString        string `capnp:"urlString" json:"url_string" mapstructure:"url_string"`
-	TLSVerify        bool   `capnp:"tlsVerify" json:"tls_verify" mapstructure:"tls_verify"`
-	OriginCAPool     string `json:"origin_ca_pool" mapstructure:"origin_ca_pool"`
-	OriginServerName string `json:"origin_server_name" mapstructure:"origin_server_name"`
+	URLString        string `capnp:"urlString"`
+	TLSVerify        bool   `capnp:"tlsVerify"`
+	OriginCAPool     string
+	OriginServerName string
 }
 
 func (wsc *WebSocketOriginConfig) Service() (originservice.OriginService, error) {
@@ -277,13 +254,11 @@ func (wsc *WebSocketOriginConfig) Service() (originservice.OriginService, error)
 	return originservice.NewWebSocketService(tlsConfig, url)
 }
 
-func (_ *WebSocketOriginConfig) jsonType() string {
-	return wsType.String()
-}
+func (*WebSocketOriginConfig) isOriginConfig() {}
 
 type HelloWorldOriginConfig struct{}
 
-func (_ *HelloWorldOriginConfig) Service() (originservice.OriginService, error) {
+func (*HelloWorldOriginConfig) Service() (originservice.OriginService, error) {
 	helloCert, err := tlsconfig.GetHelloCertificateX509()
 	if err != nil {
 		return nil, errors.Wrap(err, "Cannot get Hello World server certificate")
@@ -308,9 +283,7 @@ func (_ *HelloWorldOriginConfig) Service() (originservice.OriginService, error)
 	return originservice.NewHelloWorldService(transport)
 }
 
-func (_ *HelloWorldOriginConfig) jsonType() string {
-	return helloWorldType.String()
-}
+func (*HelloWorldOriginConfig) isOriginConfig() {}
 
 /*
  * Boilerplate to convert between these structs and the primitive structs
@@ -519,9 +492,9 @@ func UnmarshalDoHProxyConfig(s tunnelrpc.DoHProxyConfig) (*DoHProxyConfig, error
 
 func MarshalReverseProxyConfig(s tunnelrpc.ReverseProxyConfig, p *ReverseProxyConfig) error {
 	s.SetTunnelHostname(p.TunnelHostname.String())
-	switch config := p.OriginConfigJSONHandler.OriginConfig.(type) {
+	switch config := p.OriginConfig.(type) {
 	case *HTTPOriginConfig:
-		ss, err := s.Origin().NewHttp()
+		ss, err := s.OriginConfig().NewHttp()
 		if err != nil {
 			return err
 		}
@@ -529,7 +502,7 @@ func MarshalReverseProxyConfig(s tunnelrpc.ReverseProxyConfig, p *ReverseProxyCo
 			return err
 		}
 	case *WebSocketOriginConfig:
-		ss, err := s.Origin().NewWebsocket()
+		ss, err := s.OriginConfig().NewWebsocket()
 		if err != nil {
 			return err
 		}
@@ -537,7 +510,7 @@ func MarshalReverseProxyConfig(s tunnelrpc.ReverseProxyConfig, p *ReverseProxyCo
 			return err
 		}
 	case *HelloWorldOriginConfig:
-		ss, err := s.Origin().NewHelloWorld()
+		ss, err := s.OriginConfig().NewHelloWorld()
 		if err != nil {
 			return err
 		}
@@ -560,9 +533,9 @@ func UnmarshalReverseProxyConfig(s tunnelrpc.ReverseProxyConfig) (*ReverseProxyC
 		return nil, err
 	}
 	p.TunnelHostname = h2mux.TunnelHostname(tunnelHostname)
-	switch s.Origin().Which() {
-	case tunnelrpc.ReverseProxyConfig_origin_Which_http:
-		ss, err := s.Origin().Http()
+	switch s.OriginConfig().Which() {
+	case tunnelrpc.ReverseProxyConfig_originConfig_Which_http:
+		ss, err := s.OriginConfig().Http()
 		if err != nil {
 			return nil, err
 		}
@@ -570,9 +543,9 @@ func UnmarshalReverseProxyConfig(s tunnelrpc.ReverseProxyConfig) (*ReverseProxyC
 		if err != nil {
 			return nil, err
 		}
-		p.OriginConfigJSONHandler = &OriginConfigJSONHandler{config}
-	case tunnelrpc.ReverseProxyConfig_origin_Which_websocket:
-		ss, err := s.Origin().Websocket()
+		p.OriginConfig = config
+	case tunnelrpc.ReverseProxyConfig_originConfig_Which_websocket:
+		ss, err := s.OriginConfig().Websocket()
 		if err != nil {
 			return nil, err
 		}
@@ -580,9 +553,9 @@ func UnmarshalReverseProxyConfig(s tunnelrpc.ReverseProxyConfig) (*ReverseProxyC
 		if err != nil {
 			return nil, err
 		}
-		p.OriginConfigJSONHandler = &OriginConfigJSONHandler{config}
-	case tunnelrpc.ReverseProxyConfig_origin_Which_helloWorld:
-		ss, err := s.Origin().HelloWorld()
+		p.OriginConfig = config
+	case tunnelrpc.ReverseProxyConfig_originConfig_Which_helloWorld:
+		ss, err := s.OriginConfig().HelloWorld()
 		if err != nil {
 			return nil, err
 		}
@@ -590,7 +563,7 @@ func UnmarshalReverseProxyConfig(s tunnelrpc.ReverseProxyConfig) (*ReverseProxyC
 		if err != nil {
 			return nil, err
 		}
-		p.OriginConfigJSONHandler = &OriginConfigJSONHandler{config}
+		p.OriginConfig = config
 	}
 	p.Retries = s.Retries()
 	p.ConnectionTimeout = time.Duration(s.ConnectionTimeout())
@@ -690,13 +663,13 @@ func (i ClientService_PogsImpl) UseConfiguration(p tunnelrpc.ClientService_useCo
 }
 
 type UseConfigurationResult struct {
-	Success       bool            `json:"success"`
-	FailedConfigs []*FailedConfig `json:"failed_configs"`
+	Success       bool
+	FailedConfigs []*FailedConfig
 }
 
 type FailedConfig struct {
-	Config FallibleConfig `json:"config"`
-	Reason string         `json:"reason"`
+	Config FallibleConfig
+	Reason string
 }
 
 func MarshalFailedConfig(s tunnelrpc.FailedConfig, p *FailedConfig) error {

tunnelrpc/pogs/config_test.go

@@ -61,13 +61,13 @@ func ClientConfigTestCases() []*ClientConfig {
 			sampleReverseProxyConfig(func(c *ReverseProxyConfig) {
 			}),
 			sampleReverseProxyConfig(func(c *ReverseProxyConfig) {
-				c.OriginConfigJSONHandler = &OriginConfigJSONHandler{sampleHTTPOriginConfig()}
+				c.OriginConfig = sampleHTTPOriginConfig()
 			}),
 			sampleReverseProxyConfig(func(c *ReverseProxyConfig) {
-				c.OriginConfigJSONHandler = &OriginConfigJSONHandler{sampleHTTPOriginConfigUnixPath()}
+				c.OriginConfig = sampleHTTPOriginConfigUnixPath()
 			}),
 			sampleReverseProxyConfig(func(c *ReverseProxyConfig) {
-				c.OriginConfigJSONHandler = &OriginConfigJSONHandler{sampleWebSocketOriginConfig()}
+				c.OriginConfig = sampleWebSocketOriginConfig()
 			}),
 		}
 	}
@@ -83,21 +83,14 @@ func ClientConfigTestCases() []*ClientConfig {
 }
 
 func TestClientConfig(t *testing.T) {
-	for i, testCase := range ClientConfigTestCases() {
-		_, seg, err := capnp.NewMessage(capnp.SingleSegment(nil))
-		capnpEntity, err := tunnelrpc.NewClientConfig(seg)
-		if !assert.NoError(t, err) {
-			t.Fatal("Couldn't initialize a new message")
-		}
-		err = MarshalClientConfig(capnpEntity, testCase)
-		if !assert.NoError(t, err, "testCase index %v failed to marshal", i) {
-			continue
-		}
-		result, err := UnmarshalClientConfig(capnpEntity)
-		if !assert.NoError(t, err, "testCase index %v failed to unmarshal", i) {
-			continue
-		}
-		assert.Equal(t, testCase, result, "testCase index %v didn't preserve struct through marshalling and unmarshalling", i)
+	for _, testCase := range ClientConfigTestCases() {
+		b, err := testCase.MarshalBytes()
+		assert.NoError(t, err)
+
+		clientConfig, err := UnmarshalClientConfigFromBytes(b)
+		assert.NoError(t, err)
+
+		assert.Equal(t, testCase, clientConfig)
 	}
 }
 
@@ -167,13 +160,13 @@ func TestReverseProxyConfig(t *testing.T) {
 	testCases := []*ReverseProxyConfig{
 		sampleReverseProxyConfig(),
 		sampleReverseProxyConfig(func(c *ReverseProxyConfig) {
-			c.OriginConfigJSONHandler = &OriginConfigJSONHandler{sampleHTTPOriginConfig()}
+			c.OriginConfig = sampleHTTPOriginConfig()
 		}),
 		sampleReverseProxyConfig(func(c *ReverseProxyConfig) {
-			c.OriginConfigJSONHandler = &OriginConfigJSONHandler{sampleHTTPOriginConfigUnixPath()}
+			c.OriginConfig = sampleHTTPOriginConfigUnixPath()
 		}),
 		sampleReverseProxyConfig(func(c *ReverseProxyConfig) {
-			c.OriginConfigJSONHandler = &OriginConfigJSONHandler{sampleWebSocketOriginConfig()}
+			c.OriginConfig = sampleWebSocketOriginConfig()
 		}),
 	}
 	for i, testCase := range testCases {
@@ -323,11 +316,11 @@ func sampleDoHProxyConfig(overrides ...func(*DoHProxyConfig)) *DoHProxyConfig {
 // applies any number of overrides to it, and returns it.
 func sampleReverseProxyConfig(overrides ...func(*ReverseProxyConfig)) *ReverseProxyConfig {
 	sample := &ReverseProxyConfig{
-		TunnelHostname:          "mock-non-lb-tunnel.example.com",
-		OriginConfigJSONHandler: &OriginConfigJSONHandler{&HelloWorldOriginConfig{}},
-		Retries:                 18,
-		ConnectionTimeout:       5 * time.Second,
-		CompressionQuality:      3,
+		TunnelHostname:     "mock-non-lb-tunnel.example.com",
+		OriginConfig:       &HelloWorldOriginConfig{},
+		Retries:            18,
+		ConnectionTimeout:  5 * time.Second,
+		CompressionQuality: 3,
 	}
 	sample.ensureNoZeroFields()
 	for _, f := range overrides {

tunnelrpc/pogs/json.go

@@ -1,101 +0,0 @@
-package pogs
-
-import (
-	"encoding/json"
-	"fmt"
-
-	"github.com/mitchellh/mapstructure"
-	"github.com/pkg/errors"
-)
-
-// ScopeUnmarshaler can marshal a Scope pog from JSON.
-type ScopeUnmarshaler struct {
-	Scope Scope
-}
-
-// UnmarshalJSON takes in a JSON string, and attempts to marshal it into a Scope.
-// If successful, the Scope member of this ScopeUnmarshaler is set and nil is returned.
-// If unsuccessful, returns an error.
-func (su *ScopeUnmarshaler) UnmarshalJSON(b []byte) error {
-	var scopeJSON map[string]interface{}
-	if err := json.Unmarshal(b, &scopeJSON); err != nil {
-		return errors.Wrapf(err, "cannot unmarshal %s into scopeJSON", string(b))
-	}
-
-	if group, ok := scopeJSON["group"]; ok {
-		if val, ok := group.(string); ok {
-			su.Scope = NewGroup(val)
-			return nil
-		}
-		return fmt.Errorf("JSON should have been a Scope, but the 'group' key contained %v", group)
-	}
-
-	if systemName, ok := scopeJSON["system_name"]; ok {
-		if val, ok := systemName.(string); ok {
-			su.Scope = NewSystemName(val)
-			return nil
-		}
-		return fmt.Errorf("JSON should have been a Scope, but the 'system_name' key contained %v", systemName)
-	}
-
-	return fmt.Errorf("JSON should have been an object with one root key, either 'system_name' or 'group'")
-}
-
-// OriginConfigJSONHandler is a wrapper to serialize OriginConfig with type information, and deserialize JSON
-// into an OriginConfig.
-type OriginConfigJSONHandler struct {
-	OriginConfig OriginConfig
-}
-
-func (ocjh *OriginConfigJSONHandler) MarshalJSON() ([]byte, error) {
-	marshaler := make(map[string]OriginConfig, 1)
-	marshaler[ocjh.OriginConfig.jsonType()] = ocjh.OriginConfig
-	return json.Marshal(marshaler)
-}
-
-func (ocjh *OriginConfigJSONHandler) UnmarshalJSON(b []byte) error {
-	var originJSON map[string]interface{}
-	if err := json.Unmarshal(b, &originJSON); err != nil {
-		return errors.Wrapf(err, "cannot unmarshal %s into originJSON", string(b))
-	}
-
-	if originConfig, ok := originJSON[httpType.String()]; ok {
-		httpOriginConfig := &HTTPOriginConfig{}
-		if err := mapstructure.Decode(originConfig, httpOriginConfig); err != nil {
-			return errors.Wrapf(err, "cannot decode %+v into HTTPOriginConfig", originConfig)
-		}
-		ocjh.OriginConfig = httpOriginConfig
-		return nil
-	}
-
-	if originConfig, ok := originJSON[wsType.String()]; ok {
-		wsOriginConfig := &WebSocketOriginConfig{}
-		if err := mapstructure.Decode(originConfig, wsOriginConfig); err != nil {
-			return errors.Wrapf(err, "cannot decode %+v into WebSocketOriginConfig", originConfig)
-		}
-		ocjh.OriginConfig = wsOriginConfig
-		return nil
-	}
-
-	if originConfig, ok := originJSON[helloWorldType.String()]; ok {
-		helloWorldOriginConfig := &HelloWorldOriginConfig{}
-		if err := mapstructure.Decode(originConfig, helloWorldOriginConfig); err != nil {
-			return errors.Wrapf(err, "cannot decode %+v into HelloWorldOriginConfig", originConfig)
-		}
-		ocjh.OriginConfig = helloWorldOriginConfig
-		return nil
-	}
-
-	return fmt.Errorf("cannot unmarshal %s into OriginConfig", string(b))
-}
-
-// FallibleConfigMarshaler is a wrapper for FallibleConfig to implement custom marshal logic
-type FallibleConfigMarshaler struct {
-	FallibleConfig FallibleConfig
-}
-
-func (fcm *FallibleConfigMarshaler) MarshalJSON() ([]byte, error) {
-	marshaler := make(map[string]FallibleConfig, 1)
-	marshaler[fcm.FallibleConfig.jsonType()] = fcm.FallibleConfig
-	return json.Marshal(marshaler)
-}

tunnelrpc/pogs/json_test.go

@@ -1,342 +0,0 @@
-package pogs
-
-import (
-	"encoding/json"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestScopeUnmarshaler_UnmarshalJSON(t *testing.T) {
-	type fields struct {
-		Scope Scope
-	}
-	type args struct {
-		b []byte
-	}
-	tests := []struct {
-		name      string
-		fields    fields
-		args      args
-		wantErr   bool
-		wantScope Scope
-	}{
-		{
-			name:      "group_successful",
-			args:      args{b: []byte(`{"group": "my-group"}`)},
-			wantScope: NewGroup("my-group"),
-		},
-		{
-			name:      "system_name_successful",
-			args:      args{b: []byte(`{"system_name": "my-computer"}`)},
-			wantScope: NewSystemName("my-computer"),
-		},
-		{
-			name:    "not_a_scope",
-			args:    args{b: []byte(`{"x": "y"}`)},
-			wantErr: true,
-		},
-		{
-			name:    "malformed_group",
-			args:    args{b: []byte(`{"group": ["a", "b"]}`)},
-			wantErr: true,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			su := &ScopeUnmarshaler{
-				Scope: tt.fields.Scope,
-			}
-			err := su.UnmarshalJSON(tt.args.b)
-			if !tt.wantErr {
-				if err != nil {
-					t.Errorf("ScopeUnmarshaler.UnmarshalJSON() error = %v, wantErr %v", err, tt.wantErr)
-				}
-				if !eqScope(tt.wantScope, su.Scope) {
-					t.Errorf("Wanted scope %v but got scope %v", tt.wantScope, su.Scope)
-				}
-			}
-		})
-	}
-}
-
-func TestUnmarshalOrigin(t *testing.T) {
-	tests := []struct {
-		jsonLiteral          string
-		exceptedOriginConfig OriginConfig
-	}{
-		{
-			jsonLiteral: `{
-				"Http":{
-					"url_string":"https.example.com",
-					"tcp_keep_alive":7000000000,
-					"dial_dual_stack":true,
-					"tls_handshake_timeout":11000000000,
-					"tls_verify":true,
-					"origin_ca_pool":"/etc/cert.pem",
-					"origin_server_name":"secure.example.com",
-					"max_idle_connections":19,
-					"idle_connection_timeout":17000000000,
-					"proxy_connection_timeout":15000000000,
-					"expect_continue_timeout":21000000000,
-					"chunked_encoding":true
-				}
-			}`,
-			exceptedOriginConfig: sampleHTTPOriginConfig(),
-		},
-		{
-			jsonLiteral: `{
-				"WebSocket":{
-					"url_string":"ssh://example.com",
-					"tls_verify":true,
-					"origin_ca_pool":"/etc/cert.pem",
-					"origin_server_name":"secure.example.com"
-				}
-			}`,
-			exceptedOriginConfig: sampleWebSocketOriginConfig(),
-		},
-		{
-			jsonLiteral: `{
-				"HelloWorld": {}
-			}`,
-			exceptedOriginConfig: &HelloWorldOriginConfig{},
-		},
-	}
-
-	for _, test := range tests {
-		originConfigJSON := prettyToValidJSON(test.jsonLiteral)
-		var OriginConfigJSONHandler OriginConfigJSONHandler
-		err := json.Unmarshal([]byte(originConfigJSON), &OriginConfigJSONHandler)
-		assert.NoError(t, err)
-		assert.Equal(t, test.exceptedOriginConfig, OriginConfigJSONHandler.OriginConfig)
-	}
-}
-
-func TestUnmarshalClientConfig(t *testing.T) {
-	prettyClientConfigJSON := `{
-		"version":10,
-		"supervisor_config":{
-			"auto_update_frequency":86400000000000,
-			"metrics_update_frequency":300000000000,
-			"grace_period":30000000000
-		},
-		"edge_connection_config":{
-			"num_ha_connections":4,
-			"heartbeat_interval":5000000000,
-			"timeout":30000000000,
-			"max_failed_heartbeats":5,
-			"user_credential_path":"~/.cloudflared/cert.pem"
-		},
-		"doh_proxy_configs":[{
-			"listen_host": "localhost",
-			"listen_port": 53,
-			"upstreams": ["https://1.1.1.1/dns-query", "https://1.0.0.1/dns-query"]
-		}],
-		"reverse_proxy_configs":[{
-			"tunnel_hostname":"sdfjadk33.cftunnel.com",
-			"origin_config":{
-				"Http":{
-					"url_string":"https://127.0.0.1:8080",
-					"tcp_keep_alive":30000000000,
-					"dial_dual_stack":true,
-					"tls_handshake_timeout":10000000000,
-					"tls_verify":true,
-					"origin_ca_pool":"",
-					"origin_server_name":"",
-					"max_idle_connections":100,
-					"idle_connection_timeout":90000000000,
-					"proxy_connection_timeout":90000000000,
-					"expect_continue_timeout":90000000000,
-					"chunked_encoding":true
-				}
-			},
-			"retries":5,
-			"connection_timeout":30,
-			"compression_quality":0
-		}]
-	}`
-	// replace new line and tab
-	clientConfigJSON := prettyToValidJSON(prettyClientConfigJSON)
-
-	var clientConfig ClientConfig
-	err := json.Unmarshal([]byte(clientConfigJSON), &clientConfig)
-	assert.NoError(t, err)
-
-	assert.Equal(t, Version(10), clientConfig.Version)
-
-	supervisorConfig := SupervisorConfig{
-		AutoUpdateFrequency:    time.Hour * 24,
-		MetricsUpdateFrequency: time.Second * 300,
-		GracePeriod:            time.Second * 30,
-	}
-	assert.Equal(t, supervisorConfig, *clientConfig.SupervisorConfig)
-
-	edgeConnectionConfig := EdgeConnectionConfig{
-		NumHAConnections:    4,
-		HeartbeatInterval:   time.Second * 5,
-		Timeout:             time.Second * 30,
-		MaxFailedHeartbeats: 5,
-		UserCredentialPath:  "~/.cloudflared/cert.pem",
-	}
-	assert.Equal(t, edgeConnectionConfig, *clientConfig.EdgeConnectionConfig)
-
-	dohProxyConfig := DoHProxyConfig{
-		ListenHost: "localhost",
-		ListenPort: 53,
-		Upstreams:  []string{"https://1.1.1.1/dns-query", "https://1.0.0.1/dns-query"},
-	}
-
-	assert.Len(t, clientConfig.DoHProxyConfigs, 1)
-	assert.Equal(t, dohProxyConfig, *clientConfig.DoHProxyConfigs[0])
-
-	reverseProxyConfig := ReverseProxyConfig{
-		TunnelHostname: "sdfjadk33.cftunnel.com",
-		OriginConfigJSONHandler: &OriginConfigJSONHandler{
-			OriginConfig: &HTTPOriginConfig{
-				URLString:              "https://127.0.0.1:8080",
-				TCPKeepAlive:           time.Second * 30,
-				DialDualStack:          true,
-				TLSHandshakeTimeout:    time.Second * 10,
-				TLSVerify:              true,
-				OriginCAPool:           "",
-				OriginServerName:       "",
-				MaxIdleConnections:     100,
-				IdleConnectionTimeout:  time.Second * 90,
-				ProxyConnectionTimeout: time.Second * 90,
-				ExpectContinueTimeout:  time.Second * 90,
-				ChunkedEncoding:        true,
-			},
-		},
-		Retries:            5,
-		ConnectionTimeout:  30,
-		CompressionQuality: 0,
-	}
-
-	assert.Len(t, clientConfig.ReverseProxyConfigs, 1)
-	assert.Equal(t, reverseProxyConfig, *clientConfig.ReverseProxyConfigs[0])
-}
-
-func TestMarshalFallibleConfig(t *testing.T) {
-	tests := []struct {
-		fallibleConfig     FallibleConfig
-		expctedJSONLiteral string
-	}{
-		{
-			fallibleConfig: sampleSupervisorConfig(),
-			expctedJSONLiteral: `{
-				"supervisor_config":{
-					"auto_update_frequency":75600000000000,
-					"metrics_update_frequency":660000000000,
-					"grace_period":31000000000
-				}
-			}`,
-		},
-		{
-			fallibleConfig: sampleEdgeConnectionConfig(),
-			expctedJSONLiteral: `{
-				"edge_connection_config":{
-					"num_ha_connections":49,
-					"heartbeat_interval":5000000000,
-					"timeout":9000000000,
-					"max_failed_heartbeats":9001,
-					"user_credential_path":"/Users/example/.cloudflared/cert.pem"
-				}
-			}`,
-		},
-		{
-			fallibleConfig: sampleDoHProxyConfig(),
-			expctedJSONLiteral: `{
-				"doh_proxy_config":{
-					"listen_host":"127.0.0.1",
-					"listen_port":53,
-					"upstreams":["1.1.1.1","1.0.0.1"]
-				}
-			}`,
-		},
-		{
-			fallibleConfig: sampleReverseProxyConfig(func(c *ReverseProxyConfig) {
-				c.OriginConfigJSONHandler = &OriginConfigJSONHandler{sampleHTTPOriginConfig()}
-			}),
-			expctedJSONLiteral: `{
-				"reverse_proxy_config":{
-					"tunnel_hostname":"mock-non-lb-tunnel.example.com",
-					"origin_config":{
-						"Http":{
-							"url_string":"https.example.com",
-							"tcp_keep_alive":7000000000,
-							"dial_dual_stack":true,
-							"tls_handshake_timeout":11000000000,
-							"tls_verify":true,
-							"origin_ca_pool":"/etc/cert.pem",
-							"origin_server_name":"secure.example.com",
-							"max_idle_connections":19,
-							"idle_connection_timeout":17000000000,
-							"proxy_connection_timeout":15000000000,
-							"expect_continue_timeout":21000000000,
-							"chunked_encoding":true
-						}
-					},
-					"retries":18,
-					"connection_timeout":5000000000,
-					"compression_quality":3
-				}
-			}`,
-		},
-		{
-			fallibleConfig: sampleReverseProxyConfig(func(c *ReverseProxyConfig) {
-				c.OriginConfigJSONHandler = &OriginConfigJSONHandler{sampleWebSocketOriginConfig()}
-			}),
-			expctedJSONLiteral: `{
-				"reverse_proxy_config":{
-					"tunnel_hostname":"mock-non-lb-tunnel.example.com",
-					"origin_config":{
-						"WebSocket":{
-							"url_string":"ssh://example.com",
-							"tls_verify":true,
-							"origin_ca_pool":"/etc/cert.pem",
-							"origin_server_name":"secure.example.com"
-						}
-					},
-					"retries":18,
-					"connection_timeout":5000000000,
-					"compression_quality":3
-				}
-			}`,
-		},
-		{
-			fallibleConfig: sampleReverseProxyConfig(func(c *ReverseProxyConfig) {
-				c.OriginConfigJSONHandler = &OriginConfigJSONHandler{&HelloWorldOriginConfig{}}
-			}),
-			expctedJSONLiteral: `{
-				"reverse_proxy_config":{
-					"tunnel_hostname":"mock-non-lb-tunnel.example.com",
-					"origin_config":{
-						"HelloWorld":{}
-					},
-					"retries":18,
-					"connection_timeout":5000000000,
-					"compression_quality":3
-				}
-			}`,
-		},
-	}
-
-	for _, test := range tests {
-		b, err := json.Marshal(test.fallibleConfig)
-		assert.NoError(t, err)
-		assert.Equal(t, prettyToValidJSON(test.expctedJSONLiteral), string(b))
-	}
-
-}
-
-type prettyJSON string
-
-func prettyToValidJSON(prettyJSON string) string {
-	return strings.ReplaceAll(strings.ReplaceAll(prettyJSON, "\n", ""), "\t", "")
-}
-
-func eqScope(s1, s2 Scope) bool {
-	return s1.Value() == s2.Value() && s1.PostgresType() == s2.PostgresType()
-}

tunnelrpc/pogs/tunnelrpc.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/cloudflare/cloudflared/tunnelrpc"
 	"github.com/google/uuid"
+	"github.com/pkg/errors"
 
 	log "github.com/sirupsen/logrus"
 	capnp "zombiezen.com/go/capnproto2"
@@ -74,28 +75,129 @@ func UnmarshalRegistrationOptions(s tunnelrpc.RegistrationOptions) (*Registratio
 	return p, err
 }
 
-type ConnectResult struct {
-	Err          *ConnectError
-	ServerInfo   ServerInfo
-	ClientConfig ClientConfig
+// ConnectResult models the result of Connect RPC, implemented by ConnectError and ConnectSuccess.
+type ConnectResult interface {
+	ConnectError() *ConnectError
+	ConnectedTo() string
+	ClientConfig() *ClientConfig
+	Marshal(s tunnelrpc.ConnectResult) error
 }
 
-func MarshalConnectResult(s tunnelrpc.ConnectResult, p *ConnectResult) error {
-	return pogs.Insert(tunnelrpc.ConnectResult_TypeID, s.Struct, p)
+func MarshalConnectResult(s tunnelrpc.ConnectResult, p ConnectResult) error {
+	return p.Marshal(s)
 }
 
-func UnmarshalConnectResult(s tunnelrpc.ConnectResult) (*ConnectResult, error) {
-	p := new(ConnectResult)
-	err := pogs.Extract(p, tunnelrpc.ConnectResult_TypeID, s.Struct)
-	return p, err
+func UnmarshalConnectResult(s tunnelrpc.ConnectResult) (ConnectResult, error) {
+	switch s.Result().Which() {
+	case tunnelrpc.ConnectResult_result_Which_err:
+		capnpConnectError, err := s.Result().Err()
+		if err != nil {
+			return nil, err
+		}
+		return UnmarshalConnectError(capnpConnectError)
+	case tunnelrpc.ConnectResult_result_Which_success:
+		capnpConnectSuccess, err := s.Result().Success()
+		if err != nil {
+			return nil, err
+		}
+		return UnmarshalConnectSuccess(capnpConnectSuccess)
+	default:
+		return nil, fmt.Errorf("Unmarshal %v not implemented yet", s.Result().Which().String())
+	}
 }
 
+// ConnectSuccess is the concrete returned type when Connect RPC succeed
+type ConnectSuccess struct {
+	ServerLocationName string
+	Config             *ClientConfig
+}
+
+func (*ConnectSuccess) ConnectError() *ConnectError {
+	return nil
+}
+
+func (cs *ConnectSuccess) ConnectedTo() string {
+	return cs.ServerLocationName
+}
+
+func (cs *ConnectSuccess) ClientConfig() *ClientConfig {
+	return cs.Config
+}
+
+func (cs *ConnectSuccess) Marshal(s tunnelrpc.ConnectResult) error {
+	capnpConnectSuccess, err := s.Result().NewSuccess()
+	if err != nil {
+		return err
+	}
+
+	err = capnpConnectSuccess.SetServerLocationName(cs.ServerLocationName)
+	if err != nil {
+		return errors.Wrap(err, "failed to set ConnectSuccess.ServerLocationName")
+	}
+
+	if cs.Config != nil {
+		capnpClientConfig, err := capnpConnectSuccess.NewClientConfig()
+		if err != nil {
+			return errors.Wrap(err, "failed to initialize ConnectSuccess.ClientConfig")
+		}
+		if err := MarshalClientConfig(capnpClientConfig, cs.Config); err != nil {
+			return errors.Wrap(err, "failed to marshal ClientConfig")
+		}
+	}
+
+	return nil
+}
+
+func UnmarshalConnectSuccess(s tunnelrpc.ConnectSuccess) (*ConnectSuccess, error) {
+	p := new(ConnectSuccess)
+
+	serverLocationName, err := s.ServerLocationName()
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get tunnelrpc.ConnectSuccess.ServerLocationName")
+	}
+	p.ServerLocationName = serverLocationName
+
+	if s.HasClientConfig() {
+		capnpClientConfig, err := s.ClientConfig()
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to get tunnelrpc.ConnectSuccess.ClientConfig")
+		}
+		p.Config, err = UnmarshalClientConfig(capnpClientConfig)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to get unmarshal ClientConfig")
+		}
+	}
+
+	return p, nil
+}
+
+// ConnectError is the concrete returned type when Connect RPC encounters some error
 type ConnectError struct {
 	Cause       string
 	RetryAfter  time.Duration
 	ShouldRetry bool
 }
 
+func (ce *ConnectError) ConnectError() *ConnectError {
+	return ce
+}
+
+func (*ConnectError) ConnectedTo() string {
+	return ""
+}
+
+func (*ConnectError) ClientConfig() *ClientConfig {
+	return nil
+}
+
+func (ce *ConnectError) Marshal(s tunnelrpc.ConnectResult) error {
+	capnpConnectError, err := s.Result().NewErr()
+	if err != nil {
+		return err
+	}
+	return MarshalConnectError(capnpConnectError, ce)
+}
+
 func MarshalConnectError(s tunnelrpc.ConnectError, p *ConnectError) error {
 	return pogs.Insert(tunnelrpc.ConnectError_TypeID, s.Struct, p)
 }
@@ -129,82 +231,13 @@ func UnmarshalServerInfo(s tunnelrpc.ServerInfo) (*ServerInfo, error) {
 	return p, err
 }
 
-//go-sumtype:decl Scope
-type Scope interface {
-	Value() string
-	PostgresType() string
-	GraphQLType() string
-	isScope()
-}
-
-type SystemName struct {
-	systemName string
-}
-
-func NewSystemName(systemName string) *SystemName {
-	return &SystemName{systemName: systemName}
-}
-
-func (s *SystemName) Value() string        { return s.systemName }
-func (_ *SystemName) PostgresType() string { return "system_name" }
-func (_ *SystemName) GraphQLType() string  { return "SYSTEM_NAME" }
-
-func (_ *SystemName) isScope() {}
-
-type Group struct {
-	group string
-}
-
-func NewGroup(group string) *Group {
-	return &Group{group: group}
-}
-
-func (g *Group) Value() string        { return g.group }
-func (_ *Group) PostgresType() string { return "group" }
-func (_ *Group) GraphQLType() string  { return "GROUP" }
-
-func (_ *Group) isScope() {}
-
-func MarshalScope(s tunnelrpc.Scope, p Scope) error {
-	ss := s.Value()
-	switch scope := p.(type) {
-	case *SystemName:
-		ss.SetSystemName(scope.systemName)
-	case *Group:
-		ss.SetGroup(scope.group)
-	default:
-		return fmt.Errorf("unexpected Scope value: %v", p)
-	}
-	return nil
-}
-
-func UnmarshalScope(s tunnelrpc.Scope) (Scope, error) {
-	ss := s.Value()
-	switch ss.Which() {
-	case tunnelrpc.Scope_value_Which_systemName:
-		systemName, err := ss.SystemName()
-		if err != nil {
-			return nil, err
-		}
-		return NewSystemName(systemName), nil
-	case tunnelrpc.Scope_value_Which_group:
-		group, err := ss.Group()
-		if err != nil {
-			return nil, err
-		}
-		return NewGroup(group), nil
-	default:
-		return nil, fmt.Errorf("unexpected Scope tag: %v", ss.Which())
-	}
-}
-
 type ConnectParameters struct {
 	OriginCert          []byte
 	CloudflaredID       uuid.UUID
 	NumPreviousAttempts uint8
 	Tags                []Tag
 	CloudflaredVersion  string
-	Scope               Scope
+	IntentLabel         string
 }
 
 func MarshalConnectParameters(s tunnelrpc.CapnpConnectParameters, p *ConnectParameters) error {
@@ -237,11 +270,7 @@ func MarshalConnectParameters(s tunnelrpc.CapnpConnectParameters, p *ConnectPara
 	if err := s.SetCloudflaredVersion(p.CloudflaredVersion); err != nil {
 		return err
 	}
-	scope, err := s.NewScope()
-	if err != nil {
-		return err
-	}
-	return MarshalScope(scope, p.Scope)
+	return s.SetIntentLabel(p.IntentLabel)
 }
 
 func UnmarshalConnectParameters(s tunnelrpc.CapnpConnectParameters) (*ConnectParameters, error) {
@@ -282,22 +311,14 @@ func UnmarshalConnectParameters(s tunnelrpc.CapnpConnectParameters) (*ConnectPar
 		return nil, err
 	}
 
-	scopeCapnp, err := s.Scope()
-	if err != nil {
-		return nil, err
-	}
-	scope, err := UnmarshalScope(scopeCapnp)
-	if err != nil {
-		return nil, err
-	}
-
+	intentLabel, err := s.IntentLabel()
 	return &ConnectParameters{
 		OriginCert:          originCert,
 		CloudflaredID:       cloudflaredID,
 		NumPreviousAttempts: s.NumPreviousAttempts(),
 		Tags:                tags,
 		CloudflaredVersion:  cloudflaredVersion,
-		Scope:               scope,
+		IntentLabel:         intentLabel,
 	}, nil
 }
 
@@ -305,7 +326,7 @@ type TunnelServer interface {
 	RegisterTunnel(ctx context.Context, originCert []byte, hostname string, options *RegistrationOptions) (*TunnelRegistration, error)
 	GetServerInfo(ctx context.Context) (*ServerInfo, error)
 	UnregisterTunnel(ctx context.Context, gracePeriodNanoSec int64) error
-	Connect(ctx context.Context, paramaters *ConnectParameters) (*ConnectResult, error)
+	Connect(ctx context.Context, parameters *ConnectParameters) (ConnectResult, error)
 }
 
 func TunnelServer_ServerToClient(s TunnelServer) tunnelrpc.TunnelServer {
@@ -366,11 +387,11 @@ func (i TunnelServer_PogsImpl) UnregisterTunnel(p tunnelrpc.TunnelServer_unregis
 }
 
 func (i TunnelServer_PogsImpl) Connect(p tunnelrpc.TunnelServer_connect) error {
-	paramaters, err := p.Params.Parameters()
+	parameters, err := p.Params.Parameters()
 	if err != nil {
 		return err
 	}
-	pogsParameters, err := UnmarshalConnectParameters(paramaters)
+	pogsParameters, err := UnmarshalConnectParameters(parameters)
 	if err != nil {
 		return err
 	}
@@ -383,7 +404,7 @@ func (i TunnelServer_PogsImpl) Connect(p tunnelrpc.TunnelServer_connect) error {
 	if err != nil {
 		return err
 	}
-	return MarshalConnectResult(result, connectResult)
+	return connectResult.Marshal(result)
 }
 
 type TunnelServer_PogsClient struct {
@@ -447,7 +468,7 @@ func (c TunnelServer_PogsClient) UnregisterTunnel(ctx context.Context, gracePeri
 
 func (c TunnelServer_PogsClient) Connect(ctx context.Context,
 	parameters *ConnectParameters,
-) (*ConnectResult, error) {
+) (ConnectResult, error) {
 	client := tunnelrpc.TunnelServer{Client: c.Client}
 	promise := client.Connect(ctx, func(p tunnelrpc.TunnelServer_connect_Params) error {
 		connectParameters, err := p.NewParameters()

tunnelrpc/pogs/tunnelrpc_test.go

@@ -11,50 +11,21 @@ import (
 	capnp "zombiezen.com/go/capnproto2"
 )
 
-// Assert *SystemName implements Scope
-var _ Scope = (*SystemName)(nil)
-
-// Assert *Group implements Scope
-var _ Scope = (*Group)(nil)
-
-func TestScope(t *testing.T) {
-	testCases := []Scope{
-		&SystemName{systemName: "my_system"},
-		&Group{group: "my_group"},
-	}
-	for i, testCase := range testCases {
-		_, seg, err := capnp.NewMessage(capnp.SingleSegment(nil))
-		capnpEntity, err := tunnelrpc.NewScope(seg)
-		if !assert.NoError(t, err) {
-			t.Fatal("Couldn't initialize a new message")
-		}
-		err = MarshalScope(capnpEntity, testCase)
-		if !assert.NoError(t, err, "testCase index %v failed to marshal", i) {
-			continue
-		}
-		result, err := UnmarshalScope(capnpEntity)
-		if !assert.NoError(t, err, "testCase index %v failed to unmarshal", i) {
-			continue
-		}
-		assert.Equal(t, testCase, result, "testCase index %v didn't preserve struct through marshalling and unmarshalling", i)
-	}
-}
-
-func sampleTestConnectResult() *ConnectResult {
-	return &ConnectResult{
-		Err: &ConnectError{
+func TestConnectResult(t *testing.T) {
+	testCases := []ConnectResult{
+		&ConnectError{
 			Cause:       "it broke",
 			ShouldRetry: false,
 			RetryAfter:  2 * time.Second,
 		},
-		ServerInfo:   ServerInfo{LocationName: "computer"},
-		ClientConfig: *sampleClientConfig(),
-	}
-}
-
-func TestConnectResult(t *testing.T) {
-	testCases := []*ConnectResult{
-		sampleTestConnectResult(),
+		&ConnectSuccess{
+			ServerLocationName: "SFO",
+			Config:             sampleClientConfig(),
+		},
+		&ConnectSuccess{
+			ServerLocationName: "",
+			Config:             nil,
+		},
 	}
 	for i, testCase := range testCases {
 		_, seg, err := capnp.NewMessage(capnp.SingleSegment(nil))
@@ -78,7 +49,7 @@ func TestConnectParameters(t *testing.T) {
 	testCases := []*ConnectParameters{
 		sampleConnectParameters(),
 		sampleConnectParameters(func(c *ConnectParameters) {
-			c.Scope = &SystemName{systemName: "my_system"}
+			c.IntentLabel = "my_intent"
 		}),
 		sampleConnectParameters(func(c *ConnectParameters) {
 			c.Tags = nil
@@ -118,7 +89,7 @@ func sampleConnectParameters(overrides ...func(*ConnectParameters)) *ConnectPara
 			},
 		},
 		CloudflaredVersion: "7.0",
-		Scope:              &Group{group: "my_group"},
+		IntentLabel:        "my_intent",
 	}
 	sample.ensureNoZeroFields()
 	for _, f := range overrides {

tunnelrpc/tunnelrpc.capnp

@@ -57,25 +57,15 @@ struct CapnpConnectParameters {
     tags @3 :List(Tag);
     # release version of cloudflared
     cloudflaredVersion @4 :Text;
-    # identifier for this cloudflared instance
-    scope @5 :Scope;
-}
-
-struct Scope {
-    value :union {
-        # Standalone instance
-        systemName @0 :Text;
-        # Associated with a group of identical cloudflared instances
-        group @1 :Text;
-    }
+    # which intent this cloudflared instance should get its behaviour from
+    intentLabel @5 :Text;
 }
 
 struct ConnectResult {
-    err @0 :ConnectError;
-    # Information about the server this connection is established with
-    serverInfo @1 :ServerInfo;
-    # How this cloudflared instance should be configured
-    clientConfig @2 :ClientConfig;
+    result :union {
+        err @0 :ConnectError;
+        success @1 :ConnectSuccess;
+    }
 }
 
 struct ConnectError {
@@ -85,6 +75,13 @@ struct ConnectError {
     shouldRetry @2 :Bool;
 }
 
+struct ConnectSuccess {
+    # Information about the server this connection is established with
+    serverLocationName @0 :Text;
+    # How this cloudflared instance should be configured. This can be null if there isn't an intent for this origin yet
+    clientConfig @1 :ClientConfig;
+}
+
 struct ClientConfig {
     # Version of this configuration. This value is opaque, but is guaranteed
     # to monotonically increase in value. Any configuration supplied to
@@ -136,7 +133,7 @@ struct EdgeConnectionConfig {
 
 struct ReverseProxyConfig {
     tunnelHostname @0 :Text;
-    origin :union {
+    originConfig :union {
         http @1 :HTTPOriginConfig;
         websocket @2 :WebSocketOriginConfig;
         helloWorld @3 :HelloWorldOriginConfig;

vendor/github.com/BurntSushi/toml/cmd/toml-test-decoder/COPYING

@@ -1,21 +0,0 @@
-The MIT License (MIT)
-
-Copyright (c) 2013 TOML authors
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.

vendor/github.com/BurntSushi/toml/cmd/toml-test-encoder/COPYING

@@ -1,21 +0,0 @@
-The MIT License (MIT)
-
-Copyright (c) 2013 TOML authors
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.

vendor/github.com/BurntSushi/toml/cmd/tomlv/COPYING

@@ -1,21 +0,0 @@
-The MIT License (MIT)
-
-Copyright (c) 2013 TOML authors
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.

vendor/github.com/aws/aws-sdk-go/aws/client/client.go

@@ -64,7 +64,7 @@ func New(cfg aws.Config, info metadata.ClientInfo, handlers request.Handlers, op
 	default:
 		maxRetries := aws.IntValue(cfg.MaxRetries)
 		if cfg.MaxRetries == nil || maxRetries == aws.UseServiceDefaultRetries {
-			maxRetries = 3
+			maxRetries = DefaultRetryerMaxNumRetries
 		}
 		svc.Retryer = DefaultRetryer{NumMaxRetries: maxRetries}
 	}

vendor/github.com/aws/aws-sdk-go/aws/client/default_retryer.go

@@ -1,6 +1,7 @@
 package client
 
 import (
+	"math"
 	"strconv"
 	"time"
 
@@ -9,69 +10,142 @@ import (
 )
 
 // DefaultRetryer implements basic retry logic using exponential backoff for
-// most services. If you want to implement custom retry logic, implement the
-// request.Retryer interface or create a structure type that composes this
-// struct and override the specific methods. For example, to override only
-// the MaxRetries method:
+// most services. If you want to implement custom retry logic, you can implement the
+// request.Retryer interface.
 //
-//   type retryer struct {
-//       client.DefaultRetryer
-//   }
-//
-//   // This implementation always has 100 max retries
-//   func (d retryer) MaxRetries() int { return 100 }
 type DefaultRetryer struct {
-	NumMaxRetries int
+	// Num max Retries is the number of max retries that will be performed.
+	// By default, this is zero.
+	NumMaxRetries    int
+
+	// MinRetryDelay is the minimum retry delay after which retry will be performed.
+	// If not set, the value is 0ns.
+	MinRetryDelay    time.Duration
+
+	// MinThrottleRetryDelay is the minimum retry delay when throttled.
+	// If not set, the value is 0ns.
+	MinThrottleDelay time.Duration
+
+	// MaxRetryDelay is the maximum retry delay before which retry must be performed.
+	// If not set, the value is 0ns.
+	MaxRetryDelay    time.Duration
+
+	// MaxThrottleDelay is the maximum retry delay when throttled.
+	// If not set, the value is 0ns.
+	MaxThrottleDelay time.Duration
 }
 
+const (
+	// DefaultRetryerMaxNumRetries sets maximum number of retries
+	DefaultRetryerMaxNumRetries = 3
+
+	// DefaultRetryerMinRetryDelay sets minimum retry delay
+	DefaultRetryerMinRetryDelay = 30 * time.Millisecond
+
+	// DefaultRetryerMinThrottleDelay sets minimum delay when throttled
+	DefaultRetryerMinThrottleDelay = 500 * time.Millisecond
+
+	// DefaultRetryerMaxRetryDelay sets maximum retry delay
+	DefaultRetryerMaxRetryDelay = 300 * time.Second
+
+	// DefaultRetryerMaxThrottleDelay sets maximum delay when throttled
+	DefaultRetryerMaxThrottleDelay = 300 * time.Second
+)
+
 // MaxRetries returns the number of maximum returns the service will use to make
 // an individual API request.
 func (d DefaultRetryer) MaxRetries() int {
 	return d.NumMaxRetries
 }
 
+// setRetryerDefaults sets the default values of the retryer if not set
+func (d *DefaultRetryer) setRetryerDefaults() {
+	if d.MinRetryDelay == 0 {
+		d.MinRetryDelay = DefaultRetryerMinRetryDelay
+	}
+	if d.MaxRetryDelay == 0 {
+		d.MaxRetryDelay = DefaultRetryerMaxRetryDelay
+	}
+	if d.MinThrottleDelay == 0 {
+		d.MinThrottleDelay = DefaultRetryerMinThrottleDelay
+	}
+	if d.MaxThrottleDelay == 0 {
+		d.MaxThrottleDelay = DefaultRetryerMaxThrottleDelay
+	}
+}
+
 // RetryRules returns the delay duration before retrying this request again
 func (d DefaultRetryer) RetryRules(r *request.Request) time.Duration {
-	// Set the upper limit of delay in retrying at ~five minutes
-	minTime := 30
+
+	// if number of max retries is zero, no retries will be performed.
+	if d.NumMaxRetries == 0 {
+		return 0
+	}
+
+	// Sets default value for retryer members
+	d.setRetryerDefaults()
+
+	// minDelay is the minimum retryer delay
+	minDelay := d.MinRetryDelay
+
+	var initialDelay time.Duration
+
 	isThrottle := r.IsErrorThrottle()
 	if isThrottle {
-		if delay, ok := getRetryDelay(r); ok {
-			return delay
+		if delay, ok := getRetryAfterDelay(r); ok {
+			initialDelay = delay
 		}
-
-		minTime = 500
+		minDelay = d.MinThrottleDelay
 	}
 
 	retryCount := r.RetryCount
-	if isThrottle && retryCount > 8 {
-		retryCount = 8
-	} else if retryCount > 13 {
-		retryCount = 13
+
+	// maxDelay the maximum retryer delay
+	maxDelay := d.MaxRetryDelay
+
+	if isThrottle {
+		maxDelay = d.MaxThrottleDelay
 	}
 
-	delay := (1 << uint(retryCount)) * (sdkrand.SeededRand.Intn(minTime) + minTime)
-	return time.Duration(delay) * time.Millisecond
+	var delay time.Duration
+
+	// Logic to cap the retry count based on the minDelay provided
+	actualRetryCount := int(math.Log2(float64(minDelay))) + 1
+	if actualRetryCount < 63-retryCount {
+		delay = time.Duration(1<<uint64(retryCount)) * getJitterDelay(minDelay)
+		if delay > maxDelay {
+			delay = getJitterDelay(maxDelay / 2)
+		}
+	} else {
+		delay = getJitterDelay(maxDelay / 2)
+	}
+	return delay + initialDelay
+}
+
+// getJitterDelay returns a jittered delay for retry
+func getJitterDelay(duration time.Duration) time.Duration {
+	return time.Duration(sdkrand.SeededRand.Int63n(int64(duration)) + int64(duration))
 }
 
 // ShouldRetry returns true if the request should be retried.
 func (d DefaultRetryer) ShouldRetry(r *request.Request) bool {
+
+	// ShouldRetry returns false if number of max retries is 0.
+	if d.NumMaxRetries == 0 {
+		return false
+	}
+
 	// If one of the other handlers already set the retry state
 	// we don't want to override it based on the service's state
 	if r.Retryable != nil {
 		return *r.Retryable
 	}
-
-	if r.HTTPResponse.StatusCode >= 500 && r.HTTPResponse.StatusCode != 501 {
-		return true
-	}
-
 	return r.IsErrorRetryable() || r.IsErrorThrottle()
 }
 
 // This will look in the Retry-After header, RFC 7231, for how long
 // it will wait before attempting another request
-func getRetryDelay(r *request.Request) (time.Duration, bool) {
+func getRetryAfterDelay(r *request.Request) (time.Duration, bool) {
 	if !canUseRetryAfterHeader(r) {
 		return 0, false
 	}

vendor/github.com/aws/aws-sdk-go/aws/client/no_op_retryer.go

@@ -0,0 +1,28 @@
+package client
+
+import (
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws/request"
+)
+
+// NoOpRetryer provides a retryer that performs no retries.
+// It should be used when we do not want retries to be performed.
+type NoOpRetryer struct{}
+
+// MaxRetries returns the number of maximum returns the service will use to make
+// an individual API; For NoOpRetryer the MaxRetries will always be zero.
+func (d NoOpRetryer) MaxRetries() int {
+	return 0
+}
+
+// ShouldRetry will always return false for NoOpRetryer, as it should never retry.
+func (d NoOpRetryer) ShouldRetry(_ *request.Request) bool {
+	return false
+}
+
+// RetryRules returns the delay duration before retrying this request again;
+// since NoOpRetryer does not retry, RetryRules always returns 0.
+func (d NoOpRetryer) RetryRules(_ *request.Request) time.Duration {
+	return 0
+}

vendor/github.com/aws/aws-sdk-go/aws/convert_types.go

@@ -179,6 +179,242 @@ func IntValueMap(src map[string]*int) map[string]int {
 	return dst
 }
 
+// Uint returns a pointer to the uint value passed in.
+func Uint(v uint) *uint {
+	return &v
+}
+
+// UintValue returns the value of the uint pointer passed in or
+// 0 if the pointer is nil.
+func UintValue(v *uint) uint {
+	if v != nil {
+		return *v
+	}
+	return 0
+}
+
+// UintSlice converts a slice of uint values uinto a slice of
+// uint pointers
+func UintSlice(src []uint) []*uint {
+	dst := make([]*uint, len(src))
+	for i := 0; i < len(src); i++ {
+		dst[i] = &(src[i])
+	}
+	return dst
+}
+
+// UintValueSlice converts a slice of uint pointers uinto a slice of
+// uint values
+func UintValueSlice(src []*uint) []uint {
+	dst := make([]uint, len(src))
+	for i := 0; i < len(src); i++ {
+		if src[i] != nil {
+			dst[i] = *(src[i])
+		}
+	}
+	return dst
+}
+
+// UintMap converts a string map of uint values uinto a string
+// map of uint pointers
+func UintMap(src map[string]uint) map[string]*uint {
+	dst := make(map[string]*uint)
+	for k, val := range src {
+		v := val
+		dst[k] = &v
+	}
+	return dst
+}
+
+// UintValueMap converts a string map of uint pointers uinto a string
+// map of uint values
+func UintValueMap(src map[string]*uint) map[string]uint {
+	dst := make(map[string]uint)
+	for k, val := range src {
+		if val != nil {
+			dst[k] = *val
+		}
+	}
+	return dst
+}
+
+// Int8 returns a pointer to the int8 value passed in.
+func Int8(v int8) *int8 {
+	return &v
+}
+
+// Int8Value returns the value of the int8 pointer passed in or
+// 0 if the pointer is nil.
+func Int8Value(v *int8) int8 {
+	if v != nil {
+		return *v
+	}
+	return 0
+}
+
+// Int8Slice converts a slice of int8 values into a slice of
+// int8 pointers
+func Int8Slice(src []int8) []*int8 {
+	dst := make([]*int8, len(src))
+	for i := 0; i < len(src); i++ {
+		dst[i] = &(src[i])
+	}
+	return dst
+}
+
+// Int8ValueSlice converts a slice of int8 pointers into a slice of
+// int8 values
+func Int8ValueSlice(src []*int8) []int8 {
+	dst := make([]int8, len(src))
+	for i := 0; i < len(src); i++ {
+		if src[i] != nil {
+			dst[i] = *(src[i])
+		}
+	}
+	return dst
+}
+
+// Int8Map converts a string map of int8 values into a string
+// map of int8 pointers
+func Int8Map(src map[string]int8) map[string]*int8 {
+	dst := make(map[string]*int8)
+	for k, val := range src {
+		v := val
+		dst[k] = &v
+	}
+	return dst
+}
+
+// Int8ValueMap converts a string map of int8 pointers into a string
+// map of int8 values
+func Int8ValueMap(src map[string]*int8) map[string]int8 {
+	dst := make(map[string]int8)
+	for k, val := range src {
+		if val != nil {
+			dst[k] = *val
+		}
+	}
+	return dst
+}
+
+// Int16 returns a pointer to the int16 value passed in.
+func Int16(v int16) *int16 {
+	return &v
+}
+
+// Int16Value returns the value of the int16 pointer passed in or
+// 0 if the pointer is nil.
+func Int16Value(v *int16) int16 {
+	if v != nil {
+		return *v
+	}
+	return 0
+}
+
+// Int16Slice converts a slice of int16 values into a slice of
+// int16 pointers
+func Int16Slice(src []int16) []*int16 {
+	dst := make([]*int16, len(src))
+	for i := 0; i < len(src); i++ {
+		dst[i] = &(src[i])
+	}
+	return dst
+}
+
+// Int16ValueSlice converts a slice of int16 pointers into a slice of
+// int16 values
+func Int16ValueSlice(src []*int16) []int16 {
+	dst := make([]int16, len(src))
+	for i := 0; i < len(src); i++ {
+		if src[i] != nil {
+			dst[i] = *(src[i])
+		}
+	}
+	return dst
+}
+
+// Int16Map converts a string map of int16 values into a string
+// map of int16 pointers
+func Int16Map(src map[string]int16) map[string]*int16 {
+	dst := make(map[string]*int16)
+	for k, val := range src {
+		v := val
+		dst[k] = &v
+	}
+	return dst
+}
+
+// Int16ValueMap converts a string map of int16 pointers into a string
+// map of int16 values
+func Int16ValueMap(src map[string]*int16) map[string]int16 {
+	dst := make(map[string]int16)
+	for k, val := range src {
+		if val != nil {
+			dst[k] = *val
+		}
+	}
+	return dst
+}
+
+// Int32 returns a pointer to the int32 value passed in.
+func Int32(v int32) *int32 {
+	return &v
+}
+
+// Int32Value returns the value of the int32 pointer passed in or
+// 0 if the pointer is nil.
+func Int32Value(v *int32) int32 {
+	if v != nil {
+		return *v
+	}
+	return 0
+}
+
+// Int32Slice converts a slice of int32 values into a slice of
+// int32 pointers
+func Int32Slice(src []int32) []*int32 {
+	dst := make([]*int32, len(src))
+	for i := 0; i < len(src); i++ {
+		dst[i] = &(src[i])
+	}
+	return dst
+}
+
+// Int32ValueSlice converts a slice of int32 pointers into a slice of
+// int32 values
+func Int32ValueSlice(src []*int32) []int32 {
+	dst := make([]int32, len(src))
+	for i := 0; i < len(src); i++ {
+		if src[i] != nil {
+			dst[i] = *(src[i])
+		}
+	}
+	return dst
+}
+
+// Int32Map converts a string map of int32 values into a string
+// map of int32 pointers
+func Int32Map(src map[string]int32) map[string]*int32 {
+	dst := make(map[string]*int32)
+	for k, val := range src {
+		v := val
+		dst[k] = &v
+	}
+	return dst
+}
+
+// Int32ValueMap converts a string map of int32 pointers into a string
+// map of int32 values
+func Int32ValueMap(src map[string]*int32) map[string]int32 {
+	dst := make(map[string]int32)
+	for k, val := range src {
+		if val != nil {
+			dst[k] = *val
+		}
+	}
+	return dst
+}
+
 // Int64 returns a pointer to the int64 value passed in.
 func Int64(v int64) *int64 {
 	return &v
@@ -238,6 +474,301 @@ func Int64ValueMap(src map[string]*int64) map[string]int64 {
 	return dst
 }
 
+// Uint8 returns a pointer to the uint8 value passed in.
+func Uint8(v uint8) *uint8 {
+	return &v
+}
+
+// Uint8Value returns the value of the uint8 pointer passed in or
+// 0 if the pointer is nil.
+func Uint8Value(v *uint8) uint8 {
+	if v != nil {
+		return *v
+	}
+	return 0
+}
+
+// Uint8Slice converts a slice of uint8 values into a slice of
+// uint8 pointers
+func Uint8Slice(src []uint8) []*uint8 {
+	dst := make([]*uint8, len(src))
+	for i := 0; i < len(src); i++ {
+		dst[i] = &(src[i])
+	}
+	return dst
+}
+
+// Uint8ValueSlice converts a slice of uint8 pointers into a slice of
+// uint8 values
+func Uint8ValueSlice(src []*uint8) []uint8 {
+	dst := make([]uint8, len(src))
+	for i := 0; i < len(src); i++ {
+		if src[i] != nil {
+			dst[i] = *(src[i])
+		}
+	}
+	return dst
+}
+
+// Uint8Map converts a string map of uint8 values into a string
+// map of uint8 pointers
+func Uint8Map(src map[string]uint8) map[string]*uint8 {
+	dst := make(map[string]*uint8)
+	for k, val := range src {
+		v := val
+		dst[k] = &v
+	}
+	return dst
+}
+
+// Uint8ValueMap converts a string map of uint8 pointers into a string
+// map of uint8 values
+func Uint8ValueMap(src map[string]*uint8) map[string]uint8 {
+	dst := make(map[string]uint8)
+	for k, val := range src {
+		if val != nil {
+			dst[k] = *val
+		}
+	}
+	return dst
+}
+
+// Uint16 returns a pointer to the uint16 value passed in.
+func Uint16(v uint16) *uint16 {
+	return &v
+}
+
+// Uint16Value returns the value of the uint16 pointer passed in or
+// 0 if the pointer is nil.
+func Uint16Value(v *uint16) uint16 {
+	if v != nil {
+		return *v
+	}
+	return 0
+}
+
+// Uint16Slice converts a slice of uint16 values into a slice of
+// uint16 pointers
+func Uint16Slice(src []uint16) []*uint16 {
+	dst := make([]*uint16, len(src))
+	for i := 0; i < len(src); i++ {
+		dst[i] = &(src[i])
+	}
+	return dst
+}
+
+// Uint16ValueSlice converts a slice of uint16 pointers into a slice of
+// uint16 values
+func Uint16ValueSlice(src []*uint16) []uint16 {
+	dst := make([]uint16, len(src))
+	for i := 0; i < len(src); i++ {
+		if src[i] != nil {
+			dst[i] = *(src[i])
+		}
+	}
+	return dst
+}
+
+// Uint16Map converts a string map of uint16 values into a string
+// map of uint16 pointers
+func Uint16Map(src map[string]uint16) map[string]*uint16 {
+	dst := make(map[string]*uint16)
+	for k, val := range src {
+		v := val
+		dst[k] = &v
+	}
+	return dst
+}
+
+// Uint16ValueMap converts a string map of uint16 pointers into a string
+// map of uint16 values
+func Uint16ValueMap(src map[string]*uint16) map[string]uint16 {
+	dst := make(map[string]uint16)
+	for k, val := range src {
+		if val != nil {
+			dst[k] = *val
+		}
+	}
+	return dst
+}
+
+// Uint32 returns a pointer to the uint32 value passed in.
+func Uint32(v uint32) *uint32 {
+	return &v
+}
+
+// Uint32Value returns the value of the uint32 pointer passed in or
+// 0 if the pointer is nil.
+func Uint32Value(v *uint32) uint32 {
+	if v != nil {
+		return *v
+	}
+	return 0
+}
+
+// Uint32Slice converts a slice of uint32 values into a slice of
+// uint32 pointers
+func Uint32Slice(src []uint32) []*uint32 {
+	dst := make([]*uint32, len(src))
+	for i := 0; i < len(src); i++ {
+		dst[i] = &(src[i])
+	}
+	return dst
+}
+
+// Uint32ValueSlice converts a slice of uint32 pointers into a slice of
+// uint32 values
+func Uint32ValueSlice(src []*uint32) []uint32 {
+	dst := make([]uint32, len(src))
+	for i := 0; i < len(src); i++ {
+		if src[i] != nil {
+			dst[i] = *(src[i])
+		}
+	}
+	return dst
+}
+
+// Uint32Map converts a string map of uint32 values into a string
+// map of uint32 pointers
+func Uint32Map(src map[string]uint32) map[string]*uint32 {
+	dst := make(map[string]*uint32)
+	for k, val := range src {
+		v := val
+		dst[k] = &v
+	}
+	return dst
+}
+
+// Uint32ValueMap converts a string map of uint32 pointers into a string
+// map of uint32 values
+func Uint32ValueMap(src map[string]*uint32) map[string]uint32 {
+	dst := make(map[string]uint32)
+	for k, val := range src {
+		if val != nil {
+			dst[k] = *val
+		}
+	}
+	return dst
+}
+
+// Uint64 returns a pointer to the uint64 value passed in.
+func Uint64(v uint64) *uint64 {
+	return &v
+}
+
+// Uint64Value returns the value of the uint64 pointer passed in or
+// 0 if the pointer is nil.
+func Uint64Value(v *uint64) uint64 {
+	if v != nil {
+		return *v
+	}
+	return 0
+}
+
+// Uint64Slice converts a slice of uint64 values into a slice of
+// uint64 pointers
+func Uint64Slice(src []uint64) []*uint64 {
+	dst := make([]*uint64, len(src))
+	for i := 0; i < len(src); i++ {
+		dst[i] = &(src[i])
+	}
+	return dst
+}
+
+// Uint64ValueSlice converts a slice of uint64 pointers into a slice of
+// uint64 values
+func Uint64ValueSlice(src []*uint64) []uint64 {
+	dst := make([]uint64, len(src))
+	for i := 0; i < len(src); i++ {
+		if src[i] != nil {
+			dst[i] = *(src[i])
+		}
+	}
+	return dst
+}
+
+// Uint64Map converts a string map of uint64 values into a string
+// map of uint64 pointers
+func Uint64Map(src map[string]uint64) map[string]*uint64 {
+	dst := make(map[string]*uint64)
+	for k, val := range src {
+		v := val
+		dst[k] = &v
+	}
+	return dst
+}
+
+// Uint64ValueMap converts a string map of uint64 pointers into a string
+// map of uint64 values
+func Uint64ValueMap(src map[string]*uint64) map[string]uint64 {
+	dst := make(map[string]uint64)
+	for k, val := range src {
+		if val != nil {
+			dst[k] = *val
+		}
+	}
+	return dst
+}
+
+// Float32 returns a pointer to the float32 value passed in.
+func Float32(v float32) *float32 {
+	return &v
+}
+
+// Float32Value returns the value of the float32 pointer passed in or
+// 0 if the pointer is nil.
+func Float32Value(v *float32) float32 {
+	if v != nil {
+		return *v
+	}
+	return 0
+}
+
+// Float32Slice converts a slice of float32 values into a slice of
+// float32 pointers
+func Float32Slice(src []float32) []*float32 {
+	dst := make([]*float32, len(src))
+	for i := 0; i < len(src); i++ {
+		dst[i] = &(src[i])
+	}
+	return dst
+}
+
+// Float32ValueSlice converts a slice of float32 pointers into a slice of
+// float32 values
+func Float32ValueSlice(src []*float32) []float32 {
+	dst := make([]float32, len(src))
+	for i := 0; i < len(src); i++ {
+		if src[i] != nil {
+			dst[i] = *(src[i])
+		}
+	}
+	return dst
+}
+
+// Float32Map converts a string map of float32 values into a string
+// map of float32 pointers
+func Float32Map(src map[string]float32) map[string]*float32 {
+	dst := make(map[string]*float32)
+	for k, val := range src {
+		v := val
+		dst[k] = &v
+	}
+	return dst
+}
+
+// Float32ValueMap converts a string map of float32 pointers into a string
+// map of float32 values
+func Float32ValueMap(src map[string]*float32) map[string]float32 {
+	dst := make(map[string]float32)
+	for k, val := range src {
+		if val != nil {
+			dst[k] = *val
+		}
+	}
+	return dst
+}
+
 // Float64 returns a pointer to the float64 value passed in.
 func Float64(v float64) *float64 {
 	return &v

vendor/github.com/aws/aws-sdk-go/aws/csm/metric_chan.go

@@ -16,25 +16,26 @@ var (
 
 type metricChan struct {
 	ch     chan metric
-	paused int64
+	paused *int64
 }
 
 func newMetricChan(size int) metricChan {
 	return metricChan{
-		ch: make(chan metric, size),
+		ch:     make(chan metric, size),
+		paused: new(int64),
 	}
 }
 
 func (ch *metricChan) Pause() {
-	atomic.StoreInt64(&ch.paused, pausedEnum)
+	atomic.StoreInt64(ch.paused, pausedEnum)
 }
 
 func (ch *metricChan) Continue() {
-	atomic.StoreInt64(&ch.paused, runningEnum)
+	atomic.StoreInt64(ch.paused, runningEnum)
 }
 
 func (ch *metricChan) IsPaused() bool {
-	v := atomic.LoadInt64(&ch.paused)
+	v := atomic.LoadInt64(ch.paused)
 	return v == pausedEnum
 }
 

vendor/github.com/aws/aws-sdk-go/aws/ec2metadata/api.go

@@ -152,18 +152,19 @@ type EC2IAMInfo struct {
 // An EC2InstanceIdentityDocument provides the shape for unmarshaling
 // an instance identity document
 type EC2InstanceIdentityDocument struct {
-	DevpayProductCodes []string  `json:"devpayProductCodes"`
-	AvailabilityZone   string    `json:"availabilityZone"`
-	PrivateIP          string    `json:"privateIp"`
-	Version            string    `json:"version"`
-	Region             string    `json:"region"`
-	InstanceID         string    `json:"instanceId"`
-	BillingProducts    []string  `json:"billingProducts"`
-	InstanceType       string    `json:"instanceType"`
-	AccountID          string    `json:"accountId"`
-	PendingTime        time.Time `json:"pendingTime"`
-	ImageID            string    `json:"imageId"`
-	KernelID           string    `json:"kernelId"`
-	RamdiskID          string    `json:"ramdiskId"`
-	Architecture       string    `json:"architecture"`
+	DevpayProductCodes      []string  `json:"devpayProductCodes"`
+	MarketplaceProductCodes []string  `json:"marketplaceProductCodes"`
+	AvailabilityZone        string    `json:"availabilityZone"`
+	PrivateIP               string    `json:"privateIp"`
+	Version                 string    `json:"version"`
+	Region                  string    `json:"region"`
+	InstanceID              string    `json:"instanceId"`
+	BillingProducts         []string  `json:"billingProducts"`
+	InstanceType            string    `json:"instanceType"`
+	AccountID               string    `json:"accountId"`
+	PendingTime             time.Time `json:"pendingTime"`
+	ImageID                 string    `json:"imageId"`
+	KernelID                string    `json:"kernelId"`
+	RamdiskID               string    `json:"ramdiskId"`
+	Architecture            string    `json:"architecture"`
 }

vendor/github.com/aws/aws-sdk-go/aws/endpoints/defaults.go

@@ -11,6 +11,8 @@ const (
 	AwsPartitionID      = "aws"        // AWS Standard partition.
 	AwsCnPartitionID    = "aws-cn"     // AWS China partition.
 	AwsUsGovPartitionID = "aws-us-gov" // AWS GovCloud (US) partition.
+	AwsIsoPartitionID   = "aws-iso"    // AWS ISO (US) partition.
+	AwsIsoBPartitionID  = "aws-iso-b"  // AWS ISOB (US) partition.
 )
 
 // AWS Standard partition's regions.
@@ -47,8 +49,18 @@ const (
 	UsGovWest1RegionID = "us-gov-west-1" // AWS GovCloud (US).
 )
 
+// AWS ISO (US) partition's regions.
+const (
+	UsIsoEast1RegionID = "us-iso-east-1" // US ISO East.
+)
+
+// AWS ISOB (US) partition's regions.
+const (
+	UsIsobEast1RegionID = "us-isob-east-1" // US ISOB East (Ohio).
+)
+
 // DefaultResolver returns an Endpoint resolver that will be able
-// to resolve endpoints for: AWS Standard, AWS China, and AWS GovCloud (US).
+// to resolve endpoints for: AWS Standard, AWS China, AWS GovCloud (US), AWS ISO (US), and AWS ISOB (US).
 //
 // Use DefaultPartitions() to get the list of the default partitions.
 func DefaultResolver() Resolver {
@@ -56,7 +68,7 @@ func DefaultResolver() Resolver {
 }
 
 // DefaultPartitions returns a list of the partitions the SDK is bundled
-// with. The available partitions are: AWS Standard, AWS China, and AWS GovCloud (US).
+// with. The available partitions are: AWS Standard, AWS China, AWS GovCloud (US), AWS ISO (US), and AWS ISOB (US).
 //
 //    partitions := endpoints.DefaultPartitions
 //    for _, p := range partitions {
@@ -70,6 +82,8 @@ var defaultPartitions = partitions{
 	awsPartition,
 	awscnPartition,
 	awsusgovPartition,
+	awsisoPartition,
+	awsisobPartition,
 }
 
 // AwsPartition returns the Resolver for AWS Standard.
@@ -467,8 +481,14 @@ var awsPartition = partition{
 				"ap-southeast-2": endpoint{},
 				"eu-central-1":   endpoint{},
 				"eu-west-1":      endpoint{},
-				"us-east-1":      endpoint{},
-				"us-west-2":      endpoint{},
+				"fips": endpoint{
+					Hostname: "appstream2-fips.us-west-2.amazonaws.com",
+					CredentialScope: credentialScope{
+						Region: "us-west-2",
+					},
+				},
+				"us-east-1": endpoint{},
+				"us-west-2": endpoint{},
 			},
 		},
 		"appsync": service{
@@ -490,6 +510,7 @@ var awsPartition = partition{
 		"athena": service{
 
 			Endpoints: endpoints{
+				"ap-east-1":      endpoint{},
 				"ap-northeast-1": endpoint{},
 				"ap-northeast-2": endpoint{},
 				"ap-south-1":     endpoint{},
@@ -910,6 +931,7 @@ var awsPartition = partition{
 				"ap-southeast-2": endpoint{},
 				"ca-central-1":   endpoint{},
 				"eu-central-1":   endpoint{},
+				"eu-north-1":     endpoint{},
 				"eu-west-1":      endpoint{},
 				"eu-west-2":      endpoint{},
 				"eu-west-3":      endpoint{},
@@ -1038,6 +1060,16 @@ var awsPartition = partition{
 				"us-west-2":      endpoint{},
 			},
 		},
+		"connect": service{
+
+			Endpoints: endpoints{
+				"ap-northeast-1": endpoint{},
+				"ap-southeast-2": endpoint{},
+				"eu-central-1":   endpoint{},
+				"us-east-1":      endpoint{},
+				"us-west-2":      endpoint{},
+			},
+		},
 		"cur": service{
 
 			Endpoints: endpoints{
@@ -1074,8 +1106,11 @@ var awsPartition = partition{
 				"ap-northeast-2": endpoint{},
 				"ap-southeast-1": endpoint{},
 				"ap-southeast-2": endpoint{},
+				"ca-central-1":   endpoint{},
 				"eu-central-1":   endpoint{},
 				"eu-west-1":      endpoint{},
+				"eu-west-2":      endpoint{},
+				"eu-west-3":      endpoint{},
 				"fips-us-east-1": endpoint{
 					Hostname: "datasync-fips.us-east-1.amazonaws.com",
 					CredentialScope: credentialScope{
@@ -1100,10 +1135,11 @@ var awsPartition = partition{
 						Region: "us-west-2",
 					},
 				},
-				"us-east-1": endpoint{},
-				"us-east-2": endpoint{},
-				"us-west-1": endpoint{},
-				"us-west-2": endpoint{},
+				"me-south-1": endpoint{},
+				"us-east-1":  endpoint{},
+				"us-east-2":  endpoint{},
+				"us-west-1":  endpoint{},
+				"us-west-2":  endpoint{},
 			},
 		},
 		"dax": service{
@@ -1131,6 +1167,7 @@ var awsPartition = partition{
 		"directconnect": service{
 
 			Endpoints: endpoints{
+				"ap-east-1":      endpoint{},
 				"ap-northeast-1": endpoint{},
 				"ap-northeast-2": endpoint{},
 				"ap-south-1":     endpoint{},
@@ -1706,6 +1743,7 @@ var awsPartition = partition{
 				"eu-west-1":      endpoint{},
 				"eu-west-2":      endpoint{},
 				"eu-west-3":      endpoint{},
+				"me-south-1":     endpoint{},
 				"sa-east-1":      endpoint{},
 				"us-east-1":      endpoint{},
 				"us-east-2":      endpoint{},
@@ -1757,6 +1795,7 @@ var awsPartition = partition{
 				"eu-west-1":      endpoint{},
 				"eu-west-2":      endpoint{},
 				"eu-west-3":      endpoint{},
+				"me-south-1":     endpoint{},
 				"sa-east-1":      endpoint{},
 				"us-east-1":      endpoint{},
 				"us-east-1-fips": endpoint{
@@ -1846,6 +1885,7 @@ var awsPartition = partition{
 				},
 			},
 			Endpoints: endpoints{
+				"ap-east-1":      endpoint{},
 				"ap-northeast-1": endpoint{},
 				"ap-northeast-2": endpoint{},
 				"ap-south-1":     endpoint{},
@@ -1857,6 +1897,7 @@ var awsPartition = partition{
 				"eu-west-1":      endpoint{},
 				"eu-west-2":      endpoint{},
 				"eu-west-3":      endpoint{},
+				"me-south-1":     endpoint{},
 				"sa-east-1":      endpoint{},
 				"us-east-1":      endpoint{},
 				"us-east-2":      endpoint{},
@@ -1942,6 +1983,7 @@ var awsPartition = partition{
 			},
 			Endpoints: endpoints{
 				"ap-northeast-1": endpoint{},
+				"ap-northeast-2": endpoint{},
 				"ap-southeast-2": endpoint{},
 				"eu-west-1":      endpoint{},
 				"us-east-1":      endpoint{},
@@ -2217,6 +2259,7 @@ var awsPartition = partition{
 				"ap-southeast-2": endpoint{},
 				"eu-central-1":   endpoint{},
 				"eu-west-1":      endpoint{},
+				"eu-west-2":      endpoint{},
 				"eu-west-3":      endpoint{},
 				"sa-east-1":      endpoint{},
 				"us-east-1":      endpoint{},
@@ -2514,6 +2557,16 @@ var awsPartition = partition{
 				"us-west-2":      endpoint{},
 			},
 		},
+		"qldb": service{
+
+			Endpoints: endpoints{
+				"ap-northeast-1": endpoint{},
+				"eu-west-1":      endpoint{},
+				"us-east-1":      endpoint{},
+				"us-east-2":      endpoint{},
+				"us-west-2":      endpoint{},
+			},
+		},
 		"ram": service{
 
 			Endpoints: endpoints{
@@ -2614,12 +2667,36 @@ var awsPartition = partition{
 				"eu-west-1":      endpoint{},
 				"eu-west-2":      endpoint{},
 				"eu-west-3":      endpoint{},
-				"me-south-1":     endpoint{},
-				"sa-east-1":      endpoint{},
-				"us-east-1":      endpoint{},
-				"us-east-2":      endpoint{},
-				"us-west-1":      endpoint{},
-				"us-west-2":      endpoint{},
+				"fips-us-east-1": endpoint{
+					Hostname: "resource-groups-fips.us-east-1.amazonaws.com",
+					CredentialScope: credentialScope{
+						Region: "us-east-1",
+					},
+				},
+				"fips-us-east-2": endpoint{
+					Hostname: "resource-groups-fips.us-east-2.amazonaws.com",
+					CredentialScope: credentialScope{
+						Region: "us-east-2",
+					},
+				},
+				"fips-us-west-1": endpoint{
+					Hostname: "resource-groups-fips.us-west-1.amazonaws.com",
+					CredentialScope: credentialScope{
+						Region: "us-west-1",
+					},
+				},
+				"fips-us-west-2": endpoint{
+					Hostname: "resource-groups-fips.us-west-2.amazonaws.com",
+					CredentialScope: credentialScope{
+						Region: "us-west-2",
+					},
+				},
+				"me-south-1": endpoint{},
+				"sa-east-1":  endpoint{},
+				"us-east-1":  endpoint{},
+				"us-east-2":  endpoint{},
+				"us-west-1":  endpoint{},
+				"us-west-2":  endpoint{},
 			},
 		},
 		"robomaker": service{
@@ -3023,6 +3100,7 @@ var awsPartition = partition{
 				"eu-west-1":      endpoint{},
 				"eu-west-2":      endpoint{},
 				"eu-west-3":      endpoint{},
+				"me-south-1":     endpoint{},
 				"sa-east-1":      endpoint{},
 				"us-east-1":      endpoint{},
 				"us-east-2":      endpoint{},
@@ -3133,6 +3211,7 @@ var awsPartition = partition{
 		"servicediscovery": service{
 
 			Endpoints: endpoints{
+				"ap-east-1":      endpoint{},
 				"ap-northeast-1": endpoint{},
 				"ap-northeast-2": endpoint{},
 				"ap-south-1":     endpoint{},
@@ -3140,9 +3219,11 @@ var awsPartition = partition{
 				"ap-southeast-2": endpoint{},
 				"ca-central-1":   endpoint{},
 				"eu-central-1":   endpoint{},
+				"eu-north-1":     endpoint{},
 				"eu-west-1":      endpoint{},
 				"eu-west-2":      endpoint{},
 				"eu-west-3":      endpoint{},
+				"me-south-1":     endpoint{},
 				"sa-east-1":      endpoint{},
 				"us-east-1":      endpoint{},
 				"us-east-2":      endpoint{},
@@ -3150,6 +3231,16 @@ var awsPartition = partition{
 				"us-west-2":      endpoint{},
 			},
 		},
+		"session.qldb": service{
+
+			Endpoints: endpoints{
+				"ap-northeast-1": endpoint{},
+				"eu-west-1":      endpoint{},
+				"us-east-1":      endpoint{},
+				"us-east-2":      endpoint{},
+				"us-west-2":      endpoint{},
+			},
+		},
 		"shield": service{
 			IsRegionalized: boxedFalse,
 			Defaults: endpoint{
@@ -4164,7 +4255,7 @@ var awscnPartition = partition{
 
 			Endpoints: endpoints{
 				"aws-cn-global": endpoint{
-					Hostname: "support.cn-north-1.amazonaws.com",
+					Hostname: "support.cn-north-1.amazonaws.com.cn",
 					CredentialScope: credentialScope{
 						Region: "cn-north-1",
 					},
@@ -4698,6 +4789,7 @@ var awsusgovPartition = partition{
 		"ram": service{
 
 			Endpoints: endpoints{
+				"us-gov-east-1": endpoint{},
 				"us-gov-west-1": endpoint{},
 			},
 		},
@@ -4721,6 +4813,25 @@ var awsusgovPartition = partition{
 				"us-gov-west-1": endpoint{},
 			},
 		},
+		"resource-groups": service{
+
+			Endpoints: endpoints{
+				"fips-us-gov-east-1": endpoint{
+					Hostname: "resource-groups.us-gov-east-1.amazonaws.com",
+					CredentialScope: credentialScope{
+						Region: "us-gov-east-1",
+					},
+				},
+				"fips-us-gov-west-1": endpoint{
+					Hostname: "resource-groups.us-gov-west-1.amazonaws.com",
+					CredentialScope: credentialScope{
+						Region: "us-gov-west-1",
+					},
+				},
+				"us-gov-east-1": endpoint{},
+				"us-gov-west-1": endpoint{},
+			},
+		},
 		"route53": service{
 			PartitionEndpoint: "aws-us-gov-global",
 			IsRegionalized:    boxedFalse,
@@ -4822,6 +4933,18 @@ var awsusgovPartition = partition{
 				},
 			},
 		},
+		"servicecatalog": service{
+
+			Endpoints: endpoints{
+				"us-gov-west-1": endpoint{},
+				"us-gov-west-1-fips": endpoint{
+					Hostname: "servicecatalog-fips.us-gov-west-1.amazonaws.com",
+					CredentialScope: credentialScope{
+						Region: "us-gov-west-1",
+					},
+				},
+			},
+		},
 		"sms": service{
 
 			Endpoints: endpoints{
@@ -4947,3 +5070,612 @@ var awsusgovPartition = partition{
 		},
 	},
 }
+
+// AwsIsoPartition returns the Resolver for AWS ISO (US).
+func AwsIsoPartition() Partition {
+	return awsisoPartition.Partition()
+}
+
+var awsisoPartition = partition{
+	ID:        "aws-iso",
+	Name:      "AWS ISO (US)",
+	DNSSuffix: "c2s.ic.gov",
+	RegionRegex: regionRegex{
+		Regexp: func() *regexp.Regexp {
+			reg, _ := regexp.Compile("^us\\-iso\\-\\w+\\-\\d+$")
+			return reg
+		}(),
+	},
+	Defaults: endpoint{
+		Hostname:          "{service}.{region}.{dnsSuffix}",
+		Protocols:         []string{"https"},
+		SignatureVersions: []string{"v4"},
+	},
+	Regions: regions{
+		"us-iso-east-1": region{
+			Description: "US ISO East",
+		},
+	},
+	Services: services{
+		"api.ecr": service{
+
+			Endpoints: endpoints{
+				"us-iso-east-1": endpoint{
+					Hostname: "api.ecr.us-iso-east-1.c2s.ic.gov",
+					CredentialScope: credentialScope{
+						Region: "us-iso-east-1",
+					},
+				},
+			},
+		},
+		"application-autoscaling": service{
+			Defaults: endpoint{
+				Hostname:  "autoscaling.{region}.amazonaws.com",
+				Protocols: []string{"http", "https"},
+				CredentialScope: credentialScope{
+					Service: "application-autoscaling",
+				},
+			},
+			Endpoints: endpoints{
+				"us-iso-east-1": endpoint{},
+			},
+		},
+		"autoscaling": service{
+
+			Endpoints: endpoints{
+				"us-iso-east-1": endpoint{
+					Protocols: []string{"http", "https"},
+				},
+			},
+		},
+		"cloudformation": service{
+
+			Endpoints: endpoints{
+				"us-iso-east-1": endpoint{},
+			},
+		},
+		"cloudtrail": service{
+
+			Endpoints: endpoints{
+				"us-iso-east-1": endpoint{},
+			},
+		},
+		"codedeploy": service{
+
+			Endpoints: endpoints{
+				"us-iso-east-1": endpoint{},
+			},
+		},
+		"config": service{
+
+			Endpoints: endpoints{
+				"us-iso-east-1": endpoint{},
+			},
+		},
+		"datapipeline": service{
+
+			Endpoints: endpoints{
+				"us-iso-east-1": endpoint{},
+			},
+		},
+		"directconnect": service{
+
+			Endpoints: endpoints{
+				"us-iso-east-1": endpoint{},
+			},
+		},
+		"dms": service{
+
+			Endpoints: endpoints{
+				"us-iso-east-1": endpoint{},
+			},
+		},
+		"ds": service{
+
+			Endpoints: endpoints{
+				"us-iso-east-1": endpoint{},
+			},
+		},
+		"dynamodb": service{
+
+			Endpoints: endpoints{
+				"us-iso-east-1": endpoint{
+					Protocols: []string{"http", "https"},
+				},
+			},
+		},
+		"ec2": service{
+
+			Endpoints: endpoints{
+				"us-iso-east-1": endpoint{},
+			},
+		},
+		"ec2metadata": service{
+			PartitionEndpoint: "aws-global",
+			IsRegionalized:    boxedFalse,
+
+			Endpoints: endpoints{
+				"aws-global": endpoint{
+					Hostname:  "169.254.169.254/latest",
+					Protocols: []string{"http"},
+				},
+			},
+		},
+		"ecs": service{
+
+			Endpoints: endpoints{
+				"us-iso-east-1": endpoint{},
+			},
+		},
+		"elasticache": service{
+
+			Endpoints: endpoints{
+				"us-iso-east-1": endpoint{},
+			},
+		},
+		"elasticloadbalancing": service{
+
+			Endpoints: endpoints{
+				"us-iso-east-1": endpoint{
+					Protocols: []string{"http", "https"},
+				},
+			},
+		},
+		"elasticmapreduce": service{
+
+			Endpoints: endpoints{
+				"us-iso-east-1": endpoint{
+					Protocols: []string{"https"},
+				},
+			},
+		},
+		"events": service{
+
+			Endpoints: endpoints{
+				"us-iso-east-1": endpoint{},
+			},
+		},
+		"glacier": service{
+
+			Endpoints: endpoints{
+				"us-iso-east-1": endpoint{
+					Protocols: []string{"http", "https"},
+				},
+			},
+		},
+		"health": service{
+
+			Endpoints: endpoints{
+				"us-iso-east-1": endpoint{},
+			},
+		},
+		"iam": service{
+			PartitionEndpoint: "aws-iso-global",
+			IsRegionalized:    boxedFalse,
+
+			Endpoints: endpoints{
+				"aws-iso-global": endpoint{
+					Hostname: "iam.us-iso-east-1.c2s.ic.gov",
+					CredentialScope: credentialScope{
+						Region: "us-iso-east-1",
+					},
+				},
+			},
+		},
+		"kinesis": service{
+
+			Endpoints: endpoints{
+				"us-iso-east-1": endpoint{},
+			},
+		},
+		"kms": service{
+
+			Endpoints: endpoints{
+				"ProdFips": endpoint{
+					Hostname: "kms-fips.us-iso-east-1.c2s.ic.gov",
+					CredentialScope: credentialScope{
+						Region: "us-iso-east-1",
+					},
+				},
+				"us-iso-east-1": endpoint{},
+			},
+		},
+		"lambda": service{
+
+			Endpoints: endpoints{
+				"us-iso-east-1": endpoint{},
+			},
+		},
+		"logs": service{
+
+			Endpoints: endpoints{
+				"us-iso-east-1": endpoint{},
+			},
+		},
+		"monitoring": service{
+
+			Endpoints: endpoints{
+				"us-iso-east-1": endpoint{},
+			},
+		},
+		"rds": service{
+
+			Endpoints: endpoints{
+				"us-iso-east-1": endpoint{},
+			},
+		},
+		"redshift": service{
+
+			Endpoints: endpoints{
+				"us-iso-east-1": endpoint{},
+			},
+		},
+		"route53": service{
+			PartitionEndpoint: "aws-iso-global",
+			IsRegionalized:    boxedFalse,
+
+			Endpoints: endpoints{
+				"aws-iso-global": endpoint{
+					Hostname: "route53.c2s.ic.gov",
+					CredentialScope: credentialScope{
+						Region: "us-iso-east-1",
+					},
+				},
+			},
+		},
+		"s3": service{
+			Defaults: endpoint{
+				SignatureVersions: []string{"s3v4"},
+			},
+			Endpoints: endpoints{
+				"us-iso-east-1": endpoint{
+					Protocols:         []string{"http", "https"},
+					SignatureVersions: []string{"s3v4"},
+				},
+			},
+		},
+		"snowball": service{
+
+			Endpoints: endpoints{
+				"us-iso-east-1": endpoint{},
+			},
+		},
+		"sns": service{
+
+			Endpoints: endpoints{
+				"us-iso-east-1": endpoint{
+					Protocols: []string{"http", "https"},
+				},
+			},
+		},
+		"sqs": service{
+
+			Endpoints: endpoints{
+				"us-iso-east-1": endpoint{
+					Protocols: []string{"http", "https"},
+				},
+			},
+		},
+		"states": service{
+
+			Endpoints: endpoints{
+				"us-iso-east-1": endpoint{},
+			},
+		},
+		"streams.dynamodb": service{
+			Defaults: endpoint{
+				Protocols: []string{"http", "https"},
+				CredentialScope: credentialScope{
+					Service: "dynamodb",
+				},
+			},
+			Endpoints: endpoints{
+				"us-iso-east-1": endpoint{
+					Protocols: []string{"http", "https"},
+				},
+			},
+		},
+		"sts": service{
+
+			Endpoints: endpoints{
+				"us-iso-east-1": endpoint{},
+			},
+		},
+		"support": service{
+			PartitionEndpoint: "aws-iso-global",
+
+			Endpoints: endpoints{
+				"aws-iso-global": endpoint{
+					Hostname: "support.us-iso-east-1.c2s.ic.gov",
+					CredentialScope: credentialScope{
+						Region: "us-iso-east-1",
+					},
+				},
+			},
+		},
+		"swf": service{
+
+			Endpoints: endpoints{
+				"us-iso-east-1": endpoint{},
+			},
+		},
+		"workspaces": service{
+
+			Endpoints: endpoints{
+				"us-iso-east-1": endpoint{},
+			},
+		},
+	},
+}
+
+// AwsIsoBPartition returns the Resolver for AWS ISOB (US).
+func AwsIsoBPartition() Partition {
+	return awsisobPartition.Partition()
+}
+
+var awsisobPartition = partition{
+	ID:        "aws-iso-b",
+	Name:      "AWS ISOB (US)",
+	DNSSuffix: "sc2s.sgov.gov",
+	RegionRegex: regionRegex{
+		Regexp: func() *regexp.Regexp {
+			reg, _ := regexp.Compile("^us\\-isob\\-\\w+\\-\\d+$")
+			return reg
+		}(),
+	},
+	Defaults: endpoint{
+		Hostname:          "{service}.{region}.{dnsSuffix}",
+		Protocols:         []string{"https"},
+		SignatureVersions: []string{"v4"},
+	},
+	Regions: regions{
+		"us-isob-east-1": region{
+			Description: "US ISOB East (Ohio)",
+		},
+	},
+	Services: services{
+		"application-autoscaling": service{
+			Defaults: endpoint{
+				Hostname:  "autoscaling.{region}.amazonaws.com",
+				Protocols: []string{"http", "https"},
+				CredentialScope: credentialScope{
+					Service: "application-autoscaling",
+				},
+			},
+			Endpoints: endpoints{
+				"us-isob-east-1": endpoint{},
+			},
+		},
+		"autoscaling": service{
+			Defaults: endpoint{
+				Protocols: []string{"http", "https"},
+			},
+			Endpoints: endpoints{
+				"us-isob-east-1": endpoint{},
+			},
+		},
+		"cloudformation": service{
+
+			Endpoints: endpoints{
+				"us-isob-east-1": endpoint{},
+			},
+		},
+		"cloudtrail": service{
+
+			Endpoints: endpoints{
+				"us-isob-east-1": endpoint{},
+			},
+		},
+		"config": service{
+
+			Endpoints: endpoints{
+				"us-isob-east-1": endpoint{},
+			},
+		},
+		"directconnect": service{
+
+			Endpoints: endpoints{
+				"us-isob-east-1": endpoint{},
+			},
+		},
+		"dms": service{
+
+			Endpoints: endpoints{
+				"us-isob-east-1": endpoint{},
+			},
+		},
+		"dynamodb": service{
+			Defaults: endpoint{
+				Protocols: []string{"http", "https"},
+			},
+			Endpoints: endpoints{
+				"us-isob-east-1": endpoint{},
+			},
+		},
+		"ec2": service{
+			Defaults: endpoint{
+				Protocols: []string{"http", "https"},
+			},
+			Endpoints: endpoints{
+				"us-isob-east-1": endpoint{},
+			},
+		},
+		"ec2metadata": service{
+			PartitionEndpoint: "aws-global",
+			IsRegionalized:    boxedFalse,
+
+			Endpoints: endpoints{
+				"aws-global": endpoint{
+					Hostname:  "169.254.169.254/latest",
+					Protocols: []string{"http"},
+				},
+			},
+		},
+		"elasticache": service{
+
+			Endpoints: endpoints{
+				"us-isob-east-1": endpoint{},
+			},
+		},
+		"elasticloadbalancing": service{
+
+			Endpoints: endpoints{
+				"us-isob-east-1": endpoint{
+					Protocols: []string{"https"},
+				},
+			},
+		},
+		"elasticmapreduce": service{
+
+			Endpoints: endpoints{
+				"us-isob-east-1": endpoint{},
+			},
+		},
+		"events": service{
+
+			Endpoints: endpoints{
+				"us-isob-east-1": endpoint{},
+			},
+		},
+		"glacier": service{
+
+			Endpoints: endpoints{
+				"us-isob-east-1": endpoint{},
+			},
+		},
+		"health": service{
+
+			Endpoints: endpoints{
+				"us-isob-east-1": endpoint{},
+			},
+		},
+		"iam": service{
+			PartitionEndpoint: "aws-iso-b-global",
+			IsRegionalized:    boxedFalse,
+
+			Endpoints: endpoints{
+				"aws-iso-b-global": endpoint{
+					Hostname: "iam.us-isob-east-1.sc2s.sgov.gov",
+					CredentialScope: credentialScope{
+						Region: "us-isob-east-1",
+					},
+				},
+			},
+		},
+		"kinesis": service{
+
+			Endpoints: endpoints{
+				"us-isob-east-1": endpoint{},
+			},
+		},
+		"kms": service{
+
+			Endpoints: endpoints{
+				"ProdFips": endpoint{
+					Hostname: "kms-fips.us-isob-east-1.sc2s.sgov.gov",
+					CredentialScope: credentialScope{
+						Region: "us-isob-east-1",
+					},
+				},
+				"us-isob-east-1": endpoint{},
+			},
+		},
+		"logs": service{
+
+			Endpoints: endpoints{
+				"us-isob-east-1": endpoint{},
+			},
+		},
+		"monitoring": service{
+
+			Endpoints: endpoints{
+				"us-isob-east-1": endpoint{},
+			},
+		},
+		"rds": service{
+
+			Endpoints: endpoints{
+				"us-isob-east-1": endpoint{},
+			},
+		},
+		"redshift": service{
+
+			Endpoints: endpoints{
+				"us-isob-east-1": endpoint{},
+			},
+		},
+		"s3": service{
+			Defaults: endpoint{
+				Protocols:         []string{"http", "https"},
+				SignatureVersions: []string{"s3v4"},
+			},
+			Endpoints: endpoints{
+				"us-isob-east-1": endpoint{},
+			},
+		},
+		"snowball": service{
+
+			Endpoints: endpoints{
+				"us-isob-east-1": endpoint{},
+			},
+		},
+		"sns": service{
+			Defaults: endpoint{
+				Protocols: []string{"http", "https"},
+			},
+			Endpoints: endpoints{
+				"us-isob-east-1": endpoint{},
+			},
+		},
+		"sqs": service{
+			Defaults: endpoint{
+				SSLCommonName: "{region}.queue.{dnsSuffix}",
+				Protocols:     []string{"http", "https"},
+			},
+			Endpoints: endpoints{
+				"us-isob-east-1": endpoint{},
+			},
+		},
+		"states": service{
+
+			Endpoints: endpoints{
+				"us-isob-east-1": endpoint{},
+			},
+		},
+		"streams.dynamodb": service{
+			Defaults: endpoint{
+				Protocols: []string{"http", "https"},
+				CredentialScope: credentialScope{
+					Service: "dynamodb",
+				},
+			},
+			Endpoints: endpoints{
+				"us-isob-east-1": endpoint{},
+			},
+		},
+		"sts": service{
+
+			Endpoints: endpoints{
+				"us-isob-east-1": endpoint{},
+			},
+		},
+		"support": service{
+			PartitionEndpoint: "aws-iso-b-global",
+
+			Endpoints: endpoints{
+				"aws-iso-b-global": endpoint{
+					Hostname: "support.us-isob-east-1.sc2s.sgov.gov",
+					CredentialScope: credentialScope{
+						Region: "us-isob-east-1",
+					},
+				},
+			},
+		},
+		"swf": service{
+
+			Endpoints: endpoints{
+				"us-isob-east-1": endpoint{},
+			},
+		},
+	},
+}

vendor/github.com/aws/aws-sdk-go/aws/request/handlers.go

@@ -23,7 +23,7 @@ type Handlers struct {
 	Complete         HandlerList
 }
 
-// Copy returns of this handler's lists.
+// Copy returns a copy of this handler's lists.
 func (h *Handlers) Copy() Handlers {
 	return Handlers{
 		Validate:         h.Validate.copy(),
@@ -42,7 +42,7 @@ func (h *Handlers) Copy() Handlers {
 	}
 }
 
-// Clear removes callback functions for all handlers
+// Clear removes callback functions for all handlers.
 func (h *Handlers) Clear() {
 	h.Validate.Clear()
 	h.Build.Clear()

vendor/github.com/aws/aws-sdk-go/aws/request/retryer.go

@@ -94,10 +94,6 @@ var validParentCodes = map[string]struct{}{
 	ErrCodeRead:          {},
 }
 
-type temporaryError interface {
-	Temporary() bool
-}
-
 func isNestedErrorRetryable(parentErr awserr.Error) bool {
 	if parentErr == nil {
 		return false
@@ -116,7 +112,7 @@ func isNestedErrorRetryable(parentErr awserr.Error) bool {
 		return isCodeRetryable(aerr.Code())
 	}
 
-	if t, ok := err.(temporaryError); ok {
+	if t, ok := err.(temporary); ok {
 		return t.Temporary() || isErrConnectionReset(err)
 	}
 
@@ -223,6 +219,16 @@ func (r *Request) IsErrorRetryable() bool {
 		return true
 	}
 
+	// HTTP response status code 501 should not be retried.
+	// 501 represents Not Implemented which means the request method is not
+	// supported by the server and cannot be handled.
+	if r.HTTPResponse != nil {
+		// HTTP response status code 500 represents internal server error and
+		// should be retried without any throttle.
+		if r.HTTPResponse.StatusCode == 500 {
+			return true
+		}
+	}
 	return IsErrorRetryable(r.Error)
 }
 
@@ -237,7 +243,11 @@ func (r *Request) IsErrorThrottle() bool {
 
 	if r.HTTPResponse != nil {
 		switch r.HTTPResponse.StatusCode {
-		case 429, 502, 503, 504:
+		case
+			429, // error caused due to too many requests
+			502, // Bad Gateway error should be throttled
+			503, // caused when service is unavailable
+			504: // error occurred due to gateway timeout
 			return true
 		}
 	}

vendor/github.com/aws/aws-sdk-go/aws/session/session.go

@@ -136,7 +136,7 @@ func New(cfgs ...*aws.Config) *Session {
 // to be built with retrieving credentials with AssumeRole set in the config.
 //
 // See the NewSessionWithOptions func for information on how to override or
-// control through code how the Session will be created. Such as specifying the
+// control through code how the Session will be created, such as specifying the
 // config profile, and controlling if shared config is enabled or not.
 func NewSession(cfgs ...*aws.Config) (*Session, error) {
 	opts := Options{}
@@ -571,7 +571,7 @@ func initHandlers(s *Session) {
 	}
 }
 
-// Copy creates and returns a copy of the current Session, coping the config
+// Copy creates and returns a copy of the current Session, copying the config
 // and handlers. If any additional configs are provided they will be merged
 // on top of the Session's copied config.
 //

vendor/github.com/aws/aws-sdk-go/aws/version.go

@@ -5,4 +5,4 @@ package aws
 const SDKName = "aws-sdk-go"
 
 // SDKVersion is the version of this SDK
-const SDKVersion = "1.23.9"
+const SDKVersion = "1.25.8"

vendor/github.com/aws/aws-sdk-go/internal/ini/ini_parser.go

@@ -162,7 +162,7 @@ loop:
 			if len(tokens) == 0 {
 				break loop
 			}
-
+			// if should skip is true, we skip the tokens until should skip is set to false.
 			step = SkipTokenState
 		}
 
@@ -218,7 +218,7 @@ loop:
 			// S -> equal_expr' expr_stmt'
 			switch k.Kind {
 			case ASTKindEqualExpr:
-				// assiging a value to some key
+				// assigning a value to some key
 				k.AppendChild(newExpression(tok))
 				stack.Push(newExprStatement(k))
 			case ASTKindExpr:
@@ -250,6 +250,13 @@ loop:
 			if !runeCompare(tok.Raw(), openBrace) {
 				return nil, NewParseError("expected '['")
 			}
+			// If OpenScopeState is not at the start, we must mark the previous ast as complete
+			//
+			// for example: if previous ast was a skip statement;
+			// we should mark it as complete before we create a new statement
+			if k.Kind != ASTKindStart {
+				stack.MarkComplete(k)
+			}
 
 			stmt := newStatement()
 			stack.Push(stmt)

vendor/github.com/aws/aws-sdk-go/internal/ini/skipper.go

@@ -22,24 +22,24 @@ func newSkipper() skipper {
 }
 
 func (s *skipper) ShouldSkip(tok Token) bool {
+	// should skip state will be modified only if previous token was new line (NL);
+	// and the current token is not WhiteSpace (WS).
 	if s.shouldSkip &&
 		s.prevTok.Type() == TokenNL &&
 		tok.Type() != TokenWS {
-
 		s.Continue()
 		return false
 	}
 	s.prevTok = tok
-
 	return s.shouldSkip
 }
 
 func (s *skipper) Skip() {
 	s.shouldSkip = true
-	s.prevTok = emptyToken
 }
 
 func (s *skipper) Continue() {
 	s.shouldSkip = false
+	// empty token is assigned as we return to default state, when should skip is false
 	s.prevTok = emptyToken
 }

vendor/github.com/aws/aws-sdk-go/internal/sdkio/byte.go

@@ -0,0 +1,12 @@
+package sdkio
+
+const (
+	// Byte is 8 bits
+	Byte int64 = 1
+	// KibiByte (KiB) is 1024 Bytes
+	KibiByte = Byte * 1024
+	// MebiByte (MiB) is 1024 KiB
+	MebiByte = KibiByte * 1024
+	// GibiByte (GiB) is 1024 MiB
+	GibiByte = MebiByte * 1024
+)

vendor/github.com/aws/aws-sdk-go/internal/sdkmath/floor.go

@@ -0,0 +1,15 @@
+// +build go1.10
+
+package sdkmath
+
+import "math"
+
+// Round returns the nearest integer, rounding half away from zero.
+//
+// Special cases are:
+//	Round(±0) = ±0
+//	Round(±Inf) = ±Inf
+//	Round(NaN) = NaN
+func Round(x float64) float64 {
+	return math.Round(x)
+}

vendor/github.com/aws/aws-sdk-go/internal/sdkmath/floor_go1.9.go

@@ -0,0 +1,56 @@
+// +build !go1.10
+
+package sdkmath
+
+import "math"
+
+// Copied from the Go standard library's (Go 1.12) math/floor.go for use in
+// Go version prior to Go 1.10.
+const (
+	uvone    = 0x3FF0000000000000
+	mask     = 0x7FF
+	shift    = 64 - 11 - 1
+	bias     = 1023
+	signMask = 1 << 63
+	fracMask = 1<<shift - 1
+)
+
+// Round returns the nearest integer, rounding half away from zero.
+//
+// Special cases are:
+//	Round(±0) = ±0
+//	Round(±Inf) = ±Inf
+//	Round(NaN) = NaN
+//
+// Copied from the Go standard library's (Go 1.12) math/floor.go for use in
+// Go version prior to Go 1.10.
+func Round(x float64) float64 {
+	// Round is a faster implementation of:
+	//
+	// func Round(x float64) float64 {
+	//   t := Trunc(x)
+	//   if Abs(x-t) >= 0.5 {
+	//     return t + Copysign(1, x)
+	//   }
+	//   return t
+	// }
+	bits := math.Float64bits(x)
+	e := uint(bits>>shift) & mask
+	if e < bias {
+		// Round abs(x) < 1 including denormals.
+		bits &= signMask // +-0
+		if e == bias-1 {
+			bits |= uvone // +-1
+		}
+	} else if e < bias+shift {
+		// Round any abs(x) >= 1 containing a fractional component [0,1).
+		//
+		// Numbers with larger exponents are returned unchanged since they
+		// must be either an integer, infinity, or NaN.
+		const half = 1 << (shift - 1)
+		e -= bias
+		bits += half >> e
+		bits &^= fracMask >> e
+	}
+	return math.Float64frombits(bits)
+}

vendor/github.com/aws/aws-sdk-go/internal/sdkrand/read.go

@@ -0,0 +1,11 @@
+// +build go1.6
+
+package sdkrand
+
+import "math/rand"
+
+// Read provides the stub for math.Rand.Read method support for go version's
+// 1.6 and greater.
+func Read(r *rand.Rand, p []byte) (int, error) {
+	return r.Read(p)
+}

vendor/github.com/aws/aws-sdk-go/internal/sdkrand/read_1_5.go

@@ -0,0 +1,24 @@
+// +build !go1.6
+
+package sdkrand
+
+import "math/rand"
+
+// Read backfills Go 1.6's math.Rand.Reader for Go 1.5
+func Read(r *rand.Rand, p []byte) (n int, err error) {
+	// Copy of Go standard libraries math package's read function not added to
+	// standard library until Go 1.6.
+	var pos int8
+	var val int64
+	for n = 0; n < len(p); n++ {
+		if pos == 0 {
+			val = r.Int63()
+			pos = 7
+		}
+		p[n] = byte(val)
+		val >>= 8
+		pos--
+	}
+
+	return n, err
+}

vendor/github.com/aws/aws-sdk-go/private/protocol/timestamp.go

@@ -1,8 +1,11 @@
 package protocol
 
 import (
+	"math"
 	"strconv"
 	"time"
+
+	"github.com/aws/aws-sdk-go/internal/sdkmath"
 )
 
 // Names of time formats supported by the SDK
@@ -13,12 +16,19 @@ const (
 )
 
 // Time formats supported by the SDK
+// Output time is intended to not contain decimals
 const (
 	// RFC 7231#section-7.1.1.1 timetamp format. e.g Tue, 29 Apr 2014 18:30:38 GMT
 	RFC822TimeFormat = "Mon, 2 Jan 2006 15:04:05 GMT"
 
+	// This format is used for output time without seconds precision
+	RFC822OutputTimeFormat = "Mon, 02 Jan 2006 15:04:05 GMT"
+
 	// RFC3339 a subset of the ISO8601 timestamp format. e.g 2014-04-29T18:30:38Z
-	ISO8601TimeFormat = "2006-01-02T15:04:05Z"
+	ISO8601TimeFormat = "2006-01-02T15:04:05.999999999Z"
+
+	// This format is used for output time without seconds precision
+	ISO8601OutputTimeFormat = "2006-01-02T15:04:05Z"
 )
 
 // IsKnownTimestampFormat returns if the timestamp format name
@@ -42,9 +52,9 @@ func FormatTime(name string, t time.Time) string {
 
 	switch name {
 	case RFC822TimeFormatName:
-		return t.Format(RFC822TimeFormat)
+		return t.Format(RFC822OutputTimeFormat)
 	case ISO8601TimeFormatName:
-		return t.Format(ISO8601TimeFormat)
+		return t.Format(ISO8601OutputTimeFormat)
 	case UnixTimeFormatName:
 		return strconv.FormatInt(t.Unix(), 10)
 	default:
@@ -62,10 +72,12 @@ func ParseTime(formatName, value string) (time.Time, error) {
 		return time.Parse(ISO8601TimeFormat, value)
 	case UnixTimeFormatName:
 		v, err := strconv.ParseFloat(value, 64)
+		_, dec := math.Modf(v)
+		dec = sdkmath.Round(dec*1e3) / 1e3 //Rounds 0.1229999 to 0.123
 		if err != nil {
 			return time.Time{}, err
 		}
-		return time.Unix(int64(v), 0), nil
+		return time.Unix(int64(v), int64(dec*(1e9))), nil
 	default:
 		panic("unknown timestamp format name, " + formatName)
 	}

vendor/github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil/sort.go

@@ -0,0 +1,32 @@
+package xmlutil
+
+import (
+	"encoding/xml"
+	"strings"
+)
+
+type xmlAttrSlice []xml.Attr
+
+func (x xmlAttrSlice) Len() int {
+	return len(x)
+}
+
+func (x xmlAttrSlice) Less(i, j int) bool {
+	spaceI, spaceJ := x[i].Name.Space, x[j].Name.Space
+	localI, localJ := x[i].Name.Local, x[j].Name.Local
+	valueI, valueJ := x[i].Value, x[j].Value
+
+	spaceCmp := strings.Compare(spaceI, spaceJ)
+	localCmp := strings.Compare(localI, localJ)
+	valueCmp := strings.Compare(valueI, valueJ)
+
+	if spaceCmp == -1 || (spaceCmp == 0 && (localCmp == -1 || (localCmp == 0 && valueCmp == -1))) {
+		return true
+	}
+
+	return false
+}
+
+func (x xmlAttrSlice) Swap(i, j int) {
+	x[i], x[j] = x[j], x[i]
+}

vendor/github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil/xml_to_struct.go

@@ -119,7 +119,18 @@ func (n *XMLNode) findElem(name string) (string, bool) {
 
 // StructToXML writes an XMLNode to a xml.Encoder as tokens.
 func StructToXML(e *xml.Encoder, node *XMLNode, sorted bool) error {
-	e.EncodeToken(xml.StartElement{Name: node.Name, Attr: node.Attr})
+	// Sort Attributes
+	attrs := node.Attr
+	if sorted {
+		sortedAttrs := make([]xml.Attr, len(attrs))
+		for _, k := range node.Attr {
+			sortedAttrs = append(sortedAttrs, k)
+		}
+		sort.Sort(xmlAttrSlice(sortedAttrs))
+		attrs = sortedAttrs
+	}
+
+	e.EncodeToken(xml.StartElement{Name: node.Name, Attr: attrs})
 
 	if node.Text != "" {
 		e.EncodeToken(xml.CharData([]byte(node.Text)))

vendor/github.com/aws/aws-sdk-go/service/s3/api.go

@@ -7043,7 +7043,7 @@ func (s *AbortIncompleteMultipartUpload) SetDaysAfterInitiation(v int64) *AbortI
 }
 
 type AbortMultipartUploadInput struct {
-	_ struct{} `type:"structure"`
+	_ struct{} `locationName:"AbortMultipartUploadRequest" type:"structure"`
 
 	// Name of the bucket to which the multipart upload was initiated.
 	//
@@ -8084,7 +8084,7 @@ func (s *CommonPrefix) SetPrefix(v string) *CommonPrefix {
 }
 
 type CompleteMultipartUploadInput struct {
-	_ struct{} `type:"structure" payload:"MultipartUpload"`
+	_ struct{} `locationName:"CompleteMultipartUploadRequest" type:"structure" payload:"MultipartUpload"`
 
 	// Bucket is a required field
 	Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"`
@@ -8404,7 +8404,7 @@ func (s *ContinuationEvent) UnmarshalEvent(
 }
 
 type CopyObjectInput struct {
-	_ struct{} `type:"structure"`
+	_ struct{} `locationName:"CopyObjectRequest" type:"structure"`
 
 	// The canned ACL to apply to the object.
 	ACL *string `location:"header" locationName:"x-amz-acl" type:"string" enum:"ObjectCannedACL"`
@@ -9025,7 +9025,7 @@ func (s *CreateBucketConfiguration) SetLocationConstraint(v string) *CreateBucke
 }
 
 type CreateBucketInput struct {
-	_ struct{} `type:"structure" payload:"CreateBucketConfiguration"`
+	_ struct{} `locationName:"CreateBucketRequest" type:"structure" payload:"CreateBucketConfiguration"`
 
 	// The canned ACL to apply to the bucket.
 	ACL *string `location:"header" locationName:"x-amz-acl" type:"string" enum:"BucketCannedACL"`
@@ -9166,7 +9166,7 @@ func (s *CreateBucketOutput) SetLocation(v string) *CreateBucketOutput {
 }
 
 type CreateMultipartUploadInput struct {
-	_ struct{} `type:"structure"`
+	_ struct{} `locationName:"CreateMultipartUploadRequest" type:"structure"`
 
 	// The canned ACL to apply to the object.
 	ACL *string `location:"header" locationName:"x-amz-acl" type:"string" enum:"ObjectCannedACL"`
@@ -9708,7 +9708,7 @@ func (s *Delete) SetQuiet(v bool) *Delete {
 }
 
 type DeleteBucketAnalyticsConfigurationInput struct {
-	_ struct{} `type:"structure"`
+	_ struct{} `locationName:"DeleteBucketAnalyticsConfigurationRequest" type:"structure"`
 
 	// The name of the bucket from which an analytics configuration is deleted.
 	//
@@ -9784,7 +9784,7 @@ func (s DeleteBucketAnalyticsConfigurationOutput) GoString() string {
 }
 
 type DeleteBucketCorsInput struct {
-	_ struct{} `type:"structure"`
+	_ struct{} `locationName:"DeleteBucketCorsRequest" type:"structure"`
 
 	// Bucket is a required field
 	Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"`
@@ -9844,7 +9844,7 @@ func (s DeleteBucketCorsOutput) GoString() string {
 }
 
 type DeleteBucketEncryptionInput struct {
-	_ struct{} `type:"structure"`
+	_ struct{} `locationName:"DeleteBucketEncryptionRequest" type:"structure"`
 
 	// The name of the bucket containing the server-side encryption configuration
 	// to delete.
@@ -9907,7 +9907,7 @@ func (s DeleteBucketEncryptionOutput) GoString() string {
 }
 
 type DeleteBucketInput struct {
-	_ struct{} `type:"structure"`
+	_ struct{} `locationName:"DeleteBucketRequest" type:"structure"`
 
 	// Bucket is a required field
 	Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"`
@@ -9953,7 +9953,7 @@ func (s *DeleteBucketInput) getBucket() (v string) {
 }
 
 type DeleteBucketInventoryConfigurationInput struct {
-	_ struct{} `type:"structure"`
+	_ struct{} `locationName:"DeleteBucketInventoryConfigurationRequest" type:"structure"`
 
 	// The name of the bucket containing the inventory configuration to delete.
 	//
@@ -10029,7 +10029,7 @@ func (s DeleteBucketInventoryConfigurationOutput) GoString() string {
 }
 
 type DeleteBucketLifecycleInput struct {
-	_ struct{} `type:"structure"`
+	_ struct{} `locationName:"DeleteBucketLifecycleRequest" type:"structure"`
 
 	// Bucket is a required field
 	Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"`
@@ -10089,7 +10089,7 @@ func (s DeleteBucketLifecycleOutput) GoString() string {
 }
 
 type DeleteBucketMetricsConfigurationInput struct {
-	_ struct{} `type:"structure"`
+	_ struct{} `locationName:"DeleteBucketMetricsConfigurationRequest" type:"structure"`
 
 	// The name of the bucket containing the metrics configuration to delete.
 	//
@@ -10179,7 +10179,7 @@ func (s DeleteBucketOutput) GoString() string {
 }
 
 type DeleteBucketPolicyInput struct {
-	_ struct{} `type:"structure"`
+	_ struct{} `locationName:"DeleteBucketPolicyRequest" type:"structure"`
 
 	// Bucket is a required field
 	Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"`
@@ -10239,7 +10239,7 @@ func (s DeleteBucketPolicyOutput) GoString() string {
 }
 
 type DeleteBucketReplicationInput struct {
-	_ struct{} `type:"structure"`
+	_ struct{} `locationName:"DeleteBucketReplicationRequest" type:"structure"`
 
 	// The bucket name.
 	//
@@ -10304,7 +10304,7 @@ func (s DeleteBucketReplicationOutput) GoString() string {
 }
 
 type DeleteBucketTaggingInput struct {
-	_ struct{} `type:"structure"`
+	_ struct{} `locationName:"DeleteBucketTaggingRequest" type:"structure"`
 
 	// Bucket is a required field
 	Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"`
@@ -10364,7 +10364,7 @@ func (s DeleteBucketTaggingOutput) GoString() string {
 }
 
 type DeleteBucketWebsiteInput struct {
-	_ struct{} `type:"structure"`
+	_ struct{} `locationName:"DeleteBucketWebsiteRequest" type:"structure"`
 
 	// Bucket is a required field
 	Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"`
@@ -10510,7 +10510,7 @@ func (s *DeleteMarkerReplication) SetStatus(v string) *DeleteMarkerReplication {
 }
 
 type DeleteObjectInput struct {
-	_ struct{} `type:"structure"`
+	_ struct{} `locationName:"DeleteObjectRequest" type:"structure"`
 
 	// Bucket is a required field
 	Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"`
@@ -10656,7 +10656,7 @@ func (s *DeleteObjectOutput) SetVersionId(v string) *DeleteObjectOutput {
 }
 
 type DeleteObjectTaggingInput struct {
-	_ struct{} `type:"structure"`
+	_ struct{} `locationName:"DeleteObjectTaggingRequest" type:"structure"`
 
 	// Bucket is a required field
 	Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"`
@@ -10749,7 +10749,7 @@ func (s *DeleteObjectTaggingOutput) SetVersionId(v string) *DeleteObjectTaggingO
 }
 
 type DeleteObjectsInput struct {
-	_ struct{} `type:"structure" payload:"Delete"`
+	_ struct{} `locationName:"DeleteObjectsRequest" type:"structure" payload:"Delete"`
 
 	// Bucket is a required field
 	Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"`
@@ -10885,7 +10885,7 @@ func (s *DeleteObjectsOutput) SetRequestCharged(v string) *DeleteObjectsOutput {
 }
 
 type DeletePublicAccessBlockInput struct {
-	_ struct{} `type:"structure"`
+	_ struct{} `locationName:"DeletePublicAccessBlockRequest" type:"structure"`
 
 	// The Amazon S3 bucket whose PublicAccessBlock configuration you want to delete.
 	//
@@ -11341,7 +11341,7 @@ func (s *FilterRule) SetValue(v string) *FilterRule {
 }
 
 type GetBucketAccelerateConfigurationInput struct {
-	_ struct{} `type:"structure"`
+	_ struct{} `locationName:"GetBucketAccelerateConfigurationRequest" type:"structure"`
 
 	// Name of the bucket for which the accelerate configuration is retrieved.
 	//
@@ -11412,7 +11412,7 @@ func (s *GetBucketAccelerateConfigurationOutput) SetStatus(v string) *GetBucketA
 }
 
 type GetBucketAclInput struct {
-	_ struct{} `type:"structure"`
+	_ struct{} `locationName:"GetBucketAclRequest" type:"structure"`
 
 	// Bucket is a required field
 	Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"`
@@ -11489,7 +11489,7 @@ func (s *GetBucketAclOutput) SetOwner(v *Owner) *GetBucketAclOutput {
 }
 
 type GetBucketAnalyticsConfigurationInput struct {
-	_ struct{} `type:"structure"`
+	_ struct{} `locationName:"GetBucketAnalyticsConfigurationRequest" type:"structure"`
 
 	// The name of the bucket from which an analytics configuration is retrieved.
 	//
@@ -11574,7 +11574,7 @@ func (s *GetBucketAnalyticsConfigurationOutput) SetAnalyticsConfiguration(v *Ana
 }
 
 type GetBucketCorsInput struct {
-	_ struct{} `type:"structure"`
+	_ struct{} `locationName:"GetBucketCorsRequest" type:"structure"`
 
 	// Bucket is a required field
 	Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"`
@@ -11642,7 +11642,7 @@ func (s *GetBucketCorsOutput) SetCORSRules(v []*CORSRule) *GetBucketCorsOutput {
 }
 
 type GetBucketEncryptionInput struct {
-	_ struct{} `type:"structure"`
+	_ struct{} `locationName:"GetBucketEncryptionRequest" type:"structure"`
 
 	// The name of the bucket from which the server-side encryption configuration
 	// is retrieved.
@@ -11714,7 +11714,7 @@ func (s *GetBucketEncryptionOutput) SetServerSideEncryptionConfiguration(v *Serv
 }
 
 type GetBucketInventoryConfigurationInput struct {
-	_ struct{} `type:"structure"`
+	_ struct{} `locationName:"GetBucketInventoryConfigurationRequest" type:"structure"`
 
 	// The name of the bucket containing the inventory configuration to retrieve.
 	//
@@ -11799,7 +11799,7 @@ func (s *GetBucketInventoryConfigurationOutput) SetInventoryConfiguration(v *Inv
 }
 
 type GetBucketLifecycleConfigurationInput struct {
-	_ struct{} `type:"structure"`
+	_ struct{} `locationName:"GetBucketLifecycleConfigurationRequest" type:"structure"`
 
 	// Bucket is a required field
 	Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"`
@@ -11867,7 +11867,7 @@ func (s *GetBucketLifecycleConfigurationOutput) SetRules(v []*LifecycleRule) *Ge
 }
 
 type GetBucketLifecycleInput struct {
-	_ struct{} `type:"structure"`
+	_ struct{} `locationName:"GetBucketLifecycleRequest" type:"structure"`
 
 	// Bucket is a required field
 	Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"`
@@ -11935,7 +11935,7 @@ func (s *GetBucketLifecycleOutput) SetRules(v []*Rule) *GetBucketLifecycleOutput
 }
 
 type GetBucketLocationInput struct {
-	_ struct{} `type:"structure"`
+	_ struct{} `locationName:"GetBucketLocationRequest" type:"structure"`
 
 	// Bucket is a required field
 	Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"`
@@ -12003,7 +12003,7 @@ func (s *GetBucketLocationOutput) SetLocationConstraint(v string) *GetBucketLoca
 }
 
 type GetBucketLoggingInput struct {
-	_ struct{} `type:"structure"`
+	_ struct{} `locationName:"GetBucketLoggingRequest" type:"structure"`
 
 	// Bucket is a required field
 	Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"`
@@ -12075,7 +12075,7 @@ func (s *GetBucketLoggingOutput) SetLoggingEnabled(v *LoggingEnabled) *GetBucket
 }
 
 type GetBucketMetricsConfigurationInput struct {
-	_ struct{} `type:"structure"`
+	_ struct{} `locationName:"GetBucketMetricsConfigurationRequest" type:"structure"`
 
 	// The name of the bucket containing the metrics configuration to retrieve.
 	//
@@ -12160,7 +12160,7 @@ func (s *GetBucketMetricsConfigurationOutput) SetMetricsConfiguration(v *Metrics
 }
 
 type GetBucketNotificationConfigurationRequest struct {
-	_ struct{} `type:"structure"`
+	_ struct{} `locationName:"GetBucketNotificationConfigurationRequest" type:"structure"`
 
 	// Name of the bucket to get the notification configuration for.
 	//
@@ -12208,7 +12208,7 @@ func (s *GetBucketNotificationConfigurationRequest) getBucket() (v string) {
 }
 
 type GetBucketPolicyInput struct {
-	_ struct{} `type:"structure"`
+	_ struct{} `locationName:"GetBucketPolicyRequest" type:"structure"`
 
 	// Bucket is a required field
 	Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"`
@@ -12277,7 +12277,7 @@ func (s *GetBucketPolicyOutput) SetPolicy(v string) *GetBucketPolicyOutput {
 }
 
 type GetBucketPolicyStatusInput struct {
-	_ struct{} `type:"structure"`
+	_ struct{} `locationName:"GetBucketPolicyStatusRequest" type:"structure"`
 
 	// The name of the Amazon S3 bucket whose policy status you want to retrieve.
 	//
@@ -12348,7 +12348,7 @@ func (s *GetBucketPolicyStatusOutput) SetPolicyStatus(v *PolicyStatus) *GetBucke
 }
 
 type GetBucketReplicationInput struct {
-	_ struct{} `type:"structure"`
+	_ struct{} `locationName:"GetBucketReplicationRequest" type:"structure"`
 
 	// Bucket is a required field
 	Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"`
@@ -12418,7 +12418,7 @@ func (s *GetBucketReplicationOutput) SetReplicationConfiguration(v *ReplicationC
 }
 
 type GetBucketRequestPaymentInput struct {
-	_ struct{} `type:"structure"`
+	_ struct{} `locationName:"GetBucketRequestPaymentRequest" type:"structure"`
 
 	// Bucket is a required field
 	Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"`
@@ -12487,7 +12487,7 @@ func (s *GetBucketRequestPaymentOutput) SetPayer(v string) *GetBucketRequestPaym
 }
 
 type GetBucketTaggingInput struct {
-	_ struct{} `type:"structure"`
+	_ struct{} `locationName:"GetBucketTaggingRequest" type:"structure"`
 
 	// Bucket is a required field
 	Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"`
@@ -12556,7 +12556,7 @@ func (s *GetBucketTaggingOutput) SetTagSet(v []*Tag) *GetBucketTaggingOutput {
 }
 
 type GetBucketVersioningInput struct {
-	_ struct{} `type:"structure"`
+	_ struct{} `locationName:"GetBucketVersioningRequest" type:"structure"`
 
 	// Bucket is a required field
 	Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"`
@@ -12636,7 +12636,7 @@ func (s *GetBucketVersioningOutput) SetStatus(v string) *GetBucketVersioningOutp
 }
 
 type GetBucketWebsiteInput struct {
-	_ struct{} `type:"structure"`
+	_ struct{} `locationName:"GetBucketWebsiteRequest" type:"structure"`
 
 	// Bucket is a required field
 	Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"`
@@ -12730,7 +12730,7 @@ func (s *GetBucketWebsiteOutput) SetRoutingRules(v []*RoutingRule) *GetBucketWeb
 }
 
 type GetObjectAclInput struct {
-	_ struct{} `type:"structure"`
+	_ struct{} `locationName:"GetObjectAclRequest" type:"structure"`
 
 	// Bucket is a required field
 	Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"`
@@ -12853,7 +12853,7 @@ func (s *GetObjectAclOutput) SetRequestCharged(v string) *GetObjectAclOutput {
 }
 
 type GetObjectInput struct {
-	_ struct{} `type:"structure"`
+	_ struct{} `locationName:"GetObjectRequest" type:"structure"`
 
 	// Bucket is a required field
 	Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"`
@@ -13090,7 +13090,7 @@ func (s *GetObjectInput) SetVersionId(v string) *GetObjectInput {
 }
 
 type GetObjectLegalHoldInput struct {
-	_ struct{} `type:"structure"`
+	_ struct{} `locationName:"GetObjectLegalHoldRequest" type:"structure"`
 
 	// The bucket containing the object whose Legal Hold status you want to retrieve.
 	//
@@ -13199,7 +13199,7 @@ func (s *GetObjectLegalHoldOutput) SetLegalHold(v *ObjectLockLegalHold) *GetObje
 }
 
 type GetObjectLockConfigurationInput struct {
-	_ struct{} `type:"structure"`
+	_ struct{} `locationName:"GetObjectLockConfigurationRequest" type:"structure"`
 
 	// The bucket whose object lock configuration you want to retrieve.
 	//
@@ -13581,7 +13581,7 @@ func (s *GetObjectOutput) SetWebsiteRedirectLocation(v string) *GetObjectOutput
 }
 
 type GetObjectRetentionInput struct {
-	_ struct{} `type:"structure"`
+	_ struct{} `locationName:"GetObjectRetentionRequest" type:"structure"`
 
 	// The bucket containing the object whose retention settings you want to retrieve.
 	//
@@ -13690,7 +13690,7 @@ func (s *GetObjectRetentionOutput) SetRetention(v *ObjectLockRetention) *GetObje
 }
 
 type GetObjectTaggingInput struct {
-	_ struct{} `type:"structure"`
+	_ struct{} `locationName:"GetObjectTaggingRequest" type:"structure"`
 
 	// Bucket is a required field
 	Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"`
@@ -13790,7 +13790,7 @@ func (s *GetObjectTaggingOutput) SetVersionId(v string) *GetObjectTaggingOutput
 }
 
 type GetObjectTorrentInput struct {
-	_ struct{} `type:"structure"`
+	_ struct{} `locationName:"GetObjectTorrentRequest" type:"structure"`
 
 	// Bucket is a required field
 	Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"`
@@ -13895,7 +13895,7 @@ func (s *GetObjectTorrentOutput) SetRequestCharged(v string) *GetObjectTorrentOu
 }
 
 type GetPublicAccessBlockInput struct {
-	_ struct{} `type:"structure"`
+	_ struct{} `locationName:"GetPublicAccessBlockRequest" type:"structure"`
 
 	// The name of the Amazon S3 bucket whose PublicAccessBlock configuration you
 	// want to retrieve.
@@ -14126,7 +14126,7 @@ func (s *Grantee) SetURI(v string) *Grantee {
 }
 
 type HeadBucketInput struct {
-	_ struct{} `type:"structure"`
+	_ struct{} `locationName:"HeadBucketRequest" type:"structure"`
 
 	// Bucket is a required field
 	Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"`
@@ -14186,7 +14186,7 @@ func (s HeadBucketOutput) GoString() string {
 }
 
 type HeadObjectInput struct {
-	_ struct{} `type:"structure"`
+	_ struct{} `locationName:"HeadObjectRequest" type:"structure"`
 
 	// Bucket is a required field
 	Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"`
@@ -15661,7 +15661,7 @@ func (s *LifecycleRuleFilter) SetTag(v *Tag) *LifecycleRuleFilter {
 }
 
 type ListBucketAnalyticsConfigurationsInput struct {
-	_ struct{} `type:"structure"`
+	_ struct{} `locationName:"ListBucketAnalyticsConfigurationsRequest" type:"structure"`
 
 	// The name of the bucket from which analytics configurations are retrieved.
 	//
@@ -15773,7 +15773,7 @@ func (s *ListBucketAnalyticsConfigurationsOutput) SetNextContinuationToken(v str
 }
 
 type ListBucketInventoryConfigurationsInput struct {
-	_ struct{} `type:"structure"`
+	_ struct{} `locationName:"ListBucketInventoryConfigurationsRequest" type:"structure"`
 
 	// The name of the bucket containing the inventory configurations to retrieve.
 	//
@@ -15887,7 +15887,7 @@ func (s *ListBucketInventoryConfigurationsOutput) SetNextContinuationToken(v str
 }
 
 type ListBucketMetricsConfigurationsInput struct {
-	_ struct{} `type:"structure"`
+	_ struct{} `locationName:"ListBucketMetricsConfigurationsRequest" type:"structure"`
 
 	// The name of the bucket containing the metrics configurations to retrieve.
 	//
@@ -16047,7 +16047,7 @@ func (s *ListBucketsOutput) SetOwner(v *Owner) *ListBucketsOutput {
 }
 
 type ListMultipartUploadsInput struct {
-	_ struct{} `type:"structure"`
+	_ struct{} `locationName:"ListMultipartUploadsRequest" type:"structure"`
 
 	// Bucket is a required field
 	Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"`
@@ -16291,7 +16291,7 @@ func (s *ListMultipartUploadsOutput) SetUploads(v []*MultipartUpload) *ListMulti
 }
 
 type ListObjectVersionsInput struct {
-	_ struct{} `type:"structure"`
+	_ struct{} `locationName:"ListObjectVersionsRequest" type:"structure"`
 
 	// Bucket is a required field
 	Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"`
@@ -16524,7 +16524,7 @@ func (s *ListObjectVersionsOutput) SetVersions(v []*ObjectVersion) *ListObjectVe
 }
 
 type ListObjectsInput struct {
-	_ struct{} `type:"structure"`
+	_ struct{} `locationName:"ListObjectsRequest" type:"structure"`
 
 	// Bucket is a required field
 	Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"`
@@ -16736,7 +16736,7 @@ func (s *ListObjectsOutput) SetPrefix(v string) *ListObjectsOutput {
 }
 
 type ListObjectsV2Input struct {
-	_ struct{} `type:"structure"`
+	_ struct{} `locationName:"ListObjectsV2Request" type:"structure"`
 
 	// Name of the bucket to list.
 	//
@@ -16997,7 +16997,7 @@ func (s *ListObjectsV2Output) SetStartAfter(v string) *ListObjectsV2Output {
 }
 
 type ListPartsInput struct {
-	_ struct{} `type:"structure"`
+	_ struct{} `locationName:"ListPartsRequest" type:"structure"`
 
 	// Bucket is a required field
 	Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"`
@@ -18622,7 +18622,7 @@ func (s *PublicAccessBlockConfiguration) SetRestrictPublicBuckets(v bool) *Publi
 }
 
 type PutBucketAccelerateConfigurationInput struct {
-	_ struct{} `type:"structure" payload:"AccelerateConfiguration"`
+	_ struct{} `locationName:"PutBucketAccelerateConfigurationRequest" type:"structure" payload:"AccelerateConfiguration"`
 
 	// Specifies the Accelerate Configuration you want to set for the bucket.
 	//
@@ -18698,7 +18698,7 @@ func (s PutBucketAccelerateConfigurationOutput) GoString() string {
 }
 
 type PutBucketAclInput struct {
-	_ struct{} `type:"structure" payload:"AccessControlPolicy"`
+	_ struct{} `locationName:"PutBucketAclRequest" type:"structure" payload:"AccessControlPolicy"`
 
 	// The canned ACL to apply to the bucket.
 	ACL *string `location:"header" locationName:"x-amz-acl" type:"string" enum:"BucketCannedACL"`
@@ -18827,7 +18827,7 @@ func (s PutBucketAclOutput) GoString() string {
 }
 
 type PutBucketAnalyticsConfigurationInput struct {
-	_ struct{} `type:"structure" payload:"AnalyticsConfiguration"`
+	_ struct{} `locationName:"PutBucketAnalyticsConfigurationRequest" type:"structure" payload:"AnalyticsConfiguration"`
 
 	// The configuration and any analyses for the analytics filter.
 	//
@@ -18922,7 +18922,7 @@ func (s PutBucketAnalyticsConfigurationOutput) GoString() string {
 }
 
 type PutBucketCorsInput struct {
-	_ struct{} `type:"structure" payload:"CORSConfiguration"`
+	_ struct{} `locationName:"PutBucketCorsRequest" type:"structure" payload:"CORSConfiguration"`
 
 	// Bucket is a required field
 	Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"`
@@ -19004,7 +19004,7 @@ func (s PutBucketCorsOutput) GoString() string {
 }
 
 type PutBucketEncryptionInput struct {
-	_ struct{} `type:"structure" payload:"ServerSideEncryptionConfiguration"`
+	_ struct{} `locationName:"PutBucketEncryptionRequest" type:"structure" payload:"ServerSideEncryptionConfiguration"`
 
 	// Specifies default encryption for a bucket using server-side encryption with
 	// Amazon S3-managed keys (SSE-S3) or AWS KMS-managed keys (SSE-KMS). For information
@@ -19089,7 +19089,7 @@ func (s PutBucketEncryptionOutput) GoString() string {
 }
 
 type PutBucketInventoryConfigurationInput struct {
-	_ struct{} `type:"structure" payload:"InventoryConfiguration"`
+	_ struct{} `locationName:"PutBucketInventoryConfigurationRequest" type:"structure" payload:"InventoryConfiguration"`
 
 	// The name of the bucket where the inventory configuration will be stored.
 	//
@@ -19184,7 +19184,7 @@ func (s PutBucketInventoryConfigurationOutput) GoString() string {
 }
 
 type PutBucketLifecycleConfigurationInput struct {
-	_ struct{} `type:"structure" payload:"LifecycleConfiguration"`
+	_ struct{} `locationName:"PutBucketLifecycleConfigurationRequest" type:"structure" payload:"LifecycleConfiguration"`
 
 	// Bucket is a required field
 	Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"`
@@ -19260,7 +19260,7 @@ func (s PutBucketLifecycleConfigurationOutput) GoString() string {
 }
 
 type PutBucketLifecycleInput struct {
-	_ struct{} `type:"structure" payload:"LifecycleConfiguration"`
+	_ struct{} `locationName:"PutBucketLifecycleRequest" type:"structure" payload:"LifecycleConfiguration"`
 
 	// Bucket is a required field
 	Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"`
@@ -19333,7 +19333,7 @@ func (s PutBucketLifecycleOutput) GoString() string {
 }
 
 type PutBucketLoggingInput struct {
-	_ struct{} `type:"structure" payload:"BucketLoggingStatus"`
+	_ struct{} `locationName:"PutBucketLoggingRequest" type:"structure" payload:"BucketLoggingStatus"`
 
 	// Bucket is a required field
 	Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"`
@@ -19410,7 +19410,7 @@ func (s PutBucketLoggingOutput) GoString() string {
 }
 
 type PutBucketMetricsConfigurationInput struct {
-	_ struct{} `type:"structure" payload:"MetricsConfiguration"`
+	_ struct{} `locationName:"PutBucketMetricsConfigurationRequest" type:"structure" payload:"MetricsConfiguration"`
 
 	// The name of the bucket for which the metrics configuration is set.
 	//
@@ -19505,7 +19505,7 @@ func (s PutBucketMetricsConfigurationOutput) GoString() string {
 }
 
 type PutBucketNotificationConfigurationInput struct {
-	_ struct{} `type:"structure" payload:"NotificationConfiguration"`
+	_ struct{} `locationName:"PutBucketNotificationConfigurationRequest" type:"structure" payload:"NotificationConfiguration"`
 
 	// Bucket is a required field
 	Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"`
@@ -19585,7 +19585,7 @@ func (s PutBucketNotificationConfigurationOutput) GoString() string {
 }
 
 type PutBucketNotificationInput struct {
-	_ struct{} `type:"structure" payload:"NotificationConfiguration"`
+	_ struct{} `locationName:"PutBucketNotificationRequest" type:"structure" payload:"NotificationConfiguration"`
 
 	// Bucket is a required field
 	Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"`
@@ -19657,7 +19657,7 @@ func (s PutBucketNotificationOutput) GoString() string {
 }
 
 type PutBucketPolicyInput struct {
-	_ struct{} `type:"structure" payload:"Policy"`
+	_ struct{} `locationName:"PutBucketPolicyRequest" type:"structure" payload:"Policy"`
 
 	// Bucket is a required field
 	Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"`
@@ -19741,7 +19741,7 @@ func (s PutBucketPolicyOutput) GoString() string {
 }
 
 type PutBucketReplicationInput struct {
-	_ struct{} `type:"structure" payload:"ReplicationConfiguration"`
+	_ struct{} `locationName:"PutBucketReplicationRequest" type:"structure" payload:"ReplicationConfiguration"`
 
 	// Bucket is a required field
 	Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"`
@@ -19830,7 +19830,7 @@ func (s PutBucketReplicationOutput) GoString() string {
 }
 
 type PutBucketRequestPaymentInput struct {
-	_ struct{} `type:"structure" payload:"RequestPaymentConfiguration"`
+	_ struct{} `locationName:"PutBucketRequestPaymentRequest" type:"structure" payload:"RequestPaymentConfiguration"`
 
 	// Bucket is a required field
 	Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"`
@@ -19907,7 +19907,7 @@ func (s PutBucketRequestPaymentOutput) GoString() string {
 }
 
 type PutBucketTaggingInput struct {
-	_ struct{} `type:"structure" payload:"Tagging"`
+	_ struct{} `locationName:"PutBucketTaggingRequest" type:"structure" payload:"Tagging"`
 
 	// Bucket is a required field
 	Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"`
@@ -19984,7 +19984,7 @@ func (s PutBucketTaggingOutput) GoString() string {
 }
 
 type PutBucketVersioningInput struct {
-	_ struct{} `type:"structure" payload:"VersioningConfiguration"`
+	_ struct{} `locationName:"PutBucketVersioningRequest" type:"structure" payload:"VersioningConfiguration"`
 
 	// Bucket is a required field
 	Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"`
@@ -20070,7 +20070,7 @@ func (s PutBucketVersioningOutput) GoString() string {
 }
 
 type PutBucketWebsiteInput struct {
-	_ struct{} `type:"structure" payload:"WebsiteConfiguration"`
+	_ struct{} `locationName:"PutBucketWebsiteRequest" type:"structure" payload:"WebsiteConfiguration"`
 
 	// Bucket is a required field
 	Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"`
@@ -20149,7 +20149,7 @@ func (s PutBucketWebsiteOutput) GoString() string {
 }
 
 type PutObjectAclInput struct {
-	_ struct{} `type:"structure" payload:"AccessControlPolicy"`
+	_ struct{} `locationName:"PutObjectAclRequest" type:"structure" payload:"AccessControlPolicy"`
 
 	// The canned ACL to apply to the object.
 	ACL *string `location:"header" locationName:"x-amz-acl" type:"string" enum:"ObjectCannedACL"`
@@ -20324,7 +20324,7 @@ func (s *PutObjectAclOutput) SetRequestCharged(v string) *PutObjectAclOutput {
 }
 
 type PutObjectInput struct {
-	_ struct{} `type:"structure" payload:"Body"`
+	_ struct{} `locationName:"PutObjectRequest" type:"structure" payload:"Body"`
 
 	// The canned ACL to apply to the object.
 	ACL *string `location:"header" locationName:"x-amz-acl" type:"string" enum:"ObjectCannedACL"`
@@ -20671,7 +20671,7 @@ func (s *PutObjectInput) SetWebsiteRedirectLocation(v string) *PutObjectInput {
 }
 
 type PutObjectLegalHoldInput struct {
-	_ struct{} `type:"structure" payload:"LegalHold"`
+	_ struct{} `locationName:"PutObjectLegalHoldRequest" type:"structure" payload:"LegalHold"`
 
 	// The bucket containing the object that you want to place a Legal Hold on.
 	//
@@ -20791,7 +20791,7 @@ func (s *PutObjectLegalHoldOutput) SetRequestCharged(v string) *PutObjectLegalHo
 }
 
 type PutObjectLockConfigurationInput struct {
-	_ struct{} `type:"structure" payload:"ObjectLockConfiguration"`
+	_ struct{} `locationName:"PutObjectLockConfigurationRequest" type:"structure" payload:"ObjectLockConfiguration"`
 
 	// The bucket whose object lock configuration you want to create or replace.
 	//
@@ -20998,7 +20998,7 @@ func (s *PutObjectOutput) SetVersionId(v string) *PutObjectOutput {
 }
 
 type PutObjectRetentionInput struct {
-	_ struct{} `type:"structure" payload:"Retention"`
+	_ struct{} `locationName:"PutObjectRetentionRequest" type:"structure" payload:"Retention"`
 
 	// The bucket that contains the object you want to apply this Object Retention
 	// configuration to.
@@ -21129,7 +21129,7 @@ func (s *PutObjectRetentionOutput) SetRequestCharged(v string) *PutObjectRetenti
 }
 
 type PutObjectTaggingInput struct {
-	_ struct{} `type:"structure" payload:"Tagging"`
+	_ struct{} `locationName:"PutObjectTaggingRequest" type:"structure" payload:"Tagging"`
 
 	// Bucket is a required field
 	Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"`
@@ -21237,7 +21237,7 @@ func (s *PutObjectTaggingOutput) SetVersionId(v string) *PutObjectTaggingOutput
 }
 
 type PutPublicAccessBlockInput struct {
-	_ struct{} `type:"structure" payload:"PublicAccessBlockConfiguration"`
+	_ struct{} `locationName:"PutPublicAccessBlockRequest" type:"structure" payload:"PublicAccessBlockConfiguration"`
 
 	// The name of the Amazon S3 bucket whose PublicAccessBlock configuration you
 	// want to set.
@@ -21999,7 +21999,7 @@ func (s *RequestProgress) SetEnabled(v bool) *RequestProgress {
 }
 
 type RestoreObjectInput struct {
-	_ struct{} `type:"structure" payload:"RestoreRequest"`
+	_ struct{} `locationName:"RestoreObjectRequest" type:"structure" payload:"RestoreRequest"`
 
 	// Bucket is a required field
 	Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"`
@@ -23715,7 +23715,7 @@ func (s *Transition) SetStorageClass(v string) *Transition {
 }
 
 type UploadPartCopyInput struct {
-	_ struct{} `type:"structure"`
+	_ struct{} `locationName:"UploadPartCopyRequest" type:"structure"`
 
 	// Bucket is a required field
 	Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"`
@@ -24045,7 +24045,7 @@ func (s *UploadPartCopyOutput) SetServerSideEncryption(v string) *UploadPartCopy
 }
 
 type UploadPartInput struct {
-	_ struct{} `type:"structure" payload:"Body"`
+	_ struct{} `locationName:"UploadPartRequest" type:"structure" payload:"Body"`
 
 	// Object data.
 	Body io.ReadSeeker `type:"blob"`