Compare commits

...

29 Commits

Author SHA1 Message Date
Sudarsan Reddy ec2d18ea4f Release 2023.3.1 2023-03-13 11:30:44 +00:00
Sudarsan Reddy 1742379ba4 TUN-7271: Return 503 status code when no ingress rules configured 2023-03-13 09:25:34 +00:00
Sudarsan Reddy 9c15f31d00 TUN-7268: Default to Program Files as location for win32
The previous logic of var == x86 never fired for 386 arch windows
systems causing us to set ProgramFiles64Folder for the older windows
versions causing downloads to default to a different location. This
change fixes that.
2023-03-10 12:37:59 +00:00
João Oliveirinha 53fb50960d TUN-7272: Fix cloudflared returning non supported status service which breaks configuration migration 2023-03-10 10:42:37 +00:00
Devin Carr 7b8b3f73e7 TUN-7259: Add warning for missing ingress rules
Providing no ingress rules in the configuration file or via the CLI will now provide a warning and return 502 for all incoming HTTP requests.
2023-03-10 01:49:54 +00:00
Robert Dinh ede3c8e056 EDGESTORE-108: Remove deprecated s3v2 signature
https://wiki.cfdata.org/display/OPS/2021/10/14/Ceph+cluster+news
s3.cfdata.org now supports s3v4. Therefore host mangling and s3v2 signature is no longer required.
2023-03-09 18:24:42 +00:00
Devin Carr 93f8f6b55c TUN-7245: Add bastion flag to origin service check 2023-03-09 17:09:21 +00:00
Devin Carr bf3136debb TUN-7253: Adopt http.ResponseWriter for connection.ResponseWriter 2023-03-08 09:56:47 -08:00
Devin Carr 27f88ae209 TUN-7252: Remove h2mux connection 2023-03-07 13:51:37 -08:00
Sudarsan Reddy 7080b8b2e6 TUN-7226: Fixed a missed rename 2023-03-02 10:59:35 +00:00
Sudarsan Reddy 4c3417fedd Release 2023.3.0 2023-03-02 08:48:05 +00:00
Bas Westerbaan 354281fc6a RTG-2476 Add qtls override for Go 1.20 2023-03-02 08:34:51 +00:00
Spencer Comfort b6d1daaf20 check.yaml: update actions to v3 (#876) 2023-02-28 16:18:14 +00:00
Jake Edwards 844b4938ca Fixed WIX template to allow MSI upgrades (#838) 2023-02-28 16:12:23 +00:00
iBug fed60ae4c3 GH-352: Add Tunnel CLI option "edge-bind-address" (#870)
* Add Tunnel CLI option "edge-bind-address"
2023-02-28 16:11:42 +00:00
Sudarsan Reddy b97979487e TUN-7213: Decode Base64 encoded key before writing it 2023-02-28 12:54:30 +00:00
Sudarsan Reddy 2221325f3d TUN-7213: Debug homebrew-cloudflare build 2023-02-27 20:48:43 +00:00
Sudarsan Reddy 2bb054c4bf Release 2023.2.2 2023-02-27 09:05:00 +00:00
João Oliveirinha 68ef4ab2a8 TUN-7197: Add connIndex tag to debug messages of incoming requests 2023-02-22 16:08:24 +00:00
Devin Carr ea6fe121f8 TUN-7167: Respect protocol overrides with --token
Previously, if run with both `--protocol` and `--token` the protocol
would be incorrectly overridden to QUIC.
2023-02-08 11:03:04 -08:00
João Oliveirinha 079631ccea TUN-7151: Update changes file with latest release notices 2023-02-07 19:24:07 +00:00
Devin Carr 8cf2d319ca TUN-6938: Provide QUIC as first in protocol list 2023-02-06 20:05:48 -08:00
Devin Carr 0f95f8bae5 TUN-6938: Force h2mux protocol to http2 for named tunnels
Going forward, the only protocols supported will be QUIC and HTTP2,
defaulting to QUIC for "auto". Selecting h2mux protocol will be forcibly
upgraded to http2 internally.
2023-02-06 11:06:02 -08:00
Devin Carr ae46af9236 TUN-7065: Remove classic tunnel creation 2023-02-06 18:19:22 +00:00
Devin Carr bd046677e5 TUN-7158: Correct TCP tracing propagation
Previously QUIC would send TCP tracing response header that was empty regardless if prompted from origintunneld.
2023-02-03 18:01:27 -08:00
João Oliveirinha 8a9f076a26 Release 2023.2.1 2023-02-03 09:31:11 +00:00
João Oliveirinha 62dcb8a1d1 Revert "TUN-7065: Remove classic tunnel creation"
This reverts commit c24f275981.
2023-02-01 14:01:59 +00:00
João Oliveirinha 90d710e3ec Revert "TUN-7065: Revert Ingress Rule check for named tunnel configurations"
This reverts commit b8e610a067.
2023-02-01 14:01:46 +00:00
Sudarsan Reddy b8e610a067 TUN-7065: Revert Ingress Rule check for named tunnel configurations
Named Tunnels can exist without Ingress rules (They would default to
8080). Moreover, having this check also prevents warp tunnels from
starting since they do not need ingress rules.
2023-02-01 10:08:10 +00:00
43 changed files with 802 additions and 1387 deletions
+2 -2
View File
@@ -9,10 +9,10 @@ jobs:
runs-on: ${{ matrix.os }}
steps:
- name: Install Go
uses: actions/setup-go@v2
uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go-version }}
- name: Checkout code
uses: actions/checkout@v2
uses: actions/checkout@v3
- name: Test
run: make test
+5 -6
View File
@@ -3,7 +3,6 @@
set -euo pipefail
FILENAME="${PWD}/artifacts/cloudflared-darwin-amd64.tgz"
if ! VERSION="$(git describe --tags --exact-match 2>/dev/null)" ; then
echo "Skipping public release for an untagged commit."
echo "##teamcity[buildStatus status='SUCCESS' text='Skipped due to lack of tag']"
@@ -15,24 +14,24 @@ if [[ ! -f "$FILENAME" ]] ; then
exit 1
fi
if [[ "${GITHUB_PRIVATE_KEY:-}" == "" ]] ; then
echo "Missing GITHUB_PRIVATE_KEY"
if [[ "${GITHUB_PRIVATE_KEY_B64:-}" == "" ]] ; then
echo "Missing GITHUB_PRIVATE_KEY_B64"
exit 1
fi
# upload to s3 bucket for use by Homebrew formula
s3cmd \
--acl-public --signature-v2 --access_key="$AWS_ACCESS_KEY_ID" --secret_key="$AWS_SECRET_ACCESS_KEY" --host-bucket="%(bucket)s.s3.cfdata.org" \
--acl-public --access_key="$AWS_ACCESS_KEY_ID" --secret_key="$AWS_SECRET_ACCESS_KEY" --host-bucket="%(bucket)s.s3.cfdata.org" \
put "$FILENAME" "s3://cftunnel-docs/dl/cloudflared-$VERSION-darwin-amd64.tgz"
s3cmd \
--acl-public --signature-v2 --access_key="$AWS_ACCESS_KEY_ID" --secret_key="$AWS_SECRET_ACCESS_KEY" --host-bucket="%(bucket)s.s3.cfdata.org" \
--acl-public --access_key="$AWS_ACCESS_KEY_ID" --secret_key="$AWS_SECRET_ACCESS_KEY" --host-bucket="%(bucket)s.s3.cfdata.org" \
cp "s3://cftunnel-docs/dl/cloudflared-$VERSION-darwin-amd64.tgz" "s3://cftunnel-docs/dl/cloudflared-stable-darwin-amd64.tgz"
SHA256=$(sha256sum "$FILENAME" | cut -b1-64)
# set up git (note that UserKnownHostsFile is an absolute path so we can cd wherever)
mkdir -p tmp
ssh-keyscan -t rsa github.com > tmp/github.txt
echo "$GITHUB_PRIVATE_KEY" > tmp/private.key
echo "$GITHUB_PRIVATE_KEY_B64" | base64 --decode > tmp/private.key
chmod 0400 tmp/private.key
export GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=$PWD/tmp/github.txt -i $PWD/tmp/private.key -o IdentitiesOnly=yes"
+15
View File
@@ -1,3 +1,18 @@
## 2023.3.1
### Breaking Change
- Running a tunnel without ingress rules defined in configuration file nor from the CLI flags will no longer provide a default ingress rule to localhost:8080 and instead will return HTTP response code 503 for all incoming HTTP requests.
## 2023.2.2
### Notices
- Legacy tunnels were officially deprecated on December 1, 2022. Starting with this version, cloudflared no longer supports connecting legacy tunnels.
- h2mux tunnel connection protocol is no longer supported. Any tunnels still configured to use this protocol will alert and use http2 tunnel protocol instead. We recommend using quic protocol for all tunnels going forward.
## 2023.2.1
### Bug fixes
- Fixed a bug in TCP connection proxy that could result in the connection being closed before all data was written.
- cloudflared now correctly aborts body write if connection to origin service fails after response headers were sent already.
- Fixed a bug introduced in the previous release where debug endpoints were removed.
## 2022.12.0
### Improvements
- cloudflared now attempts to try other edge addresses before falling back to a lower protoocol.
+32
View File
@@ -1,3 +1,35 @@
2023.3.1
- 2023-03-13 TUN-7271: Return 503 status code when no ingress rules configured
- 2023-03-10 TUN-7272: Fix cloudflared returning non supported status service which breaks configuration migration
- 2023-03-09 TUN-7259: Add warning for missing ingress rules
- 2023-03-09 TUN-7268: Default to Program Files as location for win32
- 2023-03-07 TUN-7252: Remove h2mux connection
- 2023-03-07 TUN-7253: Adopt http.ResponseWriter for connection.ResponseWriter
- 2023-03-06 TUN-7245: Add bastion flag to origin service check
- 2023-03-06 EDGESTORE-108: Remove deprecated s3v2 signature
- 2023-03-02 TUN-7226: Fixed a missed rename
2023.3.0
- 2023-03-01 GH-352: Add Tunnel CLI option "edge-bind-address" (#870)
- 2023-03-01 Fixed WIX template to allow MSI upgrades (#838)
- 2023-02-28 TUN-7213: Decode Base64 encoded key before writing it
- 2023-02-28 check.yaml: update actions to v3 (#876)
- 2023-02-27 TUN-7213: Debug homebrew-cloudflare build
- 2023-02-15 RTG-2476 Add qtls override for Go 1.20
2023.2.2
- 2023-02-22 TUN-7197: Add connIndex tag to debug messages of incoming requests
- 2023-02-08 TUN-7167: Respect protocol overrides with --token
- 2023-02-06 TUN-7065: Remove classic tunnel creation
- 2023-02-06 TUN-6938: Force h2mux protocol to http2 for named tunnels
- 2023-02-06 TUN-6938: Provide QUIC as first in protocol list
- 2023-02-03 TUN-7158: Correct TCP tracing propagation
- 2023-02-01 TUN-7151: Update changes file with latest release notices
2023.2.1
- 2023-02-01 TUN-7065: Revert Ingress Rule check for named tunnel configurations
- 2023-02-01 Revert "TUN-7065: Revert Ingress Rule check for named tunnel configurations"
- 2023-02-01 Revert "TUN-7065: Remove classic tunnel creation"
2023.1.0
- 2023-01-10 TUN-7064: RPM digests are now sha256 instead of md5sum
- 2023-01-04 RTG-2418 Update qtls
+39 -37
View File
@@ -1,62 +1,64 @@
<?xml version="1.0"?>
<?if $(var.Platform)="x86"?>
<?define Program_Files="ProgramFilesFolder"?>
<?else?>
<?if $(var.Platform)="x64" ?>
<?define Program_Files="ProgramFiles64Folder"?>
<?endif?>
<?else ?>
<?define Program_Files="ProgramFilesFolder"?>
<?endif ?>
<?ifndef var.Version?>
<?error Undefined Version variable?>
<?endif?>
<?endif ?>
<?ifndef var.Path?>
<?error Undefined Path variable?>
<?endif?>
<?endif ?>
<Wix xmlns="http://schemas.microsoft.com/wix/2006/wi">
<Product Id="35e5e858-9372-4449-bf73-1cd6f7267128"
<Product Id="*"
UpgradeCode="23f90fdd-9328-47ea-ab52-5380855a4b12"
Name="cloudflared"
Version="$(var.Version)"
Manufacturer="cloudflare"
Language="1033">
<Package InstallerVersion="200" Compressed="yes" Comments="Windows Installer Package" InstallScope="perMachine"/>
<Package InstallerVersion="200" Compressed="yes" Comments="Windows Installer Package" InstallScope="perMachine" />
<Media Id="1" Cabinet="product.cab" EmbedCab="yes"/>
<Media Id="1" Cabinet="product.cab" EmbedCab="yes" />
<Upgrade Id="23f90fdd-9328-47ea-ab52-5380855a4b12">
<UpgradeVersion Minimum="$(var.Version)" OnlyDetect="yes" Property="NEWERVERSIONDETECTED"/>
<UpgradeVersion Minimum="2020.8.0" Maximum="$(var.Version)" IncludeMinimum="yes" IncludeMaximum="no"
Property="OLDERVERSIONBEINGUPGRADED"/>
</Upgrade>
<Condition Message="A newer version of this software is already installed.">NOT NEWERVERSIONDETECTED</Condition>
<MajorUpgrade DowngradeErrorMessage="A later version of [ProductName] is already installed. Setup will now exit." />
<Directory Id="TARGETDIR" Name="SourceDir">
<!--This specifies where the cloudflared.exe is moved to in the windows Operation System-->
<Directory Id="$(var.Program_Files)">
<Directory Id="INSTALLDIR" Name="cloudflared">
<Component Id="ApplicationFiles" Guid="35e5e858-9372-4449-bf73-1cd6f7267128">
<File Id="ApplicationFile0" Source="$(var.Path)"/>
</Component>
<Upgrade Id="23f90fdd-9328-47ea-ab52-5380855a4b12">
<UpgradeVersion Minimum="$(var.Version)" OnlyDetect="yes" Property="NEWERVERSIONDETECTED" />
<UpgradeVersion Minimum="2020.8.0" Maximum="$(var.Version)" IncludeMinimum="yes" IncludeMaximum="no"
Property="OLDERVERSIONBEINGUPGRADED" />
</Upgrade>
<Condition Message="A newer version of this software is already installed.">NOT NEWERVERSIONDETECTED</Condition>
<Directory Id="TARGETDIR" Name="SourceDir">
<!--This specifies where the cloudflared.exe is moved to in the windows Operation System-->
<Directory Id="$(var.Program_Files)">
<Directory Id="INSTALLDIR" Name="cloudflared">
<Component Id="ApplicationFiles" Guid="35e5e858-9372-4449-bf73-1cd6f7267128">
<File Id="ApplicationFile0" Source="$(var.Path)" />
</Component>
</Directory>
</Directory>
<Component Id="ENVS" Guid="6bb74449-d10d-4f4a-933e-6fc9fa006eae">
<!--Set the cloudflared bin location to the Path Environment Variable-->
<Environment Id="ENV0"
Name="PATH"
Value="[INSTALLDIR]."
Permanent="no"
Part="last"
Action="create"
System="yes" />
</Component>
</Directory>
<Component Id="ENVS" Guid="6bb74449-d10d-4f4a-933e-6fc9fa006eae">
<!--Set the cloudflared bin location to the Path Environment Variable-->
<Environment Id="ENV0"
Name="PATH"
Value="[INSTALLDIR]."
Permanent="no"
Part="last"
Action="create"
System="yes" />
</Component>
</Directory>
<Feature Id='Complete' Level='1'>
<ComponentRef Id="ENVS"/>
<ComponentRef Id='ApplicationFiles' />
</Feature>
<Feature Id='Complete' Level='1'>
<ComponentRef Id="ENVS" />
<ComponentRef Id='ApplicationFiles' />
</Feature>
</Product>
</Wix>
+39 -12
View File
@@ -36,6 +36,7 @@ import (
"github.com/cloudflare/cloudflared/supervisor"
"github.com/cloudflare/cloudflared/tlsconfig"
"github.com/cloudflare/cloudflared/tunneldns"
"github.com/cloudflare/cloudflared/validation"
)
const (
@@ -83,12 +84,12 @@ const (
LogFieldTmpTraceFilename = "tmpTraceFilename"
LogFieldTraceOutputFilepath = "traceOutputFilepath"
tunnelCmdErrorMessage = `You did not specify any valid additional argument to the cloudflared tunnel command.
tunnelCmdErrorMessage = `You did not specify any valid additional argument to the cloudflared tunnel command.
If you are trying to run a Quick Tunnel then you need to explicitly pass the --url flag.
Eg. cloudflared tunnel --url localhost:8080/.
If you are trying to run a Quick Tunnel then you need to explicitly pass the --url flag.
Eg. cloudflared tunnel --url localhost:8080/.
Please note that Quick Tunnels are meant to be ephemeral and should only be used for testing purposes.
Please note that Quick Tunnels are meant to be ephemeral and should only be used for testing purposes.
For production usage, we recommend creating Named Tunnels. (https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/install-and-setup/tunnel-guide/)
`
)
@@ -177,26 +178,46 @@ func TunnelCommand(c *cli.Context) error {
return err
}
if name := c.String("name"); name != "" { // Start a named tunnel
// Run a adhoc named tunnel
// Allows for the creation, routing (optional), and startup of a tunnel in one command
// --name required
// --url or --hello-world required
// --hostname optional
if name := c.String("name"); name != "" {
hostname, err := validation.ValidateHostname(c.String("hostname"))
if err != nil {
return errors.Wrap(err, "Invalid hostname provided")
}
url := c.String("url")
if url == hostname && url != "" && hostname != "" {
return fmt.Errorf("hostname and url shouldn't match. See --help for more information")
}
return runAdhocNamedTunnel(sc, name, c.String(CredFileFlag))
}
// Unauthenticated named tunnel on <random>.<quick-tunnels-service>.com
shouldRunQuickTunnel := c.IsSet("url") || c.IsSet("hello-world")
if !dnsProxyStandAlone(c, nil) && c.String("quick-service") != "" && shouldRunQuickTunnel {
// Run a quick tunnel
// A unauthenticated named tunnel hosted on <random>.<quick-tunnels-service>.com
// We don't support running proxy-dns and a quick tunnel at the same time as the same process
shouldRunQuickTunnel := c.IsSet("url") || c.IsSet(ingress.HelloWorldFlag)
if !c.IsSet("proxy-dns") && c.String("quick-service") != "" && shouldRunQuickTunnel {
return RunQuickTunnel(sc)
}
// If user provides a config, check to see if they meant to use `tunnel run` instead
if ref := config.GetConfiguration().TunnelID; ref != "" {
return fmt.Errorf("Use `cloudflared tunnel run` to start tunnel %s", ref)
}
// classic tunnel usage is no longer supported
// Classic tunnel usage is no longer supported
if c.String("hostname") != "" {
return deprecatedClassicTunnelErr
}
if c.IsSet("proxy-dns") {
if shouldRunQuickTunnel {
return fmt.Errorf("running a quick tunnel with `proxy-dns` is not supported")
}
// NamedTunnelProperties are nil since proxy dns server does not need it.
// This is supported for legacy reasons: dns proxy server is not a tunnel and ideally should
// not run as part of cloudflared tunnel.
@@ -339,7 +360,7 @@ func StartServer(
errC <- autoupdater.Run(ctx)
}()
// Serve DNS proxy stand-alone if no hostname or tag or app is going to run
// Serve DNS proxy stand-alone if no tunnel type (quick, adhoc, named) is going to run
if dnsProxyStandAlone(c, namedTunnel) {
connectedSignal.Notify()
// no grace period, handle SIGINT/SIGTERM immediately
@@ -533,11 +554,17 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
}),
altsrc.NewStringFlag(&cli.StringFlag{
Name: "edge-ip-version",
Usage: "Cloudflare Edge ip address version to connect with. {4, 6, auto}",
Usage: "Cloudflare Edge IP address version to connect with. {4, 6, auto}",
EnvVars: []string{"TUNNEL_EDGE_IP_VERSION"},
Value: "4",
Hidden: false,
}),
altsrc.NewStringFlag(&cli.StringFlag{
Name: "edge-bind-address",
Usage: "Bind to IP address for outgoing connections to Cloudflare Edge.",
EnvVars: []string{"TUNNEL_EDGE_BIND_ADDRESS"},
Hidden: false,
}),
altsrc.NewStringFlag(&cli.StringFlag{
Name: tlsconfig.CaCertFlag,
Usage: "Certificate Authority authenticating connections with Cloudflare's edge network.",
@@ -762,7 +789,7 @@ func configureProxyFlags(shouldHide bool) []cli.Flag {
Hidden: shouldHide,
}),
altsrc.NewBoolFlag(&cli.BoolFlag{
Name: "hello-world",
Name: ingress.HelloWorldFlag,
Value: false,
Usage: "Run Hello World Server",
EnvVars: []string{"TUNNEL_HELLO_WORLD"},
+69 -50
View File
@@ -21,12 +21,10 @@ import (
"golang.org/x/crypto/ssh/terminal"
"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
"github.com/cloudflare/cloudflared/edgediscovery/allregions"
"github.com/cloudflare/cloudflared/config"
"github.com/cloudflare/cloudflared/connection"
"github.com/cloudflare/cloudflared/edgediscovery"
"github.com/cloudflare/cloudflared/h2mux"
"github.com/cloudflare/cloudflared/edgediscovery/allregions"
"github.com/cloudflare/cloudflared/ingress"
"github.com/cloudflare/cloudflared/orchestration"
"github.com/cloudflare/cloudflared/supervisor"
@@ -45,7 +43,7 @@ var (
secretFlags = [2]*altsrc.StringFlag{credentialsContentsFlag, tunnelTokenFlag}
defaultFeatures = []string{supervisor.FeatureAllowRemoteConfig, supervisor.FeatureSerializedHeaders, supervisor.FeatureDatagramV2, supervisor.FeatureQUICSupportEOF}
configFlags = []string{"autoupdate-freq", "no-autoupdate", "retries", "protocol", "loglevel", "transport-loglevel", "origincert", "metrics", "metrics-update-freq", "edge-ip-version"}
configFlags = []string{"autoupdate-freq", "no-autoupdate", "retries", "protocol", "loglevel", "transport-loglevel", "origincert", "metrics", "metrics-update-freq", "edge-ip-version", "edge-bind-address"}
)
// returns the first path that contains a cert.pem file. If none of the DefaultConfigSearchDirectories
@@ -124,7 +122,10 @@ func isSecretEnvVar(key string) bool {
}
func dnsProxyStandAlone(c *cli.Context, namedTunnel *connection.NamedTunnelProperties) bool {
return c.IsSet("proxy-dns") && (!c.IsSet("tag") && !c.IsSet("hello-world") && namedTunnel == nil)
return c.IsSet("proxy-dns") &&
!(c.IsSet("name") || // adhoc-named tunnel
c.IsSet(ingress.HelloWorldFlag) || // quick or named tunnel
namedTunnel != nil) // named tunnel
}
func findOriginCert(originCertPath string, log *zerolog.Logger) (string, error) {
@@ -215,54 +216,23 @@ func prepareTunnelConfig(
transportProtocol = connection.QUIC.String()
}
protocolFetcher := edgediscovery.ProtocolPercentage
features := append(c.StringSlice("features"), defaultFeatures...)
features := dedup(append(c.StringSlice("features"), defaultFeatures...))
if needPQ {
features = append(features, supervisor.FeaturePostQuantum)
}
if c.IsSet(TunnelTokenFlag) {
if transportProtocol == connection.AutoSelectFlag {
protocolFetcher = func() (edgediscovery.ProtocolPercents, error) {
// If the Tunnel is remotely managed and no protocol is set, we prefer QUIC, but still allow fall-back.
preferQuic := []edgediscovery.ProtocolPercent{
{
Protocol: connection.QUIC.String(),
Percentage: 100,
},
{
Protocol: connection.HTTP2.String(),
Percentage: 100,
},
}
return preferQuic, nil
}
}
log.Info().Msg("Will be fetching remotely managed configuration from Cloudflare API. Defaulting to protocol: quic")
}
namedTunnel.Client = tunnelpogs.ClientInfo{
ClientID: clientID[:],
Features: dedup(features),
Features: features,
Version: info.Version(),
Arch: info.OSArch(),
}
cfg := config.GetConfiguration()
ingressRules, err := ingress.ParseIngress(cfg)
if err != nil && err != ingress.ErrNoIngressRules {
ingressRules, err := ingress.ParseIngressFromConfigAndCLI(cfg, c, log)
if err != nil {
return nil, nil, err
}
// Only for quick tunnels will we attempt to parse the --url flag for a tunnel ingress rule
if ingressRules.IsEmpty() && c.IsSet("url") && namedTunnel.QuickTunnelUrl != "" {
ingressRules, err = ingress.NewSingleOrigin(c, true)
if err != nil {
return nil, nil, err
}
}
if ingressRules.IsEmpty() {
return nil, nil, ingress.ErrNoIngressRules
}
protocolSelector, err := connection.NewProtocolSelector(transportProtocol, cfg.WarpRouting.Enabled, namedTunnel, protocolFetcher, supervisor.ResolveTTL, log, c.Bool("post-quantum"))
protocolSelector, err := connection.NewProtocolSelector(transportProtocol, namedTunnel.Credentials.AccountTag, c.IsSet(TunnelTokenFlag), c.Bool("post-quantum"), edgediscovery.ProtocolPercentage, connection.ResolveTTL, log)
if err != nil {
return nil, nil, err
}
@@ -288,18 +258,22 @@ func prepareTunnelConfig(
if err != nil {
return nil, nil, err
}
muxerConfig := &connection.MuxerConfig{
HeartbeatInterval: c.Duration("heartbeat-interval"),
// Note TUN-3758 , we use Int because UInt is not supported with altsrc
MaxHeartbeats: uint64(c.Int("heartbeat-count")),
// Note TUN-3758 , we use Int because UInt is not supported with altsrc
CompressionSetting: h2mux.CompressionSetting(uint64(c.Int("compression-quality"))),
MetricsUpdateFreq: c.Duration("metrics-update-freq"),
}
edgeIPVersion, err := parseConfigIPVersion(c.String("edge-ip-version"))
if err != nil {
return nil, nil, err
}
edgeBindAddr, err := parseConfigBindAddress(c.String("edge-bind-address"))
if err != nil {
return nil, nil, err
}
if err := testIPBindable(edgeBindAddr); err != nil {
return nil, nil, fmt.Errorf("invalid edge-bind-address %s: %v", edgeBindAddr, err)
}
edgeIPVersion, err = adjustIPVersionByBindAddress(edgeIPVersion, edgeBindAddr)
if err != nil {
// This is not a fatal error, we just overrode edgeIPVersion
log.Warn().Str("edgeIPVersion", edgeIPVersion.String()).Err(err).Msg("Overriding edge-ip-version")
}
var pqKexIdx int
if needPQ {
@@ -318,6 +292,7 @@ func prepareTunnelConfig(
EdgeAddrs: c.StringSlice("edge"),
Region: c.String("region"),
EdgeIPVersion: edgeIPVersion,
EdgeBindAddr: edgeBindAddr,
HAConnections: c.Int("ha-connections"),
IncidentLookup: supervisor.NewIncidentLookup(),
IsAutoupdated: c.Bool("is-autoupdated"),
@@ -331,7 +306,6 @@ func prepareTunnelConfig(
Retries: uint(c.Int("retries")),
RunFromTerminal: isRunningFromTerminal(),
NamedTunnel: namedTunnel,
MuxerConfig: muxerConfig,
ProtocolSelector: protocolSelector,
EdgeTLSConfigs: edgeTLSConfigs,
NeedPQ: needPQ,
@@ -410,6 +384,51 @@ func parseConfigIPVersion(version string) (v allregions.ConfigIPVersion, err err
return
}
func parseConfigBindAddress(ipstr string) (net.IP, error) {
// Unspecified - it's fine
if ipstr == "" {
return nil, nil
}
ip := net.ParseIP(ipstr)
if ip == nil {
return nil, fmt.Errorf("invalid value for edge-bind-address: %s", ipstr)
}
return ip, nil
}
func testIPBindable(ip net.IP) error {
// "Unspecified" = let OS choose, so always bindable
if ip == nil {
return nil
}
addr := &net.UDPAddr{IP: ip, Port: 0}
listener, err := net.ListenUDP("udp", addr)
if err != nil {
return err
}
listener.Close()
return nil
}
func adjustIPVersionByBindAddress(ipVersion allregions.ConfigIPVersion, ip net.IP) (allregions.ConfigIPVersion, error) {
if ip == nil {
return ipVersion, nil
}
// https://pkg.go.dev/net#IP.To4: "If ip is not an IPv4 address, To4 returns nil."
if ip.To4() != nil {
if ipVersion == allregions.IPv6Only {
return allregions.IPv4Only, fmt.Errorf("IPv4 bind address is specified, but edge-ip-version is IPv6")
}
return allregions.IPv4Only, nil
} else {
if ipVersion == allregions.IPv4Only {
return allregions.IPv6Only, fmt.Errorf("IPv6 bind address is specified, but edge-ip-version is IPv4")
}
return allregions.IPv6Only, nil
}
}
func newPacketConfig(c *cli.Context, logger *zerolog.Logger) (*ingress.GlobalRouterConfig, error) {
ipv4Src, err := determineICMPv4Src(c.String("icmpv4-src"), logger)
if err != nil {
@@ -9,6 +9,7 @@ import (
"crypto/x509"
"crypto/x509/pkix"
"encoding/asn1"
"net"
"os"
"testing"
@@ -214,3 +215,23 @@ func getCertPoolSubjects(certPool *x509.CertPool) ([]*pkix.Name, error) {
func isUnrecoverableError(err error) bool {
return err != nil && err.Error() != "crypto/x509: system root pool is not available on Windows"
}
func TestTestIPBindable(t *testing.T) {
assert.Nil(t, testIPBindable(nil))
// Public services - if one of these IPs is on the machine, the test environment is too weird
assert.NotNil(t, testIPBindable(net.ParseIP("8.8.8.8")))
assert.NotNil(t, testIPBindable(net.ParseIP("1.1.1.1")))
addrs, err := net.InterfaceAddrs()
if err != nil {
t.Fatal(err)
}
for i, addr := range addrs {
if i >= 3 {
break
}
ip := addr.(*net.IPNet).IP
assert.Nil(t, testIPBindable(ip))
}
}
+8 -28
View File
@@ -30,6 +30,7 @@ class NamedTunnelBaseConfig(BaseConfig):
tunnel: str = None
credentials_file: str = None
ingress: list = None
hostname: str = None
def __post_init__(self):
if self.tunnel is None:
@@ -41,8 +42,10 @@ class NamedTunnelBaseConfig(BaseConfig):
def merge_config(self, additional):
config = super(NamedTunnelBaseConfig, self).merge_config(additional)
config['tunnel'] = self.tunnel
config['credentials-file'] = self.credentials_file
if 'tunnel' not in config:
config['tunnel'] = self.tunnel
if 'credentials-file' not in config:
config['credentials-file'] = self.credentials_file
# In some cases we want to override default ingress, such as in config tests
if 'ingress' not in config:
config['ingress'] = self.ingress
@@ -61,7 +64,7 @@ class NamedTunnelConfig(NamedTunnelBaseConfig):
self.merge_config(additional_config))
def get_url(self):
return "https://" + self.ingress[0]['hostname']
return "https://" + self.hostname
def base_config(self):
config = self.full_config.copy()
@@ -84,28 +87,9 @@ class NamedTunnelConfig(NamedTunnelBaseConfig):
def get_credentials_json(self):
with open(self.credentials_file) as json_file:
return json.load(json_file)
@dataclass(frozen=True)
class ClassicTunnelBaseConfig(BaseConfig):
hostname: str = None
origincert: str = None
def __post_init__(self):
if self.hostname is None:
raise TypeError("Field tunnel is not set")
if self.origincert is None:
raise TypeError("Field credentials_file is not set")
def merge_config(self, additional):
config = super(ClassicTunnelBaseConfig, self).merge_config(additional)
config['hostname'] = self.hostname
config['origincert'] = self.origincert
return config
@dataclass(frozen=True)
class ClassicTunnelConfig(ClassicTunnelBaseConfig):
class QuickTunnelConfig(BaseConfig):
full_config: dict = None
additional_config: InitVar[dict] = {}
@@ -115,10 +99,6 @@ class ClassicTunnelConfig(ClassicTunnelBaseConfig):
object.__setattr__(self, 'full_config',
self.merge_config(additional_config))
def get_url(self):
return "https://" + self.hostname
@dataclass(frozen=True)
class ProxyDnsConfig(BaseConfig):
full_config = {
+15 -8
View File
@@ -5,14 +5,14 @@ from time import sleep
import pytest
import yaml
from config import NamedTunnelConfig, ClassicTunnelConfig, ProxyDnsConfig
from config import NamedTunnelConfig, ProxyDnsConfig, QuickTunnelConfig
from constants import BACKOFF_SECS, PROXY_DNS_PORT
from util import LOGGER
class CfdModes(Enum):
NAMED = auto()
CLASSIC = auto()
QUICK = auto()
PROXY_DNS = auto()
@@ -26,7 +26,7 @@ def component_tests_config():
config = yaml.safe_load(stream)
LOGGER.info(f"component tests base config {config}")
def _component_tests_config(additional_config={}, cfd_mode=CfdModes.NAMED, run_proxy_dns=True):
def _component_tests_config(additional_config={}, cfd_mode=CfdModes.NAMED, run_proxy_dns=True, provide_ingress=True):
if run_proxy_dns:
# Regression test for TUN-4177, running with proxy-dns should not prevent tunnels from running.
# So we run all tests with it.
@@ -36,18 +36,25 @@ def component_tests_config():
additional_config.pop("proxy-dns", None)
additional_config.pop("proxy-dns-port", None)
# Allows the ingress rules to be omitted from the provided config
ingress = []
if provide_ingress:
ingress = config['ingress']
# Provide the hostname to allow routing to the tunnel even if the ingress rule isn't defined in the config
hostname = config['ingress'][0]['hostname']
if cfd_mode is CfdModes.NAMED:
return NamedTunnelConfig(additional_config=additional_config,
cloudflared_binary=config['cloudflared_binary'],
tunnel=config['tunnel'],
credentials_file=config['credentials_file'],
ingress=config['ingress'])
elif cfd_mode is CfdModes.CLASSIC:
return ClassicTunnelConfig(
additional_config=additional_config, cloudflared_binary=config['cloudflared_binary'],
hostname=config['classic_hostname'], origincert=config['origincert'])
ingress=ingress,
hostname=hostname)
elif cfd_mode is CfdModes.PROXY_DNS:
return ProxyDnsConfig(cloudflared_binary=config['cloudflared_binary'])
elif cfd_mode is CfdModes.QUICK:
return QuickTunnelConfig(additional_config=additional_config, cloudflared_binary=config['cloudflared_binary'])
else:
raise Exception(f"Unknown cloudflared mode {cfd_mode}")
+1 -1
View File
@@ -7,4 +7,4 @@ PROXY_DNS_PORT = 9053
def protocols():
return ["h2mux", "http2", "quic"]
return ["http2", "quic"]
+33
View File
@@ -0,0 +1,33 @@
#!/usr/bin/env python
from conftest import CfdModes
from constants import METRICS_PORT
from util import LOGGER, start_cloudflared, wait_tunnel_ready, get_quicktunnel_url, send_requests
class TestQuickTunnels:
def test_quick_tunnel(self, tmp_path, component_tests_config):
config = component_tests_config(cfd_mode=CfdModes.QUICK, run_proxy_dns=False)
LOGGER.debug(config)
with start_cloudflared(tmp_path, config, cfd_args=["--hello-world"], new_process=True):
wait_tunnel_ready(require_min_connections=4)
url = get_quicktunnel_url()
send_requests(url, 3, True)
def test_quick_tunnel_url(self, tmp_path, component_tests_config):
config = component_tests_config(cfd_mode=CfdModes.QUICK, run_proxy_dns=False)
LOGGER.debug(config)
with start_cloudflared(tmp_path, config, cfd_args=["--url", f"http://localhost:{METRICS_PORT}/"], new_process=True):
wait_tunnel_ready()
url = get_quicktunnel_url()
send_requests(url+"/ready", 3, True)
def test_quick_tunnel_proxy_dns_url(self, tmp_path, component_tests_config):
config = component_tests_config(cfd_mode=CfdModes.QUICK, run_proxy_dns=True)
LOGGER.debug(config)
failed_start = start_cloudflared(tmp_path, config, cfd_args=["--url", f"http://localhost:{METRICS_PORT}/"], expect_success=False)
assert failed_start.returncode == 1, "Expected cloudflared to fail to run with `proxy-dns` and `hello-world`"
def test_quick_tunnel_proxy_dns_hello_world(self, tmp_path, component_tests_config):
config = component_tests_config(cfd_mode=CfdModes.QUICK, run_proxy_dns=True)
LOGGER.debug(config)
failed_start = start_cloudflared(tmp_path, config, cfd_args=["--hello-world"], expect_success=False)
assert failed_start.returncode == 1, "Expected cloudflared to fail to run with `proxy-dns` and `url`"
+43
View File
@@ -0,0 +1,43 @@
#!/usr/bin/env python
import requests
from conftest import CfdModes
from constants import METRICS_PORT, MAX_RETRIES, BACKOFF_SECS
from retrying import retry
from util import LOGGER, start_cloudflared, wait_tunnel_ready, send_requests
class TestTunnel:
'''Test tunnels with no ingress rules from config.yaml but ingress rules from CLI only'''
def test_tunnel_hello_world(self, tmp_path, component_tests_config):
config = component_tests_config(cfd_mode=CfdModes.NAMED, run_proxy_dns=False, provide_ingress=False)
LOGGER.debug(config)
with start_cloudflared(tmp_path, config, cfd_args=["run", "--hello-world"], new_process=True):
wait_tunnel_ready(tunnel_url=config.get_url(),
require_min_connections=4)
def test_tunnel_url(self, tmp_path, component_tests_config):
config = component_tests_config(cfd_mode=CfdModes.NAMED, run_proxy_dns=False, provide_ingress=False)
LOGGER.debug(config)
with start_cloudflared(tmp_path, config, cfd_args=["run", "--url", f"http://localhost:{METRICS_PORT}/"], new_process=True):
wait_tunnel_ready(require_min_connections=4)
send_requests(config.get_url()+"/ready", 3, True)
def test_tunnel_no_ingress(self, tmp_path, component_tests_config):
'''
Running a tunnel with no ingress rules provided from either config.yaml or CLI will still work but return 503
for all incoming requests.
'''
config = component_tests_config(cfd_mode=CfdModes.NAMED, run_proxy_dns=False, provide_ingress=False)
LOGGER.debug(config)
with start_cloudflared(tmp_path, config, cfd_args=["run"], new_process=True):
wait_tunnel_ready(require_min_connections=4)
resp = send_request(config.get_url()+"/")
assert resp.status_code == 503, "Expected cloudflared to return 503 for all requests with no ingress defined"
resp = send_request(config.get_url()+"/test")
assert resp.status_code == 503, "Expected cloudflared to return 503 for all requests with no ingress defined"
@retry(stop_max_attempt_number=MAX_RETRIES, wait_fixed=BACKOFF_SECS * 1000)
def send_request(url):
with requests.Session() as s:
return s.get(url, timeout=BACKOFF_SECS)
+13 -2
View File
@@ -35,7 +35,7 @@ def write_config(directory, config):
def start_cloudflared(directory, config, cfd_args=["run"], cfd_pre_args=["tunnel"], new_process=False,
allow_input=False, capture_output=True, root=False, skip_config_flag=False):
allow_input=False, capture_output=True, root=False, skip_config_flag=False, expect_success=True):
config_path = None
if not skip_config_flag:
@@ -46,7 +46,7 @@ def start_cloudflared(directory, config, cfd_args=["run"], cfd_pre_args=["tunnel
if new_process:
return run_cloudflared_background(cmd, allow_input, capture_output)
# By setting check=True, it will raise an exception if the process exits with non-zero exit code
return subprocess.run(cmd, check=True, capture_output=capture_output)
return subprocess.run(cmd, check=expect_success, capture_output=capture_output)
def cloudflared_cmd(config, config_path, cfd_args, cfd_pre_args, root):
@@ -77,7 +77,18 @@ def run_cloudflared_background(cmd, allow_input, capture_output):
cfd.terminate()
if capture_output:
LOGGER.info(f"cloudflared log: {cfd.stderr.read()}")
def get_quicktunnel_url():
quicktunnel_url = f'http://localhost:{METRICS_PORT}/quicktunnel'
with requests.Session() as s:
resp = send_request(s, quicktunnel_url, True)
hostname = resp.json()["hostname"]
assert hostname, \
f"Quicktunnel endpoint returned {hostname} but we expected a url"
return f"https://{hostname}"
def wait_tunnel_ready(tunnel_url=None, require_min_connections=1, cfd_logs=None):
try:
+2
View File
@@ -142,6 +142,7 @@ type TCPRequest struct {
LBProbe bool
FlowID string
CfTraceID string
ConnIndex uint8
}
// ReadWriteAcker is a readwriter with the ability to Acknowledge to the downstream (edge) that the origin has
@@ -199,6 +200,7 @@ func (h *HTTPResponseReadWriteAcker) AckConnection(tracePropagation string) erro
type ResponseWriter interface {
WriteRespHeaders(status int, header http.Header) error
AddTrailer(trailerName, trailerValue string)
http.ResponseWriter
io.Writer
}
+1 -207
View File
@@ -1,46 +1,17 @@
package connection
import (
"context"
"io"
"net"
"net/http"
"time"
"github.com/pkg/errors"
"github.com/rs/zerolog"
"golang.org/x/sync/errgroup"
"github.com/cloudflare/cloudflared/h2mux"
"github.com/cloudflare/cloudflared/tracing"
tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
"github.com/cloudflare/cloudflared/websocket"
)
const (
muxerTimeout = 5 * time.Second
openStreamTimeout = 30 * time.Second
muxerTimeout = 5 * time.Second
)
type h2muxConnection struct {
orchestrator Orchestrator
gracePeriod time.Duration
muxerConfig *MuxerConfig
muxer *h2mux.Muxer
// connectionID is only used by metrics, and prometheus requires labels to be string
connIndexStr string
connIndex uint8
observer *Observer
gracefulShutdownC <-chan struct{}
stoppedGracefully bool
log *zerolog.Logger
// newRPCClientFunc allows us to mock RPCs during testing
newRPCClientFunc func(context.Context, io.ReadWriteCloser, *zerolog.Logger) NamedTunnelRPCClient
}
type MuxerConfig struct {
HeartbeatInterval time.Duration
MaxHeartbeats uint64
@@ -59,180 +30,3 @@ func (mc *MuxerConfig) H2MuxerConfig(h h2mux.MuxedStreamHandler, log *zerolog.Lo
CompressionQuality: mc.CompressionSetting,
}
}
// NewTunnelHandler returns a TunnelHandler, origin LAN IP and error
func NewH2muxConnection(
orchestrator Orchestrator,
gracePeriod time.Duration,
muxerConfig *MuxerConfig,
edgeConn net.Conn,
connIndex uint8,
observer *Observer,
gracefulShutdownC <-chan struct{},
log *zerolog.Logger,
) (*h2muxConnection, error, bool) {
h := &h2muxConnection{
orchestrator: orchestrator,
gracePeriod: gracePeriod,
muxerConfig: muxerConfig,
connIndexStr: uint8ToString(connIndex),
connIndex: connIndex,
observer: observer,
gracefulShutdownC: gracefulShutdownC,
newRPCClientFunc: newRegistrationRPCClient,
log: log,
}
// Establish a muxed connection with the edge
// Client mux handshake with agent server
muxer, err := h2mux.Handshake(edgeConn, edgeConn, *muxerConfig.H2MuxerConfig(h, observer.logTransport), h2mux.ActiveStreams)
if err != nil {
recoverable := isHandshakeErrRecoverable(err, connIndex, observer)
return nil, err, recoverable
}
h.muxer = muxer
return h, nil, false
}
func (h *h2muxConnection) ServeNamedTunnel(ctx context.Context, namedTunnel *NamedTunnelProperties, connOptions *tunnelpogs.ConnectionOptions, connectedFuse ConnectedFuse) error {
errGroup, serveCtx := errgroup.WithContext(ctx)
errGroup.Go(func() error {
return h.serveMuxer(serveCtx)
})
errGroup.Go(func() error {
if err := h.registerNamedTunnel(serveCtx, namedTunnel, connOptions); err != nil {
return err
}
connectedFuse.Connected()
return nil
})
errGroup.Go(func() error {
h.controlLoop(serveCtx, connectedFuse, true)
return nil
})
err := errGroup.Wait()
if err == errMuxerStopped {
if h.stoppedGracefully {
return nil
}
h.observer.log.Info().Uint8(LogFieldConnIndex, h.connIndex).Msg("Unexpected muxer shutdown")
}
return err
}
func (h *h2muxConnection) serveMuxer(ctx context.Context) error {
// All routines should stop when muxer finish serving. When muxer is shutdown
// gracefully, it doesn't return an error, so we need to return errMuxerShutdown
// here to notify other routines to stop
err := h.muxer.Serve(ctx)
if err == nil {
return errMuxerStopped
}
return err
}
func (h *h2muxConnection) controlLoop(ctx context.Context, connectedFuse ConnectedFuse, isNamedTunnel bool) {
updateMetricsTicker := time.NewTicker(h.muxerConfig.MetricsUpdateFreq)
defer updateMetricsTicker.Stop()
var shutdownCompleted <-chan struct{}
for {
select {
case <-h.gracefulShutdownC:
if connectedFuse.IsConnected() {
h.unregister(isNamedTunnel)
}
h.stoppedGracefully = true
h.gracefulShutdownC = nil
shutdownCompleted = h.muxer.Shutdown()
case <-shutdownCompleted:
return
case <-ctx.Done():
// UnregisterTunnel blocks until the RPC call returns
if !h.stoppedGracefully && connectedFuse.IsConnected() {
h.unregister(isNamedTunnel)
}
h.muxer.Shutdown()
// don't wait for shutdown to finish when context is closed, this is the hard termination path
return
case <-updateMetricsTicker.C:
h.observer.metrics.updateMuxerMetrics(h.connIndexStr, h.muxer.Metrics())
}
}
}
func (h *h2muxConnection) newRPCStream(ctx context.Context, rpcName rpcName) (*h2mux.MuxedStream, error) {
openStreamCtx, openStreamCancel := context.WithTimeout(ctx, openStreamTimeout)
defer openStreamCancel()
stream, err := h.muxer.OpenRPCStream(openStreamCtx)
if err != nil {
return nil, err
}
return stream, nil
}
func (h *h2muxConnection) ServeStream(stream *h2mux.MuxedStream) error {
respWriter := &h2muxRespWriter{stream}
req, reqErr := h.newRequest(stream)
if reqErr != nil {
respWriter.WriteErrorResponse()
return reqErr
}
var sourceConnectionType = TypeHTTP
if websocket.IsWebSocketUpgrade(req) {
sourceConnectionType = TypeWebsocket
}
originProxy, err := h.orchestrator.GetOriginProxy()
if err != nil {
respWriter.WriteErrorResponse()
return err
}
err = originProxy.ProxyHTTP(respWriter, tracing.NewTracedHTTPRequest(req, h.log), sourceConnectionType == TypeWebsocket)
if err != nil {
respWriter.WriteErrorResponse()
}
return err
}
func (h *h2muxConnection) newRequest(stream *h2mux.MuxedStream) (*http.Request, error) {
req, err := http.NewRequest("GET", "http://localhost:8080", h2mux.MuxedStreamReader{MuxedStream: stream})
if err != nil {
return nil, errors.Wrap(err, "Unexpected error from http.NewRequest")
}
err = H2RequestHeadersToH1Request(stream.Headers, req)
if err != nil {
return nil, errors.Wrap(err, "invalid request received")
}
return req, nil
}
type h2muxRespWriter struct {
*h2mux.MuxedStream
}
func (rp *h2muxRespWriter) AddTrailer(trailerName, trailerValue string) {
// do nothing. we don't support trailers over h2mux
}
func (rp *h2muxRespWriter) WriteRespHeaders(status int, header http.Header) error {
headers := H1ResponseToH2ResponseHeaders(status, header)
headers = append(headers, h2mux.Header{Name: ResponseMetaHeader, Value: responseMetaHeaderOrigin})
return rp.WriteHeaders(headers)
}
func (rp *h2muxRespWriter) WriteErrorResponse() {
_ = rp.WriteHeaders([]h2mux.Header{
{Name: ":status", Value: "502"},
{Name: ResponseMetaHeader, Value: responseMetaHeaderCfd},
})
_, _ = rp.Write([]byte("502 Bad Gateway"))
}
-301
View File
@@ -1,301 +0,0 @@
package connection
import (
"context"
"fmt"
"io"
"net"
"net/http"
"strconv"
"sync"
"testing"
"time"
"github.com/gobwas/ws/wsutil"
"github.com/rs/zerolog"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/cloudflare/cloudflared/h2mux"
)
var (
testMuxerConfig = &MuxerConfig{
HeartbeatInterval: time.Second * 5,
MaxHeartbeats: 5,
CompressionSetting: 0,
MetricsUpdateFreq: time.Second * 5,
}
)
func newH2MuxConnection(t require.TestingT) (*h2muxConnection, *h2mux.Muxer) {
edgeConn, originConn := net.Pipe()
edgeMuxChan := make(chan *h2mux.Muxer)
go func() {
edgeMuxConfig := h2mux.MuxerConfig{
Log: &log,
Handler: h2mux.MuxedStreamFunc(func(stream *h2mux.MuxedStream) error {
// we only expect RPC traffic in client->edge direction, provide minimal support for mocking
require.True(t, stream.IsRPCStream())
return stream.WriteHeaders([]h2mux.Header{
{Name: ":status", Value: "200"},
})
}),
}
edgeMux, err := h2mux.Handshake(edgeConn, edgeConn, edgeMuxConfig, h2mux.ActiveStreams)
require.NoError(t, err)
edgeMuxChan <- edgeMux
}()
var connIndex = uint8(0)
testObserver := NewObserver(&log, &log)
h2muxConn, err, _ := NewH2muxConnection(testOrchestrator, testGracePeriod, testMuxerConfig, originConn, connIndex, testObserver, nil, &log)
require.NoError(t, err)
return h2muxConn, <-edgeMuxChan
}
func TestServeStreamHTTP(t *testing.T) {
tests := []testRequest{
{
name: "ok",
endpoint: "/ok",
expectedStatus: http.StatusOK,
expectedBody: []byte(http.StatusText(http.StatusOK)),
},
{
name: "large_file",
endpoint: "/large_file",
expectedStatus: http.StatusOK,
expectedBody: testLargeResp,
},
{
name: "Bad request",
endpoint: "/400",
expectedStatus: http.StatusBadRequest,
expectedBody: []byte(http.StatusText(http.StatusBadRequest)),
},
{
name: "Internal server error",
endpoint: "/500",
expectedStatus: http.StatusInternalServerError,
expectedBody: []byte(http.StatusText(http.StatusInternalServerError)),
},
{
name: "Proxy error",
endpoint: "/error",
expectedStatus: http.StatusBadGateway,
expectedBody: nil,
isProxyError: true,
},
}
ctx, cancel := context.WithCancel(context.Background())
h2muxConn, edgeMux := newH2MuxConnection(t)
var wg sync.WaitGroup
wg.Add(2)
go func() {
defer wg.Done()
_ = edgeMux.Serve(ctx)
}()
go func() {
defer wg.Done()
err := h2muxConn.serveMuxer(ctx)
require.Error(t, err)
}()
for _, test := range tests {
headers := []h2mux.Header{
{
Name: ":path",
Value: test.endpoint,
},
}
stream, err := edgeMux.OpenStream(ctx, headers, nil)
require.NoError(t, err)
require.True(t, hasHeader(stream, ":status", strconv.Itoa(test.expectedStatus)))
if test.isProxyError {
assert.True(t, hasHeader(stream, ResponseMetaHeader, responseMetaHeaderCfd))
} else {
assert.True(t, hasHeader(stream, ResponseMetaHeader, responseMetaHeaderOrigin))
body := make([]byte, len(test.expectedBody))
_, err = stream.Read(body)
require.NoError(t, err)
require.Equal(t, test.expectedBody, body)
}
}
cancel()
wg.Wait()
}
func TestServeStreamWS(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
h2muxConn, edgeMux := newH2MuxConnection(t)
var wg sync.WaitGroup
wg.Add(2)
go func() {
defer wg.Done()
edgeMux.Serve(ctx)
}()
go func() {
defer wg.Done()
err := h2muxConn.serveMuxer(ctx)
require.Error(t, err)
}()
headers := []h2mux.Header{
{
Name: ":path",
Value: "/ws/echo",
},
{
Name: "connection",
Value: "upgrade",
},
{
Name: "upgrade",
Value: "websocket",
},
}
readPipe, writePipe := io.Pipe()
stream, err := edgeMux.OpenStream(ctx, headers, readPipe)
require.NoError(t, err)
require.True(t, hasHeader(stream, ":status", strconv.Itoa(http.StatusSwitchingProtocols)))
assert.True(t, hasHeader(stream, ResponseMetaHeader, responseMetaHeaderOrigin))
data := []byte("test websocket")
err = wsutil.WriteClientBinary(writePipe, data)
require.NoError(t, err)
respBody, err := wsutil.ReadServerBinary(stream)
require.NoError(t, err)
require.Equal(t, data, respBody, fmt.Sprintf("Expect %s, got %s", string(data), string(respBody)))
cancel()
wg.Wait()
}
func TestGracefulShutdownH2Mux(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
h2muxConn, edgeMux := newH2MuxConnection(t)
shutdownC := make(chan struct{})
unregisteredC := make(chan struct{})
h2muxConn.gracefulShutdownC = shutdownC
h2muxConn.newRPCClientFunc = func(_ context.Context, _ io.ReadWriteCloser, _ *zerolog.Logger) NamedTunnelRPCClient {
return &mockNamedTunnelRPCClient{
registered: nil,
unregistered: unregisteredC,
}
}
var wg sync.WaitGroup
wg.Add(3)
go func() {
defer wg.Done()
_ = edgeMux.Serve(ctx)
}()
go func() {
defer wg.Done()
_ = h2muxConn.serveMuxer(ctx)
}()
go func() {
defer wg.Done()
h2muxConn.controlLoop(ctx, &mockConnectedFuse{}, true)
}()
time.Sleep(100 * time.Millisecond)
close(shutdownC)
select {
case <-unregisteredC:
break // ok
case <-time.Tick(time.Second):
assert.Fail(t, "timed out waiting for control loop to unregister")
}
cancel()
wg.Wait()
assert.True(t, h2muxConn.stoppedGracefully)
assert.Nil(t, h2muxConn.gracefulShutdownC)
}
func hasHeader(stream *h2mux.MuxedStream, name, val string) bool {
for _, header := range stream.Headers {
if header.Name == name && header.Value == val {
return true
}
}
return false
}
func benchmarkServeStreamHTTPSimple(b *testing.B, test testRequest) {
ctx, cancel := context.WithCancel(context.Background())
h2muxConn, edgeMux := newH2MuxConnection(b)
var wg sync.WaitGroup
wg.Add(2)
go func() {
defer wg.Done()
edgeMux.Serve(ctx)
}()
go func() {
defer wg.Done()
err := h2muxConn.serveMuxer(ctx)
require.Error(b, err)
}()
headers := []h2mux.Header{
{
Name: ":path",
Value: test.endpoint,
},
}
body := make([]byte, len(test.expectedBody))
b.ResetTimer()
for i := 0; i < b.N; i++ {
b.StartTimer()
stream, openstreamErr := edgeMux.OpenStream(ctx, headers, nil)
_, readBodyErr := stream.Read(body)
b.StopTimer()
require.NoError(b, openstreamErr)
assert.True(b, hasHeader(stream, ResponseMetaHeader, responseMetaHeaderOrigin))
require.True(b, hasHeader(stream, ":status", strconv.Itoa(http.StatusOK)))
require.NoError(b, readBodyErr)
require.Equal(b, test.expectedBody, body)
}
cancel()
wg.Wait()
}
func BenchmarkServeStreamHTTPSimple(b *testing.B) {
test := testRequest{
name: "ok",
endpoint: "/ok",
expectedStatus: http.StatusOK,
expectedBody: []byte(http.StatusText(http.StatusOK)),
}
benchmarkServeStreamHTTPSimple(b, test)
}
func BenchmarkServeStreamHTTPLargeFile(b *testing.B) {
test := testRequest{
name: "large_file",
endpoint: "/large_file",
expectedStatus: http.StatusOK,
expectedBody: testLargeResp,
}
benchmarkServeStreamHTTPSimple(b, test)
}
+11 -1
View File
@@ -129,7 +129,7 @@ func (c *HTTP2Connection) ServeHTTP(w http.ResponseWriter, r *http.Request) {
case TypeWebsocket, TypeHTTP:
stripWebsocketUpgradeHeader(r)
// Check for tracing on request
tr := tracing.NewTracedHTTPRequest(r, c.log)
tr := tracing.NewTracedHTTPRequest(r, c.connIndex, c.log)
if err := originProxy.ProxyHTTP(respWriter, tr, connType == TypeWebsocket); err != nil {
requestErr = fmt.Errorf("Failed to proxy HTTP: %w", err)
}
@@ -147,6 +147,7 @@ func (c *HTTP2Connection) ServeHTTP(w http.ResponseWriter, r *http.Request) {
CFRay: FindCfRayHeader(r),
LBProbe: IsLBProbeRequest(r),
CfTraceID: r.Header.Get(tracing.TracerContextName),
ConnIndex: c.connIndex,
})
default:
@@ -196,6 +197,7 @@ type http2RespWriter struct {
flusher http.Flusher
shouldFlush bool
statusWritten bool
respHeaders http.Header
log *zerolog.Logger
}
@@ -275,6 +277,14 @@ func (rp *http2RespWriter) WriteRespHeaders(status int, header http.Header) erro
return nil
}
func (rp *http2RespWriter) Header() http.Header {
return rp.respHeaders
}
func (rp *http2RespWriter) WriteHeader(status int) {
rp.WriteRespHeaders(status, rp.respHeaders)
}
func (rp *http2RespWriter) WriteErrorResponse() bool {
if rp.statusWritten {
return false
+79 -126
View File
@@ -1,7 +1,6 @@
package connection
import (
"errors"
"fmt"
"hash/fnv"
"sync"
@@ -13,7 +12,7 @@ import (
)
const (
AvailableProtocolFlagMessage = "Available protocols: 'auto' - automatically chooses the best protocol over time (the default; and also the recommended one); 'quic' - based on QUIC, relying on UDP egress to Cloudflare edge; 'http2' - using Go's HTTP2 library, relying on TCP egress to Cloudflare edge; 'h2mux' - Cloudflare's implementation of HTTP/2, deprecated"
AvailableProtocolFlagMessage = "Available protocols: 'auto' - automatically chooses the best protocol over time (the default; and also the recommended one); 'quic' - based on QUIC, relying on UDP egress to Cloudflare edge; 'http2' - using Go's HTTP2 library, relying on TCP egress to Cloudflare edge"
// edgeH2muxTLSServerName is the server name to establish h2mux connection with edge
edgeH2muxTLSServerName = "cftunnel.com"
// edgeH2TLSServerName is the server name to establish http2 connection with edge
@@ -21,43 +20,32 @@ const (
// edgeQUICServerName is the server name to establish quic connection with edge.
edgeQUICServerName = "quic.cftunnel.com"
AutoSelectFlag = "auto"
// SRV and TXT record resolution TTL
ResolveTTL = time.Hour
)
var (
// ProtocolList represents a list of supported protocols for communication with the edge.
ProtocolList = []Protocol{H2mux, HTTP2, HTTP2Warp, QUIC, QUICWarp}
// ProtocolList represents a list of supported protocols for communication with the edge
// in order of precedence for remote percentage fetcher.
ProtocolList = []Protocol{QUIC, HTTP2}
)
type Protocol int64
const (
// H2mux protocol can be used both with Classic and Named Tunnels. .
H2mux Protocol = iota
// HTTP2 is used only with named tunnels. It's more efficient than H2mux for L4 proxying.
HTTP2
// QUIC is used only with named tunnels.
// HTTP2 using golang HTTP2 library for edge connections.
HTTP2 Protocol = iota
// QUIC using quic-go for edge connections.
QUIC
// HTTP2Warp is used only with named tunnels. It's useful for warp-routing where we don't want to fallback to
// H2mux on HTTP2 failure to connect.
HTTP2Warp
//QUICWarp is used only with named tunnels. It's useful for warp-routing where we want to fallback to HTTP2 but
// don't want HTTP2 to fallback to H2mux
QUICWarp
)
// Fallback returns the fallback protocol and whether the protocol has a fallback
func (p Protocol) fallback() (Protocol, bool) {
switch p {
case H2mux:
return 0, false
case HTTP2:
return H2mux, true
case HTTP2Warp:
return 0, false
case QUIC:
return HTTP2, true
case QUICWarp:
return HTTP2Warp, true
default:
return 0, false
}
@@ -65,11 +53,9 @@ func (p Protocol) fallback() (Protocol, bool) {
func (p Protocol) String() string {
switch p {
case H2mux:
return "h2mux"
case HTTP2, HTTP2Warp:
case HTTP2:
return "http2"
case QUIC, QUICWarp:
case QUIC:
return "quic"
default:
return fmt.Sprintf("unknown protocol")
@@ -78,15 +64,11 @@ func (p Protocol) String() string {
func (p Protocol) TLSSettings() *TLSSettings {
switch p {
case H2mux:
return &TLSSettings{
ServerName: edgeH2muxTLSServerName,
}
case HTTP2, HTTP2Warp:
case HTTP2:
return &TLSSettings{
ServerName: edgeH2TLSServerName,
}
case QUIC, QUICWarp:
case QUIC:
return &TLSSettings{
ServerName: edgeQUICServerName,
NextProtos: []string{"argotunnel"},
@@ -106,6 +88,7 @@ type ProtocolSelector interface {
Fallback() (Protocol, bool)
}
// staticProtocolSelector will not provide a different protocol for Fallback
type staticProtocolSelector struct {
current Protocol
}
@@ -115,10 +98,11 @@ func (s *staticProtocolSelector) Current() Protocol {
}
func (s *staticProtocolSelector) Fallback() (Protocol, bool) {
return 0, false
return s.current, false
}
type autoProtocolSelector struct {
// remoteProtocolSelector will fetch a list of remote protocols to provide for edge discovery
type remoteProtocolSelector struct {
lock sync.RWMutex
current Protocol
@@ -127,23 +111,21 @@ type autoProtocolSelector struct {
protocolPool []Protocol
switchThreshold int32
fetchFunc PercentageFetcher
fetchFunc edgediscovery.PercentageFetcher
refreshAfter time.Time
ttl time.Duration
log *zerolog.Logger
needPQ bool
}
func newAutoProtocolSelector(
func newRemoteProtocolSelector(
current Protocol,
protocolPool []Protocol,
switchThreshold int32,
fetchFunc PercentageFetcher,
fetchFunc edgediscovery.PercentageFetcher,
ttl time.Duration,
log *zerolog.Logger,
needPQ bool,
) *autoProtocolSelector {
return &autoProtocolSelector{
) *remoteProtocolSelector {
return &remoteProtocolSelector{
current: current,
protocolPool: protocolPool,
switchThreshold: switchThreshold,
@@ -151,11 +133,10 @@ func newAutoProtocolSelector(
refreshAfter: time.Now().Add(ttl),
ttl: ttl,
log: log,
needPQ: needPQ,
}
}
func (s *autoProtocolSelector) Current() Protocol {
func (s *remoteProtocolSelector) Current() Protocol {
s.lock.Lock()
defer s.lock.Unlock()
if time.Now().Before(s.refreshAfter) {
@@ -173,7 +154,13 @@ func (s *autoProtocolSelector) Current() Protocol {
return s.current
}
func getProtocol(protocolPool []Protocol, fetchFunc PercentageFetcher, switchThreshold int32) (Protocol, error) {
func (s *remoteProtocolSelector) Fallback() (Protocol, bool) {
s.lock.RLock()
defer s.lock.RUnlock()
return s.current.fallback()
}
func getProtocol(protocolPool []Protocol, fetchFunc edgediscovery.PercentageFetcher, switchThreshold int32) (Protocol, error) {
protocolPercentages, err := fetchFunc()
if err != nil {
return 0, err
@@ -185,112 +172,78 @@ func getProtocol(protocolPool []Protocol, fetchFunc PercentageFetcher, switchThr
}
}
return protocolPool[len(protocolPool)-1], nil
// Default to first index in protocolPool list
return protocolPool[0], nil
}
func (s *autoProtocolSelector) Fallback() (Protocol, bool) {
// defaultProtocolSelector will allow for a protocol to have a fallback
type defaultProtocolSelector struct {
lock sync.RWMutex
current Protocol
}
func newDefaultProtocolSelector(
current Protocol,
) *defaultProtocolSelector {
return &defaultProtocolSelector{
current: current,
}
}
func (s *defaultProtocolSelector) Current() Protocol {
s.lock.Lock()
defer s.lock.Unlock()
return s.current
}
func (s *defaultProtocolSelector) Fallback() (Protocol, bool) {
s.lock.RLock()
defer s.lock.RUnlock()
if s.needPQ {
return 0, false
}
return s.current.fallback()
}
type PercentageFetcher func() (edgediscovery.ProtocolPercents, error)
func NewProtocolSelector(
protocolFlag string,
warpRoutingEnabled bool,
namedTunnel *NamedTunnelProperties,
fetchFunc PercentageFetcher,
ttl time.Duration,
log *zerolog.Logger,
accountTag string,
tunnelTokenProvided bool,
needPQ bool,
protocolFetcher edgediscovery.PercentageFetcher,
resolveTTL time.Duration,
log *zerolog.Logger,
) (ProtocolSelector, error) {
// Classic tunnel is only supported with h2mux
if namedTunnel == nil {
if needPQ {
return nil, errors.New("Classic tunnel does not support post-quantum")
}
// With --post-quantum, we force quic
if needPQ {
return &staticProtocolSelector{
current: H2mux,
current: QUIC,
}, nil
}
threshold := switchThreshold(namedTunnel.Credentials.AccountTag)
fetchedProtocol, err := getProtocol([]Protocol{QUIC, HTTP2}, fetchFunc, threshold)
if err != nil && protocolFlag == "auto" {
log.Err(err).Msg("Unable to lookup protocol. Defaulting to `http2`. If this fails, you can attempt `--protocol quic` instead.")
if needPQ {
return nil, errors.New("http2 does not support post-quantum")
}
return &staticProtocolSelector{
current: HTTP2,
}, nil
}
if warpRoutingEnabled {
if protocolFlag == H2mux.String() || fetchedProtocol == H2mux {
log.Warn().Msg("Warp routing is not supported in h2mux protocol. Upgrading to http2 to allow it.")
protocolFlag = HTTP2.String()
fetchedProtocol = HTTP2Warp
}
return selectWarpRoutingProtocols(protocolFlag, fetchFunc, ttl, log, threshold, fetchedProtocol, needPQ)
threshold := switchThreshold(accountTag)
fetchedProtocol, err := getProtocol(ProtocolList, protocolFetcher, threshold)
log.Debug().Msgf("Fetched protocol: %s", fetchedProtocol)
if err != nil {
log.Warn().Msg("Unable to lookup protocol percentage.")
// Falling through here since 'auto' is handled in the switch and failing
// to do the protocol lookup isn't a failure since it can be triggered again
// after the TTL.
}
return selectNamedTunnelProtocols(protocolFlag, fetchFunc, ttl, log, threshold, fetchedProtocol, needPQ)
}
func selectNamedTunnelProtocols(
protocolFlag string,
fetchFunc PercentageFetcher,
ttl time.Duration,
log *zerolog.Logger,
threshold int32,
protocol Protocol,
needPQ bool,
) (ProtocolSelector, error) {
// If the user picks a protocol, then we stick to it no matter what.
switch protocolFlag {
case H2mux.String():
return &staticProtocolSelector{current: H2mux}, nil
case "h2mux":
// Any users still requesting h2mux will be upgraded to http2 instead
log.Warn().Msg("h2mux is no longer a supported protocol: upgrading edge connection to http2. Please remove '--protocol h2mux' from runtime arguments to remove this warning.")
return &staticProtocolSelector{current: HTTP2}, nil
case QUIC.String():
return &staticProtocolSelector{current: QUIC}, nil
case HTTP2.String():
return &staticProtocolSelector{current: HTTP2}, nil
}
// If the user does not pick (hopefully the majority) then we use the one derived from the TXT DNS record and
// fallback on failures.
if protocolFlag == AutoSelectFlag {
return newAutoProtocolSelector(protocol, []Protocol{QUIC, HTTP2, H2mux}, threshold, fetchFunc, ttl, log, needPQ), nil
}
return nil, fmt.Errorf("Unknown protocol %s, %s", protocolFlag, AvailableProtocolFlagMessage)
}
func selectWarpRoutingProtocols(
protocolFlag string,
fetchFunc PercentageFetcher,
ttl time.Duration,
log *zerolog.Logger,
threshold int32,
protocol Protocol,
needPQ bool,
) (ProtocolSelector, error) {
// If the user picks a protocol, then we stick to it no matter what.
switch protocolFlag {
case QUIC.String():
return &staticProtocolSelector{current: QUICWarp}, nil
case HTTP2.String():
return &staticProtocolSelector{current: HTTP2Warp}, nil
}
// If the user does not pick (hopefully the majority) then we use the one derived from the TXT DNS record and
// fallback on failures.
if protocolFlag == AutoSelectFlag {
return newAutoProtocolSelector(protocol, []Protocol{QUICWarp, HTTP2Warp}, threshold, fetchFunc, ttl, log, needPQ), nil
case AutoSelectFlag:
// When a --token is provided, we want to start with QUIC but have fallback to HTTP2
if tunnelTokenProvided {
return newDefaultProtocolSelector(QUIC), nil
}
return newRemoteProtocolSelector(fetchedProtocol, ProtocolList, threshold, protocolFetcher, resolveTTL, log), nil
}
return nil, fmt.Errorf("Unknown protocol %s, %s", protocolFlag, AvailableProtocolFlagMessage)
+47 -180
View File
@@ -3,7 +3,6 @@ package connection
import (
"fmt"
"testing"
"time"
"github.com/stretchr/testify/assert"
@@ -11,19 +10,11 @@ import (
)
const (
testNoTTL = 0
noWarpRoutingEnabled = false
testNoTTL = 0
testAccountTag = "testAccountTag"
)
var (
testNamedTunnelProperties = &NamedTunnelProperties{
Credentials: Credentials{
AccountTag: "testAccountTag",
},
}
)
func mockFetcher(getError bool, protocolPercent ...edgediscovery.ProtocolPercent) PercentageFetcher {
func mockFetcher(getError bool, protocolPercent ...edgediscovery.ProtocolPercent) edgediscovery.PercentageFetcher {
return func() (edgediscovery.ProtocolPercents, error) {
if getError {
return nil, fmt.Errorf("failed to fetch percentage")
@@ -37,7 +28,7 @@ type dynamicMockFetcher struct {
err error
}
func (dmf *dynamicMockFetcher) fetch() PercentageFetcher {
func (dmf *dynamicMockFetcher) fetch() edgediscovery.PercentageFetcher {
return func() (edgediscovery.ProtocolPercents, error) {
return dmf.protocolPercents, dmf.err
}
@@ -45,181 +36,58 @@ func (dmf *dynamicMockFetcher) fetch() PercentageFetcher {
func TestNewProtocolSelector(t *testing.T) {
tests := []struct {
name string
protocol string
expectedProtocol Protocol
hasFallback bool
expectedFallback Protocol
warpRoutingEnabled bool
namedTunnelConfig *NamedTunnelProperties
fetchFunc PercentageFetcher
wantErr bool
name string
protocol string
tunnelTokenProvided bool
needPQ bool
expectedProtocol Protocol
hasFallback bool
expectedFallback Protocol
wantErr bool
}{
{
name: "classic tunnel",
protocol: "h2mux",
expectedProtocol: H2mux,
namedTunnelConfig: nil,
name: "named tunnel with unknown protocol",
protocol: "unknown",
wantErr: true,
},
{
name: "named tunnel over h2mux",
protocol: "h2mux",
expectedProtocol: H2mux,
fetchFunc: func() (edgediscovery.ProtocolPercents, error) { return nil, nil },
namedTunnelConfig: testNamedTunnelProperties,
},
{
name: "named tunnel over http2",
protocol: "http2",
expectedProtocol: HTTP2,
fetchFunc: mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 0}),
namedTunnelConfig: testNamedTunnelProperties,
},
{
name: "named tunnel http2 disabled still gets http2 because it is manually picked",
protocol: "http2",
expectedProtocol: HTTP2,
fetchFunc: mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: -1}),
namedTunnelConfig: testNamedTunnelProperties,
},
{
name: "named tunnel quic disabled still gets quic because it is manually picked",
protocol: "quic",
expectedProtocol: QUIC,
fetchFunc: mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 100}, edgediscovery.ProtocolPercent{Protocol: "quic", Percentage: -1}),
namedTunnelConfig: testNamedTunnelProperties,
},
{
name: "named tunnel quic and http2 disabled",
protocol: AutoSelectFlag,
expectedProtocol: H2mux,
fetchFunc: mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: -1}, edgediscovery.ProtocolPercent{Protocol: "quic", Percentage: -1}),
namedTunnelConfig: testNamedTunnelProperties,
},
{
name: "named tunnel quic disabled",
protocol: AutoSelectFlag,
name: "named tunnel with h2mux: force to http2",
protocol: "h2mux",
expectedProtocol: HTTP2,
// Hasfallback true is because if http2 fails, then we further fallback to h2mux.
hasFallback: true,
expectedFallback: H2mux,
fetchFunc: mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 100}, edgediscovery.ProtocolPercent{Protocol: "quic", Percentage: -1}),
namedTunnelConfig: testNamedTunnelProperties,
},
{
name: "named tunnel auto all http2 disabled",
protocol: AutoSelectFlag,
expectedProtocol: H2mux,
fetchFunc: mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: -1}),
namedTunnelConfig: testNamedTunnelProperties,
name: "named tunnel with http2: no fallback",
protocol: "http2",
expectedProtocol: HTTP2,
},
{
name: "named tunnel auto to h2mux",
protocol: AutoSelectFlag,
expectedProtocol: H2mux,
fetchFunc: mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 0}),
namedTunnelConfig: testNamedTunnelProperties,
name: "named tunnel with auto: quic",
protocol: AutoSelectFlag,
expectedProtocol: QUIC,
hasFallback: true,
expectedFallback: HTTP2,
},
{
name: "named tunnel auto to http2",
protocol: AutoSelectFlag,
expectedProtocol: HTTP2,
hasFallback: true,
expectedFallback: H2mux,
fetchFunc: mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 100}),
namedTunnelConfig: testNamedTunnelProperties,
name: "named tunnel (post quantum)",
protocol: AutoSelectFlag,
needPQ: true,
expectedProtocol: QUIC,
},
{
name: "named tunnel auto to quic",
protocol: AutoSelectFlag,
expectedProtocol: QUIC,
hasFallback: true,
expectedFallback: HTTP2,
fetchFunc: mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "quic", Percentage: 100}),
namedTunnelConfig: testNamedTunnelProperties,
},
{
name: "warp routing requesting h2mux",
protocol: "h2mux",
expectedProtocol: HTTP2Warp,
hasFallback: false,
fetchFunc: mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 100}),
warpRoutingEnabled: true,
namedTunnelConfig: testNamedTunnelProperties,
},
{
name: "warp routing requesting h2mux picks HTTP2 even if http2 percent is -1",
protocol: "h2mux",
expectedProtocol: HTTP2Warp,
hasFallback: false,
fetchFunc: mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: -1}),
warpRoutingEnabled: true,
namedTunnelConfig: testNamedTunnelProperties,
},
{
name: "warp routing http2",
protocol: "http2",
expectedProtocol: HTTP2Warp,
hasFallback: false,
fetchFunc: mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 100}),
warpRoutingEnabled: true,
namedTunnelConfig: testNamedTunnelProperties,
},
{
name: "warp routing quic",
protocol: AutoSelectFlag,
expectedProtocol: QUICWarp,
hasFallback: true,
expectedFallback: HTTP2Warp,
fetchFunc: mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "quic", Percentage: 100}),
warpRoutingEnabled: true,
namedTunnelConfig: testNamedTunnelProperties,
},
{
name: "warp routing auto",
protocol: AutoSelectFlag,
expectedProtocol: HTTP2Warp,
hasFallback: false,
fetchFunc: mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 100}),
warpRoutingEnabled: true,
namedTunnelConfig: testNamedTunnelProperties,
},
{
name: "warp routing auto- quic",
protocol: AutoSelectFlag,
expectedProtocol: QUICWarp,
hasFallback: true,
expectedFallback: HTTP2Warp,
fetchFunc: mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 100}, edgediscovery.ProtocolPercent{Protocol: "quic", Percentage: 100}),
warpRoutingEnabled: true,
namedTunnelConfig: testNamedTunnelProperties,
},
{
// None named tunnel can only use h2mux, so specifying an unknown protocol is not an error
name: "classic tunnel unknown protocol",
protocol: "unknown",
expectedProtocol: H2mux,
},
{
name: "named tunnel unknown protocol",
protocol: "unknown",
fetchFunc: mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 100}),
namedTunnelConfig: testNamedTunnelProperties,
wantErr: true,
},
{
name: "named tunnel fetch error",
protocol: AutoSelectFlag,
fetchFunc: mockFetcher(true),
namedTunnelConfig: testNamedTunnelProperties,
expectedProtocol: HTTP2,
wantErr: false,
name: "named tunnel (post quantum) w/http2",
protocol: "http2",
needPQ: true,
expectedProtocol: QUIC,
},
}
fetcher := dynamicMockFetcher{
protocolPercents: edgediscovery.ProtocolPercents{},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
selector, err := NewProtocolSelector(test.protocol, test.warpRoutingEnabled, test.namedTunnelConfig, test.fetchFunc, testNoTTL, &log, false)
selector, err := NewProtocolSelector(test.protocol, testAccountTag, test.tunnelTokenProvided, test.needPQ, fetcher.fetch(), ResolveTTL, &log)
if test.wantErr {
assert.Error(t, err, fmt.Sprintf("test %s failed", test.name))
} else {
@@ -237,15 +105,15 @@ func TestNewProtocolSelector(t *testing.T) {
func TestAutoProtocolSelectorRefresh(t *testing.T) {
fetcher := dynamicMockFetcher{}
selector, err := NewProtocolSelector(AutoSelectFlag, noWarpRoutingEnabled, testNamedTunnelProperties, fetcher.fetch(), testNoTTL, &log, false)
selector, err := NewProtocolSelector(AutoSelectFlag, testAccountTag, false, false, fetcher.fetch(), testNoTTL, &log)
assert.NoError(t, err)
assert.Equal(t, H2mux, selector.Current())
assert.Equal(t, QUIC, selector.Current())
fetcher.protocolPercents = edgediscovery.ProtocolPercents{edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 100}}
assert.Equal(t, HTTP2, selector.Current())
fetcher.protocolPercents = edgediscovery.ProtocolPercents{edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 0}}
assert.Equal(t, H2mux, selector.Current())
assert.Equal(t, QUIC, selector.Current())
fetcher.protocolPercents = edgediscovery.ProtocolPercents{edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 100}}
assert.Equal(t, HTTP2, selector.Current())
@@ -255,10 +123,10 @@ func TestAutoProtocolSelectorRefresh(t *testing.T) {
fetcher.protocolPercents = edgediscovery.ProtocolPercents{edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: -1}}
fetcher.err = nil
assert.Equal(t, H2mux, selector.Current())
assert.Equal(t, QUIC, selector.Current())
fetcher.protocolPercents = edgediscovery.ProtocolPercents{edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 0}}
assert.Equal(t, H2mux, selector.Current())
assert.Equal(t, QUIC, selector.Current())
fetcher.protocolPercents = edgediscovery.ProtocolPercents{edgediscovery.ProtocolPercent{Protocol: "quic", Percentage: 100}}
assert.Equal(t, QUIC, selector.Current())
@@ -267,7 +135,7 @@ func TestAutoProtocolSelectorRefresh(t *testing.T) {
func TestHTTP2ProtocolSelectorRefresh(t *testing.T) {
fetcher := dynamicMockFetcher{}
// Since the user chooses http2 on purpose, we always stick to it.
selector, err := NewProtocolSelector("http2", noWarpRoutingEnabled, testNamedTunnelProperties, fetcher.fetch(), testNoTTL, &log, false)
selector, err := NewProtocolSelector(HTTP2.String(), testAccountTag, false, false, fetcher.fetch(), testNoTTL, &log)
assert.NoError(t, err)
assert.Equal(t, HTTP2, selector.Current())
@@ -294,13 +162,12 @@ func TestHTTP2ProtocolSelectorRefresh(t *testing.T) {
assert.Equal(t, HTTP2, selector.Current())
}
func TestProtocolSelectorRefreshTTL(t *testing.T) {
func TestAutoProtocolSelectorNoRefreshWithToken(t *testing.T) {
fetcher := dynamicMockFetcher{}
fetcher.protocolPercents = edgediscovery.ProtocolPercents{edgediscovery.ProtocolPercent{Protocol: "quic", Percentage: 100}}
selector, err := NewProtocolSelector(AutoSelectFlag, noWarpRoutingEnabled, testNamedTunnelProperties, fetcher.fetch(), time.Hour, &log, false)
selector, err := NewProtocolSelector(AutoSelectFlag, testAccountTag, true, false, fetcher.fetch(), testNoTTL, &log)
assert.NoError(t, err)
assert.Equal(t, QUIC, selector.Current())
fetcher.protocolPercents = edgediscovery.ProtocolPercents{edgediscovery.ProtocolPercent{Protocol: "quic", Percentage: 0}}
fetcher.protocolPercents = edgediscovery.ProtocolPercents{edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 100}}
assert.Equal(t, QUIC, selector.Current())
}
+32 -10
View File
@@ -60,12 +60,14 @@ type QUICConnection struct {
packetRouter *ingress.PacketRouter
controlStreamHandler ControlStreamHandler
connOptions *tunnelpogs.ConnectionOptions
connIndex uint8
}
// NewQUICConnection returns a new instance of QUICConnection.
func NewQUICConnection(
quicConfig *quic.Config,
edgeAddr net.Addr,
localAddr net.IP,
connIndex uint8,
tlsConfig *tls.Config,
orchestrator Orchestrator,
@@ -74,7 +76,7 @@ func NewQUICConnection(
logger *zerolog.Logger,
packetRouterConfig *ingress.GlobalRouterConfig,
) (*QUICConnection, error) {
udpConn, err := createUDPConnForConnIndex(connIndex, logger)
udpConn, err := createUDPConnForConnIndex(connIndex, localAddr, logger)
if err != nil {
return nil, err
}
@@ -106,6 +108,7 @@ func NewQUICConnection(
packetRouter: packetRouter,
controlStreamHandler: controlStreamHandler,
connOptions: connOptions,
connIndex: connIndex,
}, nil
}
@@ -258,7 +261,7 @@ func (q *QUICConnection) dispatchRequest(ctx context.Context, stream *quicpogs.R
switch request.Type {
case quicpogs.ConnectionTypeHTTP, quicpogs.ConnectionTypeWebsocket:
tracedReq, err := buildHTTPRequest(ctx, request, stream, q.logger)
tracedReq, err := buildHTTPRequest(ctx, request, stream, q.connIndex, q.logger)
if err != nil {
return err, false
}
@@ -272,6 +275,7 @@ func (q *QUICConnection) dispatchRequest(ctx context.Context, stream *quicpogs.R
Dest: request.Dest,
FlowID: metadata[QUICMetadataFlowID],
CfTraceID: metadata[tracing.TracerContextName],
ConnIndex: q.connIndex,
}), rwa.connectResponseSent
default:
return errors.Errorf("unsupported error type: %s", request.Type), false
@@ -383,17 +387,22 @@ type streamReadWriteAcker struct {
// AckConnection acks response back to the proxy.
func (s *streamReadWriteAcker) AckConnection(tracePropagation string) error {
metadata := quicpogs.Metadata{
Key: tracing.CanonicalCloudflaredTracingHeader,
Val: tracePropagation,
metadata := []quicpogs.Metadata{}
// Only add tracing if provided by origintunneld
if tracePropagation != "" {
metadata = append(metadata, quicpogs.Metadata{
Key: tracing.CanonicalCloudflaredTracingHeader,
Val: tracePropagation,
})
}
s.connectResponseSent = true
return s.WriteConnectResponseData(nil, metadata)
return s.WriteConnectResponseData(nil, metadata...)
}
// httpResponseAdapter translates responses written by the HTTP Proxy into ones that can be used in QUIC.
type httpResponseAdapter struct {
*quicpogs.RequestServerStream
headers http.Header
connectResponseSent bool
}
@@ -418,6 +427,14 @@ func (hrw *httpResponseAdapter) WriteRespHeaders(status int, header http.Header)
return hrw.WriteConnectResponseData(nil, metadata...)
}
func (hrw *httpResponseAdapter) Header() http.Header {
return hrw.headers
}
func (hrw *httpResponseAdapter) WriteHeader(status int) {
hrw.WriteRespHeaders(status, hrw.headers)
}
func (hrw *httpResponseAdapter) WriteErrorResponse(err error) {
hrw.WriteConnectResponseData(err, quicpogs.Metadata{Key: "HttpStatus", Val: strconv.Itoa(http.StatusBadGateway)})
}
@@ -431,6 +448,7 @@ func buildHTTPRequest(
ctx context.Context,
connectRequest *quicpogs.ConnectRequest,
body io.ReadCloser,
connIndex uint8,
log *zerolog.Logger,
) (*tracing.TracedHTTPRequest, error) {
metadata := connectRequest.MetadataMap()
@@ -474,7 +492,7 @@ func buildHTTPRequest(
stripWebsocketUpgradeHeader(req)
// Check for tracing on request
tracedReq := tracing.NewTracedHTTPRequest(req, log)
tracedReq := tracing.NewTracedHTTPRequest(req, connIndex, log)
return tracedReq, err
}
@@ -555,13 +573,17 @@ func (rp *muxerWrapper) Close() error {
return nil
}
func createUDPConnForConnIndex(connIndex uint8, logger *zerolog.Logger) (*net.UDPConn, error) {
func createUDPConnForConnIndex(connIndex uint8, localIP net.IP, logger *zerolog.Logger) (*net.UDPConn, error) {
portMapMutex.Lock()
defer portMapMutex.Unlock()
if localIP == nil {
localIP = net.IPv4zero
}
// if port was not set yet, it will be zero, so bind will randomly allocate one.
if port, ok := portForConnIndex[connIndex]; ok {
udpConn, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.IPv4zero, Port: port})
udpConn, err := net.ListenUDP("udp", &net.UDPAddr{IP: localIP, Port: port})
// if there wasn't an error, or if port was 0 (independently of error or not, just return)
if err == nil {
return udpConn, nil
@@ -571,7 +593,7 @@ func createUDPConnForConnIndex(connIndex uint8, logger *zerolog.Logger) (*net.UD
}
// if we reached here, then there was an error or port as not been allocated it.
udpConn, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.IPv4zero, Port: 0})
udpConn, err := net.ListenUDP("udp", &net.UDPAddr{IP: localIP, Port: 0})
if err == nil {
udpAddr, ok := (udpConn.LocalAddr()).(*net.UDPAddr)
if !ok {
+6 -5
View File
@@ -485,7 +485,7 @@ func TestBuildHTTPRequest(t *testing.T) {
for _, test := range tests {
test := test // capture range variable
t.Run(test.name, func(t *testing.T) {
req, err := buildHTTPRequest(context.Background(), test.connectRequest, test.body, &log)
req, err := buildHTTPRequest(context.Background(), test.connectRequest, test.body, 0, &log)
assert.NoError(t, err)
test.req = test.req.WithContext(req.Context())
assert.Equal(t, test.req, req.Request)
@@ -572,7 +572,7 @@ func TestNopCloserReadWriterCloseAfterEOF(t *testing.T) {
func TestCreateUDPConnReuseSourcePort(t *testing.T) {
logger := zerolog.Nop()
conn, err := createUDPConnForConnIndex(0, &logger)
conn, err := createUDPConnForConnIndex(0, nil, &logger)
require.NoError(t, err)
getPortFunc := func(conn *net.UDPConn) int {
@@ -586,17 +586,17 @@ func TestCreateUDPConnReuseSourcePort(t *testing.T) {
conn.Close()
// should get the same port as before.
conn, err = createUDPConnForConnIndex(0, &logger)
conn, err = createUDPConnForConnIndex(0, nil, &logger)
require.NoError(t, err)
require.Equal(t, initialPort, getPortFunc(conn))
// new index, should get a different port
conn1, err := createUDPConnForConnIndex(1, &logger)
conn1, err := createUDPConnForConnIndex(1, nil, &logger)
require.NoError(t, err)
require.NotEqual(t, initialPort, getPortFunc(conn1))
// not closing the conn and trying to obtain a new conn for same index should give a different random port
conn, err = createUDPConnForConnIndex(0, &logger)
conn, err = createUDPConnForConnIndex(0, nil, &logger)
require.NoError(t, err)
require.NotEqual(t, initialPort, getPortFunc(conn))
}
@@ -716,6 +716,7 @@ func testQUICConnection(udpListenerAddr net.Addr, t *testing.T, index uint8) *QU
qc, err := NewQUICConnection(
testQUICConfig,
udpListenerAddr,
nil,
index,
tlsClientConfig,
&mockOrchestrator{originProxy: &mockOriginProxyWithRequest{}},
-176
View File
@@ -2,7 +2,6 @@ package connection
import (
"context"
"fmt"
"io"
"net"
"time"
@@ -152,178 +151,3 @@ const (
unregister rpcName = "unregister"
authenticate rpcName = " authenticate"
)
func (h *h2muxConnection) registerTunnel(ctx context.Context, credentialSetter CredentialManager, classicTunnel *ClassicTunnelProperties, registrationOptions *tunnelpogs.RegistrationOptions) error {
h.observer.sendRegisteringEvent(registrationOptions.ConnectionID)
stream, err := h.newRPCStream(ctx, register)
if err != nil {
return err
}
rpcClient := NewTunnelServerClient(ctx, stream, h.observer.log)
defer rpcClient.Close()
_ = h.logServerInfo(ctx, rpcClient)
registration := rpcClient.client.RegisterTunnel(
ctx,
classicTunnel.OriginCert,
classicTunnel.Hostname,
registrationOptions,
)
if registrationErr := registration.DeserializeError(); registrationErr != nil {
// RegisterTunnel RPC failure
return h.processRegisterTunnelError(registrationErr, register)
}
credentialSetter.SetEventDigest(h.connIndex, registration.EventDigest)
return h.processRegistrationSuccess(registration, register, credentialSetter, classicTunnel)
}
type CredentialManager interface {
ReconnectToken() ([]byte, error)
EventDigest(connID uint8) ([]byte, error)
SetEventDigest(connID uint8, digest []byte)
ConnDigest(connID uint8) ([]byte, error)
SetConnDigest(connID uint8, digest []byte)
}
func (h *h2muxConnection) processRegistrationSuccess(
registration *tunnelpogs.TunnelRegistration,
name rpcName,
credentialManager CredentialManager, classicTunnel *ClassicTunnelProperties,
) error {
for _, logLine := range registration.LogLines {
h.observer.log.Info().Msg(logLine)
}
if registration.TunnelID != "" {
h.observer.metrics.tunnelsHA.AddTunnelID(h.connIndex, registration.TunnelID)
h.observer.log.Info().Msgf("Each HA connection's tunnel IDs: %v", h.observer.metrics.tunnelsHA.String())
}
credentialManager.SetConnDigest(h.connIndex, registration.ConnDigest)
h.observer.metrics.userHostnamesCounts.WithLabelValues(registration.Url).Inc()
h.observer.log.Info().Msgf("Route propagating, it may take up to 1 minute for your new route to become functional")
h.observer.metrics.regSuccess.WithLabelValues(string(name)).Inc()
return nil
}
func (h *h2muxConnection) processRegisterTunnelError(err tunnelpogs.TunnelRegistrationError, name rpcName) error {
if err.Error() == DuplicateConnectionError {
h.observer.metrics.regFail.WithLabelValues("dup_edge_conn", string(name)).Inc()
return errDuplicationConnection
}
h.observer.metrics.regFail.WithLabelValues("server_error", string(name)).Inc()
return ServerRegisterTunnelError{
Cause: err,
Permanent: err.IsPermanent(),
}
}
func (h *h2muxConnection) reconnectTunnel(ctx context.Context, credentialManager CredentialManager, classicTunnel *ClassicTunnelProperties, registrationOptions *tunnelpogs.RegistrationOptions) error {
token, err := credentialManager.ReconnectToken()
if err != nil {
return err
}
eventDigest, err := credentialManager.EventDigest(h.connIndex)
if err != nil {
return err
}
connDigest, err := credentialManager.ConnDigest(h.connIndex)
if err != nil {
return err
}
h.observer.log.Debug().Msg("initiating RPC stream to reconnect")
stream, err := h.newRPCStream(ctx, register)
if err != nil {
return err
}
rpcClient := NewTunnelServerClient(ctx, stream, h.observer.log)
defer rpcClient.Close()
_ = h.logServerInfo(ctx, rpcClient)
registration := rpcClient.client.ReconnectTunnel(
ctx,
token,
eventDigest,
connDigest,
classicTunnel.Hostname,
registrationOptions,
)
if registrationErr := registration.DeserializeError(); registrationErr != nil {
// ReconnectTunnel RPC failure
return h.processRegisterTunnelError(registrationErr, reconnect)
}
return h.processRegistrationSuccess(registration, reconnect, credentialManager, classicTunnel)
}
func (h *h2muxConnection) logServerInfo(ctx context.Context, rpcClient *tunnelServerClient) error {
// Request server info without blocking tunnel registration; must use capnp library directly.
serverInfoPromise := tunnelrpc.TunnelServer{Client: rpcClient.client.Client}.GetServerInfo(ctx, func(tunnelrpc.TunnelServer_getServerInfo_Params) error {
return nil
})
serverInfoMessage, err := serverInfoPromise.Result().Struct()
if err != nil {
h.observer.log.Err(err).Msg("Failed to retrieve server information")
return err
}
serverInfo, err := tunnelpogs.UnmarshalServerInfo(serverInfoMessage)
if err != nil {
h.observer.log.Err(err).Msg("Failed to retrieve server information")
return err
}
h.observer.logServerInfo(h.connIndex, serverInfo.LocationName, net.IP{}, "Connection established")
return nil
}
func (h *h2muxConnection) registerNamedTunnel(
ctx context.Context,
namedTunnel *NamedTunnelProperties,
connOptions *tunnelpogs.ConnectionOptions,
) error {
stream, err := h.newRPCStream(ctx, register)
if err != nil {
return err
}
rpcClient := h.newRPCClientFunc(ctx, stream, h.observer.log)
defer rpcClient.Close()
registrationDetails, err := rpcClient.RegisterConnection(ctx, namedTunnel, connOptions, h.connIndex, nil, h.observer)
if err != nil {
return err
}
h.observer.logServerInfo(h.connIndex, registrationDetails.Location, nil, fmt.Sprintf("Connection %s registered", registrationDetails.UUID))
h.observer.sendConnectedEvent(h.connIndex, H2mux, registrationDetails.Location)
return nil
}
func (h *h2muxConnection) unregister(isNamedTunnel bool) {
h.observer.sendUnregisteringEvent(h.connIndex)
unregisterCtx, cancel := context.WithTimeout(context.Background(), h.gracePeriod)
defer cancel()
stream, err := h.newRPCStream(unregisterCtx, unregister)
if err != nil {
return
}
defer stream.Close()
if isNamedTunnel {
rpcClient := h.newRPCClientFunc(unregisterCtx, stream, h.observer.log)
defer rpcClient.Close()
rpcClient.GracefulShutdown(unregisterCtx, h.gracePeriod)
} else {
rpcClient := NewTunnelServerClient(unregisterCtx, stream, h.observer.log)
defer rpcClient.Close()
// gracePeriod is encoded in int64 using capnproto
_ = rpcClient.client.UnregisterTunnel(unregisterCtx, h.gracePeriod.Nanoseconds())
}
h.observer.log.Info().Uint8(LogFieldConnIndex, h.connIndex).Msg("Unregistered tunnel connection")
}
+13
View File
@@ -41,6 +41,19 @@ const (
IPv6Only ConfigIPVersion = 6
)
func (c ConfigIPVersion) String() string {
switch c {
case Auto:
return "auto"
case IPv4Only:
return "4"
case IPv6Only:
return "6"
default:
return ""
}
}
// IPVersion is the IP version of an EdgeAddr
type EdgeIPVersion int8
+4
View File
@@ -15,12 +15,16 @@ func DialEdge(
timeout time.Duration,
tlsConfig *tls.Config,
edgeTCPAddr *net.TCPAddr,
localIP net.IP,
) (net.Conn, error) {
// Inherit from parent context so we can cancel (Ctrl-C) while dialing
dialCtx, dialCancel := context.WithTimeout(ctx, timeout)
defer dialCancel()
dialer := net.Dialer{}
if localIP != nil {
dialer.LocalAddr = &net.TCPAddr{IP: localIP, Port: 0}
}
edgeConn, err := dialer.DialContext(dialCtx, "tcp", edgeTCPAddr.String())
if err != nil {
return nil, newDialError(err, "DialContext error")
+2
View File
@@ -15,6 +15,8 @@ var (
errNoProtocolRecord = fmt.Errorf("No TXT record found for %s to determine connection protocol", protocolRecord)
)
type PercentageFetcher func() (ProtocolPercents, error)
// ProtocolPercent represents a single Protocol Percentage combination.
type ProtocolPercent struct {
Protocol string `json:"protocol"`
+5 -3
View File
@@ -114,9 +114,11 @@ replace gopkg.in/yaml.v3 => gopkg.in/yaml.v3 v3.0.1
// Post-quantum tunnel RTG-1339
replace (
// branch go1.18
// Branches go1.18 go1.19 go1.20 on github.com/cloudflare/qtls-pq
github.com/marten-seemann/qtls-go1-18 => github.com/cloudflare/qtls-pq v0.0.0-20230103171413-e7a2fb559a0e
// branch go1.19
github.com/marten-seemann/qtls-go1-19 => github.com/cloudflare/qtls-pq v0.0.0-20230103171656-05e84f90909e
github.com/marten-seemann/qtls-go1-20 => github.com/cloudflare/qtls-pq v0.0.0-20230215110727-8b4e1699c2a8
github.com/quic-go/qtls-go1-18 => github.com/cloudflare/qtls-pq v0.0.0-20230103171413-e7a2fb559a0e
github.com/quic-go/qtls-go1-19 => github.com/cloudflare/qtls-pq v0.0.0-20230103171656-05e84f90909e
github.com/quic-go/qtls-go1-20 => github.com/cloudflare/qtls-pq v0.0.0-20230215110727-8b4e1699c2a8
)
+1 -1
View File
@@ -113,7 +113,7 @@ func (rc *RemoteConfig) UnmarshalJSON(b []byte) error {
return nil
}
func originRequestFromSingeRule(c *cli.Context) OriginRequestConfig {
func originRequestFromSingleRule(c *cli.Context) OriginRequestConfig {
var connectTimeout = defaultHTTPConnectTimeout
var tlsTimeout = defaultTLSTimeout
var tcpKeepAlive = defaultTCPKeepAlive
+1 -1
View File
@@ -411,7 +411,7 @@ func TestDefaultConfigFromCLI(t *testing.T) {
KeepAliveTimeout: defaultKeepAliveTimeout,
ProxyAddress: defaultProxyAddress,
}
actual := originRequestFromSingeRule(c)
actual := originRequestFromSingleRule(c)
require.Equal(t, expected, actual)
}
+60 -16
View File
@@ -20,6 +20,7 @@ import (
var (
ErrNoIngressRules = errors.New("The config file doesn't contain any ingress rules")
ErrNoIngressRulesCLI = errors.New("No ingress rules were defined in provided config (if any) nor from the cli, cloudflared will return 503 for all incoming HTTP requests")
errLastRuleNotCatchAll = errors.New("The last ingress rule must match all URLs (i.e. it should not have a hostname or path filter)")
errBadWildcard = errors.New("Hostname patterns can have at most one wildcard character (\"*\") and it can only be used for subdomains, e.g. \"*.example.com\"")
errHostnameContainsPort = errors.New("Hostname cannot contain a port")
@@ -70,16 +71,52 @@ type Ingress struct {
Defaults OriginRequestConfig `json:"originRequest"`
}
// NewSingleOrigin constructs an Ingress set with only one rule, constructed from
// CLI parameters for quick tunnels like --url or --no-chunked-encoding.
func NewSingleOrigin(c *cli.Context, allowURLFromArgs bool) (Ingress, error) {
// ParseIngress parses ingress rules, but does not send HTTP requests to the origins.
func ParseIngress(conf *config.Configuration) (Ingress, error) {
if len(conf.Ingress) == 0 {
return Ingress{}, ErrNoIngressRules
}
return validateIngress(conf.Ingress, originRequestFromConfig(conf.OriginRequest))
}
// ParseIngressFromConfigAndCLI will parse the configuration rules from config files for ingress
// rules and then attempt to parse CLI for ingress rules.
// Will always return at least one valid ingress rule. If none are provided by the user, the default
// will be to return 502 status code for all incoming requests.
func ParseIngressFromConfigAndCLI(conf *config.Configuration, c *cli.Context, log *zerolog.Logger) (Ingress, error) {
// Attempt to parse ingress rules from configuration
ingressRules, err := ParseIngress(conf)
if err == nil && !ingressRules.IsEmpty() {
return ingressRules, nil
}
if err != ErrNoIngressRules {
return Ingress{}, err
}
// Attempt to parse ingress rules from CLI:
// --url or --unix-socket flag for a tunnel HTTP ingress
// --hello-world for a basic HTTP ingress self-served
// --bastion for ssh bastion service
ingressRules, err = parseCLIIngress(c, false)
if errors.Is(err, ErrNoIngressRulesCLI) {
log.Warn().Msgf(ErrNoIngressRulesCLI.Error())
return newDefaultOrigin(c, log), nil
}
if err != nil {
return Ingress{}, err
}
return ingressRules, nil
}
// parseCLIIngress constructs an Ingress set with only one rule constructed from
// CLI parameters: --url, --hello-world, --bastion, or --unix-socket
func parseCLIIngress(c *cli.Context, allowURLFromArgs bool) (Ingress, error) {
service, err := parseSingleOriginService(c, allowURLFromArgs)
if err != nil {
return Ingress{}, err
}
// Construct an Ingress with the single rule.
defaults := originRequestFromSingeRule(c)
defaults := originRequestFromSingleRule(c)
ing := Ingress{
Rules: []Rule{
{
@@ -92,6 +129,22 @@ func NewSingleOrigin(c *cli.Context, allowURLFromArgs bool) (Ingress, error) {
return ing, err
}
// newDefaultOrigin always returns a 503 response code to help indicate that there are no ingress
// rules setup, but the tunnel is reachable.
func newDefaultOrigin(c *cli.Context, log *zerolog.Logger) Ingress {
noRulesService := newDefaultStatusCode(log)
defaults := originRequestFromSingleRule(c)
ingress := Ingress{
Rules: []Rule{
{
Service: &noRulesService,
},
},
Defaults: defaults,
}
return ingress
}
// WarpRoutingService starts a tcp stream between the origin and requests from
// warp clients.
type WarpRoutingService struct {
@@ -112,7 +165,7 @@ func NewWarpRoutingService(config WarpRoutingConfig) *WarpRoutingService {
// Get a single origin service from the CLI/config.
func parseSingleOriginService(c *cli.Context, allowURLFromArgs bool) (OriginService, error) {
if c.IsSet("hello-world") {
if c.IsSet(HelloWorldFlag) {
return new(helloWorld), nil
}
if c.IsSet(config.BastionFlag) {
@@ -137,8 +190,7 @@ func parseSingleOriginService(c *cli.Context, allowURLFromArgs bool) (OriginServ
}
return &unixSocketPath{path: path, scheme: "http"}, nil
}
u, err := url.Parse("http://localhost:8080")
return &httpService{url: u}, err
return nil, ErrNoIngressRulesCLI
}
// IsEmpty checks if there are any ingress rules.
@@ -206,7 +258,7 @@ func validateIngress(ingress []config.UnvalidatedIngressRule, defaults OriginReq
}
srv := newStatusCode(statusCode)
service = &srv
} else if r.Service == HelloWorldService || r.Service == "hello-world" || r.Service == "helloworld" {
} else if r.Service == HelloWorldFlag || r.Service == HelloWorldService {
service = new(helloWorld)
} else if r.Service == ServiceSocksProxy {
rules := make([]ipaccess.Rule, len(r.OriginRequest.IPRules))
@@ -335,14 +387,6 @@ func (e errRuleShouldNotBeCatchAll) Error() string {
"will never be triggered.", e.index+1, e.hostname)
}
// ParseIngress parses ingress rules, but does not send HTTP requests to the origins.
func ParseIngress(conf *config.Configuration) (Ingress, error) {
if len(conf.Ingress) == 0 {
return Ingress{}, ErrNoIngressRules
}
return validateIngress(conf.Ingress, originRequestFromConfig(conf.OriginRequest))
}
func isHTTPService(url *url.URL) bool {
return url.Scheme == "http" || url.Scheme == "https" || url.Scheme == "ws" || url.Scheme == "wss"
}
+115 -2
View File
@@ -43,7 +43,7 @@ ingress:
require.Equal(t, "https", s.scheme)
}
func Test_parseIngress(t *testing.T) {
func TestParseIngress(t *testing.T) {
localhost8000 := MustParseURL(t, "https://localhost:8000")
localhost8001 := MustParseURL(t, "https://localhost:8001")
fourOhFour := newStatusCode(404)
@@ -517,7 +517,7 @@ func TestSingleOriginSetsConfig(t *testing.T) {
allowURLFromArgs := false
require.NoError(t, err)
ingress, err := NewSingleOrigin(cliCtx, allowURLFromArgs)
ingress, err := parseCLIIngress(cliCtx, allowURLFromArgs)
require.NoError(t, err)
assert.Equal(t, config.CustomDuration{Duration: time.Second}, ingress.Rules[0].Config.ConnectTimeout)
@@ -537,6 +537,119 @@ func TestSingleOriginSetsConfig(t *testing.T) {
assert.Equal(t, socksProxy, ingress.Rules[0].Config.ProxyType)
}
func TestSingleOriginServices(t *testing.T) {
host := "://localhost:8080"
httpURL := urlMustParse("http" + host)
tcpURL := urlMustParse("tcp" + host)
unix := "unix://service"
newCli := func(params ...string) *cli.Context {
flagSet := flag.NewFlagSet(t.Name(), flag.PanicOnError)
flagSet.Bool("hello-world", false, "")
flagSet.Bool("bastion", false, "")
flagSet.String("url", "", "")
flagSet.String("unix-socket", "", "")
cliCtx := cli.NewContext(cli.NewApp(), flagSet, nil)
for i := 0; i < len(params); i += 2 {
cliCtx.Set(params[i], params[i+1])
}
return cliCtx
}
tests := []struct {
name string
cli *cli.Context
expectedService OriginService
err error
}{
{
name: "Valid hello-world",
cli: newCli("hello-world", "true"),
expectedService: &helloWorld{},
},
{
name: "Valid bastion",
cli: newCli("bastion", "true"),
expectedService: newBastionService(),
},
{
name: "Valid http url",
cli: newCli("url", httpURL.String()),
expectedService: &httpService{url: httpURL},
},
{
name: "Valid tcp url",
cli: newCli("url", tcpURL.String()),
expectedService: newTCPOverWSService(tcpURL),
},
{
name: "Valid unix-socket",
cli: newCli("unix-socket", unix),
expectedService: &unixSocketPath{path: unix, scheme: "http"},
},
{
name: "No origins defined",
cli: newCli(),
err: ErrNoIngressRulesCLI,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
ingress, err := parseCLIIngress(test.cli, false)
require.Equal(t, err, test.err)
if test.err != nil {
return
}
require.Equal(t, 1, len(ingress.Rules))
rule := ingress.Rules[0]
require.Equal(t, test.expectedService, rule.Service)
})
}
}
func urlMustParse(s string) *url.URL {
u, err := url.Parse(s)
if err != nil {
panic(err)
}
return u
}
func TestSingleOriginServices_URL(t *testing.T) {
host := "://localhost:8080"
newCli := func(param string, value string) *cli.Context {
flagSet := flag.NewFlagSet(t.Name(), flag.PanicOnError)
flagSet.String("url", "", "")
cliCtx := cli.NewContext(cli.NewApp(), flagSet, nil)
cliCtx.Set(param, value)
return cliCtx
}
httpTests := []string{"http", "https"}
for _, test := range httpTests {
t.Run(test, func(t *testing.T) {
url := urlMustParse(test + host)
ingress, err := parseCLIIngress(newCli("url", url.String()), false)
require.NoError(t, err)
require.Equal(t, 1, len(ingress.Rules))
rule := ingress.Rules[0]
require.Equal(t, &httpService{url: url}, rule.Service)
})
}
tcpTests := []string{"ssh", "rdp", "smb", "tcp"}
for _, test := range tcpTests {
t.Run(test, func(t *testing.T) {
url := urlMustParse(test + host)
ingress, err := parseCLIIngress(newCli("url", url.String()), false)
require.NoError(t, err)
require.Equal(t, 1, len(ingress.Rules))
rule := ingress.Rules[0]
require.Equal(t, newTCPOverWSService(url), rule.Service)
})
}
}
func TestFindMatchingRule(t *testing.T) {
ingress := Ingress{
Rules: []Rule{
+3
View File
@@ -44,6 +44,9 @@ func (o *httpService) RoundTrip(req *http.Request) (*http.Response, error) {
}
func (o *statusCode) RoundTrip(_ *http.Request) (*http.Response, error) {
if o.defaultResp {
o.log.Warn().Msgf(ErrNoIngressRulesCLI.Error())
}
resp := &http.Response{
StatusCode: o.code,
Status: fmt.Sprintf("%d %s", o.code, http.StatusText(o.code)),
+10
View File
@@ -23,6 +23,7 @@ import (
const (
HelloWorldService = "hello_world"
HelloWorldFlag = "hello-world"
HttpStatusService = "http_status"
)
@@ -246,12 +247,21 @@ func (o helloWorld) MarshalJSON() ([]byte, error) {
// Typical use-case is "user wants the catch-all rule to just respond 404".
type statusCode struct {
code int
// Set only when the user has not defined any ingress rules
defaultResp bool
log *zerolog.Logger
}
func newStatusCode(status int) statusCode {
return statusCode{code: status}
}
// default status code (503) that is returned for requests to cloudflared that don't have any ingress rules setup
func newDefaultStatusCode(log *zerolog.Logger) statusCode {
return statusCode{code: 503, defaultResp: true, log: log}
}
func (o *statusCode) String() string {
return fmt.Sprintf("http_status:%d", o.code)
}
+2 -2
View File
@@ -358,7 +358,7 @@ func proxyHTTP(originProxy connection.OriginProxy, hostname string) (*http.Respo
return nil, err
}
err = originProxy.ProxyHTTP(respWriter, tracing.NewTracedHTTPRequest(req, &log), false)
err = originProxy.ProxyHTTP(respWriter, tracing.NewTracedHTTPRequest(req, 0, &log), false)
if err != nil {
return nil, err
}
@@ -608,7 +608,7 @@ func TestPersistentConnection(t *testing.T) {
respWriter, err := connection.NewHTTP2RespWriter(req, wsRespReadWriter, connection.TypeWebsocket, &log)
require.NoError(t, err)
err = originProxy.ProxyHTTP(respWriter, tracing.NewTracedHTTPRequest(req, &log), true)
err = originProxy.ProxyHTTP(respWriter, tracing.NewTracedHTTPRequest(req, 0, &log), true)
require.NoError(t, err)
}()
+19 -15
View File
@@ -28,6 +28,7 @@ const (
LogFieldRule = "ingressRule"
LogFieldOriginService = "originService"
LogFieldFlowID = "flowID"
LogFieldConnIndex = "connIndex"
trailerHeaderName = "Trailer"
)
@@ -94,9 +95,10 @@ func (p *Proxy) ProxyHTTP(
trace.WithAttributes(attribute.String("req-host", req.Host)))
rule, ruleNum := p.ingressRules.FindMatchingRule(req.Host, req.URL.Path)
logFields := logFields{
cfRay: cfRay,
lbProbe: lbProbe,
rule: ruleNum,
cfRay: cfRay,
lbProbe: lbProbe,
rule: ruleNum,
connIndex: tr.ConnIndex,
}
p.logRequest(req, logFields)
ruleSpan.SetAttributes(attribute.Int("rule-num", ruleNum))
@@ -163,14 +165,14 @@ func (p *Proxy) ProxyTCP(
tracedCtx := tracing.NewTracedContext(serveCtx, req.CfTraceID, p.log)
p.log.Debug().Str(LogFieldFlowID, req.FlowID).Msg("tcp proxy stream started")
p.log.Debug().Str(LogFieldFlowID, req.FlowID).Uint8(LogFieldConnIndex, req.ConnIndex).Msg("tcp proxy stream started")
if err := p.proxyStream(tracedCtx, rwa, req.Dest, p.warpRouting.Proxy); err != nil {
p.logRequestError(err, req.CFRay, req.FlowID, "", ingress.ServiceWarpRouting)
return err
}
p.log.Debug().Str(LogFieldFlowID, req.FlowID).Msg("tcp proxy stream finished successfully")
p.log.Debug().Str(LogFieldFlowID, req.FlowID).Uint8(LogFieldConnIndex, req.ConnIndex).Msg("tcp proxy stream finished successfully")
return nil
}
@@ -320,10 +322,11 @@ func (p *Proxy) appendTagHeaders(r *http.Request) {
}
type logFields struct {
cfRay string
lbProbe bool
rule interface{}
flowID string
cfRay string
lbProbe bool
rule interface{}
flowID string
connIndex uint8
}
func copyTrailers(w connection.ResponseWriter, response *http.Response) {
@@ -348,6 +351,7 @@ func (p *Proxy) logRequest(r *http.Request, fields logFields) {
Str("host", r.Host).
Str("path", r.URL.Path).
Interface("rule", fields.rule).
Uint8(LogFieldConnIndex, fields.connIndex).
Msg("Inbound request")
if contentLen := r.ContentLength; contentLen == -1 {
@@ -360,18 +364,18 @@ func (p *Proxy) logRequest(r *http.Request, fields logFields) {
func (p *Proxy) logOriginResponse(resp *http.Response, fields logFields) {
responseByCode.WithLabelValues(strconv.Itoa(resp.StatusCode)).Inc()
if fields.cfRay != "" {
p.log.Debug().Msgf("CF-RAY: %s Status: %s served by ingress %d", fields.cfRay, resp.Status, fields.rule)
p.log.Debug().Uint8(LogFieldConnIndex, fields.connIndex).Msgf("CF-RAY: %s Status: %s served by ingress %d", fields.cfRay, resp.Status, fields.rule)
} else if fields.lbProbe {
p.log.Debug().Msgf("Response to Load Balancer health check %s", resp.Status)
p.log.Debug().Uint8(LogFieldConnIndex, fields.connIndex).Msgf("Response to Load Balancer health check %s", resp.Status)
} else {
p.log.Debug().Msgf("Status: %s served by ingress %v", resp.Status, fields.rule)
p.log.Debug().Uint8(LogFieldConnIndex, fields.connIndex).Msgf("Status: %s served by ingress %v", resp.Status, fields.rule)
}
p.log.Debug().Msgf("CF-RAY: %s Response Headers %+v", fields.cfRay, resp.Header)
p.log.Debug().Uint8(LogFieldConnIndex, fields.connIndex).Msgf("CF-RAY: %s Response Headers %+v", fields.cfRay, resp.Header)
if contentLen := resp.ContentLength; contentLen == -1 {
p.log.Debug().Msgf("CF-RAY: %s Response content length unknown", fields.cfRay)
p.log.Debug().Uint8(LogFieldConnIndex, fields.connIndex).Msgf("CF-RAY: %s Response content length unknown", fields.cfRay)
} else {
p.log.Debug().Msgf("CF-RAY: %s Response content length %d", fields.cfRay, contentLen)
p.log.Debug().Uint8(LogFieldConnIndex, fields.connIndex).Msgf("CF-RAY: %s Response content length %d", fields.cfRay, contentLen)
}
}
+20 -19
View File
@@ -23,7 +23,6 @@ import (
"golang.org/x/sync/errgroup"
"github.com/cloudflare/cloudflared/cfio"
"github.com/cloudflare/cloudflared/config"
"github.com/cloudflare/cloudflared/connection"
"github.com/cloudflare/cloudflared/hello"
@@ -34,7 +33,7 @@ import (
)
var (
testTags = []tunnelpogs.Tag{tunnelpogs.Tag{Name: "Name", Value: "value"}}
testTags = []tunnelpogs.Tag{{Name: "Name", Value: "value"}}
noWarpRouting = ingress.WarpRoutingConfig{}
testWarpRouting = ingress.WarpRoutingConfig{
Enabled: true,
@@ -150,8 +149,7 @@ func TestProxySingleOrigin(t *testing.T) {
err := cliCtx.Set("hello-world", "true")
require.NoError(t, err)
allowURLFromArgs := false
ingressRule, err := ingress.NewSingleOrigin(cliCtx, allowURLFromArgs)
ingressRule, err := ingress.ParseIngressFromConfigAndCLI(&config.Configuration{}, cliCtx, &log)
require.NoError(t, err)
require.NoError(t, ingressRule.StartOrigins(&log, ctx.Done()))
@@ -170,7 +168,7 @@ func testProxyHTTP(proxy connection.OriginProxy) func(t *testing.T) {
require.NoError(t, err)
log := zerolog.Nop()
err = proxy.ProxyHTTP(responseWriter, tracing.NewTracedHTTPRequest(req, &log), false)
err = proxy.ProxyHTTP(responseWriter, tracing.NewTracedHTTPRequest(req, 0, &log), false)
require.NoError(t, err)
for _, tag := range testTags {
assert.Equal(t, tag.Value, req.Header.Get(TagHeaderNamePrefix+tag.Name))
@@ -198,7 +196,7 @@ func testProxyWebsocket(proxy connection.OriginProxy) func(t *testing.T) {
errGroup, ctx := errgroup.WithContext(ctx)
errGroup.Go(func() error {
log := zerolog.Nop()
err = proxy.ProxyHTTP(responseWriter, tracing.NewTracedHTTPRequest(req, &log), true)
err = proxy.ProxyHTTP(responseWriter, tracing.NewTracedHTTPRequest(req, 0, &log), true)
require.NoError(t, err)
require.Equal(t, http.StatusSwitchingProtocols, responseWriter.Code)
@@ -260,7 +258,7 @@ func testProxySSE(proxy connection.OriginProxy) func(t *testing.T) {
go func() {
defer wg.Done()
log := zerolog.Nop()
err = proxy.ProxyHTTP(responseWriter, tracing.NewTracedHTTPRequest(req, &log), false)
err = proxy.ProxyHTTP(responseWriter, tracing.NewTracedHTTPRequest(req, 0, &log), false)
require.Equal(t, err.Error(), "context canceled")
require.Equal(t, http.StatusOK, responseWriter.Code)
@@ -367,7 +365,7 @@ func runIngressTestScenarios(t *testing.T, unvalidatedIngress []config.Unvalidat
req, err := http.NewRequest(http.MethodGet, test.url, nil)
require.NoError(t, err)
err = proxy.ProxyHTTP(responseWriter, tracing.NewTracedHTTPRequest(req, &log), false)
err = proxy.ProxyHTTP(responseWriter, tracing.NewTracedHTTPRequest(req, 0, &log), false)
require.NoError(t, err)
assert.Equal(t, test.expectedStatus, responseWriter.Code)
@@ -414,7 +412,7 @@ func TestProxyError(t *testing.T) {
req, err := http.NewRequest(http.MethodGet, "http://127.0.0.1", nil)
assert.NoError(t, err)
assert.Error(t, proxy.ProxyHTTP(responseWriter, tracing.NewTracedHTTPRequest(req, &log), false))
assert.Error(t, proxy.ProxyHTTP(responseWriter, tracing.NewTracedHTTPRequest(req, 0, &log), false))
}
type replayer struct {
@@ -695,14 +693,13 @@ func TestConnections(t *testing.T) {
err = proxy.ProxyTCP(ctx, rwa, &connection.TCPRequest{Dest: dest})
} else {
log := zerolog.Nop()
err = proxy.ProxyHTTP(respWriter, tracing.NewTracedHTTPRequest(req, &log), test.args.connectionType == connection.TypeWebsocket)
err = proxy.ProxyHTTP(respWriter, tracing.NewTracedHTTPRequest(req, 0, &log), test.args.connectionType == connection.TypeWebsocket)
}
cancel()
assert.Equal(t, test.want.err, err != nil)
assert.Equal(t, test.want.message, replayer.Bytes())
respPrinter := respWriter.(responsePrinter)
assert.Equal(t, test.want.headers, respPrinter.headers())
assert.Equal(t, test.want.headers, respWriter.Header())
replayer.rw.Reset()
})
}
@@ -795,10 +792,6 @@ func (p *pipedRequestBody) Close() error {
return nil
}
type responsePrinter interface {
headers() http.Header
}
type wsRespWriter struct {
w io.Writer
responseHeaders http.Header
@@ -837,12 +830,16 @@ func (w *wsRespWriter) AddTrailer(trailerName, trailerValue string) {
}
// respHeaders is a test function to read respHeaders
func (w *wsRespWriter) headers() http.Header {
func (w *wsRespWriter) Header() http.Header {
// Removing indeterminstic header because it cannot be asserted.
w.responseHeaders.Del("Date")
return w.responseHeaders
}
func (w *wsRespWriter) WriteHeader(status int) {
// unused
}
type mockTCPRespWriter struct {
w io.Writer
responseHeaders http.Header
@@ -863,7 +860,7 @@ func (m *mockTCPRespWriter) Write(p []byte) (n int, err error) {
return m.w.Write(p)
}
func (w *mockTCPRespWriter) AddTrailer(trailerName, trailerValue string) {
func (m *mockTCPRespWriter) AddTrailer(trailerName, trailerValue string) {
// do nothing
}
@@ -874,10 +871,14 @@ func (m *mockTCPRespWriter) WriteRespHeaders(status int, header http.Header) err
}
// respHeaders is a test function to read respHeaders
func (m *mockTCPRespWriter) headers() http.Header {
func (m *mockTCPRespWriter) Header() http.Header {
return m.responseHeaders
}
func (m *mockTCPRespWriter) WriteHeader(status int) {
// do nothing
}
func createSingleIngressConfig(t *testing.T, service string) ingress.Ingress {
ingressConfig := &config.Configuration{
Ingress: []config.UnvalidatedIngressRule{
+3 -12
View File
@@ -3,12 +3,10 @@ package supervisor
import (
"context"
"errors"
"fmt"
"net"
"strings"
"time"
"github.com/google/uuid"
"github.com/lucas-clemente/quic-go"
"github.com/rs/zerolog"
@@ -21,8 +19,6 @@ import (
)
const (
// SRV and TXT record resolution TTL
ResolveTTL = time.Hour
// Waiting time before retrying a failed tunnel connection
tunnelRetryDuration = time.Second * 10
// Interval between registering new tunnels
@@ -38,7 +34,6 @@ const (
// Supervisor manages non-declarative tunnels. Establishes TCP connections with the edge, and
// reconnects them if they disconnect.
type Supervisor struct {
cloudflaredUUID uuid.UUID
config *TunnelConfig
orchestrator *orchestration.Orchestrator
edgeIPs *edgediscovery.Edge
@@ -68,13 +63,9 @@ type tunnelError struct {
}
func NewSupervisor(config *TunnelConfig, orchestrator *orchestration.Orchestrator, reconnectCh chan ReconnectSignal, gracefulShutdownC <-chan struct{}) (*Supervisor, error) {
cloudflaredUUID, err := uuid.NewRandom()
if err != nil {
return nil, fmt.Errorf("failed to generate cloudflared instance ID: %w", err)
}
isStaticEdge := len(config.EdgeAddrs) > 0
var err error
var edgeIPs *edgediscovery.Edge
if isStaticEdge { // static edge addresses
edgeIPs, err = edgediscovery.StaticEdge(config.Log, config.EdgeAddrs)
@@ -91,14 +82,15 @@ func NewSupervisor(config *TunnelConfig, orchestrator *orchestration.Orchestrato
log := NewConnAwareLogger(config.Log, tracker, config.Observer)
edgeAddrHandler := NewIPAddrFallback(config.MaxEdgeAddrRetries)
edgeBindAddr := config.EdgeBindAddr
edgeTunnelServer := EdgeTunnelServer{
config: config,
cloudflaredUUID: cloudflaredUUID,
orchestrator: orchestrator,
credentialManager: reconnectCredentialManager,
edgeAddrs: edgeIPs,
edgeAddrHandler: edgeAddrHandler,
edgeBindAddr: edgeBindAddr,
tracker: tracker,
reconnectCh: reconnectCh,
gracefulShutdownC: gracefulShutdownC,
@@ -106,7 +98,6 @@ func NewSupervisor(config *TunnelConfig, orchestrator *orchestration.Orchestrato
}
return &Supervisor{
cloudflaredUUID: cloudflaredUUID,
config: config,
orchestrator: orchestrator,
edgeIPs: edgeIPs,
-74
View File
@@ -1,74 +0,0 @@
package supervisor
import (
"context"
"testing"
"github.com/rs/zerolog"
"github.com/stretchr/testify/assert"
"github.com/cloudflare/cloudflared/connection"
"github.com/cloudflare/cloudflared/edgediscovery"
"github.com/cloudflare/cloudflared/edgediscovery/allregions"
"github.com/cloudflare/cloudflared/signal"
)
type mockProtocolSelector struct {
protocols []connection.Protocol
index int
}
func (m *mockProtocolSelector) Current() connection.Protocol {
return m.protocols[m.index]
}
func (m *mockProtocolSelector) Fallback() (connection.Protocol, bool) {
m.index++
if m.index == len(m.protocols) {
return m.protocols[len(m.protocols)-1], false
}
return m.protocols[m.index], true
}
type mockEdgeTunnelServer struct {
config *TunnelConfig
}
func (m *mockEdgeTunnelServer) Serve(ctx context.Context, connIndex uint8, protocolFallback *protocolFallback, connectedSignal *signal.Signal) error {
// This is to mock the first connection falling back because of connectivity issues.
protocolFallback.protocol, _ = m.config.ProtocolSelector.Fallback()
connectedSignal.Notify()
return nil
}
// Test to check if initialize sets all the different connections to the same protocol should the first
// tunnel fall back.
func Test_Initialize_Same_Protocol(t *testing.T) {
edgeIPs, err := edgediscovery.ResolveEdge(&zerolog.Logger{}, "us", allregions.Auto)
assert.Nil(t, err)
s := Supervisor{
edgeIPs: edgeIPs,
config: &TunnelConfig{
ProtocolSelector: &mockProtocolSelector{protocols: []connection.Protocol{connection.QUIC, connection.HTTP2, connection.H2mux}},
},
tunnelsProtocolFallback: make(map[int]*protocolFallback),
edgeTunnelServer: &mockEdgeTunnelServer{
config: &TunnelConfig{
ProtocolSelector: &mockProtocolSelector{protocols: []connection.Protocol{connection.QUIC, connection.HTTP2, connection.H2mux}},
},
},
}
ctx := context.Background()
connectedSignal := signal.New(make(chan struct{}))
s.initialize(ctx, connectedSignal)
// Make sure we fell back to http2 as the mock Serve is wont to do.
assert.Equal(t, s.tunnelsProtocolFallback[0].protocol, connection.HTTP2)
// Ensure all the protocols we set to try are the same as what the first tunnel has fallen back to.
for _, protocolFallback := range s.tunnelsProtocolFallback {
assert.Equal(t, protocolFallback.protocol, s.tunnelsProtocolFallback[0].protocol)
}
}
+9 -71
View File
@@ -49,6 +49,7 @@ type TunnelConfig struct {
EdgeAddrs []string
Region string
EdgeIPVersion allregions.ConfigIPVersion
EdgeBindAddr net.IP
HAConnections int
IncidentLookup IncidentLookup
IsAutoupdated bool
@@ -68,7 +69,6 @@ type TunnelConfig struct {
PQKexIdx int
NamedTunnel *connection.NamedTunnelProperties
MuxerConfig *connection.MuxerConfig
ProtocolSelector connection.ProtocolSelector
EdgeTLSConfigs map[connection.Protocol]*tls.Config
PacketConfig *ingress.GlobalRouterConfig
@@ -90,7 +90,7 @@ func (c *TunnelConfig) registrationOptions(connectionID uint8, OriginLocalIP str
OriginLocalIP: OriginLocalIP,
IsAutoupdated: c.IsAutoupdated,
RunFromTerminal: c.RunFromTerminal,
CompressionQuality: uint64(c.MuxerConfig.CompressionSetting),
CompressionQuality: 0,
UUID: uuid.String(),
Features: c.SupportedFeatures(),
}
@@ -105,7 +105,7 @@ func (c *TunnelConfig) connectionOptions(originLocalAddr string, numPreviousAtte
Client: c.NamedTunnel.Client,
OriginLocalIP: originIP,
ReplaceExisting: c.ReplaceExisting,
CompressionQuality: uint8(c.MuxerConfig.CompressionSetting),
CompressionQuality: 0,
NumPreviousAttempts: numPreviousAttempts,
}
}
@@ -203,11 +203,11 @@ func (f *ipAddrFallback) ShouldGetNewAddress(connIndex uint8, err error) (needsN
type EdgeTunnelServer struct {
config *TunnelConfig
cloudflaredUUID uuid.UUID
orchestrator *orchestration.Orchestrator
credentialManager *reconnectCredentialManager
edgeAddrHandler EdgeAddrHandler
edgeAddrs *edgediscovery.Edge
edgeBindAddr net.IP
reconnectCh chan ReconnectSignal
gracefulShutdownC <-chan struct{}
tracker *tunnelstate.ConnTracker
@@ -488,7 +488,7 @@ func (e *EdgeTunnelServer) serveConnection(
)
switch protocol {
case connection.QUIC, connection.QUICWarp:
case connection.QUIC:
connOptions := e.config.connectionOptions(addr.UDP.String(), uint8(backoff.Retries()))
return e.serveQUIC(ctx,
addr.UDP,
@@ -497,8 +497,8 @@ func (e *EdgeTunnelServer) serveConnection(
controlStream,
connIndex)
case connection.HTTP2, connection.HTTP2Warp:
edgeConn, err := edgediscovery.DialEdge(ctx, dialTimeout, e.config.EdgeTLSConfigs[protocol], addr.TCP)
case connection.HTTP2:
edgeConn, err := edgediscovery.DialEdge(ctx, dialTimeout, e.config.EdgeTLSConfigs[protocol], addr.TCP, e.edgeBindAddr)
if err != nil {
connLog.ConnAwareLogger().Err(err).Msg("Unable to establish connection with Cloudflare edge")
return err, true
@@ -517,21 +517,7 @@ func (e *EdgeTunnelServer) serveConnection(
}
default:
edgeConn, err := edgediscovery.DialEdge(ctx, dialTimeout, e.config.EdgeTLSConfigs[protocol], addr.TCP)
if err != nil {
connLog.ConnAwareLogger().Err(err).Msg("Unable to establish connection with Cloudflare edge")
return err, true
}
if err := e.serveH2mux(
ctx,
connLog,
edgeConn,
connIndex,
connectedFuse,
); err != nil {
return err, false
}
return fmt.Errorf("invalid protocol selected: %s", protocol), false
}
return
}
@@ -544,55 +530,6 @@ func (r unrecoverableError) Error() string {
return r.err.Error()
}
func (e *EdgeTunnelServer) serveH2mux(
ctx context.Context,
connLog *ConnAwareLogger,
edgeConn net.Conn,
connIndex uint8,
connectedFuse *connectedFuse,
) error {
if e.config.NeedPQ {
return unrecoverableError{errors.New("H2Mux transport does not support post-quantum")}
}
connLog.Logger().Debug().Msgf("Connecting via h2mux")
// Returns error from parsing the origin URL or handshake errors
handler, err, recoverable := connection.NewH2muxConnection(
e.orchestrator,
e.config.GracePeriod,
e.config.MuxerConfig,
edgeConn,
connIndex,
e.config.Observer,
e.gracefulShutdownC,
e.config.Log,
)
if err != nil {
if !recoverable {
return unrecoverableError{err}
}
return err
}
errGroup, serveCtx := errgroup.WithContext(ctx)
errGroup.Go(func() error {
connOptions := e.config.connectionOptions(edgeConn.LocalAddr().String(), uint8(connectedFuse.backoff.Retries()))
return handler.ServeNamedTunnel(serveCtx, e.config.NamedTunnel, connOptions, connectedFuse)
})
errGroup.Go(func() error {
err := listenReconnect(serveCtx, e.reconnectCh, e.gracefulShutdownC)
if err != nil {
// forcefully break the connection (this is only used for testing)
// errgroup will return context canceled for the handler.ServeClassicTunnel
connLog.Logger().Debug().Msg("Forcefully breaking h2mux connection")
}
return err
})
return errGroup.Wait()
}
func (e *EdgeTunnelServer) serveHTTP2(
ctx context.Context,
connLog *ConnAwareLogger,
@@ -673,6 +610,7 @@ func (e *EdgeTunnelServer) serveQUIC(
quicConn, err := connection.NewQUICConnection(
quicConfig,
edgeAddr,
e.edgeBindAddr,
connIndex,
tlsConfig,
e.orchestrator,
+9 -11
View File
@@ -18,7 +18,7 @@ type dynamicMockFetcher struct {
err error
}
func (dmf *dynamicMockFetcher) fetch() connection.PercentageFetcher {
func (dmf *dynamicMockFetcher) fetch() edgediscovery.PercentageFetcher {
return func() (edgediscovery.ProtocolPercents, error) {
return dmf.protocolPercents, dmf.err
}
@@ -32,24 +32,22 @@ func TestWaitForBackoffFallback(t *testing.T) {
}
log := zerolog.Nop()
resolveTTL := time.Duration(0)
namedTunnel := &connection.NamedTunnelProperties{}
mockFetcher := dynamicMockFetcher{
protocolPercents: edgediscovery.ProtocolPercents{edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 100}},
protocolPercents: edgediscovery.ProtocolPercents{edgediscovery.ProtocolPercent{Protocol: "quic", Percentage: 100}},
}
warpRoutingEnabled := false
protocolSelector, err := connection.NewProtocolSelector(
"auto",
warpRoutingEnabled,
namedTunnel,
"",
false,
false,
mockFetcher.fetch(),
resolveTTL,
&log,
false,
)
assert.NoError(t, err)
initProtocol := protocolSelector.Current()
assert.Equal(t, connection.HTTP2, initProtocol)
assert.Equal(t, connection.QUIC, initProtocol)
protoFallback := &protocolFallback{
backoff,
@@ -100,12 +98,12 @@ func TestWaitForBackoffFallback(t *testing.T) {
// The reason why there's no fallback available is because we pick a specific protocol instead of letting it be auto.
protocolSelector, err = connection.NewProtocolSelector(
"quic",
warpRoutingEnabled,
namedTunnel,
"",
false,
false,
mockFetcher.fetch(),
resolveTTL,
&log,
false,
)
assert.NoError(t, err)
protoFallback = &protocolFallback{backoff, protocolSelector.Current(), false}
+4 -3
View File
@@ -73,15 +73,16 @@ func Init(version string) {
type TracedHTTPRequest struct {
*http.Request
*cfdTracer
ConnIndex uint8 // The connection index used to proxy the request
}
// NewTracedHTTPRequest creates a new tracer for the current HTTP request context.
func NewTracedHTTPRequest(req *http.Request, log *zerolog.Logger) *TracedHTTPRequest {
func NewTracedHTTPRequest(req *http.Request, connIndex uint8, log *zerolog.Logger) *TracedHTTPRequest {
ctx, exists := extractTrace(req)
if !exists {
return &TracedHTTPRequest{req, &cfdTracer{trace.NewNoopTracerProvider(), &NoopOtlpClient{}, log}}
return &TracedHTTPRequest{req, &cfdTracer{trace.NewNoopTracerProvider(), &NoopOtlpClient{}, log}, connIndex}
}
return &TracedHTTPRequest{req.WithContext(ctx), newCfdTracer(ctx, log)}
return &TracedHTTPRequest{req.WithContext(ctx), newCfdTracer(ctx, log), connIndex}
}
func (tr *TracedHTTPRequest) ToTracedContext() *TracedContext {
+5 -5
View File
@@ -17,7 +17,7 @@ func TestNewCfTracer(t *testing.T) {
log := zerolog.Nop()
req := httptest.NewRequest("GET", "http://localhost", nil)
req.Header.Add(TracerContextName, "14cb070dde8e51fc5ae8514e69ba42ca:b38f1bf5eae406f3:0:1")
tr := NewTracedHTTPRequest(req, &log)
tr := NewTracedHTTPRequest(req, 0, &log)
assert.NotNil(t, tr)
assert.IsType(t, tracesdk.NewTracerProvider(), tr.TracerProvider)
assert.IsType(t, &InMemoryOtlpClient{}, tr.exporter)
@@ -28,7 +28,7 @@ func TestNewCfTracerMultiple(t *testing.T) {
req := httptest.NewRequest("GET", "http://localhost", nil)
req.Header.Add(TracerContextName, "1241ce3ecdefc68854e8514e69ba42ca:b38f1bf5eae406f3:0:1")
req.Header.Add(TracerContextName, "14cb070dde8e51fc5ae8514e69ba42ca:b38f1bf5eae406f3:0:1")
tr := NewTracedHTTPRequest(req, &log)
tr := NewTracedHTTPRequest(req, 0, &log)
assert.NotNil(t, tr)
assert.IsType(t, tracesdk.NewTracerProvider(), tr.TracerProvider)
assert.IsType(t, &InMemoryOtlpClient{}, tr.exporter)
@@ -38,7 +38,7 @@ func TestNewCfTracerNilHeader(t *testing.T) {
log := zerolog.Nop()
req := httptest.NewRequest("GET", "http://localhost", nil)
req.Header[http.CanonicalHeaderKey(TracerContextName)] = nil
tr := NewTracedHTTPRequest(req, &log)
tr := NewTracedHTTPRequest(req, 0, &log)
assert.NotNil(t, tr)
assert.IsType(t, trace.NewNoopTracerProvider(), tr.TracerProvider)
assert.IsType(t, &NoopOtlpClient{}, tr.exporter)
@@ -49,7 +49,7 @@ func TestNewCfTracerInvalidHeaders(t *testing.T) {
req := httptest.NewRequest("GET", "http://localhost", nil)
for _, test := range [][]string{nil, {""}} {
req.Header[http.CanonicalHeaderKey(TracerContextName)] = test
tr := NewTracedHTTPRequest(req, &log)
tr := NewTracedHTTPRequest(req, 0, &log)
assert.NotNil(t, tr)
assert.IsType(t, trace.NewNoopTracerProvider(), tr.TracerProvider)
assert.IsType(t, &NoopOtlpClient{}, tr.exporter)
@@ -60,7 +60,7 @@ func TestAddingSpansWithNilMap(t *testing.T) {
log := zerolog.Nop()
req := httptest.NewRequest("GET", "http://localhost", nil)
req.Header.Add(TracerContextName, "14cb070dde8e51fc5ae8514e69ba42ca:b38f1bf5eae406f3:0:1")
tr := NewTracedHTTPRequest(req, &log)
tr := NewTracedHTTPRequest(req, 0, &log)
exporter := tr.exporter.(*InMemoryOtlpClient)
+4
View File
@@ -601,3 +601,7 @@ zombiezen.com/go/capnproto2/std/capnp/rpc
# gopkg.in/yaml.v3 => gopkg.in/yaml.v3 v3.0.1
# github.com/marten-seemann/qtls-go1-18 => github.com/cloudflare/qtls-pq v0.0.0-20230103171413-e7a2fb559a0e
# github.com/marten-seemann/qtls-go1-19 => github.com/cloudflare/qtls-pq v0.0.0-20230103171656-05e84f90909e
# github.com/marten-seemann/qtls-go1-20 => github.com/cloudflare/qtls-pq v0.0.0-20230215110727-8b4e1699c2a8
# github.com/quic-go/qtls-go1-18 => github.com/cloudflare/qtls-pq v0.0.0-20230103171413-e7a2fb559a0e
# github.com/quic-go/qtls-go1-19 => github.com/cloudflare/qtls-pq v0.0.0-20230103171656-05e84f90909e
# github.com/quic-go/qtls-go1-20 => github.com/cloudflare/qtls-pq v0.0.0-20230215110727-8b4e1699c2a8