Release 2021.6.0

CHANGES.md

@@ -1,5 +1,15 @@
 **Experimental**: This is a new format for release notes. The format and availability is subject to change.
 
+## 2021.6.0
+### Bug Fixes
+- Fixes a http2 transport (the new default for Named Tunnels) to work with unix socket origins.
+
+
+## 2021.5.10
+### Bug Fixes
+- Fixes a memory leak in h2mux transport that connects cloudflared to Cloudflare edge.
+
+
 ## 2021.5.9
 ### New Features
 - Uses new Worker based login helper service to facilitate token exchange in cloudflared flows.
@@ -10,9 +20,10 @@
 ## 2021.5.8
 ### New Features
 - When creating a DNS record to point a hostname at a tunnel, you can now use --overwrite-dns to overwrite any existing
-  DNS records with that hostname. This works when using the CLI to provision DNS, and when starting an adhoc tunnel, e.g.
-  - `cloudflared route --overwrite-dns dns foo foo.example.com`
-  - `cloudflared tunnel --overwrite-dns --name foo --hostname foo.example.com`
+  DNS records with that hostname. This works both when using the CLI to provision DNS, as well as when starting an adhoc
+  named tunnel, e.g.:
+  - `cloudflared tunnel route dns --overwrite-dns foo-tunnel foo.example.com`
+  - `cloudflared tunnel --overwrite-dns --name foo-tunnel --hostname foo.example.com`
 
 ## 2021.5.7
 ### New Features

RELEASE_NOTES

@@ -1,3 +1,10 @@
+2021.6.0
+- 2021-06-21 TUN-4571: Changelog for 2021.6.0
+- 2021-06-18 TUN-4571: Fix proxying to unix sockets when using HTTP2 transport to Cloudflare Edge
+- 2021-06-07 TUN-4502: Make `cloudflared tunnel route` subcommands described consistently
+- 2021-06-08 TUN-4504: Fix component tests in windows
+- 2021-05-27 TUN-4461: Log resulting DNS hostname if one is received from Cloudflare API
+
 2021.5.10
 - 2021-05-25 TUN-4456: Replaced instances of Tick() with Ticker() in h2mux paths
 

cmd/cloudflared/tunnel/subcommands.go

@@ -655,7 +655,6 @@ func cleanupCommand(c *cli.Context) error {
 func buildRouteCommand() *cli.Command {
 	return &cli.Command{
 		Name:      "route",
-		Action:    cliutil.ConfiguredAction(routeCommand),
 		Usage:     "Define which traffic routed from Cloudflare edge to this tunnel: requests to a DNS hostname, to a Cloudflare Load Balancer, or traffic originating from Cloudflare WARP clients",
 		UsageText: "cloudflared tunnel [tunnel command options] route [subcommand options] [dns TUNNEL HOSTNAME]|[lb TUNNEL HOSTNAME LB-POOL]|[ip NETWORK TUNNEL]",
 		Description: `The route command defines how Cloudflare will proxy requests to this tunnel.
@@ -675,16 +674,30 @@ Further information about managing Cloudflare WARP traffic to your tunnel is ava
 `,
 		CustomHelpTemplate: commandHelpTemplate(),
 		Subcommands: []*cli.Command{
+			{
+				Name:        "dns",
+				Action:      cliutil.ConfiguredAction(routeDnsCommand),
+				Usage:       "Route a hostname by creating a DNS CNAME record to a tunnel",
+				UsageText:   "cloudflared tunnel route dns [TUNNEL] [HOSTNAME]",
+				Description: `Creates a DNS CNAME record hostname that points to the tunnel.`,
+				Flags:       []cli.Flag{overwriteDNSFlag},
+			},
+			{
+				Name:        "lb",
+				Action:      cliutil.ConfiguredAction(routeLbCommand),
+				Usage:       "Use this tunnel as a load balancer origin, creating pool and load balancer if necessary",
+				UsageText:   "cloudflared tunnel route dns [TUNNEL] [HOSTNAME] [LB-POOL]",
+				Description: `Creates Load Balancer with an origin pool that points to the tunnel.`,
+			},
 			buildRouteIPSubcommand(),
 		},
-		Flags: []cli.Flag{overwriteDNSFlag},
 	}
 }
 
 func dnsRouteFromArg(c *cli.Context, overwriteExisting bool) (tunnelstore.Route, error) {
 	const (
-		userHostnameIndex = 2
-		expectedNArgs     = 3
+		userHostnameIndex = 1
+		expectedNArgs     = 2
 	)
 	if c.NArg() != expectedNArgs {
 		return nil, cliutil.UsageError("Expected %d arguments, got %d", expectedNArgs, c.NArg())
@@ -700,9 +713,9 @@ func dnsRouteFromArg(c *cli.Context, overwriteExisting bool) (tunnelstore.Route,
 
 func lbRouteFromArg(c *cli.Context) (tunnelstore.Route, error) {
 	const (
-		lbNameIndex   = 2
-		lbPoolIndex   = 3
-		expectedNArgs = 4
+		lbNameIndex   = 1
+		lbPoolIndex   = 2
+		expectedNArgs = 3
 	)
 	if c.NArg() != expectedNArgs {
 		return nil, cliutil.UsageError("Expected %d arguments, got %d", expectedNArgs, c.NArg())
@@ -744,41 +757,39 @@ func validateHostname(s string, allowWildcardSubdomain bool) bool {
 	return err == nil && validateName(puny, allowWildcardSubdomain)
 }
 
-func routeCommand(c *cli.Context) error {
-	if c.NArg() < 2 {
-		return cliutil.UsageError(`"cloudflared tunnel route" requires the first argument to be the route type(dns or lb), followed by the ID or name of the tunnel`)
+func routeDnsCommand(c *cli.Context) error {
+	if c.NArg() != 2 {
+		return cliutil.UsageError(`This command expects the format "cloudflared tunnel route dns <tunnel name/id> <hostname>"`)
 	}
+	return routeCommand(c, "dns")
+}
+
+func routeLbCommand(c *cli.Context) error {
+	if c.NArg() != 3 {
+		return cliutil.UsageError(`This command expects the format "cloudflared tunnel route lb <tunnel name/id> <hostname> <load balancer pool>"`)
+	}
+	return routeCommand(c, "lb")
+}
+
+func routeCommand(c *cli.Context, routeType string) error {
 	sc, err := newSubcommandContext(c)
 	if err != nil {
 		return err
 	}
 
-	const tunnelIDIndex = 1
-
-	routeType := c.Args().First()
+	tunnelID, err := sc.findID(c.Args().Get(0))
+	if err != nil {
+		return err
+	}
 	var route tunnelstore.Route
-	var tunnelID uuid.UUID
 	switch routeType {
 	case "dns":
-		tunnelID, err = sc.findID(c.Args().Get(tunnelIDIndex))
-		if err != nil {
-			return err
-		}
 		route, err = dnsRouteFromArg(c, c.Bool(overwriteDNSFlagName))
-		if err != nil {
-			return err
-		}
 	case "lb":
-		tunnelID, err = sc.findID(c.Args().Get(tunnelIDIndex))
-		if err != nil {
-			return err
-		}
 		route, err = lbRouteFromArg(c)
-		if err != nil {
-			return err
-		}
-	default:
-		return cliutil.UsageError("%s is not a recognized route type. Supported route types are dns and lb", routeType)
+	}
+	if err != nil {
+		return err
 	}
 
 	res, err := sc.route(tunnelID, route)

component-tests/test_logging.py

@@ -7,7 +7,7 @@ from util import start_cloudflared, wait_tunnel_ready, send_requests
 # Rolling logger rotate log files after 1 MB
 rotate_after_size = 1000 * 1000
 default_log_file = "cloudflared.log"
-expect_message = "Starting Hello World server"
+expect_message = "Starting Hello"
 
 
 def assert_log_to_terminal(cloudflared):

component-tests/test_reconnect.py

@@ -1,20 +1,23 @@
 #!/usr/bin/env python
 import copy
+import platform
 from time import sleep
 
+import pytest
 from flaky import flaky
 
 from util import start_cloudflared, wait_tunnel_ready, check_tunnel_not_connected
 
 
 @flaky(max_runs=3, min_passes=1)
-class TestReconnect():
+class TestReconnect:
     default_ha_conns = 4
-    default_reconnect_secs = 5
+    default_reconnect_secs = 15
     extra_config = {
         "stdin-control": True,
     }
 
+    @pytest.mark.skipif(platform.system() == "Windows", reason=f"Currently buggy on Windows TUN-4584")
     def test_named_reconnect(self, tmp_path, component_tests_config):
         config = component_tests_config(self.extra_config)
         with start_cloudflared(tmp_path, config, new_process=True, allow_input=True, capture_output=False) as cloudflared:
@@ -36,7 +39,7 @@ class TestReconnect():
         cloudflared.stdin.flush()
 
     def assert_reconnect(self, config, cloudflared, repeat):
-        wait_tunnel_ready(tunnel_url=config.get_url())
+        wait_tunnel_ready(tunnel_url=config.get_url(), require_min_connections=self.default_ha_conns)
         for _ in range(repeat):
             for i in range(self.default_ha_conns):
                 self.send_reconnect(cloudflared, self.default_reconnect_secs)
@@ -44,11 +47,9 @@ class TestReconnect():
                 if expect_connections > 0:
                     # Don't check if tunnel returns 200 here because there is a race condition between wait_tunnel_ready
                     # retrying to get 200 response and reconnecting
-                    wait_tunnel_ready(
-                        require_min_connections=expect_connections)
+                    wait_tunnel_ready(require_min_connections=expect_connections)
                 else:
                     check_tunnel_not_connected()
 
-            sleep(self.default_reconnect_secs + 10)
-            wait_tunnel_ready(tunnel_url=config.get_url(),
-                              require_min_connections=self.default_ha_conns)
+            sleep(self.default_reconnect_secs * 2)
+            wait_tunnel_ready(tunnel_url=config.get_url(), require_min_connections=self.default_ha_conns)

component-tests/util.py

@@ -34,7 +34,7 @@ def cloudflared_cmd(config, config_path, cfd_args, cfd_pre_args, root):
         cmd += ["sudo"]
     cmd += [config.cloudflared_binary]
     cmd += cfd_pre_args
-    cmd += ["--config", config_path]
+    cmd += ["--config", str(config_path)]
     cmd += cfd_args
     LOGGER.info(f"Run cmd {cmd} with config {config}")
     return cmd
@@ -50,8 +50,7 @@ def run_cloudflared_background(cmd, allow_input, capture_output):
     finally:
         cfd.terminate()
         if capture_output:
-            LOGGER.info(
-                f"cloudflared log: {cfd.stderr.read()}")
+            LOGGER.info(f"cloudflared log: {cfd.stderr.read()}")
 
 
 @retry(stop_max_attempt_number=MAX_RETRIES, wait_fixed=BACKOFF_SECS * 1000)

connection/http2.go

@@ -98,6 +98,8 @@ func (c *http2Connection) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	defer c.activeRequestsWG.Done()
 
 	connType := determineHTTP2Type(r)
+	handleMissingRequestParts(connType, r)
+
 	respWriter, err := newHTTP2RespWriter(r, w, connType)
 	if err != nil {
 		c.observer.log.Error().Msg(err.Error())
@@ -255,6 +257,20 @@ func determineHTTP2Type(r *http.Request) Type {
 	}
 }
 
+func handleMissingRequestParts(connType Type, r *http.Request) {
+	if connType == TypeHTTP {
+		// http library has no guarantees that we receive a filled URL. If not, then we fill it, as we reuse the request
+		// for proxying. We use the same values as we used to in h2mux. For proxying they should not matter since we
+		// control the dialer on every egress proxied.
+		if len(r.URL.Scheme) == 0 {
+			r.URL.Scheme = "http"
+		}
+		if len(r.URL.Host) == 0 {
+			r.URL.Host = "localhost:8080"
+		}
+	}
+}
+
 func isControlStreamUpgrade(r *http.Request) bool {
 	return r.Header.Get(InternalUpgradeHeader) == ControlStreamUpgrade
 }

origin/proxy_test.go

@@ -6,9 +6,11 @@ import (
 	"flag"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"net"
 	"net/http"
 	"net/http/httptest"
+	"os"
 	"sync"
 	"testing"
 	"time"
@@ -273,26 +275,7 @@ func TestProxyMultipleOrigins(t *testing.T) {
 		},
 	}
 
-	ingress, err := ingress.ParseIngress(&config.Configuration{
-		TunnelID: t.Name(),
-		Ingress:  unvalidatedIngress,
-	})
-	require.NoError(t, err)
-
-	log := zerolog.Nop()
-
-	ctx, cancel := context.WithCancel(context.Background())
-	errC := make(chan error)
-	var wg sync.WaitGroup
-	require.NoError(t, ingress.StartOrigins(&wg, &log, ctx.Done(), errC))
-
-	proxy := NewOriginProxy(ingress, unusedWarpRoutingService, testTags, &log)
-
-	tests := []struct {
-		url            string
-		expectedStatus int
-		expectedBody   []byte
-	}{
+	tests := []MultipleIngressTest{
 		{
 			url:            "http://api.example.com",
 			expectedStatus: http.StatusCreated,
@@ -317,6 +300,31 @@ func TestProxyMultipleOrigins(t *testing.T) {
 		},
 	}
 
+	runIngressTestScenarios(t, unvalidatedIngress, tests)
+}
+
+type MultipleIngressTest struct {
+	url            string
+	expectedStatus int
+	expectedBody   []byte
+}
+
+func runIngressTestScenarios(t *testing.T, unvalidatedIngress []config.UnvalidatedIngressRule, tests []MultipleIngressTest) {
+	ingress, err := ingress.ParseIngress(&config.Configuration{
+		TunnelID: t.Name(),
+		Ingress:  unvalidatedIngress,
+	})
+	require.NoError(t, err)
+
+	log := zerolog.Nop()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	errC := make(chan error)
+	var wg sync.WaitGroup
+	require.NoError(t, ingress.StartOrigins(&wg, &log, ctx.Done(), errC))
+
+	proxy := NewOriginProxy(ingress, unusedWarpRoutingService, testTags, &log)
+
 	for _, test := range tests {
 		responseWriter := newMockHTTPRespWriter()
 		req, err := http.NewRequest(http.MethodGet, test.url, nil)
@@ -633,6 +641,45 @@ func TestConnections(t *testing.T) {
 	}
 }
 
+func TestUnixSocketOrigin(t *testing.T) {
+	file, err := ioutil.TempFile("", "unix.sock")
+	require.NoError(t, err)
+	os.Remove(file.Name()) // remove the file since binding the socket expects to create it
+
+	l, err := net.Listen("unix", file.Name())
+	require.NoError(t, err)
+	defer l.Close()
+	defer os.Remove(file.Name())
+
+	api := &httptest.Server{
+		Listener: l,
+		Config:   &http.Server{Handler: mockAPI{}},
+	}
+	api.Start()
+	defer api.Close()
+
+	unvalidatedIngress := []config.UnvalidatedIngressRule{
+		{
+			Hostname: "unix.example.com",
+			Service:  "unix:" + file.Name(),
+		},
+		{
+			Hostname: "*",
+			Service:  "http_status:404",
+		},
+	}
+
+	tests := []MultipleIngressTest{
+		{
+			url:            "http://unix.example.com",
+			expectedStatus: http.StatusCreated,
+			expectedBody:   []byte("Created"),
+		},
+	}
+
+	runIngressTestScenarios(t, unvalidatedIngress, tests)
+}
+
 type requestBody struct {
 	pw *io.PipeWriter
 	pr *io.PipeReader

tunnelstore/client.go

@@ -88,6 +88,7 @@ type DNSRoute struct {
 type DNSRouteResult struct {
 	route *DNSRoute
 	CName Change `json:"cname"`
+	Name  string `json:"name"`
 }
 
 func NewDNSRoute(userHostname string, overwriteExisting bool) Route {
@@ -135,7 +136,16 @@ func (res *DNSRouteResult) SuccessSummary() string {
 	case ChangeUnchanged:
 		msgFmt = "%s is already configured to route to your tunnel"
 	}
-	return fmt.Sprintf(msgFmt, res.route.userHostname)
+	return fmt.Sprintf(msgFmt, res.hostname())
+}
+
+// hostname yields the resulting name for the DNS route; if that is not available from Cloudflare API, then the
+// requested name is returned instead (should not be the common path, it is just a fall-back).
+func (res *DNSRouteResult) hostname() string {
+	if res.Name != "" {
+		return res.Name
+	}
+	return res.route.userHostname
 }
 
 type LBRoute struct {