Release 2023.10.0

This changes guarantees that the coommand to report rule matches when
testing local config reports the rule number using the 0-based indexing.
This is to be consistent with the 0-based indexing on the log lines when
proxying requests.

.github/workflows/check.yaml

@@ -4,7 +4,7 @@ jobs:
   check:
     strategy:
       matrix:
-        go-version: [1.19.x]
+        go-version: [1.20.x]
         os: [ubuntu-latest, macos-latest, windows-latest]
     runs-on: ${{ matrix.os }}
     steps:

CHANGES.md

@@ -1,3 +1,6 @@
+## 2023.9.0
+### Notices
+- The `warp-routing` `enabled: boolean` flag is no longer supported in the configuration file. Warp Routing traffic (eg TCP, UDP, ICMP) traffic is proxied to cloudflared if routes to the target tunnel are configured. This change does not affect remotely managed tunnels, but for locally managed tunnels, users that might be relying on this feature flag to block traffic should instead guarantee that tunnel has no Private Routes configured for the tunnel.
 ## 2023.7.0
 ### New Features
 - You can now enable additional diagnostics over the management.argotunnel.com service for your active cloudflared connectors via a new runtime flag `--management-diagnostics` (or env `TUNNEL_MANAGEMENT_DIAGNOSTICS`). This feature is provided as opt-in and requires the flag to enable. Endpoints such as /metrics provides your prometheus metrics endpoint another mechanism to be reached. Additionally /debug/pprof/(goroutine|heap) are also introduced to allow for remotely retrieving active pprof information from a running cloudflared connector.

Dockerfile

@@ -1,7 +1,7 @@
 # use a builder image for building cloudflare
 ARG TARGET_GOOS
 ARG TARGET_GOARCH
-FROM golang:1.19 as builder
+FROM golang:1.20.6 as builder
 ENV GO111MODULE=on \
     CGO_ENABLED=0 \
     TARGET_GOOS=${TARGET_GOOS} \

Dockerfile.amd64

@@ -1,5 +1,5 @@
 # use a builder image for building cloudflare
-FROM golang:1.19 as builder
+FROM golang:1.20.6 as builder
 ENV GO111MODULE=on \
     CGO_ENABLED=0 
     

Dockerfile.arm64

@@ -1,5 +1,5 @@
 # use a builder image for building cloudflare
-FROM golang:1.19 as builder
+FROM golang:1.20.6 as builder
 ENV GO111MODULE=on \
     CGO_ENABLED=0 
     

README.md

@@ -53,9 +53,6 @@ Want to test Cloudflare Tunnel before adding a website to Cloudflare? You can do
 
 ## Deprecated versions
 
-Cloudflare currently supports versions of `cloudflared` 2020.5.1 and later. Breaking changes unrelated to feature availability may be introduced that will impact versions released prior to 2020.5.1. You can read more about upgrading `cloudflared` in our [developer documentation](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/install-and-setup/installation#updating-cloudflared).
+Cloudflare currently supports versions of cloudflared that are **within one year** of the most recent release. Breaking changes unrelated to feature availability may be introduced that will impact versions released more than one year ago. You can read more about upgrading cloudflared in our [developer documentation](https://developers.cloudflare.com/cloudflare-one/connections/connect-networks/downloads/#updating-cloudflared).
 
-| Version(s) | Deprecation status |
-|---|---|
-| 2020.5.1 and later | Supported |
-| Versions prior to 2020.5.1 | No longer supported |
+For example, as of January 2023 Cloudflare will support cloudflared version 2023.1.1 to cloudflared 2022.1.1.

RELEASE_NOTES

@@ -1,3 +1,23 @@
+2023.10.0
+- 2023-10-06 TUN-7864: Document cloudflared versions support
+- 2023-10-03 CUSTESC-33731: Make rule match test report rule in 0-index base
+- 2023-09-22 TUN-7824: Fix usage of systemctl status to detect which services are installed
+- 2023-09-20 TUN-7813: Improve tunnel delete command to use cascade delete
+- 2023-09-20 TUN-7787: cloudflared only list ip routes targeted for cfd_tunnel
+- 2023-09-15 TUN-7787: Refactor cloudflared to use new route endpoints based on route IDs
+- 2023-09-08 TUN-7776: Remove warp-routing flag from cloudflared
+- 2023-09-05 TUN-7756: Clarify that QUIC is mandatory to support ICMP proxying
+
+2023.8.2
+- 2023-08-25 TUN-7700: Implement feature selector to determine if connections will prefer post quantum cryptography
+- 2023-08-22 TUN-7707: Use X25519Kyber768Draft00 curve when post-quantum feature is enabled
+
+2023.8.1
+- 2023-08-23 TUN-7718: Update R2 Token to no longer encode secret
+
+2023.8.0
+- 2023-07-26 TUN-7584: Bump go 1.20.6
+
 2023.7.3
 - 2023-07-25 TUN-7628: Correct Host parsing for Access
 - 2023-07-24 TUN-7624: Fix flaky TestBackoffGracePeriod test in cloudflared

cfapi/client.go

@@ -9,7 +9,7 @@ type TunnelClient interface {
 	GetTunnel(tunnelID uuid.UUID) (*Tunnel, error)
 	GetTunnelToken(tunnelID uuid.UUID) (string, error)
 	GetManagementToken(tunnelID uuid.UUID) (string, error)
-	DeleteTunnel(tunnelID uuid.UUID) error
+	DeleteTunnel(tunnelID uuid.UUID, cascade bool) error
 	ListTunnels(filter *TunnelFilter) ([]*Tunnel, error)
 	ListActiveClients(tunnelID uuid.UUID) ([]*ActiveClient, error)
 	CleanupConnections(tunnelID uuid.UUID, params *CleanupParams) error
@@ -22,7 +22,7 @@ type HostnameClient interface {
 type IPRouteClient interface {
 	ListRoutes(filter *IpRouteFilter) ([]*DetailedRoute, error)
 	AddRoute(newRoute NewRoute) (Route, error)
-	DeleteRoute(params DeleteRouteParams) error
+	DeleteRoute(id uuid.UUID) error
 	GetByIP(params GetRouteByIpParams) (DetailedRoute, error)
 }
 

cfapi/ip_route.go

@@ -75,10 +75,12 @@ type NewRoute struct {
 // MarshalJSON handles fields with non-JSON types (e.g. net.IPNet).
 func (r NewRoute) MarshalJSON() ([]byte, error) {
 	return json.Marshal(&struct {
+		Network  string     `json:"network"`
 		TunnelID uuid.UUID  `json:"tunnel_id"`
 		Comment  string     `json:"comment"`
 		VNetID   *uuid.UUID `json:"virtual_network_id,omitempty"`
 	}{
+		Network:  r.Network.String(),
 		TunnelID: r.TunnelID,
 		Comment:  r.Comment,
 		VNetID:   r.VNetID,
@@ -87,6 +89,7 @@ func (r NewRoute) MarshalJSON() ([]byte, error) {
 
 // DetailedRoute is just a Route with some extra fields, e.g. TunnelName.
 type DetailedRoute struct {
+	ID       uuid.UUID `json:"id"`
 	Network  CIDR      `json:"network"`
 	TunnelID uuid.UUID `json:"tunnel_id"`
 	// Optional field. When unset, it means the DetailedRoute belongs to the default virtual network.
@@ -115,7 +118,8 @@ func (r DetailedRoute) TableString() string {
 	}
 
 	return fmt.Sprintf(
-		"%s\t%s\t%s\t%s\t%s\t%s\t%s\t",
+		"%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t",
+		r.ID,
 		r.Network.String(),
 		vnetColumn,
 		r.Comment,
@@ -126,12 +130,6 @@ func (r DetailedRoute) TableString() string {
 	)
 }
 
-type DeleteRouteParams struct {
-	Network net.IPNet
-	// Optional field. If unset, backend will assume the default vnet for the account.
-	VNetID *uuid.UUID
-}
-
 type GetRouteByIpParams struct {
 	Ip net.IP
 	// Optional field. If unset, backend will assume the default vnet for the account.
@@ -158,7 +156,7 @@ func (r *RESTClient) ListRoutes(filter *IpRouteFilter) ([]*DetailedRoute, error)
 // AddRoute calls the Tunnelstore POST endpoint for a given route.
 func (r *RESTClient) AddRoute(newRoute NewRoute) (Route, error) {
 	endpoint := r.baseEndpoints.accountRoutes
-	endpoint.Path = path.Join(endpoint.Path, "network", url.PathEscape(newRoute.Network.String()))
+	endpoint.Path = path.Join(endpoint.Path)
 	resp, err := r.sendRequest("POST", endpoint, newRoute)
 	if err != nil {
 		return Route{}, errors.Wrap(err, "REST request failed")
@@ -173,10 +171,9 @@ func (r *RESTClient) AddRoute(newRoute NewRoute) (Route, error) {
 }
 
 // DeleteRoute calls the Tunnelstore DELETE endpoint for a given route.
-func (r *RESTClient) DeleteRoute(params DeleteRouteParams) error {
+func (r *RESTClient) DeleteRoute(id uuid.UUID) error {
 	endpoint := r.baseEndpoints.accountRoutes
-	endpoint.Path = path.Join(endpoint.Path, "network", url.PathEscape(params.Network.String()))
-	setVnetParam(&endpoint, params.VNetID)
+	endpoint.Path = path.Join(endpoint.Path, url.PathEscape(id.String()))
 
 	resp, err := r.sendRequest("DELETE", endpoint, nil)
 	if err != nil {

cfapi/ip_route_filter.go

@@ -58,31 +58,29 @@ type IpRouteFilter struct {
 
 // NewIpRouteFilterFromCLI parses CLI flags to discover which filters should get applied.
 func NewIpRouteFilterFromCLI(c *cli.Context) (*IpRouteFilter, error) {
-	f := &IpRouteFilter{
-		queryParams: url.Values{},
-	}
+	f := NewIPRouteFilter()
 
 	// Set deletion filter
 	if flag := filterIpRouteDeleted.Name; c.IsSet(flag) && c.Bool(flag) {
-		f.deleted()
+		f.Deleted()
 	} else {
-		f.notDeleted()
+		f.NotDeleted()
 	}
 
 	if subset, err := cidrFromFlag(c, filterSubsetIpRoute); err != nil {
 		return nil, err
 	} else if subset != nil {
-		f.networkIsSupersetOf(*subset)
+		f.NetworkIsSupersetOf(*subset)
 	}
 
 	if superset, err := cidrFromFlag(c, filterSupersetIpRoute); err != nil {
 		return nil, err
 	} else if superset != nil {
-		f.networkIsSupersetOf(*superset)
+		f.NetworkIsSupersetOf(*superset)
 	}
 
 	if comment := c.String(filterIpRouteComment.Name); comment != "" {
-		f.commentIs(comment)
+		f.CommentIs(comment)
 	}
 
 	if tunnelID := c.String(filterIpRouteTunnelID.Name); tunnelID != "" {
@@ -90,7 +88,7 @@ func NewIpRouteFilterFromCLI(c *cli.Context) (*IpRouteFilter, error) {
 		if err != nil {
 			return nil, errors.Wrapf(err, "Couldn't parse UUID from %s", filterIpRouteTunnelID.Name)
 		}
-		f.tunnelID(u)
+		f.TunnelID(u)
 	}
 
 	if vnetId := c.String(filterIpRouteByVnet.Name); vnetId != "" {
@@ -98,7 +96,7 @@ func NewIpRouteFilterFromCLI(c *cli.Context) (*IpRouteFilter, error) {
 		if err != nil {
 			return nil, errors.Wrapf(err, "Couldn't parse UUID from %s", filterIpRouteByVnet.Name)
 		}
-		f.vnetID(u)
+		f.VNetID(u)
 	}
 
 	if maxFetch := c.Int("max-fetch-size"); maxFetch > 0 {
@@ -124,35 +122,44 @@ func cidrFromFlag(c *cli.Context, flag cli.StringFlag) (*net.IPNet, error) {
 	return subset, nil
 }
 
-func (f *IpRouteFilter) commentIs(comment string) {
+func NewIPRouteFilter() *IpRouteFilter {
+	values := &IpRouteFilter{queryParams: url.Values{}}
+
+	// always list cfd_tunnel routes only
+	values.queryParams.Set("tun_types", "cfd_tunnel")
+
+	return values
+}
+
+func (f *IpRouteFilter) CommentIs(comment string) {
 	f.queryParams.Set("comment", comment)
 }
 
-func (f *IpRouteFilter) notDeleted() {
+func (f *IpRouteFilter) NotDeleted() {
 	f.queryParams.Set("is_deleted", "false")
 }
 
-func (f *IpRouteFilter) deleted() {
+func (f *IpRouteFilter) Deleted() {
 	f.queryParams.Set("is_deleted", "true")
 }
 
-func (f *IpRouteFilter) networkIsSubsetOf(superset net.IPNet) {
+func (f *IpRouteFilter) NetworkIsSubsetOf(superset net.IPNet) {
 	f.queryParams.Set("network_subset", superset.String())
 }
 
-func (f *IpRouteFilter) networkIsSupersetOf(subset net.IPNet) {
+func (f *IpRouteFilter) NetworkIsSupersetOf(subset net.IPNet) {
 	f.queryParams.Set("network_superset", subset.String())
 }
 
-func (f *IpRouteFilter) existedAt(existedAt time.Time) {
+func (f *IpRouteFilter) ExistedAt(existedAt time.Time) {
 	f.queryParams.Set("existed_at", existedAt.Format(time.RFC3339))
 }
 
-func (f *IpRouteFilter) tunnelID(id uuid.UUID) {
+func (f *IpRouteFilter) TunnelID(id uuid.UUID) {
 	f.queryParams.Set("tunnel_id", id.String())
 }
 
-func (f *IpRouteFilter) vnetID(id uuid.UUID) {
+func (f *IpRouteFilter) VNetID(id uuid.UUID) {
 	f.queryParams.Set("virtual_network_id", id.String())
 }
 

cfapi/ip_route_test.go

@@ -69,6 +69,7 @@ func TestDetailedRouteJsonRoundtrip(t *testing.T) {
 	}{
 		{
 			`{
+				"id":"91ebc578-cc99-4641-9937-0fb630505fa0",
 				"network":"10.1.2.40/29",
 				"tunnel_id":"fba6ffea-807f-4e7a-a740-4184ee1b82c8",
 				"comment":"test",
@@ -80,6 +81,7 @@ func TestDetailedRouteJsonRoundtrip(t *testing.T) {
 		},
 		{
 			`{
+				"id":"91ebc578-cc99-4641-9937-0fb630505fa0",
 				"network":"10.1.2.40/29",
 				"tunnel_id":"fba6ffea-807f-4e7a-a740-4184ee1b82c8",
 				"virtual_network_id":"38c95083-8191-4110-8339-3f438d44fdb9",
@@ -167,9 +169,10 @@ func TestRouteTableString(t *testing.T) {
 	require.NoError(t, err)
 	require.NotNil(t, network)
 	r := DetailedRoute{
+		ID:      uuid.Nil,
 		Network: CIDR(*network),
 	}
 	row := r.TableString()
 	fmt.Println(row)
-	require.True(t, strings.HasPrefix(row, "1.2.3.4/32"))
+	require.True(t, strings.HasPrefix(row, "00000000-0000-0000-0000-000000000000\t1.2.3.4/32"))
 }

cfapi/tunnel.go

@@ -159,9 +159,14 @@ func (r *RESTClient) GetManagementToken(tunnelID uuid.UUID) (token string, err e
 	return "", r.statusCodeToError("get tunnel token", resp)
 }
 
-func (r *RESTClient) DeleteTunnel(tunnelID uuid.UUID) error {
+func (r *RESTClient) DeleteTunnel(tunnelID uuid.UUID, cascade bool) error {
 	endpoint := r.baseEndpoints.accountLevel
 	endpoint.Path = path.Join(endpoint.Path, fmt.Sprintf("%v", tunnelID))
+	// Cascade will delete all tunnel dependencies (connections, routes, etc.) that
+	// are linked to the deleted tunnel.
+	if cascade {
+		endpoint.RawQuery = "cascade=true"
+	}
 	resp, err := r.sendRequest("DELETE", endpoint, nil)
 	if err != nil {
 		return errors.Wrap(err, "REST request failed")

cfsetup.yaml

@@ -1,5 +1,5 @@
-pinned_go: &pinned_go go=1.19.6-1
-pinned_go_fips: &pinned_go_fips go-boring=1.19.6-1
+pinned_go: &pinned_go go=1.20.6-1
+pinned_go_fips: &pinned_go_fips go-boring=1.20.6-1
 
 build_dir: &build_dir /cfsetup_build
 default-flavor: bullseye

cmd/cloudflared/generic_service.go

@@ -1,5 +1,4 @@
 //go:build !windows && !darwin && !linux
-// +build !windows,!darwin,!linux
 
 package main
 

cmd/cloudflared/linux_service.go

@@ -1,5 +1,4 @@
 //go:build linux
-// +build linux
 
 package main
 
@@ -370,7 +369,7 @@ func uninstallSystemd(log *zerolog.Logger) error {
 	// Get only the installed services
 	installedServices := make(map[string]ServiceTemplate)
 	for serviceName, serviceTemplate := range systemdAllTemplates {
-		if err := runCommand("systemctl", "status", serviceName); err == nil {
+		if err := runCommand("systemctl", "list-units", "--all", "|", "grep", serviceName); err == nil {
 			installedServices[serviceName] = serviceTemplate
 		} else {
 			log.Info().Msgf("Service '%s' not installed, skipping its uninstall", serviceName)

cmd/cloudflared/macos_service.go

@@ -1,5 +1,4 @@
 //go:build darwin
-// +build darwin
 
 package main
 

cmd/cloudflared/tunnel/cmd.go

@@ -392,7 +392,7 @@ func StartServer(
 		observer.SendURL(quickTunnelURL)
 	}
 
-	tunnelConfig, orchestratorConfig, err := prepareTunnelConfig(c, info, log, logTransport, observer, namedTunnel)
+	tunnelConfig, orchestratorConfig, err := prepareTunnelConfig(ctx, c, info, log, logTransport, observer, namedTunnel)
 	if err != nil {
 		log.Err(err).Msg("Couldn't start tunnel")
 		return err

cmd/cloudflared/tunnel/config_test.go

@@ -4,10 +4,12 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/require"
+
+	"github.com/cloudflare/cloudflared/features"
 )
 
 func TestDedup(t *testing.T) {
 	expected := []string{"a", "b"}
-	actual := dedup([]string{"a", "b", "a"})
+	actual := features.Dedup([]string{"a", "b", "a"})
 	require.ElementsMatch(t, expected, actual)
 }

cmd/cloudflared/tunnel/configuration.go

@@ -1,9 +1,9 @@
 package tunnel
 
 import (
+	"context"
 	"crypto/tls"
 	"fmt"
-	mathRand "math/rand"
 	"net"
 	"net/netip"
 	"os"
@@ -113,6 +113,7 @@ func dnsProxyStandAlone(c *cli.Context, namedTunnel *connection.NamedTunnelPrope
 }
 
 func prepareTunnelConfig(
+	ctx context.Context,
 	c *cli.Context,
 	info *cliutil.BuildInfo,
 	log, logTransport *zerolog.Logger,
@@ -132,22 +133,36 @@ func prepareTunnelConfig(
 	tags = append(tags, tunnelpogs.Tag{Name: "ID", Value: clientID.String()})
 
 	transportProtocol := c.String("protocol")
-	needPQ := c.Bool("post-quantum")
-	if needPQ {
+
+	clientFeatures := features.Dedup(append(c.StringSlice("features"), features.DefaultFeatures...))
+
+	staticFeatures := features.StaticFeatures{}
+	if c.Bool("post-quantum") {
 		if FipsEnabled {
 			return nil, nil, fmt.Errorf("post-quantum not supported in FIPS mode")
 		}
+		pqMode := features.PostQuantumStrict
+		staticFeatures.PostQuantumMode = &pqMode
+	}
+	featureSelector, err := features.NewFeatureSelector(ctx, namedTunnel.Credentials.AccountTag, staticFeatures, log)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "Failed to create feature selector")
+	}
+	pqMode := featureSelector.PostQuantumMode()
+	if pqMode == features.PostQuantumStrict {
 		// Error if the user tries to force a non-quic transport protocol
 		if transportProtocol != connection.AutoSelectFlag && transportProtocol != connection.QUIC.String() {
 			return nil, nil, fmt.Errorf("post-quantum is only supported with the quic transport")
 		}
 		transportProtocol = connection.QUIC.String()
+		clientFeatures = append(clientFeatures, features.FeaturePostQuantum)
+
+		log.Info().Msgf(
+			"Using hybrid post-quantum key agreement %s",
+			supervisor.PQKexName,
+		)
 	}
 
-	clientFeatures := dedup(append(c.StringSlice("features"), features.DefaultFeatures...))
-	if needPQ {
-		clientFeatures = append(clientFeatures, features.FeaturePostQuantum)
-	}
 	namedTunnel.Client = tunnelpogs.ClientInfo{
 		ClientID: clientID[:],
 		Features: clientFeatures,
@@ -203,15 +218,6 @@ func prepareTunnelConfig(
 		log.Warn().Str("edgeIPVersion", edgeIPVersion.String()).Err(err).Msg("Overriding edge-ip-version")
 	}
 
-	var pqKexIdx int
-	if needPQ {
-		pqKexIdx = mathRand.Intn(len(supervisor.PQKexes))
-		log.Info().Msgf(
-			"Using experimental hybrid post-quantum key agreement %s",
-			supervisor.PQKexNames[supervisor.PQKexes[pqKexIdx]],
-		)
-	}
-
 	tunnelConfig := &supervisor.TunnelConfig{
 		GracePeriod:     gracePeriod,
 		ReplaceExisting: c.Bool("force"),
@@ -236,8 +242,7 @@ func prepareTunnelConfig(
 		NamedTunnel:                 namedTunnel,
 		ProtocolSelector:            protocolSelector,
 		EdgeTLSConfigs:              edgeTLSConfigs,
-		NeedPQ:                      needPQ,
-		PQKexIdx:                    pqKexIdx,
+		FeatureSelector:             featureSelector,
 		MaxEdgeAddrRetries:          uint8(c.Int("max-edge-addr-retries")),
 		UDPUnregisterSessionTimeout: c.Duration(udpUnregisterSessionTimeoutFlag),
 		DisableQUICPathMTUDiscovery: c.Bool(quicDisablePathMTUDiscovery),
@@ -280,25 +285,6 @@ func isRunningFromTerminal() bool {
 	return term.IsTerminal(int(os.Stdout.Fd()))
 }
 
-// Remove any duplicates from the slice
-func dedup(slice []string) []string {
-
-	// Convert the slice into a set
-	set := make(map[string]bool, 0)
-	for _, str := range slice {
-		set[str] = true
-	}
-
-	// Convert the set back into a slice
-	keys := make([]string, len(set))
-	i := 0
-	for str := range set {
-		keys[i] = str
-		i++
-	}
-	return keys
-}
-
 // ParseConfigIPVersion returns the IP version from possible expected values from config
 func parseConfigIPVersion(version string) (v allregions.ConfigIPVersion, err error) {
 	switch version {

cmd/cloudflared/tunnel/configuration_test.go

@@ -1,5 +1,4 @@
 //go:build ignore
-// +build ignore
 
 // TODO: Remove the above build tag and include this test when we start compiling with Golang 1.10.0+
 

cmd/cloudflared/tunnel/ingress_subcommands.go

@@ -139,7 +139,7 @@ func testURLCommand(c *cli.Context) error {
 	}
 
 	_, i := ing.FindMatchingRule(requestURL.Hostname(), requestURL.Path)
-	fmt.Printf("Matched rule #%d\n", i+1)
+	fmt.Printf("Matched rule #%d\n", i)
 	fmt.Println(ing.Rules[i].MultiLineString())
 	return nil
 }

cmd/cloudflared/tunnel/signal_test.go

@@ -1,5 +1,4 @@
 //go:build !windows
-// +build !windows
 
 package tunnel
 

cmd/cloudflared/tunnel/subcommand_context.go

@@ -156,7 +156,7 @@ func (sc *subcommandContext) create(name string, credentialsFilePath string, sec
 		var errorLines []string
 		errorLines = append(errorLines, fmt.Sprintf("Your tunnel '%v' was created with ID %v. However, cloudflared couldn't write tunnel credentials to %s.", tunnel.Name, tunnel.ID, credentialsFilePath))
 		errorLines = append(errorLines, fmt.Sprintf("The file-writing error is: %v", writeFileErr))
-		if deleteErr := client.DeleteTunnel(tunnel.ID); deleteErr != nil {
+		if deleteErr := client.DeleteTunnel(tunnel.ID, true); deleteErr != nil {
 			errorLines = append(errorLines, fmt.Sprintf("Cloudflared tried to delete the tunnel for you, but encountered an error. You should use `cloudflared tunnel delete %v` to delete the tunnel yourself, because the tunnel can't be run without the tunnelfile.", tunnel.ID))
 			errorLines = append(errorLines, fmt.Sprintf("The delete tunnel error is: %v", deleteErr))
 		} else {
@@ -206,13 +206,8 @@ func (sc *subcommandContext) delete(tunnelIDs []uuid.UUID) error {
 		if !tunnel.DeletedAt.IsZero() {
 			return fmt.Errorf("Tunnel %s has already been deleted", tunnel.ID)
 		}
-		if forceFlagSet {
-			if err := client.CleanupConnections(tunnel.ID, cfapi.NewCleanupParams()); err != nil {
-				return errors.Wrapf(err, "Error cleaning up connections for tunnel %s", tunnel.ID)
-			}
-		}
 
-		if err := client.DeleteTunnel(tunnel.ID); err != nil {
+		if err := client.DeleteTunnel(tunnel.ID, forceFlagSet); err != nil {
 			return errors.Wrapf(err, "Error deleting tunnel %s", tunnel.ID)
 		}
 

cmd/cloudflared/tunnel/subcommand_context_teamnet.go

@@ -1,6 +1,9 @@
 package tunnel
 
 import (
+	"net"
+
+	"github.com/google/uuid"
 	"github.com/pkg/errors"
 
 	"github.com/cloudflare/cloudflared/cfapi"
@@ -24,12 +27,12 @@ func (sc *subcommandContext) addRoute(newRoute cfapi.NewRoute) (cfapi.Route, err
 	return client.AddRoute(newRoute)
 }
 
-func (sc *subcommandContext) deleteRoute(params cfapi.DeleteRouteParams) error {
+func (sc *subcommandContext) deleteRoute(id uuid.UUID) error {
 	client, err := sc.client()
 	if err != nil {
 		return errors.Wrap(err, noClientMsg)
 	}
-	return client.DeleteRoute(params)
+	return client.DeleteRoute(id)
 }
 
 func (sc *subcommandContext) getRouteByIP(params cfapi.GetRouteByIpParams) (cfapi.DetailedRoute, error) {
@@ -39,3 +42,25 @@ func (sc *subcommandContext) getRouteByIP(params cfapi.GetRouteByIpParams) (cfap
 	}
 	return client.GetByIP(params)
 }
+
+func (sc *subcommandContext) getRouteId(network net.IPNet, vnetId *uuid.UUID) (uuid.UUID, error) {
+	filters := cfapi.NewIPRouteFilter()
+	filters.NotDeleted()
+	filters.NetworkIsSubsetOf(network)
+	filters.NetworkIsSupersetOf(network)
+
+	if vnetId != nil {
+		filters.VNetID(*vnetId)
+	}
+
+	result, err := sc.listRoutes(filters)
+	if err != nil {
+		return uuid.Nil, err
+	}
+
+	if len(result) != 1 {
+		return uuid.Nil, errors.New("unable to find route for provided network and vnet")
+	}
+
+	return result[0].ID, nil
+}

cmd/cloudflared/tunnel/subcommand_context_test.go

@@ -219,7 +219,7 @@ func (d *deleteMockTunnelStore) GetTunnelToken(tunnelID uuid.UUID) (string, erro
 	return "token", nil
 }
 
-func (d *deleteMockTunnelStore) DeleteTunnel(tunnelID uuid.UUID) error {
+func (d *deleteMockTunnelStore) DeleteTunnel(tunnelID uuid.UUID, cascade bool) error {
 	tunnel, ok := d.mockTunnels[tunnelID]
 	if !ok {
 		return fmt.Errorf("Couldn't find tunnel: %v", tunnelID)

cmd/cloudflared/tunnel/subcommands.go

@@ -119,8 +119,8 @@ var (
 	forceDeleteFlag = &cli.BoolFlag{
 		Name:    "force",
 		Aliases: []string{"f"},
-		Usage: "Cleans up any stale connections before the tunnel is deleted. cloudflared will not " +
-			"delete a tunnel with connections without this flag.",
+		Usage: "Deletes a tunnel even if tunnel is connected and it has dependencies associated to it. (eg. IP routes)." +
+			" It is not possible to delete tunnels that have connections or non-deleted dependencies, without this flag.",
 		EnvVars: []string{"TUNNEL_RUN_FORCE_OVERWRITE"},
 	}
 	selectProtocolFlag = altsrc.NewStringFlag(&cli.StringFlag{

cmd/cloudflared/tunnel/teamnet_subcommands.go

@@ -21,6 +21,8 @@ var (
 		Aliases: []string{"vn"},
 		Usage:   "The ID or name of the virtual network to which the route is associated to.",
 	}
+
+	routeAddError = errors.New("You must supply exactly one argument, the ID or CIDR of the route you want to delete")
 )
 
 func buildRouteIPSubcommand() *cli.Command {
@@ -68,11 +70,9 @@ which virtual network's routing table you want to add the route to with:
 				Name:      "delete",
 				Action:    cliutil.ConfiguredAction(deleteRouteCommand),
 				Usage:     "Delete a row from your organization's private routing table",
-				UsageText: "cloudflared tunnel [--config FILEPATH] route ip delete [flags] [CIDR]",
-				Description: `Deletes the row for a given CIDR from your routing table. That portion of your network
-will no longer be reachable by the WARP clients. Note that if you use virtual
-networks, then you have to tell which virtual network whose routing table you
-have a row deleted from.`,
+				UsageText: "cloudflared tunnel [--config FILEPATH] route ip delete [flags] [Route ID or CIDR]",
+				Description: `Deletes the row for the given route ID from your routing table. That portion of your network
+will no longer be reachable.`,
 				Flags: []cli.Flag{vnetFlag},
 			},
 			{
@@ -187,33 +187,36 @@ func deleteRouteCommand(c *cli.Context) error {
 	}
 
 	if c.NArg() != 1 {
-		return errors.New("You must supply exactly one argument, the network whose route you want to delete (in CIDR form e.g. 1.2.3.4/32)")
+		return routeAddError
 	}
 
-	_, network, err := net.ParseCIDR(c.Args().First())
+	var routeId uuid.UUID
+	routeId, err = uuid.Parse(c.Args().First())
 	if err != nil {
-		return errors.Wrap(err, "Invalid network CIDR")
-	}
-	if network == nil {
-		return errors.New("Invalid network CIDR")
-	}
+		_, network, err := net.ParseCIDR(c.Args().First())
+		if err != nil || network == nil {
+			return routeAddError
+		}
 
-	params := cfapi.DeleteRouteParams{
-		Network: *network,
-	}
+		var vnetId *uuid.UUID
+		if c.IsSet(vnetFlag.Name) {
+			id, err := getVnetId(sc, c.String(vnetFlag.Name))
+			if err != nil {
+				return err
+			}
+			vnetId = &id
+		}
 
-	if c.IsSet(vnetFlag.Name) {
-		vnetId, err := getVnetId(sc, c.String(vnetFlag.Name))
+		routeId, err = sc.getRouteId(*network, vnetId)
 		if err != nil {
 			return err
 		}
-		params.VNetID = &vnetId
 	}
 
-	if err := sc.deleteRoute(params); err != nil {
+	if err := sc.deleteRoute(routeId); err != nil {
 		return errors.Wrap(err, "API error")
 	}
-	fmt.Printf("Successfully deleted route for %s\n", network)
+	fmt.Printf("Successfully deleted route with ID %s\n", routeId)
 	return nil
 }
 
@@ -269,7 +272,7 @@ func formatAndPrintRouteList(routes []*cfapi.DetailedRoute) {
 	defer writer.Flush()
 
 	// Print column headers with tabbed columns
-	_, _ = fmt.Fprintln(writer, "NETWORK\tVIRTUAL NET ID\tCOMMENT\tTUNNEL ID\tTUNNEL NAME\tCREATED\tDELETED\t")
+	_, _ = fmt.Fprintln(writer, "ID\tNETWORK\tVIRTUAL NET ID\tCOMMENT\tTUNNEL ID\tTUNNEL NAME\tCREATED\tDELETED\t")
 
 	// Loop through routes, create formatted string for each, and print using tabwriter
 	for _, route := range routes {

cmd/cloudflared/updater/workers_service_test.go

@@ -1,5 +1,4 @@
 //go:build !windows
-// +build !windows
 
 package updater
 

cmd/cloudflared/windows_service.go

@@ -1,5 +1,4 @@
 //go:build windows
-// +build windows
 
 package main
 

component-tests/test_config.py

@@ -36,17 +36,17 @@ class TestConfig:
         _ = start_cloudflared(tmp_path, config, validate_args)
 
         self.match_rule(tmp_path, config,
-                        "http://example.com/index.html", 1)
+                        "http://example.com/index.html", 0)
         self.match_rule(tmp_path, config,
-                        "https://example.com/index.html", 1)
+                        "https://example.com/index.html", 0)
         self.match_rule(tmp_path, config,
-                        "https://api.example.com/login", 2)
+                        "https://api.example.com/login", 1)
         self.match_rule(tmp_path, config,
-                        "https://wss.example.com", 3)
+                        "https://wss.example.com", 2)
         self.match_rule(tmp_path, config,
-                        "https://ssh.example.com", 4)
+                        "https://ssh.example.com", 3)
         self.match_rule(tmp_path, config,
-                        "https://api.example.com", 5)
+                        "https://api.example.com", 4)
 
     # This is used to check that the command tunnel ingress url <url> matches rule number <rule_num>. Note that rule number uses 1-based indexing
 

config/configuration.go

@@ -257,7 +257,6 @@ type Configuration struct {
 }
 
 type WarpRoutingConfig struct {
-	Enabled        bool            `yaml:"enabled" json:"enabled"`
 	ConnectTimeout *CustomDuration `yaml:"connectTimeout" json:"connectTimeout,omitempty"`
 	TCPKeepAlive   *CustomDuration `yaml:"tcpKeepAlive" json:"tcpKeepAlive,omitempty"`
 }

config/configuration_test.go

@@ -23,7 +23,6 @@ func TestConfigFileSettings(t *testing.T) {
 			Service:  "https://localhost:8001",
 		}
 		warpRouting = WarpRoutingConfig{
-			Enabled:        true,
 			ConnectTimeout: &CustomDuration{Duration: 2 * time.Second},
 			TCPKeepAlive:   &CustomDuration{Duration: 10 * time.Second},
 		}

connection/connection.go

@@ -40,7 +40,6 @@ type Orchestrator interface {
 	UpdateConfig(version int32, config []byte) *pogs.UpdateConfigurationResponse
 	GetConfigJSON() ([]byte, error)
 	GetOriginProxy() (OriginProxy, error)
-	WarpRoutingEnabled() (enabled bool)
 }
 
 type NamedTunnelProperties struct {

connection/quic.go

@@ -103,7 +103,7 @@ func NewQUICConnection(
 	sessionDemuxChan := make(chan *packet.Session, demuxChanCapacity)
 	datagramMuxer := quicpogs.NewDatagramMuxerV2(session, logger, sessionDemuxChan)
 	sessionManager := datagramsession.NewManager(logger, datagramMuxer.SendToSession, sessionDemuxChan)
-	packetRouter := ingress.NewPacketRouter(packetRouterConfig, datagramMuxer, logger, orchestrator.WarpRoutingEnabled)
+	packetRouter := ingress.NewPacketRouter(packetRouterConfig, datagramMuxer, logger)
 
 	return &QUICConnection{
 		session:              session,

dev.Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.19 as builder
+FROM golang:1.20.6 as builder
 ENV GO111MODULE=on \
     CGO_ENABLED=0
 WORKDIR /go/src/github.com/cloudflare/cloudflared/

features/features.go

@@ -28,3 +28,22 @@ func Contains(feature string) bool {
 	}
 	return false
 }
+
+// Remove any duplicates from the slice
+func Dedup(slice []string) []string {
+
+	// Convert the slice into a set
+	set := make(map[string]bool, 0)
+	for _, str := range slice {
+		set[str] = true
+	}
+
+	// Convert the set back into a slice
+	keys := make([]string, len(set))
+	i := 0
+	for str := range set {
+		keys[i] = str
+		i++
+	}
+	return keys
+}

features/selector.go

@@ -0,0 +1,164 @@
+package features
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"hash/fnv"
+	"net"
+	"sync"
+	"time"
+
+	"github.com/rs/zerolog"
+)
+
+const (
+	featureSelectorHostname = "cfd-features.argotunnel.com"
+	defaultRefreshFreq      = time.Hour * 6
+	lookupTimeout           = time.Second * 10
+)
+
+type PostQuantumMode uint8
+
+const (
+	PostQuantumDisabled PostQuantumMode = iota
+	// Prefer post quantum, but fallback if connection cannot be established
+	PostQuantumPrefer
+	// If the user passes the --post-quantum flag, we override
+	// CurvePreferences to only support hybrid post-quantum key agreements.
+	PostQuantumStrict
+)
+
+// If the TXT record adds other fields, the umarshal logic will ignore those keys
+// If the TXT record is missing a key, the field will unmarshal to the default Go value
+type featuresRecord struct {
+	PostQuantumPercentage int32 `json:"pq"`
+}
+
+func NewFeatureSelector(ctx context.Context, accountTag string, staticFeatures StaticFeatures, logger *zerolog.Logger) (*FeatureSelector, error) {
+	return newFeatureSelector(ctx, accountTag, logger, newDNSResolver(), staticFeatures, defaultRefreshFreq)
+}
+
+// FeatureSelector determines if this account will try new features. It preiodically queries a DNS TXT record
+// to see which features are turned on
+type FeatureSelector struct {
+	accountHash int32
+	logger      *zerolog.Logger
+	resolver    resolver
+
+	staticFeatures StaticFeatures
+
+	// lock protects concurrent access to dynamic features
+	lock     sync.RWMutex
+	features featuresRecord
+}
+
+// Features set by user provided flags
+type StaticFeatures struct {
+	PostQuantumMode *PostQuantumMode
+}
+
+func newFeatureSelector(ctx context.Context, accountTag string, logger *zerolog.Logger, resolver resolver, staticFeatures StaticFeatures, refreshFreq time.Duration) (*FeatureSelector, error) {
+	selector := &FeatureSelector{
+		accountHash:    switchThreshold(accountTag),
+		logger:         logger,
+		resolver:       resolver,
+		staticFeatures: staticFeatures,
+	}
+
+	if err := selector.refresh(ctx); err != nil {
+		logger.Err(err).Msg("Failed to fetch features, default to disable")
+	}
+
+	go selector.refreshLoop(ctx, refreshFreq)
+
+	return selector, nil
+}
+
+func (fs *FeatureSelector) PostQuantumMode() PostQuantumMode {
+	if fs.staticFeatures.PostQuantumMode != nil {
+		return *fs.staticFeatures.PostQuantumMode
+	}
+
+	fs.lock.RLock()
+	defer fs.lock.RUnlock()
+
+	if fs.features.PostQuantumPercentage > fs.accountHash {
+		return PostQuantumPrefer
+	}
+	return PostQuantumDisabled
+}
+
+func (fs *FeatureSelector) refreshLoop(ctx context.Context, refreshFreq time.Duration) {
+	ticker := time.NewTicker(refreshFreq)
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			err := fs.refresh(ctx)
+			if err != nil {
+				fs.logger.Err(err).Msg("Failed to refresh feature selector")
+			}
+		}
+	}
+}
+
+func (fs *FeatureSelector) refresh(ctx context.Context) error {
+	record, err := fs.resolver.lookupRecord(ctx)
+	if err != nil {
+		return err
+	}
+
+	var features featuresRecord
+	if err := json.Unmarshal(record, &features); err != nil {
+		return err
+	}
+
+	pq_enabled := features.PostQuantumPercentage > fs.accountHash
+	fs.logger.Debug().Int32("account_hash", fs.accountHash).Int32("pq_perct", features.PostQuantumPercentage).Bool("pq_enabled", pq_enabled).Msg("Refreshed feature")
+
+	fs.lock.Lock()
+	defer fs.lock.Unlock()
+
+	fs.features = features
+
+	return nil
+}
+
+// resolver represents an object that can look up featuresRecord
+type resolver interface {
+	lookupRecord(ctx context.Context) ([]byte, error)
+}
+
+type dnsResolver struct {
+	resolver *net.Resolver
+}
+
+func newDNSResolver() *dnsResolver {
+	return &dnsResolver{
+		resolver: net.DefaultResolver,
+	}
+}
+
+func (dr *dnsResolver) lookupRecord(ctx context.Context) ([]byte, error) {
+	ctx, cancel := context.WithTimeout(ctx, lookupTimeout)
+	defer cancel()
+
+	records, err := dr.resolver.LookupTXT(ctx, featureSelectorHostname)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(records) == 0 {
+		return nil, fmt.Errorf("No TXT record found for %s to determine which features to opt-in", featureSelectorHostname)
+	}
+
+	return []byte(records[0]), nil
+}
+
+func switchThreshold(accountTag string) int32 {
+	h := fnv.New32a()
+	_, _ = h.Write([]byte(accountTag))
+	return int32(h.Sum32() % 100)
+}

features/selector_test.go

@@ -0,0 +1,128 @@
+package features
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/rs/zerolog"
+	"github.com/stretchr/testify/require"
+)
+
+func TestUnmarshalFeaturesRecord(t *testing.T) {
+	tests := []struct {
+		record             []byte
+		expectedPercentage int32
+		expectedErr        bool
+	}{
+		{
+			record:             []byte(`{"pq":0}`),
+			expectedPercentage: 0,
+		},
+		{
+			record:             []byte(`{"pq":39}`),
+			expectedPercentage: 39,
+		},
+		{
+			record:             []byte(`{"pq":100}`),
+			expectedPercentage: 100,
+		},
+		{
+			record:             []byte(`{}`),
+			expectedPercentage: 0, // Unmarshal to default struct if key is not present
+		},
+		{
+			record:             []byte(`{"kyber":768}`),
+			expectedPercentage: 0, // Unmarshal to default struct if key is not present
+		},
+		{
+			record:      []byte(`{"pq":"kyber768"}`),
+			expectedErr: true,
+		},
+	}
+
+	for _, test := range tests {
+		var features featuresRecord
+		err := json.Unmarshal(test.record, &features)
+		if test.expectedErr {
+			require.Error(t, err, test)
+		} else {
+			require.NoError(t, err)
+			require.Equal(t, test.expectedPercentage, features.PostQuantumPercentage, test)
+		}
+	}
+}
+
+func TestRefreshFeaturesRecord(t *testing.T) {
+	// The hash of the accountTag is 82
+	accountTag := t.Name()
+	threshold := switchThreshold(accountTag)
+
+	percentages := []int32{0, 10, 80, 83, 100}
+	refreshFreq := time.Millisecond * 10
+	selector := newTestSelector(t, percentages, nil, refreshFreq)
+
+	for _, percentage := range percentages {
+		if percentage > threshold {
+			require.Equal(t, PostQuantumPrefer, selector.PostQuantumMode())
+		} else {
+			require.Equal(t, PostQuantumDisabled, selector.PostQuantumMode())
+		}
+
+		time.Sleep(refreshFreq + time.Millisecond)
+	}
+
+	// Make sure error doesn't override the last fetched features
+	require.Equal(t, PostQuantumPrefer, selector.PostQuantumMode())
+}
+
+func TestStaticFeatures(t *testing.T) {
+	percentages := []int32{0}
+	pqMode := PostQuantumStrict
+	selector := newTestSelector(t, percentages, &pqMode, time.Millisecond*10)
+	require.Equal(t, PostQuantumStrict, selector.PostQuantumMode())
+}
+
+// Verify that if the first refresh fails, the selector will use default features
+func TestFailedRefreshInitToDefault(t *testing.T) {
+	selector := newTestSelector(t, []int32{}, nil, time.Second)
+	require.Equal(t, featuresRecord{}, selector.features)
+	require.Equal(t, PostQuantumDisabled, selector.PostQuantumMode())
+}
+
+func newTestSelector(t *testing.T, percentages []int32, pqMode *PostQuantumMode, refreshFreq time.Duration) *FeatureSelector {
+	accountTag := t.Name()
+	logger := zerolog.Nop()
+
+	resolver := &mockResolver{
+		percentages: percentages,
+	}
+
+	staticFeatures := StaticFeatures{
+		PostQuantumMode: pqMode,
+	}
+	selector, err := newFeatureSelector(context.Background(), accountTag, &logger, resolver, staticFeatures, refreshFreq)
+	require.NoError(t, err)
+
+	return selector
+}
+
+type mockResolver struct {
+	nextIndex   int
+	percentages []int32
+}
+
+func (mr *mockResolver) lookupRecord(ctx context.Context) ([]byte, error) {
+	if mr.nextIndex >= len(mr.percentages) {
+		return nil, fmt.Errorf("no more record to lookup")
+	}
+
+	record, err := json.Marshal(featuresRecord{
+		PostQuantumPercentage: mr.percentages[mr.nextIndex],
+	})
+	mr.nextIndex++
+
+	return record, err
+}

go.mod

@@ -1,6 +1,6 @@
 module github.com/cloudflare/cloudflared
 
-go 1.19
+go 1.20
 
 require (
 	github.com/cloudflare/golibs v0.0.0-20170913112048-333127dbecfc
@@ -110,8 +110,5 @@ replace gopkg.in/yaml.v3 => gopkg.in/yaml.v3 v3.0.1
 replace github.com/quic-go/quic-go => github.com/devincarr/quic-go v0.0.0-20230502200822-d1f4edacbee7
 
 // Post-quantum tunnel RTG-1339
-replace (
-	// Branches go1.19 go1.20 on github.com/cloudflare/qtls-pq
-	github.com/quic-go/qtls-go1-19 => github.com/cloudflare/qtls-pq v0.0.0-20230320123031-3faac1a945b2
-	github.com/quic-go/qtls-go1-20 => github.com/cloudflare/qtls-pq v0.0.0-20230320122459-4ed280d0d633
-)
+// Branches go1.20 on github.com/cloudflare/qtls-pq
+replace github.com/quic-go/qtls-go1-20 => github.com/cloudflare/qtls-pq v0.0.0-20230320122459-4ed280d0d633

go.sum

@@ -65,8 +65,6 @@ github.com/cloudflare/golibs v0.0.0-20170913112048-333127dbecfc h1:Dvk3ySBsOm5Ev
 github.com/cloudflare/golibs v0.0.0-20170913112048-333127dbecfc/go.mod h1:HlgKKR8V5a1wroIDDIz3/A+T+9Janfq+7n1P5sEFdi0=
 github.com/cloudflare/qtls-pq v0.0.0-20230320122459-4ed280d0d633 h1:ZTub2XMOBpxyBiJf6Q+UKqAi07yt1rZmFitriHvFd8M=
 github.com/cloudflare/qtls-pq v0.0.0-20230320122459-4ed280d0d633/go.mod h1:j/igSUc4PgBMayIsBGjAFu2i7g663rm6kZrKy4htb7E=
-github.com/cloudflare/qtls-pq v0.0.0-20230320123031-3faac1a945b2 h1:0/KuLjh9lBMiXlooAdwoo+FbLVD5DABtquB0ImEFOK0=
-github.com/cloudflare/qtls-pq v0.0.0-20230320123031-3faac1a945b2/go.mod h1:XzuZIjv4mF5cM205RHHW1d60PQtWGwMR6jx38YKuYHs=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI=
@@ -337,6 +335,8 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1
 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
 github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo=
 github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4=
+github.com/quic-go/qtls-go1-19 v0.3.2 h1:tFxjCFcTQzK+oMxG6Zcvp4Dq8dx4yD3dDiIiyc86Z5U=
+github.com/quic-go/qtls-go1-19 v0.3.2/go.mod h1:ySOI96ew8lnoKPtSqx2BlI5wCpUVPT05RMAlajtnyOI=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=

ingress/config.go

@@ -44,14 +44,12 @@ const (
 )
 
 type WarpRoutingConfig struct {
-	Enabled        bool                  `yaml:"enabled" json:"enabled"`
 	ConnectTimeout config.CustomDuration `yaml:"connectTimeout" json:"connectTimeout,omitempty"`
 	TCPKeepAlive   config.CustomDuration `yaml:"tcpKeepAlive" json:"tcpKeepAlive,omitempty"`
 }
 
 func NewWarpRoutingConfig(raw *config.WarpRoutingConfig) WarpRoutingConfig {
 	cfg := WarpRoutingConfig{
-		Enabled:        raw.Enabled,
 		ConnectTimeout: defaultWarpRoutingConnectTimeout,
 		TCPKeepAlive:   defaultTCPKeepAlive,
 	}
@@ -65,9 +63,7 @@ func NewWarpRoutingConfig(raw *config.WarpRoutingConfig) WarpRoutingConfig {
 }
 
 func (c *WarpRoutingConfig) RawConfig() config.WarpRoutingConfig {
-	raw := config.WarpRoutingConfig{
-		Enabled: c.Enabled,
-	}
+	raw := config.WarpRoutingConfig{}
 	if c.ConnectTimeout.Duration != defaultWarpRoutingConnectTimeout.Duration {
 		raw.ConnectTimeout = &c.ConnectTimeout
 	}

ingress/packet_router.go

@@ -23,12 +23,11 @@ type muxer interface {
 
 // PacketRouter routes packets between Upstream and ICMPRouter. Currently it rejects all other type of ICMP packets
 type PacketRouter struct {
-	globalConfig           *GlobalRouterConfig
-	muxer                  muxer
-	logger                 *zerolog.Logger
-	checkRouterEnabledFunc func() bool
-	icmpDecoder            *packet.ICMPDecoder
-	encoder                *packet.Encoder
+	globalConfig *GlobalRouterConfig
+	muxer        muxer
+	logger       *zerolog.Logger
+	icmpDecoder  *packet.ICMPDecoder
+	encoder      *packet.Encoder
 }
 
 // GlobalRouterConfig is the configuration shared by all instance of Router.
@@ -40,14 +39,13 @@ type GlobalRouterConfig struct {
 }
 
 // NewPacketRouter creates a PacketRouter that handles ICMP packets. Packets are read from muxer but dropped if globalConfig is nil.
-func NewPacketRouter(globalConfig *GlobalRouterConfig, muxer muxer, logger *zerolog.Logger, checkRouterEnabledFunc func() bool) *PacketRouter {
+func NewPacketRouter(globalConfig *GlobalRouterConfig, muxer muxer, logger *zerolog.Logger) *PacketRouter {
 	return &PacketRouter{
-		globalConfig:           globalConfig,
-		muxer:                  muxer,
-		logger:                 logger,
-		checkRouterEnabledFunc: checkRouterEnabledFunc,
-		icmpDecoder:            packet.NewICMPDecoder(),
-		encoder:                packet.NewEncoder(),
+		globalConfig: globalConfig,
+		muxer:        muxer,
+		logger:       logger,
+		icmpDecoder:  packet.NewICMPDecoder(),
+		encoder:      packet.NewEncoder(),
 	}
 }
 
@@ -92,10 +90,6 @@ func (r *PacketRouter) handlePacket(ctx context.Context, rawPacket packet.RawPac
 		return
 	}
 
-	if enabled := r.checkRouterEnabledFunc(); !enabled {
-		return
-	}
-
 	icmpPacket, err := r.icmpDecoder.Decode(rawPacket)
 	if err != nil {
 		r.logger.Err(err).Msg("Failed to decode ICMP packet from quic datagram")

ingress/packet_router_test.go

@@ -7,7 +7,6 @@ import (
 	"net/netip"
 	"sync/atomic"
 	"testing"
-	"time"
 
 	"github.com/google/gopacket/layers"
 	"github.com/stretchr/testify/require"
@@ -29,9 +28,7 @@ var (
 
 func TestRouterReturnTTLExceed(t *testing.T) {
 	muxer := newMockMuxer(0)
-	routerEnabled := &routerEnabledChecker{}
-	routerEnabled.set(true)
-	router := NewPacketRouter(packetConfig, muxer, &noopLogger, routerEnabled.isEnabled)
+	router := NewPacketRouter(packetConfig, muxer, &noopLogger)
 	ctx, cancel := context.WithCancel(context.Background())
 	routerStopped := make(chan struct{})
 	go func() {
@@ -80,65 +77,6 @@ func TestRouterReturnTTLExceed(t *testing.T) {
 	<-routerStopped
 }
 
-func TestRouterCheckEnabled(t *testing.T) {
-	muxer := newMockMuxer(0)
-	routerEnabled := &routerEnabledChecker{}
-	router := NewPacketRouter(packetConfig, muxer, &noopLogger, routerEnabled.isEnabled)
-	ctx, cancel := context.WithCancel(context.Background())
-	routerStopped := make(chan struct{})
-	go func() {
-		router.Serve(ctx)
-		close(routerStopped)
-	}()
-
-	pk := packet.ICMP{
-		IP: &packet.IP{
-			Src:      netip.MustParseAddr("192.168.1.1"),
-			Dst:      netip.MustParseAddr("10.0.0.1"),
-			Protocol: layers.IPProtocolICMPv4,
-			TTL:      1,
-		},
-		Message: &icmp.Message{
-			Type: ipv4.ICMPTypeEcho,
-			Code: 0,
-			Body: &icmp.Echo{
-				ID:   12481,
-				Seq:  8036,
-				Data: []byte(t.Name()),
-			},
-		},
-	}
-
-	// router is disabled
-	encoder := packet.NewEncoder()
-	encodedPacket, err := encoder.Encode(&pk)
-	require.NoError(t, err)
-	sendPacket := quicpogs.RawPacket(encodedPacket)
-
-	muxer.edgeToCfd <- sendPacket
-	select {
-	case <-time.After(time.Millisecond * 10):
-	case <-muxer.cfdToEdge:
-		t.Error("Unexpected reply when router is disabled")
-	}
-	routerEnabled.set(true)
-	// router is enabled, expects reply
-	muxer.edgeToCfd <- sendPacket
-	<-muxer.cfdToEdge
-
-	routerEnabled.set(false)
-	// router is disabled
-	muxer.edgeToCfd <- sendPacket
-	select {
-	case <-time.After(time.Millisecond * 10):
-	case <-muxer.cfdToEdge:
-		t.Error("Unexpected reply when router is disabled")
-	}
-
-	cancel()
-	<-routerStopped
-}
-
 func assertTTLExceed(t *testing.T, originalPacket *packet.ICMP, expectedSrc netip.Addr, muxer *mockMuxer) {
 	encoder := packet.NewEncoder()
 	rawPacket, err := encoder.Encode(originalPacket)

orchestration/config_test.go

@@ -60,7 +60,6 @@ func TestNewLocalConfig_MarshalJSON(t *testing.T) {
 			}
 		],
         "warp-routing": {
-			"enabled": true,
 			"connectTimeout": 1
         }
 	}
@@ -83,7 +82,6 @@ func TestNewLocalConfig_MarshalJSON(t *testing.T) {
 	require.NoError(t, err)
 
 	require.Equal(t, remoteConfig.WarpRouting, ingress.WarpRoutingConfig{
-		Enabled: true,
 		ConnectTimeout: config.CustomDuration{
 			Duration: time.Second,
 		},

orchestration/orchestrator.go

@@ -29,11 +29,10 @@ type Orchestrator struct {
 	// Underlying value is proxy.Proxy, can be read without the lock, but still needs the lock to update
 	proxy atomic.Value
 	// Set of internal ingress rules defined at cloudflared startup (separate from user-defined ingress rules)
-	internalRules      []ingress.Rule
-	warpRoutingEnabled atomic.Bool
-	config             *Config
-	tags               []tunnelpogs.Tag
-	log                *zerolog.Logger
+	internalRules []ingress.Rule
+	config        *Config
+	tags          []tunnelpogs.Tag
+	log           *zerolog.Logger
 
 	// orchestrator must not handle any more updates after shutdownC is closed
 	shutdownC <-chan struct{}
@@ -136,11 +135,6 @@ func (o *Orchestrator) updateIngress(ingressRules ingress.Ingress, warpRouting i
 	o.proxy.Store(proxy)
 	o.config.Ingress = &ingressRules
 	o.config.WarpRouting = warpRouting
-	if warpRouting.Enabled {
-		o.warpRoutingEnabled.Store(true)
-	} else {
-		o.warpRoutingEnabled.Store(false)
-	}
 
 	// If proxyShutdownC is nil, there is no previous running proxy
 	if o.proxyShutdownC != nil {
@@ -209,10 +203,6 @@ func (o *Orchestrator) GetOriginProxy() (connection.OriginProxy, error) {
 	return proxy, nil
 }
 
-func (o *Orchestrator) WarpRoutingEnabled() bool {
-	return o.warpRoutingEnabled.Load()
-}
-
 func (o *Orchestrator) waitToCloseLastProxy() {
 	<-o.shutdownC
 	o.lock.Lock()

orchestration/orchestrator_test.go

@@ -55,7 +55,6 @@ func TestUpdateConfiguration(t *testing.T) {
 	initOriginProxy, err := orchestrator.GetOriginProxy()
 	require.NoError(t, err)
 	require.Implements(t, (*connection.OriginProxy)(nil), initOriginProxy)
-	require.False(t, orchestrator.WarpRoutingEnabled())
 
 	configJSONV2 := []byte(`
 {
@@ -87,7 +86,6 @@ func TestUpdateConfiguration(t *testing.T) {
         }
     ],
     "warp-routing": {
-        "enabled": true,
         "connectTimeout": 10
     }
 }
@@ -126,8 +124,6 @@ func TestUpdateConfiguration(t *testing.T) {
 	require.Equal(t, config.CustomDuration{Duration: time.Second * 90}, configV2.Ingress.Rules[2].Config.ConnectTimeout)
 	require.Equal(t, false, configV2.Ingress.Rules[2].Config.NoTLSVerify)
 	require.Equal(t, true, configV2.Ingress.Rules[2].Config.NoHappyEyeballs)
-	require.True(t, configV2.WarpRouting.Enabled)
-	require.Equal(t, configV2.WarpRouting.Enabled, orchestrator.WarpRoutingEnabled())
 	require.Equal(t, configV2.WarpRouting.ConnectTimeout.Duration, 10*time.Second)
 
 	originProxyV2, err := orchestrator.GetOriginProxy()
@@ -162,7 +158,6 @@ func TestUpdateConfiguration(t *testing.T) {
         }
     ],
     "warp-routing": {
-        "enabled": false
     }
 }
 `)
@@ -171,8 +166,6 @@ func TestUpdateConfiguration(t *testing.T) {
 	require.Len(t, configV10.Ingress.Rules, 1)
 	require.True(t, configV10.Ingress.Rules[0].Matches("blogs.tunnel.io", "/2022/02/10"))
 	require.Equal(t, ingress.HelloWorldService, configV10.Ingress.Rules[0].Service.String())
-	require.False(t, configV10.WarpRouting.Enabled)
-	require.Equal(t, configV10.WarpRouting.Enabled, orchestrator.WarpRoutingEnabled())
 
 	originProxyV10, err := orchestrator.GetOriginProxy()
 	require.NoError(t, err)
@@ -191,7 +184,6 @@ func TestUpdateConfiguration_FromMigration(t *testing.T) {
 	initOriginProxy, err := orchestrator.GetOriginProxy()
 	require.NoError(t, err)
 	require.Implements(t, (*connection.OriginProxy)(nil), initOriginProxy)
-	require.False(t, orchestrator.WarpRoutingEnabled())
 
 	configJSONV2 := []byte(`
 {
@@ -201,7 +193,6 @@ func TestUpdateConfiguration_FromMigration(t *testing.T) {
         }
     ],
     "warp-routing": {
-        "enabled": true
     }
 }
 `)
@@ -271,7 +262,6 @@ func TestConcurrentUpdateAndRead(t *testing.T) {
         }
     ],
     "warp-routing": {
-        "enabled": true
     }
 }
 `, hostname, httpOrigin.URL, expectedHost))
@@ -283,7 +273,6 @@ func TestConcurrentUpdateAndRead(t *testing.T) {
         }
     ],
     "warp-routing": {
-        "enabled": false
     }
 }
 `)
@@ -296,7 +285,6 @@ func TestConcurrentUpdateAndRead(t *testing.T) {
         }
     ],
     "warp-routing": {
-        "enabled": true
     }
 }
 `)
@@ -516,7 +504,6 @@ func TestClosePreviousProxies(t *testing.T) {
 		}
     ],
     "warp-routing": {
-        "enabled": true
     }
 }
 `, hostname))
@@ -529,7 +516,6 @@ func TestClosePreviousProxies(t *testing.T) {
 		}
     ],
     "warp-routing": {
-        "enabled": true
     }
 }
 `)
@@ -612,7 +598,6 @@ func TestPersistentConnection(t *testing.T) {
         }
     ],
     "warp-routing": {
-        "enabled": true
     }
 }
 `, wsOrigin.URL))
@@ -679,7 +664,6 @@ func TestPersistentConnection(t *testing.T) {
         }
     ],
     "warp-routing": {
-        "enabled": false
     }
 }
 `)

proxy/proxy.go

@@ -57,10 +57,8 @@ func NewOriginProxy(
 		tags:         tags,
 		log:          log,
 	}
-	if warpRouting.Enabled {
-		proxy.warpRouting = ingress.NewWarpRoutingService(warpRouting)
-		log.Info().Msgf("Warp-routing is enabled")
-	}
+
+	proxy.warpRouting = ingress.NewWarpRoutingService(warpRouting)
 
 	return proxy
 }

proxy/proxy_posix_test.go

@@ -1,5 +1,4 @@
 //go:build !windows
-// +build !windows
 
 package proxy
 

proxy/proxy_test.go

@@ -37,7 +37,6 @@ var (
 	testTags        = []tunnelpogs.Tag{{Name: "Name", Value: "value"}}
 	noWarpRouting   = ingress.WarpRoutingConfig{}
 	testWarpRouting = ingress.WarpRoutingConfig{
-		Enabled:        true,
 		ConnectTimeout: config.CustomDuration{Duration: time.Second},
 	}
 )

release_pkgs.py

@@ -13,7 +13,6 @@ import base64
 import logging
 import os
 import shutil
-from hashlib import sha256
 from pathlib import Path
 from subprocess import Popen, PIPE
 
@@ -35,7 +34,6 @@ class PkgUploader:
 
     def upload_pkg_to_r2(self, filename, upload_file_path):
         endpoint_url = f"https://{self.account_id}.r2.cloudflarestorage.com"
-        token_secret_hash = sha256(self.client_secret.encode()).hexdigest()
 
         config = Config(
             region_name='auto',
@@ -48,7 +46,7 @@ class PkgUploader:
             "s3",
             endpoint_url=endpoint_url,
             aws_access_key_id=self.client_id,
-            aws_secret_access_key=token_secret_hash,
+            aws_secret_access_key=self.client_secret,
             config=config,
         )
 

sshgen/sshgen_test.go

@@ -1,5 +1,4 @@
 //go:build !windows
-// +build !windows
 
 package sshgen
 

supervisor/pqtunnels.go

@@ -4,24 +4,23 @@ import (
 	"bytes"
 	"crypto/tls"
 	"encoding/json"
+	"fmt"
 	"net/http"
 	"sync"
+
+	"github.com/cloudflare/cloudflared/features"
 )
 
 // When experimental post-quantum tunnels are enabled, and we're hitting an
 // issue creating the tunnel, we'll report the first error
 // to https://pqtunnels.cloudflareresearch.com.
 
-var (
-	PQKexes = [...]tls.CurveID{
-		tls.CurveID(0xfe30), // X25519Kyber512Draft00
-		tls.CurveID(0xfe31), // X25519Kyber768Draft00
-	}
-	PQKexNames map[tls.CurveID]string = map[tls.CurveID]string{
-		tls.CurveID(0xfe30): "X25519Kyber512Draft00",
-		tls.CurveID(0xfe31): "X25519Kyber768Draft00",
-	}
+const (
+	PQKex     = tls.CurveID(0xfe31) // X25519Kyber768Draft00
+	PQKexName = "X25519Kyber768Draft00"
+)
 
+var (
 	pqtMux       sync.Mutex // protects pqtSubmitted and pqtWaitForMessage
 	pqtSubmitted bool       // whether an error has already been submitted
 
@@ -70,7 +69,7 @@ func submitPQTunnelError(rep error, config *TunnelConfig) {
 		Message string `json:"m"`
 		Version string `json:"v"`
 	}{
-		Group:   int(PQKexes[config.PQKexIdx]),
+		Group:   int(PQKex),
 		Message: rep.Error(),
 		Version: config.ReportedVersion,
 	})
@@ -98,3 +97,32 @@ func submitPQTunnelError(rep error, config *TunnelConfig) {
 	}
 	resp.Body.Close()
 }
+
+func curvePreference(pqMode features.PostQuantumMode, currentCurve []tls.CurveID) ([]tls.CurveID, error) {
+	switch pqMode {
+	case features.PostQuantumStrict:
+		// If the user passes the -post-quantum flag, we override
+		// CurvePreferences to only support hybrid post-quantum key agreements.
+		return []tls.CurveID{PQKex}, nil
+	case features.PostQuantumPrefer:
+		if len(currentCurve) == 0 {
+			return []tls.CurveID{PQKex}, nil
+		}
+
+		if currentCurve[0] != PQKex {
+			return append([]tls.CurveID{PQKex}, currentCurve...), nil
+		}
+		return currentCurve, nil
+	case features.PostQuantumDisabled:
+		curvePref := currentCurve
+		// Remove PQ from curve preference
+		for i, curve := range currentCurve {
+			if curve == PQKex {
+				curvePref = append(curvePref[:i], curvePref[i+1:]...)
+			}
+		}
+		return curvePref, nil
+	default:
+		return nil, fmt.Errorf("Unexpected post quantum mode")
+	}
+}

supervisor/supervisor.go

@@ -2,6 +2,7 @@ package supervisor
 
 import (
 	"context"
+	"crypto/tls"
 	"errors"
 	"net"
 	"strings"
@@ -10,6 +11,8 @@ import (
 	"github.com/quic-go/quic-go"
 	"github.com/rs/zerolog"
 
+	qtls120 "github.com/quic-go/qtls-go1-20"
+
 	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/edgediscovery"
 	"github.com/cloudflare/cloudflared/orchestration"
@@ -78,6 +81,8 @@ func NewSupervisor(config *TunnelConfig, orchestrator *orchestration.Orchestrato
 
 	reconnectCredentialManager := newReconnectCredentialManager(connection.MetricsNamespace, connection.TunnelSubsystem, config.HAConnections)
 
+	registerTLSEventLogger(config.Log)
+
 	tracker := tunnelstate.NewConnTracker(config.Log)
 	log := NewConnAwareLogger(config.Log, tracker, config.Observer)
 
@@ -336,3 +341,26 @@ func (s *Supervisor) waitForNextTunnel(index int) bool {
 func (s *Supervisor) unusedIPs() bool {
 	return s.edgeIPs.AvailableAddrs() > s.config.HAConnections
 }
+
+func registerTLSEventLogger(logger *zerolog.Logger) {
+	qtls120.SetCFEventHandler(func(ev qtls120.CFEvent) {
+		logger.Debug().Bool("handshake", ev.IsHandshake()).Str("handshake_duration", ev.Duration().String()).Str("curve", tlsCurveName(ev.KEX())).Msg("QUIC TLS event")
+	})
+}
+
+func tlsCurveName(curve tls.CurveID) string {
+	switch curve {
+	case tls.CurveP256:
+		return "p256"
+	case tls.CurveP384:
+		return "p384"
+	case tls.CurveP521:
+		return "p521"
+	case tls.X25519:
+		return "X25519"
+	case PQKex:
+		return PQKexName
+	default:
+		return "unknown"
+	}
+}

supervisor/tunnel.go

@@ -61,9 +61,6 @@ type TunnelConfig struct {
 
 	NeedPQ bool
 
-	// Index into PQKexes of post-quantum kex to use if NeedPQ is set.
-	PQKexIdx int
-
 	NamedTunnel      *connection.NamedTunnelProperties
 	ProtocolSelector connection.ProtocolSelector
 	EdgeTLSConfigs   map[connection.Protocol]*tls.Config
@@ -72,6 +69,8 @@ type TunnelConfig struct {
 	UDPUnregisterSessionTimeout time.Duration
 
 	DisableQUICPathMTUDiscovery bool
+
+	FeatureSelector *features.FeatureSelector
 }
 
 func (c *TunnelConfig) registrationOptions(connectionID uint8, OriginLocalIP string, uuid uuid.UUID) *tunnelpogs.RegistrationOptions {
@@ -354,7 +353,7 @@ func selectNextProtocol(
 				"Cloudflare Network with `quic` protocol, then most likely your machine/network is getting its egress " +
 				"UDP to port 7844 (or others) blocked or dropped. Make sure to allow egress connectivity as per " +
 				"https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/configuration/ports-and-ips/\n" +
-				"If you are using private routing to this Tunnel, then UDP (and Private DNS Resolution) will not work " +
+				"If you are using private routing to this Tunnel, then ICMP, UDP (and Private DNS Resolution) will not work " +
 				"unless your cloudflared can connect with Cloudflare Network with `quic`.")
 		}
 
@@ -539,7 +538,8 @@ func (e *EdgeTunnelServer) serveHTTP2(
 	controlStreamHandler connection.ControlStreamHandler,
 	connIndex uint8,
 ) error {
-	if e.config.NeedPQ {
+	pqMode := e.config.FeatureSelector.PostQuantumMode()
+	if pqMode == features.PostQuantumStrict {
 		return unrecoverableError{errors.New("HTTP/2 transport does not support post-quantum")}
 	}
 
@@ -582,21 +582,18 @@ func (e *EdgeTunnelServer) serveQUIC(
 ) (err error, recoverable bool) {
 	tlsConfig := e.config.EdgeTLSConfigs[connection.QUIC]
 
-	if e.config.NeedPQ {
-		// If the user passes the -post-quantum flag, we override
-		// CurvePreferences to only support hybrid post-quantum key agreements.
-		cs := make([]tls.CurveID, len(PQKexes))
-		copy(cs, PQKexes[:])
-
-		// It is unclear whether Kyber512 or Kyber768 will become the standard.
-		// Kyber768 is a bit bigger (and doesn't fit in one initial
-		// datagram anymore). We're enabling both, but pick randomly which
-		// one to put first. (TLS will use the first one in the list
-		// and allows a fallback to the second.)
-		cs[0], cs[e.config.PQKexIdx] = cs[e.config.PQKexIdx], cs[0]
-		tlsConfig.CurvePreferences = cs
+	pqMode := e.config.FeatureSelector.PostQuantumMode()
+	if pqMode == features.PostQuantumStrict || pqMode == features.PostQuantumPrefer {
+		connOptions.Client.Features = features.Dedup(append(connOptions.Client.Features, features.FeaturePostQuantum))
 	}
 
+	curvePref, err := curvePreference(pqMode, tlsConfig.CurvePreferences)
+	if err != nil {
+		return err, true
+	}
+
+	tlsConfig.CurvePreferences = curvePref
+
 	quicConfig := &quic.Config{
 		HandshakeIdleTimeout:    quicpogs.HandshakeIdleTimeout,
 		MaxIdleTimeout:          quicpogs.MaxIdleTimeout,
@@ -624,7 +621,7 @@ func (e *EdgeTunnelServer) serveQUIC(
 		e.config.UDPUnregisterSessionTimeout,
 	)
 	if err != nil {
-		if e.config.NeedPQ {
+		if pqMode == features.PostQuantumStrict || pqMode == features.PostQuantumPrefer {
 			handlePQTunnelError(err, e.config)
 		}
 

token/launch_browser_darwin.go

@@ -1,5 +1,4 @@
 //go:build darwin
-// +build darwin
 
 package token
 

token/launch_browser_other.go

@@ -1,5 +1,4 @@
 //go:build !windows && !darwin && !linux && !netbsd && !freebsd && !openbsd
-// +build !windows,!darwin,!linux,!netbsd,!freebsd,!openbsd
 
 package token
 

token/launch_browser_unix.go

@@ -1,5 +1,4 @@
 //go:build linux || freebsd || openbsd || netbsd
-// +build linux freebsd openbsd netbsd
 
 package token
 

token/launch_browser_windows.go

@@ -1,5 +1,4 @@
 //go:build windows
-// +build windows
 
 package token
 

token/token_test.go

@@ -1,5 +1,4 @@
 //go:build linux
-// +build linux
 
 package token
 

vendor/github.com/quic-go/qtls-go1-19/cfkem.go

@@ -1,170 +0,0 @@
-// Copyright 2022 Cloudflare, Inc. All rights reserved. Use of this source code
-// is governed by a BSD-style license that can be found in the LICENSE file.
-//
-// Glue to add Circl's (post-quantum) hybrid KEMs.
-//
-// To enable set CurvePreferences with the desired scheme as the first element:
-//
-//   import (
-//      "github.com/cloudflare/circl/kem/tls"
-//      "github.com/cloudflare/circl/kem/hybrid"
-//
-//          [...]
-//
-//   config.CurvePreferences = []tls.CurveID{
-//      qtls.X25519Kyber512Draft00,
-//      qtls.X25519,
-//      qtls.P256,
-//   }
-
-package qtls
-
-import (
-	"github.com/cloudflare/circl/kem"
-	"github.com/cloudflare/circl/kem/hybrid"
-
-	"crypto/tls"
-	"fmt"
-	"io"
-	"sync"
-	"time"
-)
-
-// Either ecdheParameters or kem.PrivateKey
-type clientKeySharePrivate interface{}
-
-var (
-	X25519Kyber512Draft00 = CurveID(0xfe30)
-	X25519Kyber768Draft00 = CurveID(0xfe31)
-	invalidCurveID        = CurveID(0)
-)
-
-func kemSchemeKeyToCurveID(s kem.Scheme) CurveID {
-	switch s.Name() {
-	case "Kyber512-X25519":
-		return X25519Kyber512Draft00
-	case "Kyber768-X25519":
-		return X25519Kyber768Draft00
-	default:
-		return invalidCurveID
-	}
-}
-
-// Extract CurveID from clientKeySharePrivate
-func clientKeySharePrivateCurveID(ks clientKeySharePrivate) CurveID {
-	switch v := ks.(type) {
-	case kem.PrivateKey:
-		ret := kemSchemeKeyToCurveID(v.Scheme())
-		if ret == invalidCurveID {
-			panic("cfkem: internal error: don't know CurveID for this KEM")
-		}
-		return ret
-	case ecdheParameters:
-		return v.CurveID()
-	default:
-		panic("cfkem: internal error: unknown clientKeySharePrivate")
-	}
-}
-
-// Returns scheme by CurveID if supported by Circl
-func curveIdToCirclScheme(id CurveID) kem.Scheme {
-	switch id {
-	case X25519Kyber512Draft00:
-		return hybrid.Kyber512X25519()
-	case X25519Kyber768Draft00:
-		return hybrid.Kyber768X25519()
-	}
-	return nil
-}
-
-// Generate a new shared secret and encapsulates it for the packed
-// public key in ppk using randomness from rnd.
-func encapsulateForKem(scheme kem.Scheme, rnd io.Reader, ppk []byte) (
-	ct, ss []byte, alert alert, err error) {
-	pk, err := scheme.UnmarshalBinaryPublicKey(ppk)
-	if err != nil {
-		return nil, nil, alertIllegalParameter, fmt.Errorf("unpack pk: %w", err)
-	}
-	seed := make([]byte, scheme.EncapsulationSeedSize())
-	if _, err := io.ReadFull(rnd, seed); err != nil {
-		return nil, nil, alertInternalError, fmt.Errorf("random: %w", err)
-	}
-	ct, ss, err = scheme.EncapsulateDeterministically(pk, seed)
-	return ct, ss, alertIllegalParameter, err
-}
-
-// Generate a new keypair using randomness from rnd.
-func generateKemKeyPair(scheme kem.Scheme, rnd io.Reader) (
-	kem.PublicKey, kem.PrivateKey, error) {
-	seed := make([]byte, scheme.SeedSize())
-	if _, err := io.ReadFull(rnd, seed); err != nil {
-		return nil, nil, err
-	}
-	pk, sk := scheme.DeriveKeyPair(seed)
-	return pk, sk, nil
-}
-
-// Events. We cannot use the same approach as used in our plain Go fork
-// as we cannot change tls.Config, tls.ConnectionState, etc. Also we do
-// not want to maintain a fork of quic-go itself as well. This seems
-// the simplest option.
-
-// CFEvent. There are two events: one emitted on HRR and one emitted
-type CFEvent interface {
-	// Common to all events
-	ServerSide() bool // true if server-side; false if on client-side
-
-	// HRR event. Emitted when an HRR happened.
-	IsHRR() bool // true if this is an HRR event
-
-	// Handshake event.
-	IsHandshake() bool       // true if this is a handshake event.
-	Duration() time.Duration // how long did the handshake take?
-	KEX() tls.CurveID        // which kex was established?
-}
-
-type CFEventHandler func(CFEvent)
-
-// Registers a handler to be called when a CFEvent is emitted; returns
-// the previous handler.
-func SetCFEventHandler(handler CFEventHandler) CFEventHandler {
-	cfEventMux.Lock()
-	ret := cfEventHandler
-	cfEventHandler = handler
-	cfEventMux.Unlock()
-	return ret
-}
-
-func raiseCFEvent(ev CFEvent) {
-	cfEventMux.Lock()
-	handler := cfEventHandler
-	cfEventMux.Unlock()
-	if handler != nil {
-		handler(ev)
-	}
-}
-
-var (
-	cfEventMux     sync.Mutex
-	cfEventHandler CFEventHandler
-)
-
-type cfEventHRR struct{ serverSide bool }
-
-func (*cfEventHRR) IsHRR() bool                { return true }
-func (ev *cfEventHRR) ServerSide() bool        { return ev.serverSide }
-func (*cfEventHRR) IsHandshake() bool          { return false }
-func (ev *cfEventHRR) Duration() time.Duration { panic("wrong event") }
-func (ev *cfEventHRR) KEX() tls.CurveID        { panic("wrong event") }
-
-type cfEventHandshake struct {
-	serverSide bool
-	duration   time.Duration
-	kex        tls.CurveID
-}
-
-func (*cfEventHandshake) IsHRR() bool                { return false }
-func (ev *cfEventHandshake) ServerSide() bool        { return ev.serverSide }
-func (*cfEventHandshake) IsHandshake() bool          { return true }
-func (ev *cfEventHandshake) Duration() time.Duration { return ev.duration }
-func (ev *cfEventHandshake) KEX() tls.CurveID        { return ev.kex }

vendor/github.com/quic-go/qtls-go1-19/handshake_client.go

@@ -40,7 +40,7 @@ type clientHandshakeState struct {
 
 var testingOnlyForceClientHelloSignatureAlgorithms []SignatureScheme
 
-func (c *Conn) makeClientHello() (*clientHelloMsg, clientKeySharePrivate, error) {
+func (c *Conn) makeClientHello() (*clientHelloMsg, ecdheParameters, error) {
 	config := c.config
 	if len(config.ServerName) == 0 && !config.InsecureSkipVerify {
 		return nil, nil, errors.New("tls: either ServerName or InsecureSkipVerify must be specified in the tls.Config")
@@ -142,8 +142,11 @@ func (c *Conn) makeClientHello() (*clientHelloMsg, clientKeySharePrivate, error)
 		hello.supportedSignatureAlgorithms = testingOnlyForceClientHelloSignatureAlgorithms
 	}
 
-	var secret clientKeySharePrivate
+	var params ecdheParameters
 	if hello.supportedVersions[0] == VersionTLS13 {
+		if len(hello.supportedVersions) == 1 {
+			hello.cipherSuites = hello.cipherSuites[:0]
+		}
 		if hasAESGCMHardwareSupport {
 			hello.cipherSuites = append(hello.cipherSuites, defaultCipherSuitesTLS13...)
 		} else {
@@ -151,37 +154,21 @@ func (c *Conn) makeClientHello() (*clientHelloMsg, clientKeySharePrivate, error)
 		}
 
 		curveID := config.curvePreferences()[0]
-		if scheme := curveIdToCirclScheme(curveID); scheme != nil {
-			pk, sk, err := generateKemKeyPair(scheme, config.rand())
-			if err != nil {
-				return nil, nil, fmt.Errorf("generateKemKeyPair %s: %w",
-					scheme.Name(), err)
-			}
-			packedPk, err := pk.MarshalBinary()
-			if err != nil {
-				return nil, nil, fmt.Errorf("pack circl public key %s: %w",
-					scheme.Name(), err)
-			}
-			hello.keyShares = []keyShare{{group: curveID, data: packedPk}}
-			secret = sk
-		} else {
-			if _, ok := curveForCurveID(curveID); curveID != X25519 && !ok {
-				return nil, nil, errors.New("tls: CurvePreferences includes unsupported curve")
-			}
-			params, err := generateECDHEParameters(config.rand(), curveID)
-			if err != nil {
-				return nil, nil, err
-			}
-			hello.keyShares = []keyShare{{group: curveID, data: params.PublicKey()}}
-			secret = params
+		if _, ok := curveForCurveID(curveID); curveID != X25519 && !ok {
+			return nil, nil, errors.New("tls: CurvePreferences includes unsupported curve")
 		}
+		params, err = generateECDHEParameters(config.rand(), curveID)
+		if err != nil {
+			return nil, nil, err
+		}
+		hello.keyShares = []keyShare{{group: curveID, data: params.PublicKey()}}
 	}
 
 	if hello.supportedVersions[0] == VersionTLS13 && c.extraConfig != nil && c.extraConfig.GetExtensions != nil {
 		hello.additionalExtensions = c.extraConfig.GetExtensions(typeClientHello)
 	}
 
-	return hello, secret, nil
+	return hello, params, nil
 }
 
 func (c *Conn) clientHandshake(ctx context.Context) (err error) {
@@ -274,14 +261,14 @@ func (c *Conn) clientHandshake(ctx context.Context) (err error) {
 
 	if c.vers == VersionTLS13 {
 		hs := &clientHandshakeStateTLS13{
-			c:               c,
-			ctx:             ctx,
-			serverHello:     serverHello,
-			hello:           hello,
-			keySharePrivate: ecdheParams,
-			session:         session,
-			earlySecret:     earlySecret,
-			binderKey:       binderKey,
+			c:           c,
+			ctx:         ctx,
+			serverHello: serverHello,
+			hello:       hello,
+			ecdheParams: ecdheParams,
+			session:     session,
+			earlySecret: earlySecret,
+			binderKey:   binderKey,
 		}
 
 		// In TLS 1.3, session tickets are delivered after the handshake.

vendor/github.com/quic-go/qtls-go1-19/handshake_client_tls13.go

@@ -12,12 +12,10 @@ import (
 	"crypto/rsa"
 	"encoding/binary"
 	"errors"
-	"fmt"
 	"hash"
 	"sync/atomic"
 	"time"
 
-	circlKem "github.com/cloudflare/circl/kem"
 	"golang.org/x/crypto/cryptobyte"
 )
 
@@ -26,8 +24,7 @@ type clientHandshakeStateTLS13 struct {
 	ctx         context.Context
 	serverHello *serverHelloMsg
 	hello       *clientHelloMsg
-
-	keySharePrivate clientKeySharePrivate
+	ecdheParams ecdheParameters
 
 	session     *clientSessionState
 	earlySecret []byte
@@ -47,8 +44,6 @@ type clientHandshakeStateTLS13 struct {
 func (hs *clientHandshakeStateTLS13) handshake() error {
 	c := hs.c
 
-	startTime := time.Now()
-
 	if needFIPS() {
 		return errors.New("tls: internal error: TLS 1.3 reached in FIPS mode")
 	}
@@ -61,7 +56,7 @@ func (hs *clientHandshakeStateTLS13) handshake() error {
 	}
 
 	// Consistency check on the presence of a keyShare and its parameters.
-	if hs.keySharePrivate == nil || len(hs.hello.keyShares) != 1 {
+	if hs.ecdheParams == nil || len(hs.hello.keyShares) != 1 {
 		return c.sendAlert(alertInternalError)
 	}
 
@@ -119,12 +114,6 @@ func (hs *clientHandshakeStateTLS13) handshake() error {
 		return err
 	}
 
-	raiseCFEvent(&cfEventHandshake{
-		serverSide: false,
-		duration:   time.Since(startTime),
-		kex:        hs.serverHello.serverShare.group,
-	})
-
 	atomic.StoreUint32(&c.handshakeStatus, 1)
 	c.updateConnectionState()
 	return nil
@@ -201,8 +190,6 @@ func (hs *clientHandshakeStateTLS13) sendDummyChangeCipherSpec() error {
 func (hs *clientHandshakeStateTLS13) processHelloRetryRequest() error {
 	c := hs.c
 
-	raiseCFEvent(&cfEventHRR{serverSide: false})
-
 	// The first ClientHello gets double-hashed into the transcript upon a
 	// HelloRetryRequest. (The idea is that the server might offload transcript
 	// storage to the client in the cookie.) See RFC 8446, Section 4.4.1.
@@ -246,38 +233,21 @@ func (hs *clientHandshakeStateTLS13) processHelloRetryRequest() error {
 			c.sendAlert(alertIllegalParameter)
 			return errors.New("tls: server selected unsupported group")
 		}
-		if clientKeySharePrivateCurveID(hs.keySharePrivate) == curveID {
+		if hs.ecdheParams.CurveID() == curveID {
 			c.sendAlert(alertIllegalParameter)
 			return errors.New("tls: server sent an unnecessary HelloRetryRequest key_share")
 		}
-		if scheme := curveIdToCirclScheme(curveID); scheme != nil {
-			pk, sk, err := generateKemKeyPair(scheme, c.config.rand())
-			if err != nil {
-				c.sendAlert(alertInternalError)
-				return fmt.Errorf("HRR generateKeyPair %s: %w",
-					scheme.Name(), err)
-			}
-			packedPk, err := pk.MarshalBinary()
-			if err != nil {
-				c.sendAlert(alertInternalError)
-				return fmt.Errorf("HRR pack circl public key %s: %w",
-					scheme.Name(), err)
-			}
-			hs.keySharePrivate = sk
-			hs.hello.keyShares = []keyShare{{group: curveID, data: packedPk}}
-		} else {
-			if _, ok := curveForCurveID(curveID); curveID != X25519 && !ok {
-				c.sendAlert(alertInternalError)
-				return errors.New("tls: CurvePreferences includes unsupported curve")
-			}
-			params, err := generateECDHEParameters(c.config.rand(), curveID)
-			if err != nil {
-				c.sendAlert(alertInternalError)
-				return err
-			}
-			hs.keySharePrivate = params
-			hs.hello.keyShares = []keyShare{{group: curveID, data: params.PublicKey()}}
+		if _, ok := curveForCurveID(curveID); curveID != X25519 && !ok {
+			c.sendAlert(alertInternalError)
+			return errors.New("tls: CurvePreferences includes unsupported curve")
 		}
+		params, err := generateECDHEParameters(c.config.rand(), curveID)
+		if err != nil {
+			c.sendAlert(alertInternalError)
+			return err
+		}
+		hs.ecdheParams = params
+		hs.hello.keyShares = []keyShare{{group: curveID, data: params.PublicKey()}}
 	}
 
 	hs.hello.raw = nil
@@ -363,7 +333,7 @@ func (hs *clientHandshakeStateTLS13) processServerHello() error {
 		c.sendAlert(alertIllegalParameter)
 		return errors.New("tls: server did not send a key share")
 	}
-	if hs.serverHello.serverShare.group != clientKeySharePrivateCurveID(hs.keySharePrivate) {
+	if hs.serverHello.serverShare.group != hs.ecdheParams.CurveID() {
 		c.sendAlert(alertIllegalParameter)
 		return errors.New("tls: server selected unsupported group")
 	}
@@ -401,18 +371,7 @@ func (hs *clientHandshakeStateTLS13) processServerHello() error {
 func (hs *clientHandshakeStateTLS13) establishHandshakeKeys() error {
 	c := hs.c
 
-	var sharedKey []byte
-	if params, ok := hs.keySharePrivate.(ecdheParameters); ok {
-		sharedKey = params.SharedKey(hs.serverHello.serverShare.data)
-	} else if sk, ok := hs.keySharePrivate.(circlKem.PrivateKey); ok {
-		var err error
-		sharedKey, err = sk.Scheme().Decapsulate(sk, hs.serverHello.serverShare.data)
-		if err != nil {
-			c.sendAlert(alertIllegalParameter)
-			return fmt.Errorf("%s decaps: %w", sk.Scheme().Name(), err)
-		}
-	}
-
+	sharedKey := hs.ecdheParams.SharedKey(hs.serverHello.serverShare.data)
 	if sharedKey == nil {
 		c.sendAlert(alertIllegalParameter)
 		return errors.New("tls: invalid server key share")

vendor/github.com/quic-go/qtls-go1-19/handshake_server_tls13.go

@@ -11,7 +11,6 @@ import (
 	"crypto/hmac"
 	"crypto/rsa"
 	"errors"
-	"fmt"
 	"hash"
 	"io"
 	"sync/atomic"
@@ -47,8 +46,6 @@ type serverHandshakeStateTLS13 struct {
 func (hs *serverHandshakeStateTLS13) handshake() error {
 	c := hs.c
 
-	startTime := time.Now()
-
 	if needFIPS() {
 		return errors.New("tls: internal error: TLS 1.3 reached in FIPS mode")
 	}
@@ -88,12 +85,6 @@ func (hs *serverHandshakeStateTLS13) handshake() error {
 		return err
 	}
 
-	raiseCFEvent(&cfEventHandshake{
-		serverSide: true,
-		duration:   time.Since(startTime),
-		kex:        hs.hello.serverShare.group,
-	})
-
 	atomic.StoreUint32(&c.handshakeStatus, 1)
 	c.updateConnectionState()
 	return nil
@@ -208,27 +199,17 @@ GroupSelection:
 		clientKeyShare = &hs.clientHello.keyShares[0]
 	}
 
-	if _, ok := curveForCurveID(selectedGroup); selectedGroup != X25519 && curveIdToCirclScheme(selectedGroup) == nil && !ok {
+	if _, ok := curveForCurveID(selectedGroup); selectedGroup != X25519 && !ok {
 		c.sendAlert(alertInternalError)
 		return errors.New("tls: CurvePreferences includes unsupported curve")
 	}
-	if kem := curveIdToCirclScheme(selectedGroup); kem != nil {
-		ct, ss, alert, err := encapsulateForKem(kem, c.config.rand(), clientKeyShare.data)
-		if err != nil {
-			c.sendAlert(alert)
-			return fmt.Errorf("%s encap: %w", kem.Name(), err)
-		}
-		hs.hello.serverShare = keyShare{group: selectedGroup, data: ct}
-		hs.sharedKey = ss
-	} else {
-		params, err := generateECDHEParameters(c.config.rand(), selectedGroup)
-		if err != nil {
-			c.sendAlert(alertInternalError)
-			return err
-		}
-		hs.hello.serverShare = keyShare{group: selectedGroup, data: params.PublicKey()}
-		hs.sharedKey = params.SharedKey(clientKeyShare.data)
+	params, err := generateECDHEParameters(c.config.rand(), selectedGroup)
+	if err != nil {
+		c.sendAlert(alertInternalError)
+		return err
 	}
+	hs.hello.serverShare = keyShare{group: selectedGroup, data: params.PublicKey()}
+	hs.sharedKey = params.SharedKey(clientKeyShare.data)
 	if hs.sharedKey == nil {
 		c.sendAlert(alertIllegalParameter)
 		return errors.New("tls: invalid client key share")
@@ -458,8 +439,6 @@ func (hs *serverHandshakeStateTLS13) sendDummyChangeCipherSpec() error {
 func (hs *serverHandshakeStateTLS13) doHelloRetryRequest(selectedGroup CurveID) error {
 	c := hs.c
 
-	raiseCFEvent(&cfEventHRR{serverSide: true})
-
 	// The first ClientHello gets double-hashed into the transcript upon a
 	// HelloRetryRequest. See RFC 8446, Section 4.4.1.
 	if err := transcriptMsg(hs.clientHello, hs.transcript); err != nil {

vendor/github.com/quic-go/qtls-go1-19/key_agreement.go

@@ -168,7 +168,7 @@ type ecdheKeyAgreement struct {
 func (ka *ecdheKeyAgreement) generateServerKeyExchange(config *config, cert *Certificate, clientHello *clientHelloMsg, hello *serverHelloMsg) (*serverKeyExchangeMsg, error) {
 	var curveID CurveID
 	for _, c := range clientHello.supportedCurves {
-		if config.supportsCurve(c) && curveIdToCirclScheme(c) == nil {
+		if config.supportsCurve(c) {
 			curveID = c
 			break
 		}

vendor/modules.txt

@@ -258,7 +258,7 @@ github.com/prometheus/common/model
 github.com/prometheus/procfs
 github.com/prometheus/procfs/internal/fs
 github.com/prometheus/procfs/internal/util
-# github.com/quic-go/qtls-go1-19 v0.3.2 => github.com/cloudflare/qtls-pq v0.0.0-20230320123031-3faac1a945b2
+# github.com/quic-go/qtls-go1-19 v0.3.2
 ## explicit; go 1.19
 github.com/quic-go/qtls-go1-19
 # github.com/quic-go/qtls-go1-20 v0.2.2 => github.com/cloudflare/qtls-pq v0.0.0-20230320122459-4ed280d0d633
@@ -577,5 +577,4 @@ zombiezen.com/go/capnproto2/std/capnp/rpc
 # github.com/prometheus/golang_client => github.com/prometheus/golang_client v1.12.1
 # gopkg.in/yaml.v3 => gopkg.in/yaml.v3 v3.0.1
 # github.com/quic-go/quic-go => github.com/devincarr/quic-go v0.0.0-20230502200822-d1f4edacbee7
-# github.com/quic-go/qtls-go1-19 => github.com/cloudflare/qtls-pq v0.0.0-20230320123031-3faac1a945b2
 # github.com/quic-go/qtls-go1-20 => github.com/cloudflare/qtls-pq v0.0.0-20230320122459-4ed280d0d633

watcher/file_test.go

@@ -1,5 +1,4 @@
 //go:build !windows
-// +build !windows
 
 package watcher