Release 2022.12.0

This PR does two things:
It changes how we fallback to a lower protocol: The current state
is to try connecting with a protocol. If it fails, fall back to a
lower protocol. And try connecting with that and so on. With this PR,
if we fail to connect with a protocol, we will try to connect to other
edge addresses first. Only if we fail to connect to those will we
fall back to a lower protocol.
It fixes a behaviour where if we fail to connect to an edge addr,
we keep re-trying the same address over and over again.
This PR now switches between edge addresses on subsequent connecton attempts.
Note that through these switches, it still respects the backoff time.
(We are connecting to a different edge, but this helps to not bombard an edge
address with connect requests if a particular edge addresses stops working).

.teamcity/build-macos.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set -exo pipefail
+
 if [[ "$(uname)" != "Darwin" ]] ; then
     echo "This should be run on macOS"
     exit 1
@@ -33,7 +35,9 @@ if [[ ! -z "$CFD_CODE_SIGN_KEY" ]]; then
   if [[ ! -z "$CFD_CODE_SIGN_PASS" ]]; then
     # write private key to disk and then import it keychain
     echo -n -e ${CFD_CODE_SIGN_KEY} | base64 -D > ${CODE_SIGN_PRIV}
-    out=$(security import ${CODE_SIGN_PRIV} -A -P "${CFD_CODE_SIGN_PASS}" 2>&1)
+    # we set || true  here and for every `security import invoke` because the  "duplicate SecKeychainItemImport" error 
+    # will cause set -e to exit 1. It is okay we do this because we deliberately handle this error in the lines below.
+    out=$(security import ${CODE_SIGN_PRIV} -A -P "${CFD_CODE_SIGN_PASS}" 2>&1) || true 
     exitcode=$?
     if [ -n "$out" ]; then
       if [ $exitcode -eq 0 ]; then
@@ -53,7 +57,7 @@ fi
 if [[ ! -z "$CFD_CODE_SIGN_CERT" ]]; then
   # write certificate to disk and then import it keychain
   echo -n -e ${CFD_CODE_SIGN_CERT} | base64 -D > ${CODE_SIGN_CERT}
-  out1=$(security import ${CODE_SIGN_CERT} -A 2>&1)
+  out1=$(security import ${CODE_SIGN_CERT} -A 2>&1) || true
   exitcode1=$?
   if [ -n "$out1" ]; then
     if [ $exitcode1 -eq 0 ]; then
@@ -75,7 +79,7 @@ if [[ ! -z "$CFD_INSTALLER_KEY" ]]; then
   if [[ ! -z "$CFD_INSTALLER_PASS" ]]; then
     # write private key to disk and then import it into the keychain
     echo -n -e ${CFD_INSTALLER_KEY} | base64 -D > ${INSTALLER_PRIV}
-    out2=$(security import ${INSTALLER_PRIV} -A -P "${CFD_INSTALLER_PASS}" 2>&1)
+    out2=$(security import ${INSTALLER_PRIV} -A -P "${CFD_INSTALLER_PASS}" 2>&1) || true
     exitcode2=$?
     if [ -n "$out2" ]; then
       if [ $exitcode2 -eq 0 ]; then
@@ -95,7 +99,7 @@ fi
 if [[ ! -z "$CFD_INSTALLER_CERT" ]]; then
   # write certificate to disk and then import it keychain
   echo -n -e ${CFD_INSTALLER_CERT} | base64 -D > ${INSTALLER_CERT}
-  out3=$(security import ${INSTALLER_CERT} -A 2>&1)
+  out3=$(security import ${INSTALLER_CERT} -A 2>&1) || true
   exitcode3=$?
   if [ -n "$out3" ]; then
     if [ $exitcode3 -eq 0 ]; then
@@ -138,14 +142,10 @@ fi
 if [[ ! -z "$CODE_SIGN_NAME" ]]; then
   codesign -s "${CODE_SIGN_NAME}" -f -v --timestamp --options runtime ${BINARY_NAME}
   
-  # notarize the binary
-  if [[ ! -z "$CFD_NOTE_PASSWORD" ]]; then
-    zip "${BINARY_NAME}.zip" ${BINARY_NAME} 
-    xcrun altool --notarize-app -f "${BINARY_NAME}.zip" -t osx -u ${CFD_NOTE_USERNAME} -p ${CFD_NOTE_PASSWORD} --primary-bundle-id ${BUNDLE_ID}
-  fi
+ # notarize the binary
+ # TODO: https://jira.cfdata.org/browse/TUN-5789
 fi
 
-
 # creating build directory
 rm -rf $TARGET_DIRECTORY
 mkdir "${TARGET_DIRECTORY}"
@@ -169,10 +169,7 @@ if [[ ! -z "$PKG_SIGN_NAME" ]]; then
       ${PKGNAME}
 
       # notarize the package
-      if [[ ! -z "$CFD_NOTE_PASSWORD" ]]; then
-        xcrun altool --notarize-app -f ${PKGNAME} -t osx -u ${CFD_NOTE_USERNAME} -p ${CFD_NOTE_PASSWORD} --primary-bundle-id ${BUNDLE_ID}
-        xcrun stapler staple ${PKGNAME}
-      fi
+      # TODO: https://jira.cfdata.org/browse/TUN-5789
 else
     pkgbuild --identifier com.cloudflare.${PRODUCT} \
       --version ${VERSION} \

RELEASE_NOTES

@@ -1,3 +1,22 @@
+2022.12.0
+- 2022-12-14 TUN-6999: cloudflared should attempt other edge addresses before falling back on protocol
+- 2022-12-13 TUN-7004: Dont show local config dirs for remotely configured tuns
+- 2022-12-12 TUN-7003: Tempoarily disable erroneous notarize-app
+- 2022-12-12 TUN-7003: Add back a missing fi
+- 2022-12-07 TUN-7000: Reduce metric cardinality of closedConnections metric by removing error as tag
+- 2022-12-07 TUN-6994: Improve logging config file not found
+- 2022-12-07 TUN-7002: Randomise first region selection
+- 2022-12-07 TUN-6995: Disable quick-tunnels spin up by default
+- 2022-12-05 TUN-6984: Add bash set x to improve visibility during builds
+- 2022-12-05 TUN-6984: [CI] Ignore security import errors for code_sigining
+- 2022-12-05 TUN-6984: [CI] Don't fail on unset.
+- 2022-11-30 TUN-6984: Set euo pipefile for homebrew builds
+
+2022.11.1
+- 2022-11-29 TUN-6981: We should close UDP socket if failed to connecto to edge
+- 2022-11-25 CUSTESC-23757: Fix a bug where a wildcard ingress rule would match an host without starting with a dot
+- 2022-11-24 TUN-6970: Print newline when printing tunnel token
+- 2022-11-22 TUN-6963: Refactor Metrics service setup
 2022.11.0
 - 2022-11-16 Revert "TUN-6935: Cloudflared should use APIToken instead of serviceKey"
 - 2022-11-16 TUN-6929: Use same protocol for other connections as first one

cmd/cloudflared/proxydns/cmd.go

@@ -1,6 +1,7 @@
 package proxydns
 
 import (
+	"context"
 	"net"
 	"os"
 	"os/signal"
@@ -73,7 +74,7 @@ func Run(c *cli.Context) error {
 		log.Fatal().Err(err).Msg("Failed to open the metrics listener")
 	}
 
-	go metrics.ServeMetrics(metricsListener, nil, nil, "", nil, log)
+	go metrics.ServeMetrics(metricsListener, context.Background(), metrics.Config{}, log)
 
 	listener, err := tunneldns.CreateListener(
 		c.String("address"),

cmd/cloudflared/tunnel/cmd.go

@@ -82,6 +82,15 @@ const (
 	LogFieldPIDPathname         = "pidPathname"
 	LogFieldTmpTraceFilename    = "tmpTraceFilename"
 	LogFieldTraceOutputFilepath = "traceOutputFilepath"
+
+	tunnelCmdErrorMessage = `You did not specify any valid additional argument to the cloudflared tunnel command. 
+
+If you are trying to run a Quick Tunnel then you need to explicitly pass the --url flag. 
+Eg. cloudflared tunnel --url localhost:8080/. 
+
+Please note that Quick Tunnels are meant to be ephemeral and should only be used for testing purposes. 
+For production usage, we recommend creating Named Tunnels. (https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/install-and-setup/tunnel-guide/)
+`
 )
 
 var (
@@ -166,21 +175,27 @@ func TunnelCommand(c *cli.Context) error {
 	if err != nil {
 		return err
 	}
+
 	if name := c.String("name"); name != "" { // Start a named tunnel
 		return runAdhocNamedTunnel(sc, name, c.String(CredFileFlag))
 	}
+
+	// Unauthenticated named tunnel on <random>.<quick-tunnels-service>.com
+	shouldRunQuickTunnel := c.IsSet("url") || c.IsSet("hello-world")
+	if !dnsProxyStandAlone(c, nil) && c.String("hostname") == "" && c.String("quick-service") != "" && shouldRunQuickTunnel {
+		return RunQuickTunnel(sc)
+	}
+
 	if ref := config.GetConfiguration().TunnelID; ref != "" {
 		return fmt.Errorf("Use `cloudflared tunnel run` to start tunnel %s", ref)
 	}
 
-	// Unauthenticated named tunnel on <random>.<quick-tunnels-service>.com
-	// For now, default to legacy setup unless quick-service is specified
-	if !dnsProxyStandAlone(c, nil) && c.String("hostname") == "" && c.String("quick-service") != "" {
-		return RunQuickTunnel(sc)
+	// Start a classic tunnel if hostname is specified.
+	if c.String("hostname") != "" {
+		return runClassicTunnel(sc)
 	}
 
-	// Start a classic tunnel
-	return runClassicTunnel(sc)
+	return errors.New(tunnelCmdErrorMessage)
 }
 
 func Init(info *cliutil.BuildInfo, gracefulShutdown chan struct{}) {
@@ -241,7 +256,8 @@ func StartServer(
 	listeners := gracenet.Net{}
 	errC := make(chan error)
 
-	if config.GetConfiguration().Source() == "" {
+	// Only log for locally configured tunnels (Token is blank).
+	if config.GetConfiguration().Source() == "" && c.String(TunnelTokenFlag) == "" {
 		log.Info().Msg(config.ErrNoConfigFile.Error())
 	}
 
@@ -372,7 +388,12 @@ func StartServer(
 		defer wg.Done()
 		readinessServer := metrics.NewReadyServer(log, clientID)
 		observer.RegisterSink(readinessServer)
-		errC <- metrics.ServeMetrics(metricsListener, ctx.Done(), readinessServer, quickTunnelURL, orchestrator, log)
+		metricsConfig := metrics.Config{
+			ReadyServer:         readinessServer,
+			QuickTunnelHostname: quickTunnelURL,
+			Orchestrator:        orchestrator,
+		}
+		errC <- metrics.ServeMetrics(metricsListener, ctx, metricsConfig, log)
 	}()
 
 	reconnectCh := make(chan supervisor.ReconnectSignal, c.Int("ha-connections"))
@@ -591,6 +612,12 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			Value:  5,
 			Hidden: true,
 		}),
+		altsrc.NewIntFlag(&cli.IntFlag{
+			Name:   "max-edge-addr-retries",
+			Usage:  "Maximum number of times to retry on edge addrs before falling back to a lower protocol",
+			Value:  8,
+			Hidden: true,
+		}),
 		// Note TUN-3758 , we use Int because UInt is not supported with altsrc
 		altsrc.NewIntFlag(&cli.IntFlag{
 			Name:    "retries",

cmd/cloudflared/tunnel/configuration.go

@@ -376,15 +376,16 @@ func prepareTunnelConfig(
 		Observer:        observer,
 		ReportedVersion: info.Version(),
 		// Note TUN-3758 , we use Int because UInt is not supported with altsrc
-		Retries:          uint(c.Int("retries")),
-		RunFromTerminal:  isRunningFromTerminal(),
-		NamedTunnel:      namedTunnel,
-		ClassicTunnel:    classicTunnel,
-		MuxerConfig:      muxerConfig,
-		ProtocolSelector: protocolSelector,
-		EdgeTLSConfigs:   edgeTLSConfigs,
-		NeedPQ:           needPQ,
-		PQKexIdx:         pqKexIdx,
+		Retries:            uint(c.Int("retries")),
+		RunFromTerminal:    isRunningFromTerminal(),
+		NamedTunnel:        namedTunnel,
+		ClassicTunnel:      classicTunnel,
+		MuxerConfig:        muxerConfig,
+		ProtocolSelector:   protocolSelector,
+		EdgeTLSConfigs:     edgeTLSConfigs,
+		NeedPQ:             needPQ,
+		PQKexIdx:           pqKexIdx,
+		MaxEdgeAddrRetries: uint8(c.Int("max-edge-addr-retries")),
 	}
 	packetConfig, err := newPacketConfig(c, log)
 	if err != nil {

cmd/cloudflared/tunnel/subcommands.go

@@ -783,7 +783,7 @@ func tokenCommand(c *cli.Context) error {
 		return err
 	}
 
-	fmt.Printf("%s", encodedToken)
+	fmt.Println(encodedToken)
 	return nil
 }
 

config/configuration.go

@@ -391,7 +391,8 @@ func ReadConfigFile(c *cli.Context, log *zerolog.Logger) (settings *configFileSe
 	log.Debug().Msgf("Loading configuration from %s", configFile)
 	file, err := os.Open(configFile)
 	if err != nil {
-		if os.IsNotExist(err) {
+		// If does not exist and config file was not specificly specified then return ErrNoConfigFile found.
+		if os.IsNotExist(err) && !c.IsSet("config") {
 			err = ErrNoConfigFile
 		}
 		return nil, "", err

connection/quic.go

@@ -81,6 +81,8 @@ func NewQUICConnection(
 
 	session, err := quic.Dial(udpConn, edgeAddr, edgeAddr.String(), tlsConfig, quicConfig)
 	if err != nil {
+		// close the udp server socket in case of error connecting to the edge
+		udpConn.Close()
 		return nil, &EdgeQuicDialError{Cause: err}
 	}
 

edgediscovery/allregions/regions.go

@@ -2,6 +2,7 @@ package allregions
 
 import (
 	"fmt"
+	"math/rand"
 
 	"github.com/rs/zerolog"
 )
@@ -85,6 +86,15 @@ func (rs *Regions) AddrUsedBy(connID int) *EdgeAddr {
 // GetUnusedAddr gets an unused addr from the edge, excluding the given addr. Prefer to use addresses
 // evenly across both regions.
 func (rs *Regions) GetUnusedAddr(excluding *EdgeAddr, connID int) *EdgeAddr {
+	// If both regions have the same number of available addrs, lets randomise which one
+	// we pick. The rest of this algorithm will continue to make sure we always use addresses
+	// evenly across both regions.
+	if rs.region1.AvailableAddrs() == rs.region2.AvailableAddrs() {
+		regions := []Region{rs.region1, rs.region2}
+		firstChoice := rand.Intn(2)
+		return getAddrs(excluding, connID, &regions[firstChoice], &regions[1-firstChoice])
+	}
+
 	if rs.region1.AvailableAddrs() > rs.region2.AvailableAddrs() {
 		return getAddrs(excluding, connID, &rs.region1, &rs.region2)
 	}

ingress/ingress.go

@@ -58,7 +58,7 @@ func matchHost(ruleHost, reqHost string) bool {
 
 	// Validate hostnames that use wildcards at the start
 	if strings.HasPrefix(ruleHost, "*.") {
-		toMatch := strings.TrimPrefix(ruleHost, "*.")
+		toMatch := strings.TrimPrefix(ruleHost, "*")
 		return strings.HasSuffix(reqHost, toMatch)
 	}
 	return false

ingress/rule_test.go

@@ -148,6 +148,16 @@ func Test_rule_matches(t *testing.T) {
 			},
 			want: true,
 		},
+		{
+			name: "Hostname with wildcard should not match if no dot present",
+			rule: Rule{
+				Hostname: "*.api.abc.cloud",
+			},
+			args: args{
+				requestURL: MustParseURL(t, "https://testing-api.abc.cloud"),
+			},
+			want: false,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

metrics/metrics.go

@@ -18,18 +18,24 @@ import (
 )
 
 const (
-	shutdownTimeout = time.Second * 15
-	startupTime     = time.Millisecond * 500
+	startupTime            = time.Millisecond * 500
+	defaultShutdownTimeout = time.Second * 15
 )
 
+type Config struct {
+	ReadyServer         *ReadyServer
+	QuickTunnelHostname string
+	Orchestrator        orchestrator
+
+	ShutdownTimeout time.Duration
+}
+
 type orchestrator interface {
 	GetVersionedConfigJSON() ([]byte, error)
 }
 
 func newMetricsHandler(
-	readyServer *ReadyServer,
-	quickTunnelHostname string,
-	orchestrator orchestrator,
+	config Config,
 	log *zerolog.Logger,
 ) *mux.Router {
 	router := mux.NewRouter()
@@ -39,15 +45,15 @@ func newMetricsHandler(
 	router.HandleFunc("/healthcheck", func(w http.ResponseWriter, r *http.Request) {
 		_, _ = fmt.Fprintf(w, "OK\n")
 	})
-	if readyServer != nil {
-		router.Handle("/ready", readyServer)
+	if config.ReadyServer != nil {
+		router.Handle("/ready", config.ReadyServer)
 	}
 	router.HandleFunc("/quicktunnel", func(w http.ResponseWriter, r *http.Request) {
-		_, _ = fmt.Fprintf(w, `{"hostname":"%s"}`, quickTunnelHostname)
+		_, _ = fmt.Fprintf(w, `{"hostname":"%s"}`, config.QuickTunnelHostname)
 	})
-	if orchestrator != nil {
+	if config.Orchestrator != nil {
 		router.HandleFunc("/config", func(w http.ResponseWriter, r *http.Request) {
-			json, err := orchestrator.GetVersionedConfigJSON()
+			json, err := config.Orchestrator.GetVersionedConfigJSON()
 			if err != nil {
 				w.WriteHeader(500)
 				_, _ = fmt.Fprintf(w, "ERR: %v", err)
@@ -63,10 +69,8 @@ func newMetricsHandler(
 
 func ServeMetrics(
 	l net.Listener,
-	shutdownC <-chan struct{},
-	readyServer *ReadyServer,
-	quickTunnelHostname string,
-	orchestrator orchestrator,
+	ctx context.Context,
+	config Config,
 	log *zerolog.Logger,
 ) (err error) {
 	var wg sync.WaitGroup
@@ -74,7 +78,7 @@ func ServeMetrics(
 	trace.AuthRequest = func(*http.Request) (bool, bool) { return true, true }
 	// TODO: parameterize ReadTimeout and WriteTimeout. The maximum time we can
 	// profile CPU usage depends on WriteTimeout
-	h := newMetricsHandler(readyServer, quickTunnelHostname, orchestrator, log)
+	h := newMetricsHandler(config, log)
 	server := &http.Server{
 		ReadTimeout:  10 * time.Second,
 		WriteTimeout: 10 * time.Second,
@@ -91,7 +95,12 @@ func ServeMetrics(
 	// fully started up. So add artificial delay.
 	time.Sleep(startupTime)
 
-	<-shutdownC
+	<-ctx.Done()
+	shutdownTimeout := config.ShutdownTimeout
+	if shutdownTimeout == 0 {
+		shutdownTimeout = defaultShutdownTimeout
+	}
+
 	ctx, cancel := context.WithTimeout(context.Background(), shutdownTimeout)
 	_ = server.Shutdown(ctx)
 	cancel()

quic/metrics.go

@@ -15,7 +15,7 @@ var (
 	clientConnLabels = []string{"conn_index"}
 	clientMetrics    = struct {
 		totalConnections  prometheus.Counter
-		closedConnections *prometheus.CounterVec
+		closedConnections prometheus.Counter
 		sentPackets       *prometheus.CounterVec
 		sentBytes         *prometheus.CounterVec
 		receivePackets    *prometheus.CounterVec
@@ -30,9 +30,8 @@ var (
 		totalConnections: prometheus.NewCounter(
 			totalConnectionsOpts(logging.PerspectiveClient),
 		),
-		closedConnections: prometheus.NewCounterVec(
+		closedConnections: prometheus.NewCounter(
 			closedConnectionsOpts(logging.PerspectiveClient),
-			[]string{"error"},
 		),
 		sentPackets: prometheus.NewCounterVec(
 			sentPacketsOpts(logging.PerspectiveClient),
@@ -284,7 +283,7 @@ func (cc *clientCollector) startedConnection() {
 }
 
 func (cc *clientCollector) closedConnection(err error) {
-	clientMetrics.closedConnections.WithLabelValues(err.Error()).Inc()
+	clientMetrics.closedConnections.Inc()
 }
 
 func (cc *clientCollector) sentPackets(size logging.ByteCount) {

supervisor/supervisor.go

@@ -14,7 +14,6 @@ import (
 
 	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/edgediscovery"
-	"github.com/cloudflare/cloudflared/edgediscovery/allregions"
 	"github.com/cloudflare/cloudflared/h2mux"
 	"github.com/cloudflare/cloudflared/orchestration"
 	"github.com/cloudflare/cloudflared/retry"
@@ -94,14 +93,7 @@ func NewSupervisor(config *TunnelConfig, orchestrator *orchestration.Orchestrato
 	tracker := tunnelstate.NewConnTracker(config.Log)
 	log := NewConnAwareLogger(config.Log, tracker, config.Observer)
 
-	var edgeAddrHandler EdgeAddrHandler
-	if isStaticEdge { // static edge addresses
-		edgeAddrHandler = &IPAddrFallback{}
-	} else if config.EdgeIPVersion == allregions.IPv6Only || config.EdgeIPVersion == allregions.Auto {
-		edgeAddrHandler = &IPAddrFallback{}
-	} else { // IPv4Only
-		edgeAddrHandler = &DefaultAddrFallback{}
-	}
+	edgeAddrHandler := NewIPAddrFallback(config.MaxEdgeAddrRetries)
 
 	edgeTunnelServer := EdgeTunnelServer{
 		config:            config,

supervisor/tunnel.go

@@ -41,25 +41,26 @@ const (
 )
 
 type TunnelConfig struct {
-	GracePeriod     time.Duration
-	ReplaceExisting bool
-	OSArch          string
-	ClientID        string
-	CloseConnOnce   *sync.Once // Used to close connectedSignal no more than once
-	EdgeAddrs       []string
-	Region          string
-	EdgeIPVersion   allregions.ConfigIPVersion
-	HAConnections   int
-	IncidentLookup  IncidentLookup
-	IsAutoupdated   bool
-	LBPool          string
-	Tags            []tunnelpogs.Tag
-	Log             *zerolog.Logger
-	LogTransport    *zerolog.Logger
-	Observer        *connection.Observer
-	ReportedVersion string
-	Retries         uint
-	RunFromTerminal bool
+	GracePeriod        time.Duration
+	ReplaceExisting    bool
+	OSArch             string
+	ClientID           string
+	CloseConnOnce      *sync.Once // Used to close connectedSignal no more than once
+	EdgeAddrs          []string
+	Region             string
+	EdgeIPVersion      allregions.ConfigIPVersion
+	HAConnections      int
+	IncidentLookup     IncidentLookup
+	IsAutoupdated      bool
+	LBPool             string
+	Tags               []tunnelpogs.Tag
+	Log                *zerolog.Logger
+	LogTransport       *zerolog.Logger
+	Observer           *connection.Observer
+	ReportedVersion    string
+	Retries            uint
+	MaxEdgeAddrRetries uint8
+	RunFromTerminal    bool
 
 	NeedPQ bool
 
@@ -133,6 +134,24 @@ func StartTunnelDaemon(
 	return s.Run(ctx, connectedSignal)
 }
 
+type ConnectivityError struct {
+	reachedMaxRetries bool
+}
+
+func NewConnectivityError(hasReachedMaxRetries bool) *ConnectivityError {
+	return &ConnectivityError{
+		reachedMaxRetries: hasReachedMaxRetries,
+	}
+}
+
+func (e *ConnectivityError) Error() string {
+	return fmt.Sprintf("connectivity error - reached max retries: %t", e.HasReachedMaxRetries())
+}
+
+func (e *ConnectivityError) HasReachedMaxRetries() bool {
+	return e.reachedMaxRetries
+}
+
 // EdgeAddrHandler provides a mechanism switch between behaviors in ServeTunnel
 // for handling the errors when attempting to make edge connections.
 type EdgeAddrHandler interface {
@@ -140,56 +159,47 @@ type EdgeAddrHandler interface {
 	// the edge address should be replaced with a new one. Also, will return if the
 	// error should be recognized as a connectivity error, or otherwise, a general
 	// application error.
-	ShouldGetNewAddress(err error) (needsNewAddress bool, isConnectivityError bool)
+	ShouldGetNewAddress(connIndex uint8, err error) (needsNewAddress bool, connectivityError error)
 }
 
-// DefaultAddrFallback will always return false for isConnectivityError since this
-// handler is a way to provide the legacy behavior in the new edge discovery algorithm.
-type DefaultAddrFallback struct {
-	edgeErrors int
-}
-
-func (f DefaultAddrFallback) ShouldGetNewAddress(err error) (needsNewAddress bool, isConnectivityError bool) {
-	switch err.(type) {
-	case nil: // maintain current IP address
-	// DupConnRegisterTunnelError should indicate to get a new address immediately
-	case connection.DupConnRegisterTunnelError:
-		return true, false
-	// Try the next address if it was a quic.IdleTimeoutError
-	case *quic.IdleTimeoutError,
-		edgediscovery.DialError,
-		*connection.EdgeQuicDialError:
-		// Wait for two failures before falling back to a new address
-		f.edgeErrors++
-		if f.edgeErrors >= 2 {
-			f.edgeErrors = 0
-			return true, false
-		}
-	default: // maintain current IP address
+func NewIPAddrFallback(maxRetries uint8) *ipAddrFallback {
+	return &ipAddrFallback{
+		retriesByConnIndex: make(map[uint8]uint8),
+		maxRetries:         maxRetries,
 	}
-	return false, false
 }
 
-// IPAddrFallback will have more conditions to fall back to a new address for certain
+// ipAddrFallback will have more conditions to fall back to a new address for certain
 // edge connection errors. This means that this handler will return true for isConnectivityError
 // for more cases like duplicate connection register and edge quic dial errors.
-type IPAddrFallback struct{}
+type ipAddrFallback struct {
+	m                  sync.Mutex
+	retriesByConnIndex map[uint8]uint8
+	maxRetries         uint8
+}
 
-func (f IPAddrFallback) ShouldGetNewAddress(err error) (needsNewAddress bool, isConnectivityError bool) {
+func (f *ipAddrFallback) ShouldGetNewAddress(connIndex uint8, err error) (needsNewAddress bool, connectivityError error) {
+	f.m.Lock()
+	defer f.m.Unlock()
 	switch err.(type) {
 	case nil: // maintain current IP address
 	// Try the next address if it was a quic.IdleTimeoutError
 	// DupConnRegisterTunnelError needs to also receive a new ip address
 	case connection.DupConnRegisterTunnelError,
 		*quic.IdleTimeoutError:
-		return true, false
+		return true, nil
 	// Network problems should be retried with new address immediately and report
 	// as connectivity error
 	case edgediscovery.DialError, *connection.EdgeQuicDialError:
-		return true, true
+		if f.retriesByConnIndex[connIndex] >= f.maxRetries {
+			f.retriesByConnIndex[connIndex] = 0
+			return true, NewConnectivityError(true)
+		}
+		f.retriesByConnIndex[connIndex]++
+		return true, NewConnectivityError(false)
 	default: // maintain current IP address
 	}
-	return false, false
+	return false, nil
 }
 
 type EdgeTunnelServer struct {
@@ -238,11 +248,12 @@ func (e *EdgeTunnelServer) Serve(ctx context.Context, connIndex uint8, protocolF
 		Uint8(connection.LogFieldConnIndex, connIndex).
 		Logger()
 	connLog := e.connAwareLogger.ReplaceLogger(&logger)
+
 	// Each connection to keep its own copy of protocol, because individual connections might fallback
 	// to another protocol when a particular metal doesn't support new protocol
 	// Each connection can also have it's own IP version because individual connections might fallback
 	// to another IP version.
-	err, recoverable := e.serveTunnel(
+	err, shouldFallbackProtocol := e.serveTunnel(
 		ctx,
 		connLog,
 		addr,
@@ -252,34 +263,39 @@ func (e *EdgeTunnelServer) Serve(ctx context.Context, connIndex uint8, protocolF
 		protocolFallback.protocol,
 	)
 
-	// If the connection is recoverable, we want to maintain the same IP
-	// but backoff a reconnect with some duration.
-	if recoverable {
-		duration, ok := protocolFallback.GetMaxBackoffDuration(ctx)
-		if !ok {
-			return err
-		}
-
-		e.config.Observer.SendReconnect(connIndex)
-		connLog.Logger().Info().Msgf("Retrying connection in up to %s", duration)
-	}
-
 	// Check if the connection error was from an IP issue with the host or
 	// establishing a connection to the edge and if so, rotate the IP address.
-	yes, hasConnectivityError := e.edgeAddrHandler.ShouldGetNewAddress(err)
-	if yes {
-		if _, err := e.edgeAddrs.GetDifferentAddr(int(connIndex), hasConnectivityError); err != nil {
+	shouldRotateEdgeIP, cErr := e.edgeAddrHandler.ShouldGetNewAddress(connIndex, err)
+	if shouldRotateEdgeIP {
+		// rotate IP, but forcing internal state to assign a new IP to connection index.
+		if _, err := e.edgeAddrs.GetDifferentAddr(int(connIndex), true); err != nil {
 			return err
 		}
+
+		// In addition, if it is a connectivity error, and we have exhausted the configurable maximum edge IPs to rotate,
+		// then just fallback protocol on next iteration run.
+		connectivityErr, ok := cErr.(*ConnectivityError)
+		if ok {
+			shouldFallbackProtocol = connectivityErr.HasReachedMaxRetries()
+		}
 	}
 
+	// set connection has re-connecting and log the next retrying backoff
+	duration, ok := protocolFallback.GetMaxBackoffDuration(ctx)
+	if !ok {
+		return err
+	}
+	e.config.Observer.SendReconnect(connIndex)
+	connLog.Logger().Info().Msgf("Retrying connection in up to %s", duration)
+
 	select {
 	case <-ctx.Done():
 		return ctx.Err()
 	case <-e.gracefulShutdownC:
 		return nil
 	case <-protocolFallback.BackoffTimer():
-		if !recoverable {
+		// should we fallback protocol? If not, just return. Otherwise, set new protocol for next method call.
+		if !shouldFallbackProtocol {
 			return err
 		}
 
@@ -426,7 +442,7 @@ func (e *EdgeTunnelServer) serveTunnel(
 			}
 			return err.Cause, !err.Permanent
 		case *connection.EdgeQuicDialError:
-			return err, true
+			return err, false
 		case ReconnectSignal:
 			connLog.Logger().Info().
 				IPAddr(connection.LogFieldIPAddress, addr.UDP.IP).