Release 2021.1.5

This change is focused on fixing rotating loggers in Windows
where it was failing due to Windows file semantics disallowing
the rotation while that file was still being open (because we
had multiple lumberjacks pointing to the same file).

This is fixed by ensuring the initialization happens only once.

RELEASE_NOTES

@@ -1,3 +1,18 @@
+2021.1.5
+- 2021-01-15 TUN-3594: Log ingress response at debug level
+- 2021-01-15 TUN-3765: Fix doubly nested log output by `logfile` option
+- 2021-01-16 TUN-3767: Tolerate logging errors
+- 2021-01-17 TUN-3768: Reuse file loggers
+- 2021-01-14 TUN-3738: Refactor observer to avoid potential of blocking on tunnel notifications
+- 2021-01-15 TUN-3766: Print flags defined at all levels of command hierarchy, not just locally defined flags for a command. This fixes output of overriden settings for  subcommand.
+
+2021.1.4
+- 2021-01-14 TUN-3759: Single file logging output should always append
+
+2021.1.3
+- 2021-01-14 TUN-3756: File logging output must consider the directory
+- 2021-01-14 TUN-3757: Fix legacy Uint flags that are incorrectly handled by ufarve library
+
 2021.1.2
 - 2021-01-13 TUN-3747: Fix logging in Windows
 

cmd/cloudflared/tunnel/cmd.go

@@ -328,13 +328,9 @@ func StartServer(
 
 	transportLog := logger.CreateTransportLoggerFromContext(c, isUIEnabled)
 
-	readinessCh := make(chan connection.Event, 16)
-	uiCh := make(chan connection.Event, 16)
-	eventChannels := []chan connection.Event{
-		readinessCh,
-		uiCh,
-	}
-	tunnelConfig, ingressRules, err := prepareTunnelConfig(c, buildInfo, version, log, transportLog, namedTunnel, isUIEnabled, eventChannels)
+	observer := connection.NewObserver(log, isUIEnabled)
+
+	tunnelConfig, ingressRules, err := prepareTunnelConfig(c, buildInfo, version, log, observer, namedTunnel)
 	if err != nil {
 		log.Err(err).Msg("Couldn't start tunnel")
 		return err
@@ -349,7 +345,9 @@ func StartServer(
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
-		errC <- metrics.ServeMetrics(metricsListener, shutdownC, readinessCh, log)
+		readinessServer := metrics.NewReadyServer(log)
+		observer.RegisterSink(readinessServer)
+		errC <- metrics.ServeMetrics(metricsListener, shutdownC, readinessServer, log)
 	}()
 
 	if err := ingressRules.StartOrigins(&wg, log, shutdownC, errC); err != nil {
@@ -369,20 +367,15 @@ func StartServer(
 	}()
 
 	if isUIEnabled {
-		tunnelInfo := ui.NewUIModel(
+		tunnelUI := ui.NewUIModel(
 			version,
 			hostname,
 			metricsListener.Addr().String(),
 			&ingressRules,
 			tunnelConfig.HAConnections,
 		)
-		tunnelInfo.LaunchUI(ctx, log, transportLog, uiCh)
-	} else {
-		go func() {
-			for range uiCh {
-				// Consume UI events into a noop
-			}
-		}()
+		app := tunnelUI.Launch(ctx, log, transportLog)
+		observer.RegisterSink(app)
 	}
 
 	return waitToShutdown(&wg, errC, shutdownC, graceShutdownC, c.Duration("grace-period"), log)
@@ -622,13 +615,15 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			Value:  time.Second * 5,
 			Hidden: true,
 		}),
-		altsrc.NewUint64Flag(&cli.Uint64Flag{
+		// Note TUN-3758 , we use Int because UInt is not supported with altsrc
+		altsrc.NewIntFlag(&cli.IntFlag{
 			Name:   "heartbeat-count",
 			Usage:  "Minimum number of unacked heartbeats to send before closing the connection.",
 			Value:  5,
 			Hidden: true,
 		}),
-		altsrc.NewUintFlag(&cli.UintFlag{
+		// Note TUN-3758 , we use Int because UInt is not supported with altsrc
+		altsrc.NewIntFlag(&cli.IntFlag{
 			Name:    "retries",
 			Value:   5,
 			Usage:   "Maximum number of retries for connection/protocol errors.",
@@ -647,7 +642,8 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			EnvVars: []string{"TUNNEL_GRACE_PERIOD"},
 			Hidden:  true,
 		}),
-		altsrc.NewUintFlag(&cli.UintFlag{
+		// Note TUN-3758 , we use Int because UInt is not supported with altsrc
+		altsrc.NewIntFlag(&cli.IntFlag{
 			Name:    "compression-quality",
 			Value:   0,
 			Usage:   "(beta) Use cross-stream compression instead HTTP compression. 0-off, 1-low, 2-medium, >=3-high.",
@@ -932,6 +928,7 @@ func sshFlags(shouldHide bool) []cli.Flag {
 			EnvVars: []string{"TUNNEL_PROXY_ADDRESS"},
 			Hidden:  shouldHide,
 		}),
+		// Note TUN-3758 , we use Int because UInt is not supported with altsrc
 		altsrc.NewIntFlag(&cli.IntFlag{
 			Name:    ingress.ProxyPortFlag,
 			Usage:   "Listen port for the proxy.",

cmd/cloudflared/tunnel/configuration.go

@@ -61,19 +61,12 @@ func generateRandomClientID(log *zerolog.Logger) (string, error) {
 
 func logClientOptions(c *cli.Context, log *zerolog.Logger) {
 	flags := make(map[string]interface{})
-	for _, flag := range c.LocalFlagNames() {
+	for _, flag := range c.FlagNames() {
 		flags[flag] = c.Generic(flag)
 	}
 
-	sliceFlags := []string{"header", "tag", "proxy-dns-upstream", "upstream", "edge"}
-	for _, sliceFlag := range sliceFlags {
-		if len(c.StringSlice(sliceFlag)) > 0 {
-			flags[sliceFlag] = strings.Join(c.StringSlice(sliceFlag), ", ")
-		}
-	}
-
 	if len(flags) > 0 {
-		log.Info().Msgf("Environment variables %v", flags)
+		log.Info().Msgf("Settings: %v", flags)
 	}
 
 	envs := make(map[string]string)
@@ -157,10 +150,8 @@ func prepareTunnelConfig(
 	buildInfo *buildinfo.BuildInfo,
 	version string,
 	log *zerolog.Logger,
-	transportLogger *zerolog.Logger,
+	observer *connection.Observer,
 	namedTunnel *connection.NamedTunnelConfig,
-	isUIEnabled bool,
-	eventChans []chan connection.Event,
 ) (*origin.TunnelConfig, ingress.Ingress, error) {
 	isNamedTunnel := namedTunnel != nil
 
@@ -262,8 +253,10 @@ func prepareTunnelConfig(
 	}
 	muxerConfig := &connection.MuxerConfig{
 		HeartbeatInterval:  c.Duration("heartbeat-interval"),
-		MaxHeartbeats:      c.Uint64("heartbeat-count"),
-		CompressionSetting: h2mux.CompressionSetting(c.Uint64("compression-quality")),
+		// Note TUN-3758 , we use Int because UInt is not supported with altsrc
+		MaxHeartbeats:      uint64(c.Int("heartbeat-count")),
+		// Note TUN-3758 , we use Int because UInt is not supported with altsrc
+		CompressionSetting: h2mux.CompressionSetting(uint64(c.Int("compression-quality"))),
 		MetricsUpdateFreq:  c.Duration("metrics-update-freq"),
 	}
 
@@ -279,14 +272,14 @@ func prepareTunnelConfig(
 		LBPool:           c.String("lb-pool"),
 		Tags:             tags,
 		Log:              log,
-		Observer:         connection.NewObserver(transportLogger, eventChans, isUIEnabled),
+		Observer:         observer,
 		ReportedVersion:  version,
-		Retries:          c.Uint("retries"),
+		// Note TUN-3758 , we use Int because UInt is not supported with altsrc
+		Retries:          uint(c.Int("retries")),
 		RunFromTerminal:  isRunningFromTerminal(),
 		NamedTunnel:      namedTunnel,
 		ClassicTunnel:    classicTunnel,
 		MuxerConfig:      muxerConfig,
-		TunnelEventChans: eventChans,
 		ProtocolSelector: protocolSelector,
 		EdgeTLSConfigs:   edgeTLSConfigs,
 	}, ingressRules, nil

cmd/cloudflared/ui/launch_ui.go

@@ -48,11 +48,10 @@ func NewUIModel(version, hostname, metricsURL string, ing *ingress.Ingress, haCo
 	}
 }
 
-func (data *uiModel) LaunchUI(
+func (data *uiModel) Launch(
 	ctx context.Context,
 	log, transportLog *zerolog.Logger,
-	tunnelEventChan <-chan connection.Event,
-) {
+) connection.EventSink {
 	// Configure the logger to stream logs into the textview
 
 	// Add TextView as a group to write output to
@@ -114,28 +113,9 @@ func (data *uiModel) LaunchUI(
 	grid.AddItem(logFrame, 4, 0, 5, 2, 0, 0, false)
 
 	go func() {
-		for {
-			select {
-			case <-ctx.Done():
-				app.Stop()
-				return
-			case event := <-tunnelEventChan:
-				switch event.EventType {
-				case connection.Connected:
-					data.setConnTableCell(event, connTable, palette)
-				case connection.Disconnected, connection.Reconnecting:
-					data.changeConnStatus(event, connTable, log, palette)
-				case connection.SetURL:
-					tunnelHostText.SetText(event.URL)
-					data.edgeURL = event.URL
-				case connection.RegisteringTunnel:
-					if data.edgeURL == "" {
-						tunnelHostText.SetText("Registering tunnel...")
-					}
-				}
-			}
-			app.Draw()
-		}
+		<-ctx.Done()
+		app.Stop()
+		return
 	}()
 
 	go func() {
@@ -143,6 +123,23 @@ func (data *uiModel) LaunchUI(
 			log.Error().Msgf("Error launching UI: %s", err)
 		}
 	}()
+
+	return connection.EventSinkFunc(func(event connection.Event) {
+		switch event.EventType {
+		case connection.Connected:
+			data.setConnTableCell(event, connTable, palette)
+		case connection.Disconnected, connection.Reconnecting:
+			data.changeConnStatus(event, connTable, log, palette)
+		case connection.SetURL:
+			tunnelHostText.SetText(event.URL)
+			data.edgeURL = event.URL
+		case connection.RegisteringTunnel:
+			if data.edgeURL == "" {
+				tunnelHostText.SetText("Registering tunnel...")
+			}
+		}
+		app.Draw()
+	})
 }
 
 func NewDynamicColorTextView() *tview.TextView {

connection/connection_test.go

@@ -27,13 +27,7 @@ var (
 		Scheme: "https",
 		Host:   "connectiontest.argotunnel.com",
 	}
-	testTunnelEventChan = make(chan Event)
-	testObserver        = &Observer{
-		&log,
-		m,
-		[]chan Event{testTunnelEventChan},
-		false,
-	}
+	testObserver        = NewObserver(&log, false)
 	testLargeResp = make([]byte, largeFileSize)
 )
 

connection/metrics.go

@@ -299,7 +299,7 @@ func convertRTTMilliSec(t time.Duration) float64 {
 }
 
 // Metrics that can be collected without asking the edge
-func newTunnelMetrics() *tunnelMetrics {
+func initTunnelMetrics() *tunnelMetrics {
 	maxConcurrentRequestsPerTunnel := prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Namespace: MetricsNamespace,
@@ -403,3 +403,15 @@ func (t *tunnelMetrics) registerServerLocation(connectionID, loc string) {
 	t.serverLocations.WithLabelValues(connectionID, loc).Inc()
 	t.oldServerLocations[connectionID] = loc
 }
+
+var tunnelMetricsInternal struct {
+	sync.Once
+	metrics *tunnelMetrics
+}
+
+func newTunnelMetrics() *tunnelMetrics {
+	tunnelMetricsInternal.Do(func() {
+		tunnelMetricsInternal.metrics = initTunnelMetrics()
+	})
+	return tunnelMetricsInternal.metrics
+}

connection/observer.go

@@ -10,22 +10,37 @@ import (
 	"github.com/rs/zerolog"
 )
 
-const LogFieldLocation = "location"
+const (
+	LogFieldLocation          = "location"
+	observerChannelBufferSize = 16
+)
 
 type Observer struct {
-	log              *zerolog.Logger
-	metrics          *tunnelMetrics
-	tunnelEventChans []chan Event
-	uiEnabled        bool
+	log             *zerolog.Logger
+	metrics         *tunnelMetrics
+	tunnelEventChan chan Event
+	uiEnabled       bool
+	addSinkChan     chan EventSink
 }
 
-func NewObserver(log *zerolog.Logger, tunnelEventChans []chan Event, uiEnabled bool) *Observer {
-	return &Observer{
-		log,
-		newTunnelMetrics(),
-		tunnelEventChans,
-		uiEnabled,
+type EventSink interface {
+	OnTunnelEvent(event Event)
+}
+
+func NewObserver(log *zerolog.Logger, uiEnabled bool) *Observer {
+	o := &Observer{
+		log:             log,
+		metrics:         newTunnelMetrics(),
+		uiEnabled:       uiEnabled,
+		tunnelEventChan: make(chan Event, observerChannelBufferSize),
+		addSinkChan:     make(chan EventSink, observerChannelBufferSize),
 	}
+	go o.dispatchEvents()
+	return o
+}
+
+func (o *Observer) RegisterSink(sink EventSink) {
+	o.addSinkChan <- sink
 }
 
 func (o *Observer) logServerInfo(connIndex uint8, location, msg string) {
@@ -105,7 +120,30 @@ func (o *Observer) SendDisconnect(connIndex uint8) {
 }
 
 func (o *Observer) sendEvent(e Event) {
-	for _, ch := range o.tunnelEventChans {
-		ch <- e
+	select {
+	case o.tunnelEventChan <- e:
+		break
+	default:
+		o.log.Warn().Msg("observer channel buffer is full")
 	}
 }
+
+func (o *Observer) dispatchEvents() {
+	var sinks []EventSink
+	for {
+		select {
+		case sink := <-o.addSinkChan:
+			sinks = append(sinks, sink)
+		case evt := <-o.tunnelEventChan:
+			for _, sink := range sinks {
+				sink.OnTunnelEvent(evt)
+			}
+		}
+	}
+}
+
+type EventSinkFunc func(event Event)
+
+func (f EventSinkFunc) OnTunnelEvent(event Event) {
+	f(event)
+}

connection/observer_test.go

@@ -4,14 +4,13 @@ import (
 	"strconv"
 	"sync"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 )
 
-// can only be called once
-var m = newTunnelMetrics()
-
 func TestRegisterServerLocation(t *testing.T) {
+	m := newTunnelMetrics()
 	tunnels := 20
 	var wg sync.WaitGroup
 	wg.Add(tunnels)
@@ -43,3 +42,27 @@ func TestRegisterServerLocation(t *testing.T) {
 	}
 
 }
+
+func TestObserverEventsDontBlock(t *testing.T) {
+	observer := NewObserver(&log, false)
+	var mu sync.Mutex
+	observer.RegisterSink(EventSinkFunc(func(_ Event) {
+		// callback will block if lock is already held
+		mu.Lock()
+		mu.Unlock()
+	}))
+
+	timeout := time.AfterFunc(5*time.Second, func() {
+		mu.Unlock()  // release the callback on timer expiration
+		t.Fatal("observer is blocked")
+	})
+
+	mu.Lock()  // block the callback
+	for i := 0; i < 2 * observerChannelBufferSize; i++ {
+		observer.sendRegisteringEvent()
+	}
+	if pending := timeout.Stop(); pending {
+		// release the callback if timer hasn't expired yet
+		mu.Unlock()
+	}
+}

ingress/origin_request_config.go

@@ -92,7 +92,8 @@ func originRequestFromSingeRule(c *cli.Context) OriginRequestConfig {
 		proxyAddress = c.String(flag)
 	}
 	if flag := ProxyPortFlag; c.IsSet(flag) {
-		proxyPort = c.Uint(flag)
+		// Note TUN-3758 , we use Int because UInt is not supported with altsrc
+		proxyPort = uint(c.Int(flag))
 	}
 	if c.IsSet(Socks5Flag) {
 		proxyType = socksProxy

logger/create.go

@@ -5,6 +5,8 @@ import (
 	"io"
 	"os"
 	"path"
+	"path/filepath"
+	"sync"
 
 	"github.com/mattn/go-colorable"
 	"github.com/rs/zerolog"
@@ -36,6 +38,21 @@ func fallbackLogger(err error) *zerolog.Logger {
 	return &failLog
 }
 
+type resilientMultiWriter struct {
+	writers []io.Writer
+}
+
+// This custom resilientMultiWriter is an alternative to zerolog's so that we can make it resilient to individual
+// writer's errors. E.g., when running as a Windows service, the console writer fails, but we don't want to
+// allow that to prevent all logging to fail due to breaking the for loop upon an error.
+func (t resilientMultiWriter) Write(p []byte) (n int, err error) {
+	for _, w := range t.writers {
+		_, _ = w.Write(p)
+	}
+	return len(p), nil
+}
+
+
 func newZerolog(loggerConfig *Config) *zerolog.Logger {
 	var writers []io.Writer
 
@@ -44,7 +61,7 @@ func newZerolog(loggerConfig *Config) *zerolog.Logger {
 	}
 
 	if loggerConfig.FileConfig != nil {
-		fileLogger, err := createFileLogger(*loggerConfig.FileConfig)
+		fileLogger, err := createFileWriter(*loggerConfig.FileConfig)
 		if err != nil {
 			return fallbackLogger(err)
 		}
@@ -61,7 +78,7 @@ func newZerolog(loggerConfig *Config) *zerolog.Logger {
 		writers = append(writers, rollingLogger)
 	}
 
-	multi := zerolog.MultiLevelWriter(writers...)
+	multi := resilientMultiWriter{writers}
 
 	level, err := zerolog.ParseLevel(loggerConfig.MinLevel)
 	if err != nil {
@@ -127,28 +144,43 @@ func createConsoleLogger(config ConsoleConfig) io.Writer {
 	}
 }
 
-func createFileLogger(config FileConfig) (io.Writer, error) {
-	var logFile io.Writer
-	fullpath := config.Fullpath()
-
-	// Try to open the existing file
-	logFile, err := os.OpenFile(fullpath, os.O_APPEND|os.O_WRONLY, filePermMode)
-	if err != nil {
-		// If the existing file wasn't found, or couldn't be opened, just ignore
-		// it and recreate a new one.
-		logFile, err = createLogFile(config)
-		// If creating a new logfile fails, then we have no choice but to error out.
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	fileLogger := zerolog.New(logFile).With().Logger()
-
-	return fileLogger, nil
+type fileInitializer struct {
+	once          sync.Once
+	writer        io.Writer
+	creationError error
 }
 
-func createLogFile(config FileConfig) (io.Writer, error) {
+var (
+	singleFileInit   fileInitializer
+	rotatingFileInit fileInitializer
+)
+
+func createFileWriter(config FileConfig) (io.Writer, error) {
+	singleFileInit.once.Do(func() {
+
+		var logFile io.Writer
+		fullpath := config.Fullpath()
+
+		// Try to open the existing file
+		logFile, err := os.OpenFile(fullpath, os.O_APPEND|os.O_WRONLY, filePermMode)
+		if err != nil {
+			// If the existing file wasn't found, or couldn't be opened, just ignore
+			// it and recreate a new one.
+			logFile, err = createDirFile(config)
+			// If creating a new logfile fails, then we have no choice but to error out.
+			if err != nil {
+				singleFileInit.creationError = err
+				return
+			}
+		}
+
+		singleFileInit.writer = logFile
+	})
+
+	return singleFileInit.writer, singleFileInit.creationError
+}
+
+func createDirFile(config FileConfig) (io.Writer, error) {
 	if config.Dirname != "" {
 		err := os.MkdirAll(config.Dirname, dirPermMode)
 
@@ -159,7 +191,8 @@ func createLogFile(config FileConfig) (io.Writer, error) {
 
 	mode := os.FileMode(filePermMode)
 
-	logFile, err := os.OpenFile(config.Filename, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, mode)
+	fullPath := filepath.Join(config.Dirname, config.Filename)
+	logFile, err := os.OpenFile(fullPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, mode)
 	if err != nil {
 		return nil, fmt.Errorf("unable to create a new logfile: %s", err)
 	}
@@ -168,14 +201,19 @@ func createLogFile(config FileConfig) (io.Writer, error) {
 }
 
 func createRollingLogger(config RollingConfig) (io.Writer, error) {
-	if err := os.MkdirAll(config.Dirname, dirPermMode); err != nil {
-		return nil, err
-	}
+	rotatingFileInit.once.Do(func() {
+		if err := os.MkdirAll(config.Dirname, dirPermMode); err != nil {
+			rotatingFileInit.creationError = err
+			return
+		}
 
-	return &lumberjack.Logger{
-		Filename:   path.Join(config.Dirname, config.Filename),
-		MaxBackups: config.maxBackups,
-		MaxSize:    config.maxSize,
-		MaxAge:     config.maxAge,
-	}, nil
+		rotatingFileInit.writer = &lumberjack.Logger{
+			Filename:   path.Join(config.Dirname, config.Filename),
+			MaxBackups: config.maxBackups,
+			MaxSize:    config.maxSize,
+			MaxAge:     config.maxAge,
+		}
+	})
+
+	return rotatingFileInit.writer, rotatingFileInit.creationError
 }

logger/create_test.go

@@ -0,0 +1,89 @@
+package logger
+
+import (
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog"
+	"github.com/stretchr/testify/assert"
+	"io"
+	"testing"
+)
+
+var writeCalls int
+
+type mockedWriter struct {
+	wantErr bool
+}
+
+func (c mockedWriter) Write(p []byte) (int, error) {
+	writeCalls++
+
+	if c.wantErr {
+		return -1, errors.New("Expected error")
+	}
+
+	return len(p), nil
+}
+
+// Tests that a new writer is only used if it actually works.
+func TestResilientMultiWriter(t *testing.T) {
+	tests := []struct {
+		name    string
+		writers []io.Writer
+	}{
+		{
+			name:    "All valid writers",
+			writers: []io.Writer{
+				mockedWriter {
+					wantErr: false,
+				},
+				mockedWriter {
+					wantErr: false,
+				},
+			},
+		},
+		{
+			name:    "All invalid writers",
+			writers: []io.Writer{
+				mockedWriter {
+					wantErr: true,
+				},
+				mockedWriter {
+					wantErr: true,
+				},
+			},
+		},
+		{
+			name:    "First invalid writer",
+			writers: []io.Writer{
+				mockedWriter {
+					wantErr: true,
+				},
+				mockedWriter {
+					wantErr: false,
+				},
+			},
+		},
+		{
+			name:    "First valid writer",
+			writers: []io.Writer{
+				mockedWriter {
+					wantErr: false,
+				},
+				mockedWriter {
+					wantErr: true,
+				},
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		writers := tt.writers
+		multiWriter := resilientMultiWriter{writers}
+
+		logger := zerolog.New(multiWriter).With().Timestamp().Logger().Level(zerolog.InfoLevel)
+		logger.Info().Msg("Test msg")
+
+		assert.Equal(t, len(writers), writeCalls)
+		writeCalls = 0
+	}
+}

metrics/metrics.go

@@ -10,8 +10,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/cloudflare/cloudflared/connection"
-
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/rs/zerolog"
@@ -23,21 +21,22 @@ const (
 	startupTime     = time.Millisecond * 500
 )
 
-func newMetricsHandler(connectionEvents <-chan connection.Event, log *zerolog.Logger) *http.ServeMux {
-	readyServer := NewReadyServer(connectionEvents, log)
+func newMetricsHandler(readyServer *ReadyServer) *http.ServeMux {
 	mux := http.NewServeMux()
 	mux.Handle("/metrics", promhttp.Handler())
 	mux.HandleFunc("/healthcheck", func(w http.ResponseWriter, r *http.Request) {
 		_, _ = fmt.Fprintf(w, "OK\n")
 	})
-	mux.Handle("/ready", readyServer)
+	if readyServer != nil {
+		mux.Handle("/ready", readyServer)
+	}
 	return mux
 }
 
 func ServeMetrics(
 	l net.Listener,
 	shutdownC <-chan struct{},
-	connectionEvents <-chan connection.Event,
+	readyServer *ReadyServer,
 	log *zerolog.Logger,
 ) (err error) {
 	var wg sync.WaitGroup
@@ -45,7 +44,7 @@ func ServeMetrics(
 	trace.AuthRequest = func(*http.Request) (bool, bool) { return true, true }
 	// TODO: parameterize ReadTimeout and WriteTimeout. The maximum time we can
 	// profile CPU usage depends on WriteTimeout
-	h := newMetricsHandler(connectionEvents, log)
+	h := newMetricsHandler(readyServer)
 	server := &http.Server{
 		ReadTimeout:  10 * time.Second,
 		WriteTimeout: 10 * time.Second,

metrics/readiness.go

@@ -19,30 +19,28 @@ type ReadyServer struct {
 }
 
 // NewReadyServer initializes a ReadyServer and starts listening for dis/connection events.
-func NewReadyServer(connectionEvents <-chan conn.Event, log *zerolog.Logger) *ReadyServer {
-	rs := ReadyServer{
+func NewReadyServer(log *zerolog.Logger) *ReadyServer {
+	return &ReadyServer{
 		isConnected: make(map[int]bool, 0),
 		log:         log,
 	}
-	go func() {
-		for c := range connectionEvents {
-			switch c.EventType {
-			case conn.Connected:
-				rs.Lock()
-				rs.isConnected[int(c.Index)] = true
-				rs.Unlock()
-			case conn.Disconnected, conn.Reconnecting, conn.RegisteringTunnel:
-				rs.Lock()
-				rs.isConnected[int(c.Index)] = false
-				rs.Unlock()
-			case conn.SetURL:
-				continue
-			default:
-				rs.log.Error().Msgf("Unknown connection event case %v", c)
-			}
-		}
-	}()
-	return &rs
+}
+
+func (rs *ReadyServer) OnTunnelEvent(c conn.Event) {
+	switch c.EventType {
+	case conn.Connected:
+		rs.Lock()
+		rs.isConnected[int(c.Index)] = true
+		rs.Unlock()
+	case conn.Disconnected, conn.Reconnecting, conn.RegisteringTunnel:
+		rs.Lock()
+		rs.isConnected[int(c.Index)] = false
+		rs.Unlock()
+	case conn.SetURL:
+		break
+	default:
+		rs.log.Error().Msgf("Unknown connection event case %v", c)
+	}
 }
 
 type body struct {

metrics/readiness_test.go

@@ -3,6 +3,11 @@ package metrics
 import (
 	"net/http"
 	"testing"
+
+	"github.com/rs/zerolog"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/cloudflare/cloudflared/connection"
 )
 
 func TestReadyServer_makeResponse(t *testing.T) {
@@ -56,3 +61,49 @@ func TestReadyServer_makeResponse(t *testing.T) {
 		})
 	}
 }
+
+func TestReadinessEventHandling(t *testing.T) {
+	nopLogger := zerolog.Nop()
+	rs := NewReadyServer(&nopLogger)
+
+	// start not ok
+	code, ready := rs.makeResponse()
+	assert.NotEqualValues(t, http.StatusOK, code)
+	assert.Zero(t, ready)
+
+	// one connected => ok
+	rs.OnTunnelEvent(connection.Event{
+		Index:     1,
+		EventType: connection.Connected,
+	})
+	code, ready = rs.makeResponse()
+	assert.EqualValues(t, http.StatusOK, code)
+	assert.EqualValues(t, 1, ready)
+
+	// another connected => still ok
+	rs.OnTunnelEvent(connection.Event{
+		Index:     2,
+		EventType: connection.Connected,
+	})
+	code, ready = rs.makeResponse()
+	assert.EqualValues(t, http.StatusOK, code)
+	assert.EqualValues(t, 2,  ready)
+
+	// one reconnecting => still ok
+	rs.OnTunnelEvent(connection.Event{
+		Index:     2,
+		EventType: connection.Reconnecting,
+	})
+	code, ready = rs.makeResponse()
+	assert.EqualValues(t, http.StatusOK, code)
+	assert.EqualValues(t, 1, ready)
+
+	// other disconnected => not ok
+	rs.OnTunnelEvent(connection.Event{
+		Index:     1,
+		EventType: connection.Disconnected,
+	})
+	code, ready = rs.makeResponse()
+	assert.NotEqualValues(t, http.StatusOK, code)
+	assert.Zero(t, ready)
+}

origin/proxy.go

@@ -191,7 +191,7 @@ func (c *client) logRequest(r *http.Request, cfRay string, lbProbe bool, ruleNum
 func (c *client) logOriginResponse(r *http.Response, cfRay string, lbProbe bool, ruleNum int) {
 	responseByCode.WithLabelValues(strconv.Itoa(r.StatusCode)).Inc()
 	if cfRay != "" {
-		c.log.Info().Msgf("CF-RAY: %s Status: %s served by ingress %d", cfRay, r.Status, ruleNum)
+		c.log.Debug().Msgf("CF-RAY: %s Status: %s served by ingress %d", cfRay, r.Status, ruleNum)
 	} else if lbProbe {
 		c.log.Debug().Msgf("Response to Load Balancer health check %s", r.Status)
 	} else {

origin/tunnel.go

@@ -12,7 +12,6 @@ import (
 
 	"github.com/google/uuid"
 	"github.com/pkg/errors"
-	"github.com/prometheus/client_golang/prometheus"
 	"github.com/rs/zerolog"
 	"golang.org/x/sync/errgroup"
 
@@ -27,9 +26,7 @@ import (
 
 const (
 	dialTimeout              = 15 * time.Second
-	muxerTimeout             = 5 * time.Second
 	lbProbeUserAgentPrefix   = "Mozilla/5.0 (compatible; Cloudflare-Traffic-Manager/1.0; +https://www.cloudflare.com/traffic-manager/;"
-	DuplicateConnectionError = "EDUPCONN"
 	FeatureSerializedHeaders = "serialized_headers"
 	FeatureQuickReconnects   = "quick_reconnects"
 )
@@ -37,9 +34,7 @@ const (
 type rpcName string
 
 const (
-	register     rpcName = "register"
 	reconnect    rpcName = "reconnect"
-	unregister   rpcName = "unregister"
 	authenticate rpcName = " authenticate"
 )
 
@@ -64,7 +59,6 @@ type TunnelConfig struct {
 	NamedTunnel      *connection.NamedTunnelConfig
 	ClassicTunnel    *connection.ClassicTunnelConfig
 	MuxerConfig      *connection.MuxerConfig
-	TunnelEventChans []chan connection.Event
 	ProtocolSelector connection.ProtocolSelector
 	EdgeTLSConfigs   map[connection.Protocol]*tls.Config
 }
@@ -90,11 +84,6 @@ type clientRegisterTunnelError struct {
 	cause error
 }
 
-func newRPCError(cause error, counter *prometheus.CounterVec, name rpcName) clientRegisterTunnelError {
-	counter.WithLabelValues(cause.Error(), string(name)).Inc()
-	return clientRegisterTunnelError{cause: cause}
-}
-
 func (e clientRegisterTunnelError) Error() string {
 	return e.cause.Error()
 }
@@ -466,5 +455,4 @@ func activeIncidentsMsg(incidents []Incident) string {
 		incidentStrings = append(incidentStrings, incidentString)
 	}
 	return preamble + " " + strings.Join(incidentStrings, "; ")
-
 }

origin/tunnel_test.go

@@ -53,7 +53,7 @@ func TestWaitForBackoffFallback(t *testing.T) {
 	config := &TunnelConfig{
 		Log:              &log,
 		ProtocolSelector: protocolSelector,
-		Observer:         connection.NewObserver(nil, nil, false),
+		Observer:         connection.NewObserver(nil, false),
 	}
 	connIndex := uint8(1)
 

tunneldns/tunnel.go

@@ -50,6 +50,7 @@ func Command(hidden bool) *cli.Command {
 				Value:   "localhost",
 				EnvVars: []string{"TUNNEL_DNS_ADDRESS"},
 			},
+			// Note TUN-3758 , we use Int because UInt is not supported with altsrc
 			&cli.IntFlag{
 				Name:    "port",
 				Usage:   "Listen on given port for the DNS over HTTPS proxy server.",
@@ -87,7 +88,8 @@ func Run(c *cli.Context) error {
 
 	listener, err := CreateListener(
 		c.String("address"),
-		uint16(c.Uint("port")),
+		// Note TUN-3758 , we use Int because UInt is not supported with altsrc
+		uint16(c.Int("port")),
 		c.StringSlice("upstream"),
 		c.StringSlice("bootstrap"),
 		log,