Release 2022.8.3

Protocolbackoff arrays now have Retryforever flag set to true to enable
cloudflared to keep trying to reconnect perpetually.

.docker-images

@@ -1,6 +1,6 @@
 images:
   - name: cloudflared
-    dockerfile: Dockerfile
+    dockerfile: Dockerfile.$ARCH
     context: .
     versions: 
       - latest
@@ -8,3 +8,6 @@ images:
     - name: docker.io/cloudflare
       user: env:DOCKER_USER
       password: env:DOCKER_PASSWORD
+    architectures:
+    - amd64
+    - arm64

.github/ISSUE_TEMPLATE/---bug-report.md

@@ -3,7 +3,6 @@ name: "\U0001F41B Bug report"
 about: Create a report to help us improve cloudflared
 title: "\U0001F41B"
 labels: 'Priority: Normal, Type: Bug'
-assignees: abelinkinbio
 
 ---
 
@@ -16,6 +15,10 @@ Steps to reproduce the behavior:
 2. Run '....'
 3. See error
 
+If it's an issue with Cloudflare Tunnel:
+4. Tunnel ID : 
+5. cloudflared config: 
+
 **Expected behavior**
 A clear and concise description of what you expected to happen.
 

.github/ISSUE_TEMPLATE/---documentation.md

@@ -3,7 +3,6 @@ name: "\U0001F4DD Documentation"
 about: Request new or updated documentation for cloudflared
 title: "\U0001F4DD"
 labels: 'Priority: Normal, Type: Documentation'
-assignees: abelinkinbio, ranbel
 
 ---
 

.github/ISSUE_TEMPLATE/---feature-request.md

@@ -3,7 +3,6 @@ name: "\U0001F4A1 Feature request"
 about: Suggest a feature or enhancement for cloudflared
 title: "\U0001F4A1"
 labels: 'Priority: Normal, Type: Feature Request'
-assignees: abelinkinbio, sssilver
 
 ---
 

CHANGES.md

@@ -1,3 +1,8 @@
+## 2022.8.1
+### New Features
+- cloudflared now remembers if it connected to a certain protocol successfully. If it did, it does not fall back to a lower
+  protocol on connection failures.
+
 ## 2022.7.1
 ### New Features
 - It is now possible to connect cloudflared tunnel to Cloudflare Global Network with IPv6. See `cloudflared tunnel --help` and look for `edge-ip-version` for more information. For now, the default behavior is to still connect with IPv4 only.

Dockerfile

@@ -1,7 +1,7 @@
 # use a builder image for building cloudflare
 ARG TARGET_GOOS
 ARG TARGET_GOARCH
-FROM golang:1.17.1 as builder
+FROM golang:1.19 as builder
 ENV GO111MODULE=on \
     CGO_ENABLED=0 \
     TARGET_GOOS=${TARGET_GOOS} \
@@ -18,7 +18,7 @@ COPY . .
 RUN make cloudflared
 
 # use a distroless base image with glibc
-FROM gcr.io/distroless/base-debian10:nonroot
+FROM gcr.io/distroless/base-debian11:nonroot
 
 # copy our compiled binary
 COPY --from=builder --chown=nonroot /go/src/github.com/cloudflare/cloudflared/cloudflared /usr/local/bin/

Dockerfile.amd64

@@ -0,0 +1,27 @@
+# use a builder image for building cloudflare
+FROM golang:1.19 as builder
+ENV GO111MODULE=on \
+    CGO_ENABLED=0 
+    
+LABEL org.opencontainers.image.source="https://github.com/cloudflare/cloudflared"
+
+WORKDIR /go/src/github.com/cloudflare/cloudflared/
+
+# copy our sources into the builder image
+COPY . .
+
+# compile cloudflared
+RUN GOOS=linux GOARCH=amd64 make cloudflared
+
+# use a distroless base image with glibc
+FROM gcr.io/distroless/base-debian11:nonroot
+
+# copy our compiled binary
+COPY --from=builder --chown=nonroot /go/src/github.com/cloudflare/cloudflared/cloudflared /usr/local/bin/
+
+# run as non-privileged user
+USER nonroot
+
+# command / entrypoint of container
+ENTRYPOINT ["cloudflared", "--no-autoupdate"]
+CMD ["version"]

Dockerfile.arm64

@@ -0,0 +1,27 @@
+# use a builder image for building cloudflare
+FROM golang:1.19 as builder
+ENV GO111MODULE=on \
+    CGO_ENABLED=0 
+    
+LABEL org.opencontainers.image.source="https://github.com/cloudflare/cloudflared"
+
+WORKDIR /go/src/github.com/cloudflare/cloudflared/
+
+# copy our sources into the builder image
+COPY . .
+
+# compile cloudflared
+RUN GOOS=linux GOARCH=arm64 make cloudflared
+
+# use a distroless base image with glibc
+FROM gcr.io/distroless/base-debian11:nonroot-arm64
+
+# copy our compiled binary
+COPY --from=builder --chown=nonroot /go/src/github.com/cloudflare/cloudflared/cloudflared /usr/local/bin/
+
+# run as non-privileged user
+USER nonroot
+
+# command / entrypoint of container
+ENTRYPOINT ["cloudflared", "--no-autoupdate"]
+CMD ["version"]

RELEASE_NOTES

@@ -1,3 +1,47 @@
+2022.8.3
+- 2022-08-26 TUN-6708: Fix replace flow logic
+- 2022-08-25 TUN-6705: Tunnel should retry connections forever
+- 2022-08-25 TUN-6704: Honor protocol flag when edge discovery is unreachable
+- 2022-08-25 TUN-6699: Add metric for packet too big dropped
+- 2022-08-24 TUN-6691: Properly error check for net.ErrClosed
+- 2022-08-22 TUN-6679: Allow client side of quic request to close body
+- 2022-08-22 TUN-6586: Change ICMP proxy to only build for Darwin and use echo ID to track flows
+- 2022-08-18 TUN-6530: Implement ICMPv4 proxy
+- 2022-08-17 TUN-6666: Define packet package
+- 2022-08-17 TUN-6667: DatagramMuxerV2 provides a method to receive RawPacket
+- 2022-08-16 TUN-6657: Ask for Tunnel ID and Configuration on Bug Report
+- 2022-08-16 TUN-6676: Add suport for trailers in http2 connections
+- 2022-08-11 TUN-6575: Consume cf-trace-id from incoming http2 TCP requests
+
+2022.8.2
+- 2022-08-16 TUN-6656: Docker for arm64 should not be deployed in an amd64 container
+
+2022.8.1
+- 2022-08-15 TUN-6617: Updated CHANGES.md for protocol stickiness
+- 2022-08-12 EDGEPLAT-3918: bump go and go-boring to 1.18.5
+- 2022-08-12 TUN-6652: Publish dockerfile for both amd64 and arm64
+- 2022-08-11 TUN-6617: Dont fallback to http2 if QUIC conn was successful.
+- 2022-08-11 TUN-6617: Dont fallback to http2 if QUIC conn was successful.
+- 2022-08-11 Revert "TUN-6617: Dont fallback to http2 if QUIC conn was successful."
+- 2022-08-11 TUN-6617: Dont fallback to http2 if QUIC conn was successful.
+- 2022-08-01 TUN-6584: Define QUIC datagram v2 format to support proxying IP packets
+
+2022.8.0
+- 2022-08-10 TUN-6637: Upgrade quic-go
+- 2022-08-10 TUN-6646: Add support to SafeStreamCloser to close only write side of stream
+- 2022-08-09 TUN-6642: Fix unexpected close of quic stream triggered by upstream origin close
+- 2022-08-09 TUN-6639: Validate cyclic ingress configuration
+- 2022-08-08 TUN-6637: Upgrade go version and quic-go
+- 2022-08-08 TUN-6639: Validate cyclic ingress configuration
+- 2022-08-04 EDGEPLAT-3918: build cloudflared for Bookworm
+- 2022-08-02 Revert "TUN-6576: Consume cf-trace-id from incoming TCP requests to create root span"
+- 2022-07-27 TUN-6601: Update gopkg.in/yaml.v3 references in modules
+- 2022-07-26 TUN-6576: Consume cf-trace-id from incoming TCP requests to create root span
+- 2022-07-26 TUN-6576: Consume cf-trace-id from incoming TCP requests to create root span
+- 2022-07-25 TUN-6598: Remove auto assignees on github issues
+- 2022-07-20 TUN-6583: Remove legacy --ui flag
+- 2022-07-20 cURL supports stdin and uses os pipes directly without copying
+- 2022-07-07 TUN-6517: Use QUIC stream context while proxying HTTP requests and TCP connections
 2022.7.1
 - 2022-07-06 TUN-6503: Fix transport fallback from QUIC in face of dial error "no network activity"
 

cfsetup.yaml

@@ -1,8 +1,8 @@
-pinned_go: &pinned_go go=1.17.10-1
-pinned_go_fips: &pinned_go_fips go-boring=1.17.9-1
+pinned_go: &pinned_go go=1.18.5-1
+pinned_go_fips: &pinned_go_fips go-boring=1.18.5-2
 
 build_dir: &build_dir /cfsetup_build
-default-flavor: buster
+default-flavor: bullseye
 stretch: &stretch
   build:
     build_dir: *build_dir
@@ -152,7 +152,7 @@ stretch: &stretch
       - build-essential
       - gotest-to-teamcity
     pre-cache: &test_pre_cache
-      - go get golang.org/x/tools/cmd/goimports
+      - go install golang.org/x/tools/cmd/goimports@latest
     post-cache:
       - export GOOS=linux
       - export GOARCH=amd64
@@ -246,6 +246,7 @@ stretch: &stretch
 
 buster: *stretch
 bullseye: *stretch
+bookworm: *stretch
 centos-7:
   publish-rpm:
     build_dir: *build_dir
@@ -254,8 +255,8 @@ centos-7:
     pre-cache:
       - yum install -y fakeroot
       - yum upgrade -y binutils-2.27-44.base.el7.x86_64
-      - wget https://go.dev/dl/go1.17.10.linux-amd64.tar.gz -P /tmp/
-      - tar -C /usr/local -xzf /tmp/go1.17.10.linux-amd64.tar.gz
+      - wget https://go.dev/dl/go1.18.5.linux-amd64.tar.gz -P /tmp/
+      - tar -C /usr/local -xzf /tmp/go1.18.5.linux-amd64.tar.gz
     post-cache:
       - export PATH=$PATH:/usr/local/go/bin
       - export GOOS=linux

cmd/cloudflared/access/cmd.go

@@ -296,6 +296,7 @@ func curl(c *cli.Context) error {
 // run kicks off a shell task and pipe the results to the respective std pipes
 func run(cmd string, args ...string) error {
 	c := exec.Command(cmd, args...)
+	c.Stdin = os.Stdin
 	stderr, err := c.StderrPipe()
 	if err != nil {
 		return err

cmd/cloudflared/tunnel/cmd.go

@@ -25,7 +25,6 @@ import (
 	"github.com/cloudflare/cloudflared/cfapi"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/proxydns"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/ui"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/updater"
 	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/connection"
@@ -218,7 +217,7 @@ func runAdhocNamedTunnel(sc *subcommandContext, name, credentialsOutputPath stri
 
 // runClassicTunnel creates a "classic" non-named tunnel
 func runClassicTunnel(sc *subcommandContext) error {
-	return StartServer(sc.c, buildInfo, nil, sc.log, sc.isUIEnabled)
+	return StartServer(sc.c, buildInfo, nil, sc.log)
 }
 
 func routeFromFlag(c *cli.Context) (route cfapi.HostnameRoute, ok bool) {
@@ -236,7 +235,6 @@ func StartServer(
 	info *cliutil.BuildInfo,
 	namedTunnel *connection.NamedTunnelProperties,
 	log *zerolog.Logger,
-	isUIEnabled bool,
 ) error {
 	_ = raven.SetDSN(sentryDSN)
 	var wg sync.WaitGroup
@@ -331,9 +329,9 @@ func StartServer(
 		return fmt.Errorf(errText)
 	}
 
-	logTransport := logger.CreateTransportLoggerFromContext(c, isUIEnabled)
+	logTransport := logger.CreateTransportLoggerFromContext(c, logger.EnableTerminalLog)
 
-	observer := connection.NewObserver(log, logTransport, isUIEnabled)
+	observer := connection.NewObserver(log, logTransport)
 
 	// Send Quick Tunnel URL to UI if applicable
 	var quickTunnelURL string
@@ -392,18 +390,6 @@ func StartServer(
 		errC <- supervisor.StartTunnelDaemon(ctx, tunnelConfig, orchestrator, connectedSignal, reconnectCh, graceShutdownC)
 	}()
 
-	if isUIEnabled {
-		tunnelUI := ui.NewUIModel(
-			info.Version(),
-			hostname,
-			metricsListener.Addr().String(),
-			orchestratorConfig.Ingress,
-			tunnelConfig.HAConnections,
-		)
-		app := tunnelUI.Launch(ctx, log, logTransport)
-		observer.RegisterSink(app)
-	}
-
 	gracePeriod, err := gracePeriod(c)
 	if err != nil {
 		return err
@@ -663,9 +649,9 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 		}),
 		altsrc.NewBoolFlag(&cli.BoolFlag{
 			Name:   uiFlag,
-			Usage:  "Launch tunnel UI. Tunnel logs are scrollable via 'j', 'k', or arrow keys.",
+			Usage:  "(depreciated) Launch tunnel UI. Tunnel logs are scrollable via 'j', 'k', or arrow keys.",
 			Value:  false,
-			Hidden: shouldHide,
+			Hidden: true,
 		}),
 		altsrc.NewStringFlag(&cli.StringFlag{
 			Name:   "quick-service",

cmd/cloudflared/tunnel/quick_tunnel.go

@@ -78,7 +78,6 @@ func RunQuickTunnel(sc *subcommandContext) error {
 		buildInfo,
 		&connection.NamedTunnelProperties{Credentials: credentials, QuickTunnelUrl: data.Result.Hostname},
 		sc.log,
-		sc.isUIEnabled,
 	)
 }
 

cmd/cloudflared/tunnel/subcommand_context.go

@@ -31,10 +31,9 @@ func (e errInvalidJSONCredential) Error() string {
 // subcommandContext carries structs shared between subcommands, to reduce number of arguments needed to
 // pass between subcommands, and make sure they are only initialized once
 type subcommandContext struct {
-	c           *cli.Context
-	log         *zerolog.Logger
-	isUIEnabled bool
-	fs          fileSystem
+	c   *cli.Context
+	log *zerolog.Logger
+	fs  fileSystem
 
 	// These fields should be accessed using their respective Getter
 	tunnelstoreClient cfapi.Client
@@ -42,16 +41,10 @@ type subcommandContext struct {
 }
 
 func newSubcommandContext(c *cli.Context) (*subcommandContext, error) {
-	isUIEnabled := c.IsSet(uiFlag) && c.String("name") != ""
-
-	// If UI is enabled, terminal log output should be disabled -- log should be written into a UI log window instead
-	log := logger.CreateLoggerFromContext(c, isUIEnabled)
-
 	return &subcommandContext{
-		c:           c,
-		log:         log,
-		isUIEnabled: isUIEnabled,
-		fs:          realFileSystem{},
+		c:   c,
+		log: logger.CreateLoggerFromContext(c, logger.EnableTerminalLog),
+		fs:  realFileSystem{},
 	}, nil
 }
 
@@ -312,7 +305,6 @@ func (sc *subcommandContext) runWithCredentials(credentials connection.Credentia
 		buildInfo,
 		&connection.NamedTunnelProperties{Credentials: credentials},
 		sc.log,
-		sc.isUIEnabled,
 	)
 }
 

cmd/cloudflared/tunnel/subcommand_context_test.go

@@ -35,7 +35,6 @@ func Test_subcommandContext_findCredentials(t *testing.T) {
 	type fields struct {
 		c                 *cli.Context
 		log               *zerolog.Logger
-		isUIEnabled       bool
 		fs                fileSystem
 		tunnelstoreClient cfapi.Client
 		userCredential    *userCredential
@@ -168,7 +167,6 @@ func Test_subcommandContext_findCredentials(t *testing.T) {
 			sc := &subcommandContext{
 				c:                 tt.fields.c,
 				log:               tt.fields.log,
-				isUIEnabled:       tt.fields.isUIEnabled,
 				fs:                tt.fields.fs,
 				tunnelstoreClient: tt.fields.tunnelstoreClient,
 				userCredential:    tt.fields.userCredential,
@@ -307,7 +305,6 @@ func Test_subcommandContext_Delete(t *testing.T) {
 			sc := &subcommandContext{
 				c:                 tt.fields.c,
 				log:               tt.fields.log,
-				isUIEnabled:       tt.fields.isUIEnabled,
 				fs:                tt.fields.fs,
 				tunnelstoreClient: tt.fields.tunnelstoreClient,
 				userCredential:    tt.fields.userCredential,

cmd/cloudflared/ui/launch_ui.go

@@ -1,223 +0,0 @@
-package ui
-
-import (
-	"context"
-	"fmt"
-	"strings"
-
-	"github.com/gdamore/tcell"
-	"github.com/rivo/tview"
-	"github.com/rs/zerolog"
-
-	"github.com/cloudflare/cloudflared/connection"
-	"github.com/cloudflare/cloudflared/ingress"
-)
-
-type connState struct {
-	location string
-}
-
-type uiModel struct {
-	version       string
-	edgeURL       string
-	metricsURL    string
-	localServices []string
-	connections   []connState
-}
-
-type palette struct {
-	url          string
-	connected    string
-	defaultText  string
-	disconnected string
-	reconnecting string
-	unregistered string
-}
-
-func NewUIModel(version, hostname, metricsURL string, ing *ingress.Ingress, haConnections int) *uiModel {
-	localServices := make([]string, len(ing.Rules))
-	for i, rule := range ing.Rules {
-		localServices[i] = rule.Service.String()
-	}
-	return &uiModel{
-		version:       version,
-		edgeURL:       hostname,
-		metricsURL:    metricsURL,
-		localServices: localServices,
-		connections:   make([]connState, haConnections),
-	}
-}
-
-func (data *uiModel) Launch(
-	ctx context.Context,
-	log, transportLog *zerolog.Logger,
-) connection.EventSink {
-	// Configure the logger to stream logs into the textview
-
-	// Add TextView as a group to write output to
-	logTextView := NewDynamicColorTextView()
-	// TODO: Format log for UI
-	//log.Add(logTextView, logger.NewUIFormatter(time.RFC3339), logLevels...)
-	//transportLog.Add(logTextView, logger.NewUIFormatter(time.RFC3339), logLevels...)
-
-	// Construct the UI
-	palette := palette{
-		url:          "lightblue",
-		connected:    "lime",
-		defaultText:  "white",
-		disconnected: "red",
-		reconnecting: "orange",
-		unregistered: "orange",
-	}
-
-	app := tview.NewApplication()
-
-	grid := tview.NewGrid().SetGap(1, 0)
-	frame := tview.NewFrame(grid)
-	header := fmt.Sprintf("cloudflared [::b]%s", data.version)
-
-	frame.AddText(header, true, tview.AlignLeft, tcell.ColorWhite)
-
-	// Create table to store connection info and status
-	connTable := tview.NewTable()
-	// SetColumns takes a value for each column, representing the size of the column
-	// Numbers <= 0 represent proportional widths and positive numbers represent absolute widths
-	grid.SetColumns(20, 0)
-
-	// SetRows takes a value for each row, representing the size of the row
-	grid.SetRows(1, 1, len(data.connections), 1, 0)
-
-	// AddItem takes a primitive tview type, row, column, rowSpan, columnSpan, minGridHeight, minGridWidth, and focus
-	grid.AddItem(tview.NewTextView().SetText("Tunnel:"), 0, 0, 1, 1, 0, 0, false)
-	grid.AddItem(tview.NewTextView().SetText("Status:"), 1, 0, 1, 1, 0, 0, false)
-	grid.AddItem(tview.NewTextView().SetText("Connections:"), 2, 0, 1, 1, 0, 0, false)
-
-	grid.AddItem(tview.NewTextView().SetText("Metrics:"), 3, 0, 1, 1, 0, 0, false)
-
-	tunnelHostText := tview.NewTextView().SetText(data.edgeURL)
-
-	grid.AddItem(tunnelHostText, 0, 1, 1, 1, 0, 0, false)
-	status := fmt.Sprintf("[%s]\u2022[%s] Proxying to [%s::b]%s", palette.connected, palette.defaultText, palette.url, strings.Join(data.localServices, ", "))
-	grid.AddItem(NewDynamicColorTextView().SetText(status), 1, 1, 1, 1, 0, 0, false)
-
-	grid.AddItem(connTable, 2, 1, 1, 1, 0, 0, false)
-
-	grid.AddItem(NewDynamicColorTextView().SetText(fmt.Sprintf("Metrics at [%s::b]http://%s/metrics", palette.url, data.metricsURL)), 3, 1, 1, 1, 0, 0, false)
-
-	// Add TextView to stream logs
-	// Logs are displayed in a new grid so a border can be set around them
-	logGrid := tview.NewGrid().SetBorders(true).AddItem(logTextView.SetChangedFunc(handleNewText(app, logTextView)), 0, 0, 5, 2, 0, 0, false)
-	// LogFrame holds the Logs header as well as the grid with the textView for streamed logs
-	logFrame := tview.NewFrame(logGrid).AddText("[::b]Logs:[::-]", true, tview.AlignLeft, tcell.ColorWhite).SetBorders(0, 0, 0, 0, 0, 0)
-	// Footer for log frame
-	logFrame.AddText("[::d]Use Ctrl+C to exit[::-]", false, tview.AlignRight, tcell.ColorWhite)
-	grid.AddItem(logFrame, 4, 0, 5, 2, 0, 0, false)
-
-	go func() {
-		<-ctx.Done()
-		app.Stop()
-		return
-	}()
-
-	go func() {
-		if err := app.SetRoot(frame, true).Run(); err != nil {
-			log.Error().Msgf("Error launching UI: %s", err)
-		}
-	}()
-
-	return connection.EventSinkFunc(func(event connection.Event) {
-		switch event.EventType {
-		case connection.Connected:
-			data.setConnTableCell(event, connTable, palette)
-		case connection.Disconnected, connection.Reconnecting, connection.Unregistering:
-			data.changeConnStatus(event, connTable, log, palette)
-		case connection.SetURL:
-			tunnelHostText.SetText(event.URL)
-			data.edgeURL = event.URL
-		case connection.RegisteringTunnel:
-			if data.edgeURL == "" {
-				tunnelHostText.SetText(fmt.Sprintf("Registering tunnel connection %d...", event.Index))
-			}
-		}
-		app.Draw()
-	})
-}
-
-func NewDynamicColorTextView() *tview.TextView {
-	return tview.NewTextView().SetDynamicColors(true)
-}
-
-// Re-draws application when new logs are streamed to UI
-func handleNewText(app *tview.Application, logTextView *tview.TextView) func() {
-	return func() {
-		app.Draw()
-		// SetFocus to enable scrolling in textview
-		app.SetFocus(logTextView)
-	}
-}
-
-func (data *uiModel) changeConnStatus(event connection.Event, table *tview.Table, log *zerolog.Logger, palette palette) {
-	index := int(event.Index)
-	// Get connection location and state
-	connState := data.getConnState(index)
-	// Check if connection is already displayed in UI
-	if connState == nil {
-		log.Info().Msg("Connection is not in the UI table")
-		return
-	}
-
-	locationState := event.Location
-
-	if event.EventType == connection.Reconnecting {
-		locationState = "Reconnecting..."
-	}
-
-	connectionNum := index + 1
-	// Get table cell
-	cell := table.GetCell(index, 0)
-	// Change dot color in front of text as well as location state
-	text := newCellText(palette, connectionNum, locationState, event.EventType)
-	cell.SetText(text)
-}
-
-// Return connection location and row in UI table
-func (data *uiModel) getConnState(connID int) *connState {
-	if connID < len(data.connections) {
-		return &data.connections[connID]
-	}
-
-	return nil
-}
-
-func (data *uiModel) setConnTableCell(event connection.Event, table *tview.Table, palette palette) {
-	index := int(event.Index)
-	connectionNum := index + 1
-
-	// Update slice to keep track of connection location and state in UI table
-	data.connections[index].location = event.Location
-
-	// Update text in table cell to show disconnected state
-	text := newCellText(palette, connectionNum, event.Location, event.EventType)
-	cell := tview.NewTableCell(text)
-	table.SetCell(index, 0, cell)
-}
-
-func newCellText(palette palette, connectionNum int, location string, connectedStatus connection.Status) string {
-	// HA connection indicator formatted as: "• #<CONNECTION_INDEX>: <COLO>",
-	//  where the left middle dot's color depends on the status of the connection
-	const connFmtString = "[%s]\u2022[%s] #%d: %s"
-
-	var dotColor string
-	switch connectedStatus {
-	case connection.Connected:
-		dotColor = palette.connected
-	case connection.Disconnected:
-		dotColor = palette.disconnected
-	case connection.Reconnecting:
-		dotColor = palette.reconnecting
-	case connection.Unregistering:
-		dotColor = palette.unregistered
-	}
-
-	return fmt.Sprintf(connFmtString, dotColor, palette.defaultText, connectionNum, location)
-}

component-tests/util.py

@@ -61,13 +61,15 @@ def cloudflared_cmd(config, config_path, cfd_args, cfd_pre_args, root):
 def run_cloudflared_background(cmd, allow_input, capture_output):
     output = subprocess.PIPE if capture_output else subprocess.DEVNULL
     stdin = subprocess.PIPE if allow_input else None
+    cfd = None
     try:
         cfd = subprocess.Popen(cmd, stdin=stdin, stdout=output, stderr=output)
         yield cfd
     finally:
-        cfd.terminate()
-        if capture_output:
-            LOGGER.info(f"cloudflared log: {cfd.stderr.read()}")
+        if cfd:
+            cfd.terminate()
+            if capture_output:
+                LOGGER.info(f"cloudflared log: {cfd.stderr.read()}")
 
 
 def wait_tunnel_ready(tunnel_url=None, require_min_connections=1, cfd_logs=None):

connection/connection.go

@@ -24,9 +24,16 @@ const (
 	LogFieldConnIndex      = "connIndex"
 	MaxGracePeriod         = time.Minute * 3
 	MaxConcurrentStreams   = math.MaxUint32
+
+	contentTypeHeader = "content-type"
+	sseContentType    = "text/event-stream"
+	grpcContentType   = "application/grpc"
 )
 
-var switchingProtocolText = fmt.Sprintf("%d %s", http.StatusSwitchingProtocols, http.StatusText(http.StatusSwitchingProtocols))
+var (
+	switchingProtocolText = fmt.Sprintf("%d %s", http.StatusSwitchingProtocols, http.StatusText(http.StatusSwitchingProtocols))
+	flushableContentTypes = []string{sseContentType, grpcContentType}
+)
 
 type Orchestrator interface {
 	UpdateConfig(version int32, config []byte) *pogs.UpdateConfigurationResponse
@@ -123,23 +130,24 @@ func (t Type) String() string {
 
 // OriginProxy is how data flows from cloudflared to the origin services running behind it.
 type OriginProxy interface {
-	ProxyHTTP(w ResponseWriter, tr *tracing.TracedRequest, isWebsocket bool) error
+	ProxyHTTP(w ResponseWriter, tr *tracing.TracedHTTPRequest, isWebsocket bool) error
 	ProxyTCP(ctx context.Context, rwa ReadWriteAcker, req *TCPRequest) error
 }
 
 // TCPRequest defines the input format needed to perform a TCP proxy.
 type TCPRequest struct {
-	Dest    string
-	CFRay   string
-	LBProbe bool
-	FlowID  string
+	Dest      string
+	CFRay     string
+	LBProbe   bool
+	FlowID    string
+	CfTraceID string
 }
 
 // ReadWriteAcker is a readwriter with the ability to Acknowledge to the downstream (edge) that the origin has
 // accepted the connection.
 type ReadWriteAcker interface {
 	io.ReadWriter
-	AckConnection() error
+	AckConnection(tracePropagation string) error
 }
 
 // HTTPResponseReadWriteAcker is an HTTP implementation of ReadWriteAcker.
@@ -168,22 +176,28 @@ func (h *HTTPResponseReadWriteAcker) Write(p []byte) (int, error) {
 
 // AckConnection acks an HTTP connection by sending a switch protocols status code that enables the caller to
 // upgrade to streams.
-func (h *HTTPResponseReadWriteAcker) AckConnection() error {
+func (h *HTTPResponseReadWriteAcker) AckConnection(tracePropagation string) error {
 	resp := &http.Response{
 		Status:        switchingProtocolText,
 		StatusCode:    http.StatusSwitchingProtocols,
 		ContentLength: -1,
+		Header:        http.Header{},
 	}
 
 	if secWebsocketKey := h.req.Header.Get("Sec-WebSocket-Key"); secWebsocketKey != "" {
 		resp.Header = websocket.NewResponseHeader(h.req)
 	}
 
+	if tracePropagation != "" {
+		resp.Header.Add(tracing.CanonicalCloudflaredTracingHeader, tracePropagation)
+	}
+
 	return h.w.WriteRespHeaders(resp.StatusCode, resp.Header)
 }
 
 type ResponseWriter interface {
 	WriteRespHeaders(status int, header http.Header) error
+	AddTrailer(trailerName, trailerValue string)
 	io.Writer
 }
 
@@ -192,10 +206,18 @@ type ConnectedFuse interface {
 	IsConnected() bool
 }
 
-func IsServerSentEvent(headers http.Header) bool {
-	if contentType := headers.Get("content-type"); contentType != "" {
-		return strings.HasPrefix(strings.ToLower(contentType), "text/event-stream")
+// Helper method to let the caller know what content-types should require a flush on every
+// write to a ResponseWriter.
+func shouldFlush(headers http.Header) bool {
+	if contentType := headers.Get(contentTypeHeader); contentType != "" {
+		contentType = strings.ToLower(contentType)
+		for _, c := range flushableContentTypes {
+			if strings.HasPrefix(contentType, c) {
+				return true
+			}
+		}
 	}
+
 	return false
 }
 

connection/connection_test.go

@@ -6,11 +6,9 @@ import (
 	"io"
 	"math/rand"
 	"net/http"
-	"testing"
 	"time"
 
 	"github.com/rs/zerolog"
-	"github.com/stretchr/testify/assert"
 
 	"github.com/cloudflare/cloudflared/tracing"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
@@ -30,6 +28,8 @@ var (
 	testLargeResp = make([]byte, largeFileSize)
 )
 
+var _ ReadWriteAcker = (*HTTPResponseReadWriteAcker)(nil)
+
 type testRequest struct {
 	name           string
 	endpoint       string
@@ -60,7 +60,7 @@ type mockOriginProxy struct{}
 
 func (moc *mockOriginProxy) ProxyHTTP(
 	w ResponseWriter,
-	tr *tracing.TracedRequest,
+	tr *tracing.TracedHTTPRequest,
 	isWebsocket bool,
 ) error {
 	req := tr.Request
@@ -195,40 +195,3 @@ func (mcf mockConnectedFuse) Connected() {}
 func (mcf mockConnectedFuse) IsConnected() bool {
 	return true
 }
-
-func TestIsEventStream(t *testing.T) {
-	tests := []struct {
-		headers       http.Header
-		isEventStream bool
-	}{
-		{
-			headers:       newHeader("Content-Type", "text/event-stream"),
-			isEventStream: true,
-		},
-		{
-			headers:       newHeader("content-type", "text/event-stream"),
-			isEventStream: true,
-		},
-		{
-			headers:       newHeader("Content-Type", "text/event-stream; charset=utf-8"),
-			isEventStream: true,
-		},
-		{
-			headers:       newHeader("Content-Type", "application/json"),
-			isEventStream: false,
-		},
-		{
-			headers:       http.Header{},
-			isEventStream: false,
-		},
-	}
-	for _, test := range tests {
-		assert.Equal(t, test.isEventStream, IsServerSentEvent(test.headers))
-	}
-}
-
-func newHeader(key, value string) http.Header {
-	header := http.Header{}
-	header.Add(key, value)
-	return header
-}

connection/control.go

@@ -2,6 +2,7 @@ package connection
 
 import (
 	"context"
+	"fmt"
 	"io"
 	"net"
 	"time"
@@ -21,6 +22,7 @@ type controlStream struct {
 	namedTunnelProperties *NamedTunnelProperties
 	connIndex             uint8
 	edgeAddress           net.IP
+	protocol              Protocol
 
 	newRPCClientFunc RPCClientFunc
 
@@ -51,6 +53,7 @@ func NewControlStream(
 	newRPCClientFunc RPCClientFunc,
 	gracefulShutdownC <-chan struct{},
 	gracePeriod time.Duration,
+	protocol Protocol,
 ) ControlStreamHandler {
 	if newRPCClientFunc == nil {
 		newRPCClientFunc = newRegistrationRPCClient
@@ -64,6 +67,7 @@ func NewControlStream(
 		edgeAddress:           edgeAddress,
 		gracefulShutdownC:     gracefulShutdownC,
 		gracePeriod:           gracePeriod,
+		protocol:              protocol,
 	}
 }
 
@@ -80,6 +84,9 @@ func (c *controlStream) ServeControlStream(
 		rpcClient.Close()
 		return err
 	}
+
+	c.observer.logServerInfo(c.connIndex, registrationDetails.Location, c.edgeAddress, fmt.Sprintf("Connection %s registered", registrationDetails.UUID))
+	c.observer.sendConnectedEvent(c.connIndex, c.protocol, registrationDetails.Location)
 	c.connectedFuse.Connected()
 
 	// if conn index is 0 and tunnel is not remotely managed, then send local ingress rules configuration

connection/event.go

@@ -5,6 +5,7 @@ type Event struct {
 	Index     uint8
 	EventType Status
 	Location  string
+	Protocol  Protocol
 	URL       string
 }
 

connection/h2mux.go

@@ -69,6 +69,7 @@ func NewH2muxConnection(
 	connIndex uint8,
 	observer *Observer,
 	gracefulShutdownC <-chan struct{},
+	log *zerolog.Logger,
 ) (*h2muxConnection, error, bool) {
 	h := &h2muxConnection{
 		orchestrator:      orchestrator,
@@ -79,6 +80,7 @@ func NewH2muxConnection(
 		observer:          observer,
 		gracefulShutdownC: gracefulShutdownC,
 		newRPCClientFunc:  newRegistrationRPCClient,
+		log:               log,
 	}
 
 	// Establish a muxed connection with the edge
@@ -234,7 +236,7 @@ func (h *h2muxConnection) ServeStream(stream *h2mux.MuxedStream) error {
 		return err
 	}
 
-	err = originProxy.ProxyHTTP(respWriter, tracing.NewTracedRequest(req), sourceConnectionType == TypeWebsocket)
+	err = originProxy.ProxyHTTP(respWriter, tracing.NewTracedHTTPRequest(req, h.log), sourceConnectionType == TypeWebsocket)
 	if err != nil {
 		respWriter.WriteErrorResponse()
 	}
@@ -257,6 +259,10 @@ type h2muxRespWriter struct {
 	*h2mux.MuxedStream
 }
 
+func (rp *h2muxRespWriter) AddTrailer(trailerName, trailerValue string) {
+	// do nothing. we don't support trailers over h2mux
+}
+
 func (rp *h2muxRespWriter) WriteRespHeaders(status int, header http.Header) error {
 	headers := H1ResponseToH2ResponseHeaders(status, header)
 	headers = append(headers, h2mux.Header{Name: ResponseMetaHeader, Value: responseMetaHeaderOrigin})

connection/h2mux_test.go

@@ -47,8 +47,8 @@ func newH2MuxConnection(t require.TestingT) (*h2muxConnection, *h2mux.Muxer) {
 		edgeMuxChan <- edgeMux
 	}()
 	var connIndex = uint8(0)
-	testObserver := NewObserver(&log, &log, false)
-	h2muxConn, err, _ := NewH2muxConnection(testOrchestrator, testGracePeriod, testMuxerConfig, originConn, connIndex, testObserver, nil)
+	testObserver := NewObserver(&log, &log)
+	h2muxConn, err, _ := NewH2muxConnection(testOrchestrator, testGracePeriod, testMuxerConfig, originConn, connIndex, testObserver, nil, &log)
 	require.NoError(t, err)
 	return h2muxConn, <-edgeMuxChan
 }

connection/http2.go

@@ -132,7 +132,7 @@ func (c *HTTP2Connection) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	case TypeWebsocket, TypeHTTP:
 		stripWebsocketUpgradeHeader(r)
 		// Check for tracing on request
-		tr := tracing.NewTracedRequest(r)
+		tr := tracing.NewTracedHTTPRequest(r, c.log)
 		if err := originProxy.ProxyHTTP(respWriter, tr, connType == TypeWebsocket); err != nil {
 			err := fmt.Errorf("Failed to proxy HTTP: %w", err)
 			c.log.Error().Err(err)
@@ -149,9 +149,10 @@ func (c *HTTP2Connection) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
 		rws := NewHTTPResponseReadWriterAcker(respWriter, r)
 		if err := originProxy.ProxyTCP(r.Context(), rws, &TCPRequest{
-			Dest:    host,
-			CFRay:   FindCfRayHeader(r),
-			LBProbe: IsLBProbeRequest(r),
+			Dest:      host,
+			CFRay:     FindCfRayHeader(r),
+			LBProbe:   IsLBProbeRequest(r),
+			CfTraceID: r.Header.Get(tracing.TracerContextName),
 		}); err != nil {
 			respWriter.WriteErrorResponse()
 		}
@@ -190,11 +191,12 @@ func (c *HTTP2Connection) close() {
 }
 
 type http2RespWriter struct {
-	r           io.Reader
-	w           http.ResponseWriter
-	flusher     http.Flusher
-	shouldFlush bool
-	log         *zerolog.Logger
+	r             io.Reader
+	w             http.ResponseWriter
+	flusher       http.Flusher
+	shouldFlush   bool
+	statusWritten bool
+	log           *zerolog.Logger
 }
 
 func NewHTTP2RespWriter(r *http.Request, w http.ResponseWriter, connType Type, log *zerolog.Logger) (*http2RespWriter, error) {
@@ -218,11 +220,20 @@ func NewHTTP2RespWriter(r *http.Request, w http.ResponseWriter, connType Type, l
 	}, nil
 }
 
+func (rp *http2RespWriter) AddTrailer(trailerName, trailerValue string) {
+	if !rp.statusWritten {
+		rp.log.Warn().Msg("Tried to add Trailer to response before status written. Ignoring...")
+		return
+	}
+
+	rp.w.Header().Add(http2.TrailerPrefix+trailerName, trailerValue)
+}
+
 func (rp *http2RespWriter) WriteRespHeaders(status int, header http.Header) error {
 	dest := rp.w.Header()
 	userHeaders := make(http.Header, len(header))
 	for name, values := range header {
-		// Since these are http2 headers, they're required to be lowercase
+		// lowercase headers for simplicity check
 		h2name := strings.ToLower(name)
 
 		if h2name == "content-length" {
@@ -233,7 +244,7 @@ func (rp *http2RespWriter) WriteRespHeaders(status int, header http.Header) erro
 
 		if h2name == tracing.IntCloudflaredTracingHeader {
 			// Add cf-int-cloudflared-tracing header outside of serialized userHeaders
-			rp.w.Header()[tracing.CanonicalCloudflaredTracingHeader] = values
+			dest[tracing.CanonicalCloudflaredTracingHeader] = values
 			continue
 		}
 
@@ -246,18 +257,21 @@ func (rp *http2RespWriter) WriteRespHeaders(status int, header http.Header) erro
 
 	// Perform user header serialization and set them in the single header
 	dest.Set(CanonicalResponseUserHeaders, SerializeHeaders(userHeaders))
+
 	rp.setResponseMetaHeader(responseMetaHeaderOrigin)
 	// HTTP2 removes support for 101 Switching Protocols https://tools.ietf.org/html/rfc7540#section-8.1.1
 	if status == http.StatusSwitchingProtocols {
 		status = http.StatusOK
 	}
 	rp.w.WriteHeader(status)
-	if IsServerSentEvent(header) {
+	if shouldFlush(header) {
 		rp.shouldFlush = true
 	}
 	if rp.shouldFlush {
 		rp.flusher.Flush()
 	}
+
+	rp.statusWritten = true
 	return nil
 }
 

connection/http2_test.go

@@ -33,7 +33,7 @@ func newTestHTTP2Connection() (*HTTP2Connection, net.Conn) {
 	edgeConn, cfdConn := net.Pipe()
 	var connIndex = uint8(0)
 	log := zerolog.Nop()
-	obs := NewObserver(&log, &log, false)
+	obs := NewObserver(&log, &log)
 	controlStream := NewControlStream(
 		obs,
 		mockConnectedFuse{},
@@ -43,6 +43,7 @@ func newTestHTTP2Connection() (*HTTP2Connection, net.Conn) {
 		nil,
 		nil,
 		1*time.Second,
+		HTTP2,
 	)
 	return NewHTTP2Connection(
 		cfdConn,
@@ -356,7 +357,7 @@ func TestServeControlStream(t *testing.T) {
 		unregistered: make(chan struct{}),
 	}
 
-	obs := NewObserver(&log, &log, false)
+	obs := NewObserver(&log, &log)
 	controlStream := NewControlStream(
 		obs,
 		mockConnectedFuse{},
@@ -366,6 +367,7 @@ func TestServeControlStream(t *testing.T) {
 		rpcClientFactory.newMockRPCClient,
 		nil,
 		1*time.Second,
+		HTTP2,
 	)
 	http2Conn.controlStreamHandler = controlStream
 
@@ -407,7 +409,7 @@ func TestFailRegistration(t *testing.T) {
 		unregistered: make(chan struct{}),
 	}
 
-	obs := NewObserver(&log, &log, false)
+	obs := NewObserver(&log, &log)
 	controlStream := NewControlStream(
 		obs,
 		mockConnectedFuse{},
@@ -417,6 +419,7 @@ func TestFailRegistration(t *testing.T) {
 		rpcClientFactory.newMockRPCClient,
 		nil,
 		1*time.Second,
+		HTTP2,
 	)
 	http2Conn.controlStreamHandler = controlStream
 
@@ -453,7 +456,7 @@ func TestGracefulShutdownHTTP2(t *testing.T) {
 	events := &eventCollectorSink{}
 
 	shutdownC := make(chan struct{})
-	obs := NewObserver(&log, &log, false)
+	obs := NewObserver(&log, &log)
 	obs.RegisterSink(events)
 	controlStream := NewControlStream(
 		obs,
@@ -464,6 +467,7 @@ func TestGracefulShutdownHTTP2(t *testing.T) {
 		rpcClientFactory.newMockRPCClient,
 		shutdownC,
 		1*time.Second,
+		HTTP2,
 	)
 
 	http2Conn.controlStreamHandler = controlStream

connection/observer.go

@@ -18,7 +18,6 @@ type Observer struct {
 	logTransport    *zerolog.Logger
 	metrics         *tunnelMetrics
 	tunnelEventChan chan Event
-	uiEnabled       bool
 	addSinkChan     chan EventSink
 }
 
@@ -26,12 +25,11 @@ type EventSink interface {
 	OnTunnelEvent(event Event)
 }
 
-func NewObserver(log, logTransport *zerolog.Logger, uiEnabled bool) *Observer {
+func NewObserver(log, logTransport *zerolog.Logger) *Observer {
 	o := &Observer{
 		log:             log,
 		logTransport:    logTransport,
 		metrics:         newTunnelMetrics(),
-		uiEnabled:       uiEnabled,
 		tunnelEventChan: make(chan Event, observerChannelBufferSize),
 		addSinkChan:     make(chan EventSink, observerChannelBufferSize),
 	}
@@ -57,8 +55,8 @@ func (o *Observer) sendRegisteringEvent(connIndex uint8) {
 	o.sendEvent(Event{Index: connIndex, EventType: RegisteringTunnel})
 }
 
-func (o *Observer) sendConnectedEvent(connIndex uint8, location string) {
-	o.sendEvent(Event{Index: connIndex, EventType: Connected, Location: location})
+func (o *Observer) sendConnectedEvent(connIndex uint8, protocol Protocol, location string) {
+	o.sendEvent(Event{Index: connIndex, EventType: Connected, Protocol: protocol, Location: location})
 }
 
 func (o *Observer) SendURL(url string) {

connection/observer_test.go

@@ -12,7 +12,7 @@ import (
 )
 
 func TestSendUrl(t *testing.T) {
-	observer := NewObserver(&log, &log, false)
+	observer := NewObserver(&log, &log)
 
 	observer.SendURL("my-url.com")
 	assert.Equal(t, 1.0, getCounterValue(t, observer.metrics.userHostnamesCounts, "https://my-url.com"))
@@ -63,7 +63,7 @@ func TestRegisterServerLocation(t *testing.T) {
 }
 
 func TestObserverEventsDontBlock(t *testing.T) {
-	observer := NewObserver(&log, &log, false)
+	observer := NewObserver(&log, &log)
 	var mu sync.Mutex
 	observer.RegisterSink(EventSinkFunc(func(_ Event) {
 		// callback will block if lock is already held

connection/protocol.go

@@ -209,8 +209,8 @@ func NewProtocolSelector(
 
 	threshold := switchThreshold(namedTunnel.Credentials.AccountTag)
 	fetchedProtocol, err := getProtocol([]Protocol{QUIC, HTTP2}, fetchFunc, threshold)
-	if err != nil {
-		log.Err(err).Msg("Unable to lookup protocol. Defaulting to `http2`. If this fails, you can set `--protocol h2mux` in your cloudflared command.")
+	if err != nil && protocolFlag == "auto" {
+		log.Err(err).Msg("Unable to lookup protocol. Defaulting to `http2`. If this fails, you can attempt `--protocol quic` instead.")
 		return &staticProtocolSelector{
 			current: HTTP2,
 		}, nil

connection/quic.go

@@ -9,6 +9,7 @@ import (
 	"net/http"
 	"strconv"
 	"strings"
+	"sync/atomic"
 	"time"
 
 	"github.com/google/uuid"
@@ -19,6 +20,7 @@ import (
 
 	"github.com/cloudflare/cloudflared/datagramsession"
 	"github.com/cloudflare/cloudflared/ingress"
+	"github.com/cloudflare/cloudflared/packet"
 	quicpogs "github.com/cloudflare/cloudflared/quic"
 	"github.com/cloudflare/cloudflared/tracing"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
@@ -33,14 +35,20 @@ const (
 	HTTPHostKey = "HttpHost"
 
 	QUICMetadataFlowID = "FlowID"
+	// emperically this capacity has been working well
+	demuxChanCapacity = 16
 )
 
 // QUICConnection represents the type that facilitates Proxying via QUIC streams.
 type QUICConnection struct {
-	session              quic.Connection
-	logger               *zerolog.Logger
-	orchestrator         Orchestrator
-	sessionManager       datagramsession.Manager
+	session      quic.Connection
+	logger       *zerolog.Logger
+	orchestrator Orchestrator
+	// sessionManager tracks active sessions. It receives datagrams from quic connection via datagramMuxer
+	sessionManager datagramsession.Manager
+	// datagramMuxer mux/demux datagrams from quic connection
+	datagramMuxer        quicpogs.BaseDatagramMuxer
+	packetRouter         *packetRouter
 	controlStreamHandler ControlStreamHandler
 	connOptions          *tunnelpogs.ConnectionOptions
 }
@@ -54,24 +62,37 @@ func NewQUICConnection(
 	connOptions *tunnelpogs.ConnectionOptions,
 	controlStreamHandler ControlStreamHandler,
 	logger *zerolog.Logger,
+	icmpProxy ingress.ICMPProxy,
 ) (*QUICConnection, error) {
 	session, err := quic.DialAddr(edgeAddr.String(), tlsConfig, quicConfig)
 	if err != nil {
 		return nil, &EdgeQuicDialError{Cause: err}
 	}
 
-	datagramMuxer, err := quicpogs.NewDatagramMuxer(session, logger)
-	if err != nil {
-		return nil, err
+	sessionDemuxChan := make(chan *packet.Session, demuxChanCapacity)
+	var (
+		datagramMuxer quicpogs.BaseDatagramMuxer
+		pr            *packetRouter
+	)
+	if icmpProxy != nil {
+		pr = &packetRouter{
+			muxer:     quicpogs.NewDatagramMuxerV2(session, logger, sessionDemuxChan),
+			icmpProxy: icmpProxy,
+			logger:    logger,
+		}
+		datagramMuxer = pr.muxer
+	} else {
+		datagramMuxer = quicpogs.NewDatagramMuxer(session, logger, sessionDemuxChan)
 	}
-
-	sessionManager := datagramsession.NewManager(datagramMuxer, logger)
+	sessionManager := datagramsession.NewManager(logger, datagramMuxer.SendToSession, sessionDemuxChan)
 
 	return &QUICConnection{
 		session:              session,
 		orchestrator:         orchestrator,
 		logger:               logger,
 		sessionManager:       sessionManager,
+		datagramMuxer:        datagramMuxer,
+		packetRouter:         pr,
 		controlStreamHandler: controlStreamHandler,
 		connOptions:          connOptions,
 	}, nil
@@ -108,6 +129,17 @@ func (q *QUICConnection) Serve(ctx context.Context) error {
 		return q.sessionManager.Serve(ctx)
 	})
 
+	errGroup.Go(func() error {
+		defer cancel()
+		return q.datagramMuxer.ServeReceive(ctx)
+	})
+	if q.packetRouter != nil {
+		errGroup.Go(func() error {
+			defer cancel()
+			return q.packetRouter.serve(ctx)
+		})
+	}
+
 	return errGroup.Wait()
 }
 
@@ -143,15 +175,21 @@ func (q *QUICConnection) acceptStream(ctx context.Context) error {
 }
 
 func (q *QUICConnection) runStream(quicStream quic.Stream) {
+	ctx := quicStream.Context()
 	stream := quicpogs.NewSafeStreamCloser(quicStream)
 	defer stream.Close()
 
-	if err := q.handleStream(stream); err != nil {
+	// we are going to fuse readers/writers from stream <- cloudflared -> origin, and we want to guarantee that
+	// code executed in the code path of handleStream don't trigger an earlier close to the downstream write stream.
+	// So, we wrap the stream with a no-op write closer and only this method can actually close write side of the stream.
+	// A call to close will simulate a close to the read-side, which will fail subsequent reads.
+	noCloseStream := &nopCloserReadWriter{ReadWriteCloser: stream}
+	if err := q.handleStream(ctx, noCloseStream); err != nil {
 		q.logger.Err(err).Msg("Failed to handle QUIC stream")
 	}
 }
 
-func (q *QUICConnection) handleStream(stream io.ReadWriteCloser) error {
+func (q *QUICConnection) handleStream(ctx context.Context, stream io.ReadWriteCloser) error {
 	signature, err := quicpogs.DetermineProtocol(stream)
 	if err != nil {
 		return err
@@ -162,7 +200,7 @@ func (q *QUICConnection) handleStream(stream io.ReadWriteCloser) error {
 		if err != nil {
 			return err
 		}
-		return q.handleDataStream(reqServerStream)
+		return q.handleDataStream(ctx, reqServerStream)
 	case quicpogs.RPCStreamProtocolSignature:
 		rpcStream, err := quicpogs.NewRPCServerStream(stream, signature)
 		if err != nil {
@@ -174,13 +212,13 @@ func (q *QUICConnection) handleStream(stream io.ReadWriteCloser) error {
 	}
 }
 
-func (q *QUICConnection) handleDataStream(stream *quicpogs.RequestServerStream) error {
+func (q *QUICConnection) handleDataStream(ctx context.Context, stream *quicpogs.RequestServerStream) error {
 	request, err := stream.ReadConnectRequestData()
 	if err != nil {
 		return err
 	}
 
-	if err := q.dispatchRequest(stream, err, request); err != nil {
+	if err := q.dispatchRequest(ctx, stream, err, request); err != nil {
 		_ = stream.WriteConnectResponseData(err)
 		q.logger.Err(err).Str("type", request.Type.String()).Str("dest", request.Dest).Msg("Request failed")
 	}
@@ -188,7 +226,7 @@ func (q *QUICConnection) handleDataStream(stream *quicpogs.RequestServerStream)
 	return nil
 }
 
-func (q *QUICConnection) dispatchRequest(stream *quicpogs.RequestServerStream, err error, request *quicpogs.ConnectRequest) error {
+func (q *QUICConnection) dispatchRequest(ctx context.Context, stream *quicpogs.RequestServerStream, err error, request *quicpogs.ConnectRequest) error {
 	originProxy, err := q.orchestrator.GetOriginProxy()
 	if err != nil {
 		return err
@@ -196,7 +234,7 @@ func (q *QUICConnection) dispatchRequest(stream *quicpogs.RequestServerStream, e
 
 	switch request.Type {
 	case quicpogs.ConnectionTypeHTTP, quicpogs.ConnectionTypeWebsocket:
-		tracedReq, err := buildHTTPRequest(request, stream)
+		tracedReq, err := buildHTTPRequest(ctx, request, stream, q.logger)
 		if err != nil {
 			return err
 		}
@@ -206,9 +244,10 @@ func (q *QUICConnection) dispatchRequest(stream *quicpogs.RequestServerStream, e
 	case quicpogs.ConnectionTypeTCP:
 		rwa := &streamReadWriteAcker{stream}
 		metadata := request.MetadataMap()
-		return originProxy.ProxyTCP(context.Background(), rwa, &TCPRequest{
-			Dest:   request.Dest,
-			FlowID: metadata[QUICMetadataFlowID],
+		return originProxy.ProxyTCP(ctx, rwa, &TCPRequest{
+			Dest:      request.Dest,
+			FlowID:    metadata[QUICMetadataFlowID],
+			CfTraceID: metadata[tracing.TracerContextName],
 		})
 	}
 	return nil
@@ -288,6 +327,32 @@ func (q *QUICConnection) UpdateConfiguration(ctx context.Context, version int32,
 	return q.orchestrator.UpdateConfig(version, config)
 }
 
+type packetRouter struct {
+	muxer     *quicpogs.DatagramMuxerV2
+	icmpProxy ingress.ICMPProxy
+	logger    *zerolog.Logger
+}
+
+func (pr *packetRouter) serve(ctx context.Context) error {
+	icmpDecoder := packet.NewICMPDecoder()
+	for {
+		pk, err := pr.muxer.ReceivePacket(ctx)
+		if err != nil {
+			return err
+		}
+		icmpPacket, err := icmpDecoder.Decode(pk)
+		if err != nil {
+			pr.logger.Err(err).Msg("Failed to decode ICMP packet from quic datagram")
+			continue
+		}
+
+		if err := pr.icmpProxy.Request(icmpPacket, pr.muxer); err != nil {
+			pr.logger.Err(err).Str("src", icmpPacket.Src.String()).Str("dst", icmpPacket.Dst.String()).Msg("Failed to send ICMP packet")
+			continue
+		}
+	}
+}
+
 // streamReadWriteAcker is a light wrapper over QUIC streams with a callback to send response back to
 // the client.
 type streamReadWriteAcker struct {
@@ -295,8 +360,12 @@ type streamReadWriteAcker struct {
 }
 
 // AckConnection acks response back to the proxy.
-func (s *streamReadWriteAcker) AckConnection() error {
-	return s.WriteConnectResponseData(nil)
+func (s *streamReadWriteAcker) AckConnection(tracePropagation string) error {
+	metadata := quicpogs.Metadata{
+		Key: tracing.CanonicalCloudflaredTracingHeader,
+		Val: tracePropagation,
+	}
+	return s.WriteConnectResponseData(nil, metadata)
 }
 
 // httpResponseAdapter translates responses written by the HTTP Proxy into ones that can be used in QUIC.
@@ -308,6 +377,10 @@ func newHTTPResponseAdapter(s *quicpogs.RequestServerStream) httpResponseAdapter
 	return httpResponseAdapter{s}
 }
 
+func (hrw httpResponseAdapter) AddTrailer(trailerName, trailerValue string) {
+	// we do not support trailers over QUIC
+}
+
 func (hrw httpResponseAdapter) WriteRespHeaders(status int, header http.Header) error {
 	metadata := make([]quicpogs.Metadata, 0)
 	metadata = append(metadata, quicpogs.Metadata{Key: "HttpStatus", Val: strconv.Itoa(status)})
@@ -324,14 +397,19 @@ func (hrw httpResponseAdapter) WriteErrorResponse(err error) {
 	hrw.WriteConnectResponseData(err, quicpogs.Metadata{Key: "HttpStatus", Val: strconv.Itoa(http.StatusBadGateway)})
 }
 
-func buildHTTPRequest(connectRequest *quicpogs.ConnectRequest, body io.ReadCloser) (*tracing.TracedRequest, error) {
+func buildHTTPRequest(
+	ctx context.Context,
+	connectRequest *quicpogs.ConnectRequest,
+	body io.ReadCloser,
+	log *zerolog.Logger,
+) (*tracing.TracedHTTPRequest, error) {
 	metadata := connectRequest.MetadataMap()
 	dest := connectRequest.Dest
 	method := metadata[HTTPMethodKey]
 	host := metadata[HTTPHostKey]
 	isWebsocket := connectRequest.Type == quicpogs.ConnectionTypeWebsocket
 
-	req, err := http.NewRequest(method, dest, body)
+	req, err := http.NewRequestWithContext(ctx, method, dest, body)
 	if err != nil {
 		return nil, err
 	}
@@ -366,7 +444,7 @@ func buildHTTPRequest(connectRequest *quicpogs.ConnectRequest, body io.ReadClose
 	stripWebsocketUpgradeHeader(req)
 
 	// Check for tracing on request
-	tracedReq := tracing.NewTracedRequest(req)
+	tracedReq := tracing.NewTracedHTTPRequest(req, log)
 	return tracedReq, err
 }
 
@@ -384,3 +462,40 @@ func isTransferEncodingChunked(req *http.Request) bool {
 	// separated value as well.
 	return strings.Contains(strings.ToLower(transferEncodingVal), "chunked")
 }
+
+// A helper struct that guarantees a call to close only affects read side, but not write side.
+type nopCloserReadWriter struct {
+	io.ReadWriteCloser
+
+	// for use by Read only
+	// we don't need a memory barrier here because there is an implicit assumption that
+	// Read calls can't happen concurrently by different go-routines.
+	sawEOF bool
+	// should be updated and read using atomic primitives.
+	// value is read in Read method and written in Close method, which could be done by different
+	// go-routines.
+	closed uint32
+}
+
+func (np *nopCloserReadWriter) Read(p []byte) (n int, err error) {
+	if np.sawEOF {
+		return 0, io.EOF
+	}
+
+	if atomic.LoadUint32(&np.closed) > 0 {
+		return 0, fmt.Errorf("closed by handler")
+	}
+
+	n, err = np.ReadWriteCloser.Read(p)
+	if err == io.EOF {
+		np.sawEOF = true
+	}
+
+	return
+}
+
+func (np *nopCloserReadWriter) Close() error {
+	atomic.StoreUint32(&np.closed, 1)
+
+	return nil
+}

connection/quic_test.go

@@ -10,6 +10,7 @@ import (
 	"net/http"
 	"net/url"
 	"os"
+	"strings"
 	"sync"
 	"testing"
 	"time"
@@ -31,11 +32,14 @@ import (
 var (
 	testTLSServerConfig = quicpogs.GenerateTLSConfig()
 	testQUICConfig      = &quic.Config{
-		KeepAlivePeriod: 5 * time.Second,
-		EnableDatagrams: true,
+		ConnectionIDLength: 16,
+		KeepAlivePeriod:    5 * time.Second,
+		EnableDatagrams:    true,
 	}
 )
 
+var _ ReadWriteAcker = (*streamReadWriteAcker)(nil)
+
 // TestQUICServer tests if a quic server accepts and responds to a quic client with the acceptance protocol.
 // It also serves as a demonstration for communication with the QUIC connection started by a cloudflared.
 func TestQUICServer(t *testing.T) {
@@ -220,7 +224,7 @@ func quicServer(
 
 type mockOriginProxyWithRequest struct{}
 
-func (moc *mockOriginProxyWithRequest) ProxyHTTP(w ResponseWriter, tr *tracing.TracedRequest, isWebsocket bool) error {
+func (moc *mockOriginProxyWithRequest) ProxyHTTP(w ResponseWriter, tr *tracing.TracedHTTPRequest, isWebsocket bool) error {
 	// These are a series of crude tests to ensure the headers and http related data is transferred from
 	// metadata.
 	r := tr.Request
@@ -475,9 +479,10 @@ func TestBuildHTTPRequest(t *testing.T) {
 		},
 	}
 
+	log := zerolog.Nop()
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			req, err := buildHTTPRequest(test.connectRequest, test.body)
+			req, err := buildHTTPRequest(context.Background(), test.connectRequest, test.body, &log)
 			assert.NoError(t, err)
 			test.req = test.req.WithContext(req.Context())
 			assert.Equal(t, test.req, req.Request)
@@ -486,7 +491,7 @@ func TestBuildHTTPRequest(t *testing.T) {
 }
 
 func (moc *mockOriginProxyWithRequest) ProxyTCP(ctx context.Context, rwa ReadWriteAcker, tcpRequest *TCPRequest) error {
-	rwa.AckConnection()
+	rwa.AckConnection("")
 	io.Copy(rwa, rwa)
 	return nil
 }
@@ -500,6 +505,7 @@ func TestServeUDPSession(t *testing.T) {
 	defer udpListener.Close()
 
 	ctx, cancel := context.WithCancel(context.Background())
+	val := udpListener.LocalAddr()
 
 	// Establish QUIC connection with edge
 	edgeQUICSessionChan := make(chan quic.Connection)
@@ -512,7 +518,7 @@ func TestServeUDPSession(t *testing.T) {
 		edgeQUICSessionChan <- edgeQUICSession
 	}()
 
-	qc := testQUICConnection(udpListener.LocalAddr(), t)
+	qc := testQUICConnection(val, t)
 	go qc.Serve(ctx)
 
 	edgeQUICSession := <-edgeQUICSessionChan
@@ -522,6 +528,44 @@ func TestServeUDPSession(t *testing.T) {
 	cancel()
 }
 
+func TestNopCloserReadWriterCloseBeforeEOF(t *testing.T) {
+	readerWriter := nopCloserReadWriter{ReadWriteCloser: &mockReaderNoopWriter{Reader: strings.NewReader("123456789")}}
+	buffer := make([]byte, 5)
+
+	n, err := readerWriter.Read(buffer)
+	require.NoError(t, err)
+	require.Equal(t, n, 5)
+
+	// close
+	require.NoError(t, readerWriter.Close())
+
+	// read should get error
+	n, err = readerWriter.Read(buffer)
+	require.Equal(t, n, 0)
+	require.Equal(t, err, fmt.Errorf("closed by handler"))
+}
+
+func TestNopCloserReadWriterCloseAfterEOF(t *testing.T) {
+	readerWriter := nopCloserReadWriter{ReadWriteCloser: &mockReaderNoopWriter{Reader: strings.NewReader("123456789")}}
+	buffer := make([]byte, 20)
+
+	n, err := readerWriter.Read(buffer)
+	require.NoError(t, err)
+	require.Equal(t, n, 9)
+
+	// force another read to read eof
+	n, err = readerWriter.Read(buffer)
+	require.Equal(t, err, io.EOF)
+
+	// close
+	require.NoError(t, readerWriter.Close())
+
+	// read should get EOF still
+	n, err = readerWriter.Read(buffer)
+	require.Equal(t, n, 0)
+	require.Equal(t, err, io.EOF)
+}
+
 func serveSession(ctx context.Context, qc *QUICConnection, edgeQUICSession quic.Connection, closeType closeReason, expectedReason string, t *testing.T) {
 	var (
 		payload = []byte(t.Name())
@@ -638,7 +682,20 @@ func testQUICConnection(udpListenerAddr net.Addr, t *testing.T) *QUICConnection
 		&tunnelpogs.ConnectionOptions{},
 		fakeControlStream{},
 		&log,
+		nil,
 	)
 	require.NoError(t, err)
 	return qc
 }
+
+type mockReaderNoopWriter struct {
+	io.Reader
+}
+
+func (m *mockReaderNoopWriter) Write(p []byte) (n int, err error) {
+	return len(p), nil
+}
+
+func (m *mockReaderNoopWriter) Close() error {
+	return nil
+}

connection/rpc.go

@@ -117,9 +117,6 @@ func (rsc *registrationServerClient) RegisterConnection(
 
 	observer.metrics.regSuccess.WithLabelValues("registerConnection").Inc()
 
-	observer.logServerInfo(connIndex, conn.Location, edgeAddress, fmt.Sprintf("Connection %s registered", conn.UUID))
-	observer.sendConnectedEvent(connIndex, conn.Location)
-
 	return conn, nil
 }
 
@@ -293,9 +290,13 @@ func (h *h2muxConnection) registerNamedTunnel(
 	rpcClient := h.newRPCClientFunc(ctx, stream, h.observer.log)
 	defer rpcClient.Close()
 
-	if _, err = rpcClient.RegisterConnection(ctx, namedTunnel, connOptions, h.connIndex, nil, h.observer); err != nil {
+	registrationDetails, err := rpcClient.RegisterConnection(ctx, namedTunnel, connOptions, h.connIndex, nil, h.observer)
+	if err != nil {
 		return err
 	}
+	h.observer.logServerInfo(h.connIndex, registrationDetails.Location, nil, fmt.Sprintf("Connection %s registered", registrationDetails.UUID))
+	h.observer.sendConnectedEvent(h.connIndex, H2mux, registrationDetails.Location)
+
 	return nil
 }
 

datagramsession/event.go

@@ -42,9 +42,3 @@ func (sc *errClosedSession) Error() string {
 		return fmt.Sprintf("session closed by local due to %s", sc.message)
 	}
 }
-
-// newDatagram is an event when transport receives new datagram
-type newDatagram struct {
-	sessionID uuid.UUID
-	payload   []byte
-}

datagramsession/manager.go

@@ -7,9 +7,9 @@ import (
 	"time"
 
 	"github.com/google/uuid"
-	"github.com/lucas-clemente/quic-go"
 	"github.com/rs/zerolog"
-	"golang.org/x/sync/errgroup"
+
+	"github.com/cloudflare/cloudflared/packet"
 )
 
 const (
@@ -36,26 +36,25 @@ type Manager interface {
 type manager struct {
 	registrationChan   chan *registerSessionEvent
 	unregistrationChan chan *unregisterSessionEvent
-	datagramChan       chan *newDatagram
-	closedChan         chan struct{}
-	transport          transport
+	sendFunc           transportSender
+	receiveChan        <-chan *packet.Session
+	closedChan         <-chan struct{}
 	sessions           map[uuid.UUID]*Session
 	log                *zerolog.Logger
 	// timeout waiting for an API to finish. This can be overriden in test
 	timeout time.Duration
 }
 
-func NewManager(transport transport, log *zerolog.Logger) *manager {
+func NewManager(log *zerolog.Logger, sendF transportSender, receiveChan <-chan *packet.Session) *manager {
 	return &manager{
 		registrationChan:   make(chan *registerSessionEvent),
 		unregistrationChan: make(chan *unregisterSessionEvent),
-		// datagramChan is buffered, so it can read more datagrams from transport while the event loop is processing other events
-		datagramChan: make(chan *newDatagram, requestChanCapacity),
-		closedChan:   make(chan struct{}),
-		transport:    transport,
-		sessions:     make(map[uuid.UUID]*Session),
-		log:          log,
-		timeout:      defaultReqTimeout,
+		sendFunc:           sendF,
+		receiveChan:        receiveChan,
+		closedChan:         make(chan struct{}),
+		sessions:           make(map[uuid.UUID]*Session),
+		log:                log,
+		timeout:            defaultReqTimeout,
 	}
 }
 
@@ -65,49 +64,21 @@ func (m *manager) UpdateLogger(log *zerolog.Logger) {
 }
 
 func (m *manager) Serve(ctx context.Context) error {
-	errGroup, ctx := errgroup.WithContext(ctx)
-	errGroup.Go(func() error {
-		for {
-			sessionID, payload, err := m.transport.ReceiveFrom()
-			if err != nil {
-				if aerr, ok := err.(*quic.ApplicationError); ok && uint64(aerr.ErrorCode) == uint64(quic.NoError) {
-					return nil
-				} else {
-					return err
-				}
-			}
-			datagram := &newDatagram{
-				sessionID: sessionID,
-				payload:   payload,
-			}
-			select {
-			case <-ctx.Done():
-				return ctx.Err()
-			// Only the event loop routine can update/lookup the sessions map to avoid concurrent access
-			// Send the datagram to the event loop. It will find the session to send to
-			case m.datagramChan <- datagram:
-			}
+	for {
+		select {
+		case <-ctx.Done():
+			m.shutdownSessions(ctx.Err())
+			return ctx.Err()
+		// receiveChan is buffered, so the transport can read more datagrams from transport while the event loop is
+		// processing other events
+		case datagram := <-m.receiveChan:
+			m.sendToSession(datagram)
+		case registration := <-m.registrationChan:
+			m.registerSession(ctx, registration)
+		case unregistration := <-m.unregistrationChan:
+			m.unregisterSession(unregistration)
 		}
-	})
-	errGroup.Go(func() error {
-		for {
-			select {
-			case <-ctx.Done():
-				return nil
-			case datagram := <-m.datagramChan:
-				m.sendToSession(datagram)
-			case registration := <-m.registrationChan:
-				m.registerSession(ctx, registration)
-			// TODO: TUN-5422: Unregister inactive session upon timeout
-			case unregistration := <-m.unregistrationChan:
-				m.unregisterSession(unregistration)
-			}
-		}
-	})
-	err := errGroup.Wait()
-	close(m.closedChan)
-	m.shutdownSessions(err)
-	return err
+	}
 }
 
 func (m *manager) shutdownSessions(err error) {
@@ -149,16 +120,17 @@ func (m *manager) registerSession(ctx context.Context, registration *registerSes
 }
 
 func (m *manager) newSession(id uuid.UUID, dstConn io.ReadWriteCloser) *Session {
+	logger := m.log.With().Str("sessionID", id.String()).Logger()
 	return &Session{
-		ID:        id,
-		transport: m.transport,
-		dstConn:   dstConn,
+		ID:       id,
+		sendFunc: m.sendFunc,
+		dstConn:  dstConn,
 		// activeAtChan has low capacity. It can be full when there are many concurrent read/write. markActive() will
 		// drop instead of blocking because last active time only needs to be an approximation
 		activeAtChan: make(chan time.Time, 2),
 		// capacity is 2 because close() and dstToTransport routine in Serve() can write to this channel
 		closeChan: make(chan error, 2),
-		log:       m.log,
+		log:       &logger,
 	}
 }
 
@@ -191,16 +163,13 @@ func (m *manager) unregisterSession(unregistration *unregisterSessionEvent) {
 	}
 }
 
-func (m *manager) sendToSession(datagram *newDatagram) {
-	session, ok := m.sessions[datagram.sessionID]
+func (m *manager) sendToSession(datagram *packet.Session) {
+	session, ok := m.sessions[datagram.ID]
 	if !ok {
-		m.log.Error().Str("sessionID", datagram.sessionID.String()).Msg("session not found")
+		m.log.Error().Str("sessionID", datagram.ID.String()).Msg("session not found")
 		return
 	}
 	// session writes to destination over a connected UDP socket, which should not be blocking, so this call doesn't
 	// need to run in another go routine
-	_, err := session.transportToDst(datagram.payload)
-	if err != nil {
-		m.log.Err(err).Str("sessionID", datagram.sessionID.String()).Msg("Failed to write payload to session")
-	}
+	session.transportToDst(datagram.Payload)
 }

datagramsession/manager_test.go

@@ -14,22 +14,30 @@ import (
 	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/require"
 	"golang.org/x/sync/errgroup"
+
+	"github.com/cloudflare/cloudflared/packet"
+)
+
+var (
+	nopLogger = zerolog.Nop()
 )
 
 func TestManagerServe(t *testing.T) {
 	const (
-		sessions            = 20
-		msgs                = 50
+		sessions            = 2
+		msgs                = 5
 		remoteUnregisterMsg = "eyeball closed connection"
 	)
 
-	mg, transport := newTestManager(1)
-
-	eyeballTracker := make(map[uuid.UUID]*datagramChannel)
-	for i := 0; i < sessions; i++ {
-		sessionID := uuid.New()
-		eyeballTracker[sessionID] = newDatagramChannel(1)
+	requestChan := make(chan *packet.Session)
+	transport := mockQUICTransport{
+		sessions: make(map[uuid.UUID]chan []byte),
 	}
+	for i := 0; i < sessions; i++ {
+		transport.sessions[uuid.New()] = make(chan []byte)
+	}
+
+	mg := NewManager(&nopLogger, transport.MuxSession, requestChan)
 
 	ctx, cancel := context.WithCancel(context.Background())
 	serveDone := make(chan struct{})
@@ -38,53 +46,42 @@ func TestManagerServe(t *testing.T) {
 		close(serveDone)
 	}(ctx)
 
-	go func(ctx context.Context) {
-		for {
-			sessionID, payload, err := transport.respChan.Receive(ctx)
-			if err != nil {
-				require.Equal(t, context.Canceled, err)
-				return
-			}
-			respChan := eyeballTracker[sessionID]
-			require.NoError(t, respChan.Send(ctx, sessionID, payload))
-		}
-	}(ctx)
-
 	errGroup, ctx := errgroup.WithContext(ctx)
-	for sID, receiver := range eyeballTracker {
+	for sessionID, eyeballRespChan := range transport.sessions {
+		// Assign loop variables to local variables
+		sID := sessionID
+		payload := testPayload(sID)
+		expectResp := testResponse(payload)
+
+		cfdConn, originConn := net.Pipe()
+
+		origin := mockOrigin{
+			expectMsgCount: msgs,
+			expectedMsg:    payload,
+			expectedResp:   expectResp,
+			conn:           originConn,
+		}
+
+		eyeball := mockEyeballSession{
+			id:               sID,
+			expectedMsgCount: msgs,
+			expectedMsg:      payload,
+			expectedResponse: expectResp,
+			respReceiver:     eyeballRespChan,
+		}
+
 		// Assign loop variables to local variables
-		sessionID := sID
-		eyeballRespReceiver := receiver
 		errGroup.Go(func() error {
-			payload := testPayload(sessionID)
-			expectResp := testResponse(payload)
-
-			cfdConn, originConn := net.Pipe()
-
-			origin := mockOrigin{
-				expectMsgCount: msgs,
-				expectedMsg:    payload,
-				expectedResp:   expectResp,
-				conn:           originConn,
-			}
-			eyeball := mockEyeball{
-				expectMsgCount:  msgs,
-				expectedMsg:     expectResp,
-				expectSessionID: sessionID,
-				respReceiver:    eyeballRespReceiver,
-			}
-
+			session, err := mg.RegisterSession(ctx, sID, cfdConn)
+			require.NoError(t, err)
 			reqErrGroup, reqCtx := errgroup.WithContext(ctx)
 			reqErrGroup.Go(func() error {
 				return origin.serve()
 			})
 			reqErrGroup.Go(func() error {
-				return eyeball.serve(reqCtx)
+				return eyeball.serve(reqCtx, requestChan)
 			})
 
-			session, err := mg.RegisterSession(ctx, sessionID, cfdConn)
-			require.NoError(t, err)
-
 			sessionDone := make(chan struct{})
 			go func() {
 				closedByRemote, err := session.Serve(ctx, time.Minute*2)
@@ -97,23 +94,17 @@ func TestManagerServe(t *testing.T) {
 				close(sessionDone)
 			}()
 
-			for i := 0; i < msgs; i++ {
-				require.NoError(t, transport.newRequest(ctx, sessionID, testPayload(sessionID)))
-			}
-
 			// Make sure eyeball and origin have received all messages before unregistering the session
 			require.NoError(t, reqErrGroup.Wait())
 
-			require.NoError(t, mg.UnregisterSession(ctx, sessionID, remoteUnregisterMsg, true))
+			require.NoError(t, mg.UnregisterSession(ctx, sID, remoteUnregisterMsg, true))
 			<-sessionDone
-
 			return nil
 		})
 	}
 
 	require.NoError(t, errGroup.Wait())
 	cancel()
-	transport.close()
 	<-serveDone
 }
 
@@ -122,7 +113,7 @@ func TestTimeout(t *testing.T) {
 		testTimeout = time.Millisecond * 50
 	)
 
-	mg, _ := newTestManager(1)
+	mg := NewManager(&nopLogger, nil, nil)
 	mg.timeout = testTimeout
 	ctx := context.Background()
 	sessionID := uuid.New()
@@ -135,9 +126,51 @@ func TestTimeout(t *testing.T) {
 	require.ErrorIs(t, err, context.DeadlineExceeded)
 }
 
-func TestCloseTransportCloseSessions(t *testing.T) {
-	mg, transport := newTestManager(1)
-	ctx := context.Background()
+func TestUnregisterSessionCloseSession(t *testing.T) {
+	sessionID := uuid.New()
+	payload := []byte(t.Name())
+	sender := newMockTransportSender(sessionID, payload)
+	mg := NewManager(&nopLogger, sender.muxSession, nil)
+	ctx, cancel := context.WithCancel(context.Background())
+
+	managerDone := make(chan struct{})
+	go func() {
+		err := mg.Serve(ctx)
+		require.Error(t, err)
+		close(managerDone)
+	}()
+
+	cfdConn, originConn := net.Pipe()
+	session, err := mg.RegisterSession(ctx, sessionID, cfdConn)
+	require.NoError(t, err)
+	require.NotNil(t, session)
+
+	unregisteredChan := make(chan struct{})
+	go func() {
+		_, err := originConn.Write(payload)
+		require.NoError(t, err)
+
+		err = mg.UnregisterSession(ctx, sessionID, "eyeball closed session", true)
+		require.NoError(t, err)
+
+		close(unregisteredChan)
+	}()
+
+	closedByRemote, err := session.Serve(ctx, time.Minute)
+	require.True(t, closedByRemote)
+	require.Error(t, err)
+
+	<-unregisteredChan
+	cancel()
+	<-managerDone
+}
+
+func TestManagerCtxDoneCloseSessions(t *testing.T) {
+	sessionID := uuid.New()
+	payload := []byte(t.Name())
+	sender := newMockTransportSender(sessionID, payload)
+	mg := NewManager(&nopLogger, sender.muxSession, nil)
+	ctx, cancel := context.WithCancel(context.Background())
 
 	var wg sync.WaitGroup
 	wg.Add(1)
@@ -147,35 +180,26 @@ func TestCloseTransportCloseSessions(t *testing.T) {
 		require.Error(t, err)
 	}()
 
-	cfdConn, eyeballConn := net.Pipe()
-	session, err := mg.RegisterSession(ctx, uuid.New(), cfdConn)
+	cfdConn, originConn := net.Pipe()
+	session, err := mg.RegisterSession(ctx, sessionID, cfdConn)
 	require.NoError(t, err)
 	require.NotNil(t, session)
 
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
-		_, err := eyeballConn.Write([]byte(t.Name()))
+		_, err := originConn.Write(payload)
 		require.NoError(t, err)
-		transport.close()
+		cancel()
 	}()
 
 	closedByRemote, err := session.Serve(ctx, time.Minute)
-	require.True(t, closedByRemote)
+	require.False(t, closedByRemote)
 	require.Error(t, err)
 
 	wg.Wait()
 }
 
-func newTestManager(capacity uint) (*manager, *mockQUICTransport) {
-	log := zerolog.Nop()
-	transport := &mockQUICTransport{
-		reqChan:  newDatagramChannel(capacity),
-		respChan: newDatagramChannel(capacity),
-	}
-	return NewManager(transport, &log), transport
-}
-
 type mockOrigin struct {
 	expectMsgCount int
 	expectedMsg    []byte
@@ -197,7 +221,6 @@ func (mo *mockOrigin) serve() error {
 		if !bytes.Equal(readBuffer[:n], mo.expectedMsg) {
 			return fmt.Errorf("Expect %v, read %v", mo.expectedMsg, readBuffer[:n])
 		}
-
 		_, err = mo.conn.Write(mo.expectedResp)
 		if err != nil {
 			return err
@@ -214,72 +237,35 @@ func testResponse(msg []byte) []byte {
 	return []byte(fmt.Sprintf("Response to %v", msg))
 }
 
-type mockEyeball struct {
-	expectMsgCount  int
-	expectedMsg     []byte
-	expectSessionID uuid.UUID
-	respReceiver    *datagramChannel
+type mockQUICTransport struct {
+	sessions map[uuid.UUID]chan []byte
 }
 
-func (me *mockEyeball) serve(ctx context.Context) error {
-	for i := 0; i < me.expectMsgCount; i++ {
-		sessionID, msg, err := me.respReceiver.Receive(ctx)
-		if err != nil {
-			return err
-		}
-		if sessionID != me.expectSessionID {
-			return fmt.Errorf("Expect session %s, got %s", me.expectSessionID, sessionID)
-		}
-		if !bytes.Equal(msg, me.expectedMsg) {
-			return fmt.Errorf("Expect %v, read %v", me.expectedMsg, msg)
-		}
-	}
+func (me *mockQUICTransport) MuxSession(session *packet.Session) error {
+	s := me.sessions[session.ID]
+	s <- session.Payload
 	return nil
 }
 
-// datagramChannel is a channel for Datagram with wrapper to send/receive with context
-type datagramChannel struct {
-	datagramChan chan *newDatagram
-	closedChan   chan struct{}
+type mockEyeballSession struct {
+	id               uuid.UUID
+	expectedMsgCount int
+	expectedMsg      []byte
+	expectedResponse []byte
+	respReceiver     <-chan []byte
 }
 
-func newDatagramChannel(capacity uint) *datagramChannel {
-	return &datagramChannel{
-		datagramChan: make(chan *newDatagram, capacity),
-		closedChan:   make(chan struct{}),
-	}
-}
-
-func (rc *datagramChannel) Send(ctx context.Context, sessionID uuid.UUID, payload []byte) error {
-	select {
-	case <-ctx.Done():
-		return ctx.Err()
-	case <-rc.closedChan:
-		return &errClosedSession{
-			message:  fmt.Errorf("datagram channel closed").Error(),
-			byRemote: true,
+func (me *mockEyeballSession) serve(ctx context.Context, requestChan chan *packet.Session) error {
+	for i := 0; i < me.expectedMsgCount; i++ {
+		requestChan <- &packet.Session{
+			ID:      me.id,
+			Payload: me.expectedMsg,
 		}
-	case rc.datagramChan <- &newDatagram{sessionID: sessionID, payload: payload}:
-		return nil
-	}
-}
-
-func (rc *datagramChannel) Receive(ctx context.Context) (uuid.UUID, []byte, error) {
-	select {
-	case <-ctx.Done():
-		return uuid.Nil, nil, ctx.Err()
-	case <-rc.closedChan:
-		err := &errClosedSession{
-			message:  fmt.Errorf("datagram channel closed").Error(),
-			byRemote: true,
+		resp := <-me.respReceiver
+		if !bytes.Equal(resp, me.expectedResponse) {
+			return fmt.Errorf("Expect %v, read %v", me.expectedResponse, resp)
 		}
-		return uuid.Nil, nil, err
-	case msg := <-rc.datagramChan:
-		return msg.sessionID, msg.payload, nil
+		fmt.Println("Resp", resp)
 	}
-}
-
-func (rc *datagramChannel) Close() {
-	// No need to close msgChan, it will be garbage collect once there is no reference to it
-	close(rc.closedChan)
+	return nil
 }

datagramsession/session.go

@@ -2,12 +2,16 @@ package datagramsession
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"io"
+	"net"
 	"time"
 
 	"github.com/google/uuid"
 	"github.com/rs/zerolog"
+
+	"github.com/cloudflare/cloudflared/packet"
 )
 
 const (
@@ -18,21 +22,20 @@ func SessionIdleErr(timeout time.Duration) error {
 	return fmt.Errorf("session idle for %v", timeout)
 }
 
+type transportSender func(session *packet.Session) error
+
 // Session is a bidirectional pipe of datagrams between transport and dstConn
-// Currently the only implementation of transport is quic DatagramMuxer
 // Destination can be a connection with origin or with eyeball
 // When the destination is origin:
-// - Datagrams from edge are read by Manager from the transport. Manager finds the corresponding Session and calls the
-//   write method of the Session to send to origin
-// - Datagrams from origin are read from conn and SentTo transport. Transport will return them to eyeball
+// - Manager receives datagrams from receiveChan and calls the transportToDst method of the Session to send to origin
+// - Datagrams from origin are read from conn and Send to transport using the transportSender callback. Transport will return them to eyeball
 // When the destination is eyeball:
-// - Datagrams from eyeball are read from conn and SentTo transport. Transport will send them to cloudflared
-// - Datagrams from cloudflared are read by Manager from the transport. Manager finds the corresponding Session and calls the
-//   write method of the Session to send to eyeball
+// - Datagrams from eyeball are read from conn and Send to transport. Transport will send them to cloudflared using the transportSender callback.
+// - Manager receives datagrams from receiveChan and calls the transportToDst method of the Session to send to the eyeball
 type Session struct {
-	ID        uuid.UUID
-	transport transport
-	dstConn   io.ReadWriteCloser
+	ID       uuid.UUID
+	sendFunc transportSender
+	dstConn  io.ReadWriteCloser
 	// activeAtChan is used to communicate the last read/write time
 	activeAtChan chan time.Time
 	closeChan    chan error
@@ -46,9 +49,16 @@ func (s *Session) Serve(ctx context.Context, closeAfterIdle time.Duration) (clos
 		const maxPacketSize = 1500
 		readBuffer := make([]byte, maxPacketSize)
 		for {
-			if err := s.dstToTransport(readBuffer); err != nil {
-				s.closeChan <- err
-				return
+			if closeSession, err := s.dstToTransport(readBuffer); err != nil {
+				if errors.Is(err, net.ErrClosed) {
+					s.log.Debug().Msg("Destination connection closed")
+				} else {
+					s.log.Error().Err(err).Msg("Failed to send session payload from destination to transport")
+				}
+				if closeSession {
+					s.closeChan <- err
+					return
+				}
 			}
 		}
 	}()
@@ -89,32 +99,29 @@ func (s *Session) waitForCloseCondition(ctx context.Context, closeAfterIdle time
 	}
 }
 
-func (s *Session) dstToTransport(buffer []byte) error {
+func (s *Session) dstToTransport(buffer []byte) (closeSession bool, err error) {
 	n, err := s.dstConn.Read(buffer)
 	s.markActive()
 	// https://pkg.go.dev/io#Reader suggests caller should always process n > 0 bytes
-	if n > 0 {
-		if n <= int(s.transport.MTU()) {
-			err = s.transport.SendTo(s.ID, buffer[:n])
-		} else {
-			// drop packet for now, eventually reply with ICMP for PMTUD
-			s.log.Debug().
-				Str("session", s.ID.String()).
-				Int("len", n).
-				Int("mtu", s.transport.MTU()).
-				Msg("dropped packet exceeding MTU")
+	if n > 0 || err == nil {
+		session := packet.Session{
+			ID:      s.ID,
+			Payload: buffer[:n],
+		}
+		if sendErr := s.sendFunc(&session); sendErr != nil {
+			return false, sendErr
 		}
 	}
-	// Some UDP application might send 0-size payload.
-	if err == nil && n == 0 {
-		err = s.transport.SendTo(s.ID, []byte{})
-	}
-	return err
+	return err != nil, err
 }
 
 func (s *Session) transportToDst(payload []byte) (int, error) {
 	s.markActive()
-	return s.dstConn.Write(payload)
+	n, err := s.dstConn.Write(payload)
+	if err != nil {
+		s.log.Err(err).Msg("Failed to write payload to session")
+	}
+	return n, err
 }
 
 // Sends the last active time to the idle checker loop without blocking. activeAtChan will only be full when there

datagramsession/session_test.go

@@ -11,8 +11,11 @@ import (
 	"time"
 
 	"github.com/google/uuid"
+	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/require"
 	"golang.org/x/sync/errgroup"
+
+	"github.com/cloudflare/cloudflared/packet"
 )
 
 // TestCloseSession makes sure a session will stop after context is done
@@ -41,7 +44,8 @@ func testSessionReturns(t *testing.T, closeBy closeMethod, closeAfterIdle time.D
 	cfdConn, originConn := net.Pipe()
 	payload := testPayload(sessionID)
 
-	mg, _ := newTestManager(1)
+	log := zerolog.Nop()
+	mg := NewManager(&log, nil, nil)
 	session := mg.newSession(sessionID, cfdConn)
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -114,7 +118,9 @@ func testActiveSessionNotClosed(t *testing.T, readFromDst bool, writeToDst bool)
 	cfdConn, originConn := net.Pipe()
 	payload := testPayload(sessionID)
 
-	mg, _ := newTestManager(100)
+	respChan := make(chan *packet.Session)
+	sender := newMockTransportSender(sessionID, payload)
+	mg := NewManager(&nopLogger, sender.muxSession, respChan)
 	session := mg.newSession(sessionID, cfdConn)
 
 	startTime := time.Now()
@@ -177,7 +183,7 @@ func testActiveSessionNotClosed(t *testing.T, readFromDst bool, writeToDst bool)
 
 func TestMarkActiveNotBlocking(t *testing.T) {
 	const concurrentCalls = 50
-	mg, _ := newTestManager(1)
+	mg := NewManager(&nopLogger, nil, nil)
 	session := mg.newSession(uuid.New(), nil)
 	var wg sync.WaitGroup
 	wg.Add(concurrentCalls)
@@ -190,11 +196,16 @@ func TestMarkActiveNotBlocking(t *testing.T) {
 	wg.Wait()
 }
 
+// Some UDP application might send 0-size payload.
 func TestZeroBytePayload(t *testing.T) {
 	sessionID := uuid.New()
 	cfdConn, originConn := net.Pipe()
 
-	mg, transport := newTestManager(1)
+	sender := sendOnceTransportSender{
+		baseSender: newMockTransportSender(sessionID, make([]byte, 0)),
+		sentChan:   make(chan struct{}),
+	}
+	mg := NewManager(&nopLogger, sender.muxSession, nil)
 	session := mg.newSession(sessionID, cfdConn)
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -215,11 +226,39 @@ func TestZeroBytePayload(t *testing.T) {
 		return nil
 	})
 
-	receivedSessionID, payload, err := transport.respChan.Receive(ctx)
-	require.NoError(t, err)
-	require.Len(t, payload, 0)
-	require.Equal(t, sessionID, receivedSessionID)
-
+	<-sender.sentChan
 	cancel()
 	require.NoError(t, errGroup.Wait())
 }
+
+type mockTransportSender struct {
+	expectedSessionID uuid.UUID
+	expectedPayload   []byte
+}
+
+func newMockTransportSender(expectedSessionID uuid.UUID, expectedPayload []byte) *mockTransportSender {
+	return &mockTransportSender{
+		expectedSessionID: expectedSessionID,
+		expectedPayload:   expectedPayload,
+	}
+}
+
+func (mts *mockTransportSender) muxSession(session *packet.Session) error {
+	if session.ID != mts.expectedSessionID {
+		return fmt.Errorf("Expect session %s, got %s", mts.expectedSessionID, session.ID)
+	}
+	if !bytes.Equal(session.Payload, mts.expectedPayload) {
+		return fmt.Errorf("Expect %v, read %v", mts.expectedPayload, session.Payload)
+	}
+	return nil
+}
+
+type sendOnceTransportSender struct {
+	baseSender *mockTransportSender
+	sentChan   chan struct{}
+}
+
+func (sots *sendOnceTransportSender) muxSession(session *packet.Session) error {
+	defer close(sots.sentChan)
+	return sots.baseSender.muxSession(session)
+}

datagramsession/transport.go

@@ -1,13 +0,0 @@
-package datagramsession
-
-import "github.com/google/uuid"
-
-// Transport is a connection between cloudflared and edge that can multiplex datagrams from multiple sessions
-type transport interface {
-	// SendTo writes payload for a session to the transport
-	SendTo(sessionID uuid.UUID, payload []byte) error
-	// ReceiveFrom reads the next datagram from the transport
-	ReceiveFrom() (uuid.UUID, []byte, error)
-	// Max transmission unit to receive from the transport
-	MTU() int
-}

datagramsession/transport_test.go

@@ -1,36 +0,0 @@
-package datagramsession
-
-import (
-	"context"
-
-	"github.com/google/uuid"
-)
-
-type mockQUICTransport struct {
-	reqChan  *datagramChannel
-	respChan *datagramChannel
-}
-
-func (mt *mockQUICTransport) SendTo(sessionID uuid.UUID, payload []byte) error {
-	buf := make([]byte, len(payload))
-	// The QUIC implementation copies data to another buffer before returning https://github.com/lucas-clemente/quic-go/blob/v0.24.0/session.go#L1967-L1975
-	copy(buf, payload)
-	return mt.respChan.Send(context.Background(), sessionID, buf)
-}
-
-func (mt *mockQUICTransport) ReceiveFrom() (uuid.UUID, []byte, error) {
-	return mt.reqChan.Receive(context.Background())
-}
-
-func (mt *mockQUICTransport) MTU() int {
-	return 1280
-}
-
-func (mt *mockQUICTransport) newRequest(ctx context.Context, sessionID uuid.UUID, payload []byte) error {
-	return mt.reqChan.Send(ctx, sessionID, payload)
-}
-
-func (mt *mockQUICTransport) close() {
-	mt.reqChan.Close()
-	mt.respChan.Close()
-}

dev.Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.17.10 as builder
+FROM golang:1.18.1 as builder
 ENV GO111MODULE=on \
     CGO_ENABLED=0
 WORKDIR /go/src/github.com/cloudflare/cloudflared/

go.mod

@@ -1,6 +1,6 @@
 module github.com/cloudflare/cloudflared
 
-go 1.17
+go 1.18
 
 require (
 	github.com/cloudflare/brotli-go v0.0.0-20191101163834-d34379f7ff93
@@ -9,7 +9,6 @@ require (
 	github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf
 	github.com/facebookgo/grace v0.0.0-20180706040059-75cf19382434
 	github.com/fsnotify/fsnotify v1.4.9
-	github.com/gdamore/tcell v1.3.0
 	github.com/getsentry/raven-go v0.0.0-20180517221441-ed7bcb39ff10
 	github.com/gobwas/ws v1.0.4
 	github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3
@@ -17,14 +16,13 @@ require (
 	github.com/gorilla/mux v1.8.0
 	github.com/gorilla/websocket v1.4.2
 	github.com/json-iterator/go v1.1.12
-	github.com/lucas-clemente/quic-go v0.27.1
+	github.com/lucas-clemente/quic-go v0.28.1
 	github.com/mattn/go-colorable v0.1.8
 	github.com/miekg/dns v1.1.45
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.12.1
 	github.com/prometheus/client_model v0.2.0
-	github.com/rivo/tview v0.0.0-20200712113419-c65badfc3d92
 	github.com/rs/zerolog v1.20.0
 	github.com/stretchr/testify v1.7.1
 	github.com/urfave/cli/v2 v2.3.0
@@ -36,10 +34,10 @@ require (
 	go.opentelemetry.io/proto/otlp v0.15.0
 	go.uber.org/automaxprocs v1.4.0
 	golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f
-	golang.org/x/net v0.0.0-20220114011407-0dd24b26b47d
+	golang.org/x/net v0.0.0-20220812174116-3211cb980234
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
-	golang.org/x/sys v0.0.0-20220114195835-da31bd327af9
-	golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b
+	golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10
+	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211
 	google.golang.org/protobuf v1.28.0
 	gopkg.in/coreos/go-oidc.v2 v2.2.1
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
@@ -63,22 +61,21 @@ require (
 	github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
 	github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870 // indirect
 	github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 // indirect
-	github.com/gdamore/encoding v1.0.0 // indirect
 	github.com/go-logr/logr v1.2.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
 	github.com/gobwas/httphead v0.0.0-20200921212729-da3d93bc3c58 // indirect
 	github.com/gobwas/pool v0.2.1 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/google/gopacket v1.1.19 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect
 	github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
-	github.com/lucasb-eyer/go-colorful v1.0.3 // indirect
 	github.com/marten-seemann/qtls-go1-16 v0.1.5 // indirect
-	github.com/marten-seemann/qtls-go1-17 v0.1.1 // indirect
-	github.com/marten-seemann/qtls-go1-18 v0.1.1 // indirect
+	github.com/marten-seemann/qtls-go1-17 v0.1.2 // indirect
+	github.com/marten-seemann/qtls-go1-18 v0.1.2 // indirect
+	github.com/marten-seemann/qtls-go1-19 v0.1.0-beta.1 // indirect
 	github.com/mattn/go-isatty v0.0.12 // indirect
-	github.com/mattn/go-runewidth v0.0.8 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
@@ -89,7 +86,6 @@ require (
 	github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 // indirect
 	github.com/prometheus/common v0.32.1 // indirect
 	github.com/prometheus/procfs v0.7.3 // indirect
-	github.com/rivo/uniseg v0.1.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	golang.org/x/mod v0.4.2 // indirect
 	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
@@ -105,7 +101,9 @@ require (
 
 replace github.com/urfave/cli/v2 => github.com/ipostelnik/cli/v2 v2.3.1-0.20210324024421-b6ea8234fe3d
 
-replace github.com/lucas-clemente/quic-go => github.com/chungthuang/quic-go v0.27.1-0.20220607112311-13144fbde8da
+replace github.com/lucas-clemente/quic-go => github.com/chungthuang/quic-go v0.27.1-0.20220809135021-ca330f1dec9f
 
 // Avoid 'CVE-2022-21698'
 replace github.com/prometheus/golang_client => github.com/prometheus/golang_client v1.12.1
+
+replace gopkg.in/yaml.v3 => gopkg.in/yaml.v3 v3.0.1

go.sum

@@ -70,7 +70,6 @@ github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBp
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
-github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
 github.com/DataDog/datadog-go v4.4.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
 github.com/DataDog/gostackparse v0.5.0/go.mod h1:lTfqcJKqS9KnXQGnyQMCugq3u1FP6UZMfWR0aitKFMM=
 github.com/DataDog/sketches-go v1.0.0/go.mod h1:O+XkJHWk9w4hDwY2ZUDU31ZC9sNYlYo8DiFsxjYeo1k=
@@ -104,15 +103,14 @@ github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInq
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894 h1:JLaf/iINcLyjwbtTsCJjc6rtlASgHeIJPrB6QmwURnA=
 github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA=
-github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
 github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE=
 github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ=
-github.com/chungthuang/quic-go v0.27.1-0.20220607112311-13144fbde8da h1:FmuwbQ8RU/ftTKnfz5diawqvQFH1KDB9wN2Q8S2wqds=
-github.com/chungthuang/quic-go v0.27.1-0.20220607112311-13144fbde8da/go.mod h1:AzgQoPda7N+3IqMMMkywBKggIFo2KT6pfnlrQ2QieeI=
+github.com/chungthuang/quic-go v0.27.1-0.20220809135021-ca330f1dec9f h1:UWC3XjwZzocdNCzzXxq9j/1SdHMZXhcTOsh/+gNRBUQ=
+github.com/chungthuang/quic-go v0.27.1-0.20220809135021-ca330f1dec9f/go.mod h1:oGz5DKK41cJt5+773+BSO9BXDsREY4HLf7+0odGAPO0=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
@@ -189,10 +187,6 @@ github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
-github.com/gdamore/encoding v1.0.0 h1:+7OoQ1Bc6eTm5niUzBa0Ctsh6JbMW6Ra+YNuAtDBdko=
-github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo5dl+VrEg=
-github.com/gdamore/tcell v1.3.0 h1:r35w0JBADPZCVQijYebl6YMWWtHRqVEGt7kL2eBADRM=
-github.com/gdamore/tcell v1.3.0/go.mod h1:Hjvr+Ofd+gLglo7RYKxxnzCBmev3BzsS67MebKS4zMM=
 github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg=
 github.com/getsentry/raven-go v0.0.0-20180517221441-ed7bcb39ff10 h1:YO10pIIBftO/kkTFdWhctH96grJ7qiy7bMdiZcIvPKs=
 github.com/getsentry/raven-go v0.0.0-20180517221441-ed7bcb39ff10/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ=
@@ -298,6 +292,8 @@ github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8=
+github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
 github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
@@ -343,7 +339,6 @@ github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
-github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo=
 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 h1:BZHcxBETFHIdVyhyEfOvn/RdU/QGdLI4y34qQGjGWO0=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks=
@@ -396,9 +391,6 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
-github.com/lucasb-eyer/go-colorful v1.0.2/go.mod h1:0MS4r+7BZKSJ5mw4/S5MPN+qHFF1fYclkSPilDOKW0s=
-github.com/lucasb-eyer/go-colorful v1.0.3 h1:QIbQXiugsb+q10B+MI+7DI1oQLdmnep86tWFlaaUAac=
-github.com/lucasb-eyer/go-colorful v1.0.3/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
 github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI=
 github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
 github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
@@ -407,17 +399,16 @@ github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN
 github.com/marten-seemann/qpack v0.2.1/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc=
 github.com/marten-seemann/qtls-go1-16 v0.1.5 h1:o9JrYPPco/Nukd/HpOHMHZoBDXQqoNtUCmny98/1uqQ=
 github.com/marten-seemann/qtls-go1-16 v0.1.5/go.mod h1:gNpI2Ol+lRS3WwSOtIUUtRwZEQMXjYK+dQSBFbethAk=
-github.com/marten-seemann/qtls-go1-17 v0.1.1 h1:DQjHPq+aOzUeh9/lixAGunn6rIOQyWChPSI4+hgW7jc=
-github.com/marten-seemann/qtls-go1-17 v0.1.1/go.mod h1:C2ekUKcDdz9SDWxec1N/MvcXBpaX9l3Nx67XaR84L5s=
-github.com/marten-seemann/qtls-go1-18 v0.1.1 h1:qp7p7XXUFL7fpBvSS1sWD+uSqPvzNQK43DH+/qEkj0Y=
-github.com/marten-seemann/qtls-go1-18 v0.1.1/go.mod h1:mJttiymBAByA49mhlNZZGrH5u1uXYZJ+RW28Py7f4m4=
+github.com/marten-seemann/qtls-go1-17 v0.1.2 h1:JADBlm0LYiVbuSySCHeY863dNkcpMmDR7s0bLKJeYlQ=
+github.com/marten-seemann/qtls-go1-17 v0.1.2/go.mod h1:C2ekUKcDdz9SDWxec1N/MvcXBpaX9l3Nx67XaR84L5s=
+github.com/marten-seemann/qtls-go1-18 v0.1.2 h1:JH6jmzbduz0ITVQ7ShevK10Av5+jBEKAHMntXmIV7kM=
+github.com/marten-seemann/qtls-go1-18 v0.1.2/go.mod h1:mJttiymBAByA49mhlNZZGrH5u1uXYZJ+RW28Py7f4m4=
+github.com/marten-seemann/qtls-go1-19 v0.1.0-beta.1 h1:7m/WlWcSROrcK5NxuXaxYD32BZqe/LEEnBrWcH/cOqQ=
+github.com/marten-seemann/qtls-go1-19 v0.1.0-beta.1/go.mod h1:5HTDWtVudo/WFsHKRNuOhWlbdjrfs5JHrYb0wIJqGpI=
 github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8=
 github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
 github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
-github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
-github.com/mattn/go-runewidth v0.0.8 h1:3tS41NlGYSmhhe/8fhGRzc+z3AYCw1Fe1WAyLuujKs0=
-github.com/mattn/go-runewidth v0.0.8/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4=
@@ -513,16 +504,11 @@ github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1
 github.com/rabbitmq/amqp091-go v1.1.0/go.mod h1:ogQDLSOACsLPsIq0NpbtiifNZi2YOz0VTJ0kHRghqbM=
 github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
-github.com/rivo/tview v0.0.0-20200712113419-c65badfc3d92 h1:rqaqSUdaW+OBbjnsrOoiaJv43mSRARuvsAuirmdxu7E=
-github.com/rivo/tview v0.0.0-20200712113419-c65badfc3d92/go.mod h1:6lkG1x+13OShEf0EaOCaTQYyB7d5nSbb181KtjlS+84=
-github.com/rivo/uniseg v0.1.0 h1:+2KBaVoUmb9XzDsrx/Ct0W/EYOSFf/nWTauy++DprtY=
-github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
 github.com/rs/zerolog v1.20.0 h1:38k9hgtUBdxFwE34yS8rTHmHBa4eN16E4DJlv177LNs=
 github.com/rs/zerolog v1.20.0/go.mod h1:IzD0RJ65iWH0w97OQQebJEvTZYvsCUm9WVLWBQrJRjo=
-github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
@@ -722,11 +708,12 @@ golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qx
 golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210917221730-978cfadd31cf/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20220114011407-0dd24b26b47d h1:1n1fc535VhN8SYtD4cDUyNlfpAF2ROMM9+11equK3hs=
-golang.org/x/net v0.0.0-20220114011407-0dd24b26b47d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e h1:TsQ7F31D3bUCLeqPT0u+yjp1guoArKaNKmCr22PYgTQ=
+golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.0.0-20220812174116-3211cb980234 h1:RDqmgfe7SvlMWoqC3xwQ2blLO3fcWcxMa3eBLRdRW7E=
+golang.org/x/net v0.0.0-20220812174116-3211cb980234/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -773,7 +760,6 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190626150813-e07cf5db2756/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -831,11 +817,15 @@ golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 h1:XfKQ4OlFl8okEOr5UvAqFRVj8pY/4yfcXrddB8qAbU0=
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 h1:WIoqL4EROvwiPdUtaip4VcDdpZ4kha7wBWZrbVKCIZg=
+golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b h1:9zKuko04nR4gjZ4+DNjHqRlAJqbJETHwiNKDqTfOjfE=
 golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -1106,9 +1096,6 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o=

ingress/icmp_darwin.go

@@ -0,0 +1,231 @@
+//go:build darwin
+
+package ingress
+
+import (
+	"context"
+	"fmt"
+	"math"
+	"net"
+	"net/netip"
+	"strconv"
+	"sync"
+
+	"github.com/google/gopacket/layers"
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog"
+	"golang.org/x/net/icmp"
+
+	"github.com/cloudflare/cloudflared/packet"
+)
+
+// TODO: TUN-6654 Extend support to IPv6
+// On Darwin, a non-privileged ICMP socket can read messages from all echo IDs, so we use it for all sources.
+type icmpProxy struct {
+	// TODO: TUN-6588 clean up flows
+	srcFlowTracker *packet.FlowTracker
+	echoIDTracker  *echoIDTracker
+	conn           *icmp.PacketConn
+	logger         *zerolog.Logger
+	encoder        *packet.Encoder
+}
+
+// echoIDTracker tracks which ID has been assigned. It first loops through assignment from lastAssignment to then end,
+// then from the beginning to lastAssignment.
+// ICMP echo are short lived. By the time an ID is revisited, it should have been released.
+type echoIDTracker struct {
+	lock sync.RWMutex
+	// maps the source IP to an echo ID obtained from assignment
+	srcIPMapping map[netip.Addr]uint16
+	// assignment tracks if an ID is assigned using index as the ID
+	// The size of the array is math.MaxUint16 because echo ID is 2 bytes
+	assignment [math.MaxUint16]bool
+	// nextAssignment is the next number to check for assigment
+	nextAssignment uint16
+}
+
+func newEchoIDTracker() *echoIDTracker {
+	return &echoIDTracker{
+		srcIPMapping: make(map[netip.Addr]uint16),
+	}
+}
+
+func (eit *echoIDTracker) get(srcIP netip.Addr) (uint16, bool) {
+	eit.lock.RLock()
+	defer eit.lock.RUnlock()
+	id, ok := eit.srcIPMapping[srcIP]
+	return id, ok
+}
+
+func (eit *echoIDTracker) assign(srcIP netip.Addr) (uint16, bool) {
+	eit.lock.Lock()
+	defer eit.lock.Unlock()
+
+	if eit.nextAssignment == math.MaxUint16 {
+		eit.nextAssignment = 0
+	}
+
+	for i, assigned := range eit.assignment[eit.nextAssignment:] {
+		if !assigned {
+			echoID := uint16(i) + eit.nextAssignment
+			eit.set(srcIP, echoID)
+			return echoID, true
+		}
+	}
+	for i, assigned := range eit.assignment[0:eit.nextAssignment] {
+		if !assigned {
+			echoID := uint16(i)
+			eit.set(srcIP, echoID)
+			return echoID, true
+		}
+	}
+	return 0, false
+}
+
+// Caller should hold the lock
+func (eit *echoIDTracker) set(srcIP netip.Addr, echoID uint16) {
+	eit.assignment[echoID] = true
+	eit.srcIPMapping[srcIP] = echoID
+	eit.nextAssignment = echoID + 1
+}
+
+func (eit *echoIDTracker) release(srcIP netip.Addr, id uint16) bool {
+	eit.lock.Lock()
+	defer eit.lock.Unlock()
+
+	currentID, ok := eit.srcIPMapping[srcIP]
+	if ok && id == currentID {
+		delete(eit.srcIPMapping, srcIP)
+		eit.assignment[id] = false
+		return true
+	}
+	return false
+}
+
+type echoFlowID uint16
+
+func (snf echoFlowID) Type() string {
+	return "echoID"
+}
+
+func (snf echoFlowID) String() string {
+	return strconv.FormatUint(uint64(snf), 10)
+}
+
+func newICMPProxy(listenIP net.IP, logger *zerolog.Logger) (ICMPProxy, error) {
+	network := "udp6"
+	if listenIP.To4() != nil {
+		network = "udp4"
+	}
+	// Opens a non-privileged ICMP socket
+	conn, err := icmp.ListenPacket(network, listenIP.String())
+	if err != nil {
+		return nil, err
+	}
+	return &icmpProxy{
+		srcFlowTracker: packet.NewFlowTracker(),
+		echoIDTracker:  newEchoIDTracker(),
+		conn:           conn,
+		logger:         logger,
+		encoder:        packet.NewEncoder(),
+	}, nil
+}
+
+func (ip *icmpProxy) Request(pk *packet.ICMP, responder packet.FlowResponder) error {
+	switch body := pk.Message.Body.(type) {
+	case *icmp.Echo:
+		return ip.sendICMPEchoRequest(pk, body, responder)
+	default:
+		return fmt.Errorf("sending ICMP %s is not implemented", pk.Type)
+	}
+}
+
+func (ip *icmpProxy) ListenResponse(ctx context.Context) error {
+	go func() {
+		<-ctx.Done()
+		ip.conn.Close()
+	}()
+	buf := make([]byte, 1500)
+	for {
+		n, src, err := ip.conn.ReadFrom(buf)
+		if err != nil {
+			return err
+		}
+		// TODO: TUN-6654 Check for IPv6
+		msg, err := icmp.ParseMessage(int(layers.IPProtocolICMPv4), buf[:n])
+		if err != nil {
+			ip.logger.Error().Err(err).Str("src", src.String()).Msg("Failed to parse ICMP message")
+			continue
+		}
+		switch body := msg.Body.(type) {
+		case *icmp.Echo:
+			if err := ip.handleEchoResponse(msg, body); err != nil {
+				ip.logger.Error().Err(err).
+					Str("src", src.String()).
+					Str("flowID", echoFlowID(body.ID).String()).
+					Msg("Failed to handle ICMP response")
+				continue
+			}
+		default:
+			ip.logger.Warn().
+				Str("icmpType", fmt.Sprintf("%s", msg.Type)).
+				Msgf("Responding to this type of ICMP is not implemented")
+			continue
+		}
+	}
+}
+
+func (ip *icmpProxy) sendICMPEchoRequest(pk *packet.ICMP, echo *icmp.Echo, responder packet.FlowResponder) error {
+	echoID, ok := ip.echoIDTracker.get(pk.Src)
+	if !ok {
+		echoID, ok = ip.echoIDTracker.assign(pk.Src)
+		if !ok {
+			return fmt.Errorf("failed to assign unique echo ID")
+		}
+		flowID := echoFlowID(echoID)
+		flow := packet.Flow{
+			Src:       pk.Src,
+			Dst:       pk.Dst,
+			Responder: responder,
+		}
+		if replaced := ip.srcFlowTracker.Register(flowID, &flow, true); replaced {
+			ip.logger.Info().Str("src", flow.Src.String()).Str("dst", flow.Dst.String()).Msg("Replaced flow")
+		}
+	}
+
+	echo.ID = int(echoID)
+	var pseudoHeader []byte = nil
+	serializedMsg, err := pk.Marshal(pseudoHeader)
+	if err != nil {
+		return errors.Wrap(err, "Failed to encode ICMP message")
+	}
+	// The address needs to be of type UDPAddr when conn is created without priviledge
+	_, err = ip.conn.WriteTo(serializedMsg, &net.UDPAddr{
+		IP: pk.Dst.AsSlice(),
+	})
+	return err
+}
+
+func (ip *icmpProxy) handleEchoResponse(msg *icmp.Message, echo *icmp.Echo) error {
+	flowID := echoFlowID(echo.ID)
+	flow, ok := ip.srcFlowTracker.Get(flowID)
+	if !ok {
+		return fmt.Errorf("flow not found")
+	}
+	icmpPacket := packet.ICMP{
+		IP: &packet.IP{
+			Src:      flow.Dst,
+			Dst:      flow.Src,
+			Protocol: layers.IPProtocol(msg.Type.Protocol()),
+		},
+		Message: msg,
+	}
+	serializedPacket, err := ip.encoder.Encode(&icmpPacket)
+	if err != nil {
+		return errors.Wrap(err, "Failed to encode ICMP message")
+	}
+	if err := flow.Responder.SendPacket(serializedPacket); err != nil {
+		return errors.Wrap(err, "Failed to send packet to the edge")
+	}
+	return nil
+}

ingress/icmp_darwin_test.go

@@ -0,0 +1,91 @@
+//go:build darwin
+
+package ingress
+
+import (
+	"math"
+	"net/netip"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestSingleEchoIDTracker(t *testing.T) {
+	tracker := newEchoIDTracker()
+	srcIP := netip.MustParseAddr("127.0.0.1")
+	echoID, ok := tracker.get(srcIP)
+	require.False(t, ok)
+	require.Equal(t, uint16(0), echoID)
+
+	// not assigned yet, so nothing to release
+	require.False(t, tracker.release(srcIP, echoID))
+
+	echoID, ok = tracker.assign(srcIP)
+	require.True(t, ok)
+	require.Equal(t, uint16(0), echoID)
+
+	echoID, ok = tracker.get(srcIP)
+	require.True(t, ok)
+	require.Equal(t, uint16(0), echoID)
+
+	// releasing a different ID returns false
+	require.False(t, tracker.release(srcIP, 1999))
+	require.True(t, tracker.release(srcIP, echoID))
+	// releasing the second time returns false
+	require.False(t, tracker.release(srcIP, echoID))
+
+	echoID, ok = tracker.get(srcIP)
+	require.False(t, ok)
+	require.Equal(t, uint16(0), echoID)
+
+	// Move to the next IP
+	echoID, ok = tracker.assign(srcIP)
+	require.True(t, ok)
+	require.Equal(t, uint16(1), echoID)
+}
+
+func TestFullEchoIDTracker(t *testing.T) {
+	tracker := newEchoIDTracker()
+	firstIP := netip.MustParseAddr("172.16.0.1")
+	srcIP := firstIP
+
+	for i := uint16(0); i < math.MaxUint16; i++ {
+		echoID, ok := tracker.assign(srcIP)
+		require.True(t, ok)
+		require.Equal(t, i, echoID)
+
+		echoID, ok = tracker.get(srcIP)
+		require.True(t, ok)
+		require.Equal(t, i, echoID)
+		srcIP = srcIP.Next()
+	}
+
+	// All echo IDs are assigned
+	echoID, ok := tracker.assign(srcIP.Next())
+	require.False(t, ok)
+	require.Equal(t, uint16(0), echoID)
+
+	srcIP = firstIP
+	for i := uint16(0); i < math.MaxUint16; i++ {
+		ok := tracker.release(srcIP, i)
+		require.True(t, ok)
+
+		echoID, ok = tracker.get(srcIP)
+		require.False(t, ok)
+		require.Equal(t, uint16(0), echoID)
+		srcIP = srcIP.Next()
+	}
+
+	// The IDs are assignable again
+	srcIP = firstIP
+	for i := uint16(0); i < math.MaxUint16; i++ {
+		echoID, ok := tracker.assign(srcIP)
+		require.True(t, ok)
+		require.Equal(t, i, echoID)
+
+		echoID, ok = tracker.get(srcIP)
+		require.True(t, ok)
+		require.Equal(t, i, echoID)
+		srcIP = srcIP.Next()
+	}
+}

ingress/icmp_generic.go

@@ -0,0 +1,15 @@
+//go:build !darwin
+
+package ingress
+
+import (
+	"fmt"
+	"net"
+	"runtime"
+
+	"github.com/rs/zerolog"
+)
+
+func newICMPProxy(listenIP net.IP, logger *zerolog.Logger) (ICMPProxy, error) {
+	return nil, fmt.Errorf("ICMP proxy is not implemented on %s", runtime.GOOS)
+}

ingress/origin_icmp_proxy.go

@@ -0,0 +1,22 @@
+package ingress
+
+import (
+	"context"
+	"net"
+
+	"github.com/rs/zerolog"
+
+	"github.com/cloudflare/cloudflared/packet"
+)
+
+// ICMPProxy sends ICMP messages and listens for their responses
+type ICMPProxy interface {
+	// Request sends an ICMP message
+	Request(pk *packet.ICMP, responder packet.FlowResponder) error
+	// ListenResponse listens for responses to the requests until context is done
+	ListenResponse(ctx context.Context) error
+}
+
+func NewICMPProxy(listenIP net.IP, logger *zerolog.Logger) (ICMPProxy, error) {
+	return newICMPProxy(listenIP, logger)
+}

ingress/origin_icmp_proxy_test.go

@@ -0,0 +1,151 @@
+package ingress
+
+import (
+	"context"
+	"fmt"
+	"net/netip"
+	"runtime"
+	"testing"
+
+	"github.com/google/gopacket/layers"
+	"github.com/rs/zerolog"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/net/icmp"
+	"golang.org/x/net/ipv4"
+
+	"github.com/cloudflare/cloudflared/packet"
+)
+
+var (
+	noopLogger  = zerolog.Nop()
+	localhostIP = netip.MustParseAddr("127.0.0.1")
+)
+
+// TestICMPProxyEcho makes sure we can send ICMP echo via the Request method and receives response via the
+// ListenResponse method
+func TestICMPProxyEcho(t *testing.T) {
+	skipNonDarwin(t)
+	const (
+		echoID = 36571
+		endSeq = 100
+	)
+
+	proxy, err := NewICMPProxy(localhostIP.AsSlice(), &noopLogger)
+	require.NoError(t, err)
+
+	proxyDone := make(chan struct{})
+	ctx, cancel := context.WithCancel(context.Background())
+	go func() {
+		proxy.ListenResponse(ctx)
+		close(proxyDone)
+	}()
+
+	responder := echoFlowResponder{
+		decoder:  packet.NewICMPDecoder(),
+		respChan: make(chan []byte),
+	}
+
+	ip := packet.IP{
+		Src:      localhostIP,
+		Dst:      localhostIP,
+		Protocol: layers.IPProtocolICMPv4,
+	}
+	for i := 0; i < endSeq; i++ {
+		pk := packet.ICMP{
+			IP: &ip,
+			Message: &icmp.Message{
+				Type: ipv4.ICMPTypeEcho,
+				Code: 0,
+				Body: &icmp.Echo{
+					ID:   echoID,
+					Seq:  i,
+					Data: []byte(fmt.Sprintf("icmp echo seq %d", i)),
+				},
+			},
+		}
+		require.NoError(t, proxy.Request(&pk, &responder))
+		responder.validate(t, &pk)
+	}
+	cancel()
+	<-proxyDone
+}
+
+// TestICMPProxyRejectNotEcho makes sure it rejects messages other than echo
+func TestICMPProxyRejectNotEcho(t *testing.T) {
+	skipNonDarwin(t)
+	msgs := []icmp.Message{
+		{
+			Type: ipv4.ICMPTypeDestinationUnreachable,
+			Code: 1,
+			Body: &icmp.DstUnreach{
+				Data: []byte("original packet"),
+			},
+		},
+		{
+			Type: ipv4.ICMPTypeTimeExceeded,
+			Code: 1,
+			Body: &icmp.TimeExceeded{
+				Data: []byte("original packet"),
+			},
+		},
+		{
+			Type: ipv4.ICMPType(2),
+			Code: 0,
+			Body: &icmp.PacketTooBig{
+				MTU:  1280,
+				Data: []byte("original packet"),
+			},
+		},
+	}
+	proxy, err := NewICMPProxy(localhostIP.AsSlice(), &noopLogger)
+	require.NoError(t, err)
+
+	responder := echoFlowResponder{
+		decoder:  packet.NewICMPDecoder(),
+		respChan: make(chan []byte),
+	}
+	for _, m := range msgs {
+		pk := packet.ICMP{
+			IP: &packet.IP{
+				Src:      localhostIP,
+				Dst:      localhostIP,
+				Protocol: layers.IPProtocolICMPv4,
+			},
+			Message: &m,
+		}
+		require.Error(t, proxy.Request(&pk, &responder))
+	}
+}
+
+func skipNonDarwin(t *testing.T) {
+	if runtime.GOOS != "darwin" {
+		t.Skip("Cannot create non-privileged datagram-oriented ICMP endpoint on Windows")
+	}
+}
+
+type echoFlowResponder struct {
+	decoder  *packet.ICMPDecoder
+	respChan chan []byte
+}
+
+func (efr *echoFlowResponder) SendPacket(pk packet.RawPacket) error {
+	copiedPacket := make([]byte, len(pk.Data))
+	copy(copiedPacket, pk.Data)
+	efr.respChan <- copiedPacket
+	return nil
+}
+
+func (efr *echoFlowResponder) validate(t *testing.T, echoReq *packet.ICMP) {
+	pk := <-efr.respChan
+	decoded, err := efr.decoder.Decode(packet.RawPacket{Data: pk})
+	require.NoError(t, err)
+	require.Equal(t, decoded.Src, echoReq.Dst)
+	require.Equal(t, decoded.Dst, echoReq.Src)
+	require.Equal(t, echoReq.Protocol, decoded.Protocol)
+
+	require.Equal(t, ipv4.ICMPTypeEchoReply, decoded.Type)
+	require.Equal(t, 0, decoded.Code)
+	require.NotZero(t, decoded.Checksum)
+	// TODO: TUN-6586: Enable this validation when ICMP echo ID matches on Linux
+	require.Equal(t, echoReq.Body, decoded.Body)
+}

metrics/readiness_test.go

@@ -14,7 +14,7 @@ import (
 
 func TestReadyServer_makeResponse(t *testing.T) {
 	type fields struct {
-		isConnected map[int]bool
+		isConnected map[uint8]tunnelstate.ConnectionInfo
 	}
 	tests := []struct {
 		name                 string
@@ -25,11 +25,11 @@ func TestReadyServer_makeResponse(t *testing.T) {
 		{
 			name: "One connection online => HTTP 200",
 			fields: fields{
-				isConnected: map[int]bool{
-					0: false,
-					1: false,
-					2: true,
-					3: false,
+				isConnected: map[uint8]tunnelstate.ConnectionInfo{
+					0: {IsConnected: false},
+					1: {IsConnected: false},
+					2: {IsConnected: true},
+					3: {IsConnected: false},
 				},
 			},
 			wantOK:               true,
@@ -38,11 +38,11 @@ func TestReadyServer_makeResponse(t *testing.T) {
 		{
 			name: "No connections online => no HTTP 200",
 			fields: fields{
-				isConnected: map[int]bool{
-					0: false,
-					1: false,
-					2: false,
-					3: false,
+				isConnected: map[uint8]tunnelstate.ConnectionInfo{
+					0: {IsConnected: false},
+					1: {IsConnected: false},
+					2: {IsConnected: false},
+					3: {IsConnected: false},
 				},
 			},
 			wantReadyConnections: 0,

orchestration/orchestrator_test.go

@@ -355,7 +355,7 @@ func proxyHTTP(originProxy connection.OriginProxy, hostname string) (*http.Respo
 		return nil, err
 	}
 
-	err = originProxy.ProxyHTTP(respWriter, tracing.NewTracedRequest(req), false)
+	err = originProxy.ProxyHTTP(respWriter, tracing.NewTracedHTTPRequest(req, &log), false)
 	if err != nil {
 		return nil, err
 	}
@@ -604,7 +604,7 @@ func TestPersistentConnection(t *testing.T) {
 		respWriter, err := connection.NewHTTP2RespWriter(req, wsRespReadWriter, connection.TypeWebsocket, &log)
 		require.NoError(t, err)
 
-		err = originProxy.ProxyHTTP(respWriter, tracing.NewTracedRequest(req), true)
+		err = originProxy.ProxyHTTP(respWriter, tracing.NewTracedHTTPRequest(req, &log), true)
 		require.NoError(t, err)
 	}()
 

packet/decoder.go

@@ -0,0 +1,184 @@
+package packet
+
+import (
+	"fmt"
+
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/layers"
+	"github.com/pkg/errors"
+	"golang.org/x/net/icmp"
+)
+
+func FindProtocol(p []byte) (layers.IPProtocol, error) {
+	version, err := FindIPVersion(p)
+	if err != nil {
+		return 0, err
+	}
+	switch version {
+	case 4:
+		if len(p) < ipv4HeaderLen {
+			return 0, fmt.Errorf("IPv4 packet should have at least %d bytes, got %d bytes", ipv4HeaderLen, len(p))
+		}
+		// Protocol is in the 10th byte of IPv4 header
+		return layers.IPProtocol(p[9]), nil
+	case 6:
+		if len(p) < ipv6HeaderLen {
+			return 0, fmt.Errorf("IPv6 packet should have at least %d bytes, got %d bytes", ipv6HeaderLen, len(p))
+		}
+		// Next header is in the 7th byte of IPv6 header
+		return layers.IPProtocol(p[6]), nil
+	default:
+		return 0, fmt.Errorf("unknow ip version %d", version)
+	}
+}
+
+func FindIPVersion(p []byte) (uint8, error) {
+	if len(p) == 0 {
+		return 0, fmt.Errorf("packet length is 0")
+	}
+	return p[0] >> 4, nil
+}
+
+// IPDecoder decodes raw packets into IP. It can process packets sequentially without allocating
+// memory for the layers, so it cannot be called concurrently.
+type IPDecoder struct {
+	ipv4   *layers.IPv4
+	ipv6   *layers.IPv6
+	layers uint8
+
+	v4parser *gopacket.DecodingLayerParser
+	v6parser *gopacket.DecodingLayerParser
+}
+
+func NewIPDecoder() *IPDecoder {
+	var (
+		ipv4 layers.IPv4
+		ipv6 layers.IPv6
+	)
+	dlpv4 := gopacket.NewDecodingLayerParser(layers.LayerTypeIPv4)
+	dlpv4.SetDecodingLayerContainer(gopacket.DecodingLayerSparse(nil))
+	dlpv4.AddDecodingLayer(&ipv4)
+	// Stop parsing when it encounter a layer that it doesn't have a parser
+	dlpv4.IgnoreUnsupported = true
+
+	dlpv6 := gopacket.NewDecodingLayerParser(layers.LayerTypeIPv6)
+	dlpv6.SetDecodingLayerContainer(gopacket.DecodingLayerSparse(nil))
+	dlpv6.AddDecodingLayer(&ipv6)
+	dlpv6.IgnoreUnsupported = true
+
+	return &IPDecoder{
+		ipv4:     &ipv4,
+		ipv6:     &ipv6,
+		layers:   1,
+		v4parser: dlpv4,
+		v6parser: dlpv6,
+	}
+}
+
+func (pd *IPDecoder) Decode(packet RawPacket) (*IP, error) {
+	// Should decode to IP layer
+	decoded, err := pd.decodeByVersion(packet.Data)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, layerType := range decoded {
+		switch layerType {
+		case layers.LayerTypeIPv4:
+			return newIPv4(pd.ipv4)
+		case layers.LayerTypeIPv6:
+			return newIPv6(pd.ipv6)
+		}
+	}
+	return nil, fmt.Errorf("no ip layer is decoded")
+}
+
+func (pd *IPDecoder) decodeByVersion(packet []byte) ([]gopacket.LayerType, error) {
+	version, err := FindIPVersion(packet)
+	if err != nil {
+		return nil, err
+	}
+	decoded := make([]gopacket.LayerType, 0, pd.layers)
+	switch version {
+	case 4:
+		err = pd.v4parser.DecodeLayers(packet, &decoded)
+	case 6:
+		err = pd.v6parser.DecodeLayers(packet, &decoded)
+	default:
+		err = fmt.Errorf("unknow ip version %d", version)
+	}
+	if err != nil {
+		return nil, err
+	}
+	return decoded, nil
+}
+
+// ICMPDecoder decodes raw packets into IP and ICMP. It can process packets sequentially without allocating
+// memory for the layers, so it cannot be called concurrently.
+type ICMPDecoder struct {
+	*IPDecoder
+	icmpv4 *layers.ICMPv4
+	icmpv6 *layers.ICMPv6
+}
+
+func NewICMPDecoder() *ICMPDecoder {
+	ipDecoder := NewIPDecoder()
+
+	var (
+		icmpv4 layers.ICMPv4
+		icmpv6 layers.ICMPv6
+	)
+	ipDecoder.layers++
+	ipDecoder.v4parser.AddDecodingLayer(&icmpv4)
+	ipDecoder.v6parser.AddDecodingLayer(&icmpv6)
+
+	return &ICMPDecoder{
+		IPDecoder: ipDecoder,
+		icmpv4:    &icmpv4,
+		icmpv6:    &icmpv6,
+	}
+}
+
+func (pd *ICMPDecoder) Decode(packet RawPacket) (*ICMP, error) {
+	// Should decode to IP and optionally ICMP layer
+	decoded, err := pd.decodeByVersion(packet.Data)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, layerType := range decoded {
+		switch layerType {
+		case layers.LayerTypeICMPv4:
+			ipv4, err := newIPv4(pd.ipv4)
+			if err != nil {
+				return nil, err
+			}
+			msg, err := icmp.ParseMessage(int(layers.IPProtocolICMPv4), append(pd.icmpv4.Contents, pd.icmpv4.Payload...))
+			if err != nil {
+				return nil, errors.Wrap(err, "failed to parse ICMPv4 message")
+			}
+			return &ICMP{
+				IP:      ipv4,
+				Message: msg,
+			}, nil
+		case layers.LayerTypeICMPv6:
+			ipv6, err := newIPv6(pd.ipv6)
+			if err != nil {
+				return nil, err
+			}
+			msg, err := icmp.ParseMessage(int(layers.IPProtocolICMPv6), append(pd.icmpv6.Contents, pd.icmpv6.Payload...))
+			if err != nil {
+				return nil, errors.Wrap(err, "failed to parse ICMPv6")
+			}
+			return &ICMP{
+				IP:      ipv6,
+				Message: msg,
+			}, nil
+		}
+	}
+	layers := make([]string, len(decoded))
+	for i, l := range decoded {
+		layers[i] = l.String()
+	}
+	return nil, fmt.Errorf("Expect to decode IP and ICMP layers, got %s", layers)
+}

packet/decoder_test.go

@@ -0,0 +1,252 @@
+package packet
+
+import (
+	"net"
+	"net/netip"
+	"testing"
+
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/layers"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/net/icmp"
+	"golang.org/x/net/ipv4"
+	"golang.org/x/net/ipv6"
+)
+
+func TestDecodeIP(t *testing.T) {
+	ipDecoder := NewIPDecoder()
+	icmpDecoder := NewICMPDecoder()
+	udps := []UDP{
+		{
+			IP: IP{
+				Src:      netip.MustParseAddr("172.16.0.1"),
+				Dst:      netip.MustParseAddr("10.0.0.1"),
+				Protocol: layers.IPProtocolUDP,
+			},
+			SrcPort: 31678,
+			DstPort: 53,
+		},
+		{
+
+			IP: IP{
+				Src:      netip.MustParseAddr("fd51:2391:523:f4ee::1"),
+				Dst:      netip.MustParseAddr("fd51:2391:697:f4ee::2"),
+				Protocol: layers.IPProtocolUDP,
+			},
+			SrcPort: 52139,
+			DstPort: 1053,
+		},
+	}
+
+	encoder := NewEncoder()
+	for _, udp := range udps {
+		p, err := encoder.Encode(&udp)
+		require.NoError(t, err)
+
+		ipPacket, err := ipDecoder.Decode(p)
+		require.NoError(t, err)
+		assertIPLayer(t, &udp.IP, ipPacket)
+
+		icmpPacket, err := icmpDecoder.Decode(p)
+		require.Error(t, err)
+		require.Nil(t, icmpPacket)
+	}
+}
+
+func TestDecodeICMP(t *testing.T) {
+	ipDecoder := NewIPDecoder()
+	icmpDecoder := NewICMPDecoder()
+	var (
+		ipv4Packet = IP{
+			Src:      netip.MustParseAddr("172.16.0.1"),
+			Dst:      netip.MustParseAddr("10.0.0.1"),
+			Protocol: layers.IPProtocolICMPv4,
+		}
+		ipv6Packet = IP{
+			Src:      netip.MustParseAddr("fd51:2391:523:f4ee::1"),
+			Dst:      netip.MustParseAddr("fd51:2391:697:f4ee::2"),
+			Protocol: layers.IPProtocolICMPv6,
+		}
+		icmpID  = 100
+		icmpSeq = 52819
+	)
+	tests := []struct {
+		testCase string
+		packet   *ICMP
+	}{
+		{
+			testCase: "icmpv4 time exceed",
+			packet: &ICMP{
+				IP: &ipv4Packet,
+				Message: &icmp.Message{
+					Type: ipv4.ICMPTypeTimeExceeded,
+					Code: 0,
+					Body: &icmp.TimeExceeded{
+						Data: []byte("original packet"),
+					},
+				},
+			},
+		},
+		{
+			testCase: "icmpv4 echo",
+			packet: &ICMP{
+				IP: &ipv4Packet,
+				Message: &icmp.Message{
+					Type: ipv4.ICMPTypeEcho,
+					Code: 0,
+					Body: &icmp.Echo{
+						ID:   icmpID,
+						Seq:  icmpSeq,
+						Data: []byte("icmpv4 echo"),
+					},
+				},
+			},
+		},
+		{
+			testCase: "icmpv6 destination unreachable",
+			packet: &ICMP{
+				IP: &ipv6Packet,
+				Message: &icmp.Message{
+					Type: ipv6.ICMPTypeDestinationUnreachable,
+					Code: 4,
+					Body: &icmp.DstUnreach{
+						Data: []byte("original packet"),
+					},
+				},
+			},
+		},
+		{
+			testCase: "icmpv6 echo",
+			packet: &ICMP{
+				IP: &ipv6Packet,
+				Message: &icmp.Message{
+					Type: ipv6.ICMPTypeEchoRequest,
+					Code: 0,
+					Body: &icmp.Echo{
+						ID:   icmpID,
+						Seq:  icmpSeq,
+						Data: []byte("icmpv6 echo"),
+					},
+				},
+			},
+		},
+	}
+
+	encoder := NewEncoder()
+	for _, test := range tests {
+		p, err := encoder.Encode(test.packet)
+		require.NoError(t, err)
+
+		ipPacket, err := ipDecoder.Decode(p)
+		require.NoError(t, err)
+		if ipPacket.Src.Is4() {
+			assertIPLayer(t, &ipv4Packet, ipPacket)
+		} else {
+			assertIPLayer(t, &ipv6Packet, ipPacket)
+		}
+		icmpPacket, err := icmpDecoder.Decode(p)
+		require.NoError(t, err)
+		require.Equal(t, ipPacket, icmpPacket.IP)
+
+		require.Equal(t, test.packet.Type, icmpPacket.Type)
+		require.Equal(t, test.packet.Code, icmpPacket.Code)
+		require.Equal(t, test.packet.Body, icmpPacket.Body)
+		expectedBody, err := test.packet.Body.Marshal(test.packet.Type.Protocol())
+		require.NoError(t, err)
+		decodedBody, err := icmpPacket.Body.Marshal(test.packet.Type.Protocol())
+		require.NoError(t, err)
+		require.Equal(t, expectedBody, decodedBody)
+	}
+}
+
+// TestDecodeBadPackets makes sure decoders don't decode invalid packets
+func TestDecodeBadPackets(t *testing.T) {
+	var (
+		srcIPv4 = net.ParseIP("172.16.0.1")
+		dstIPv4 = net.ParseIP("10.0.0.1")
+	)
+
+	ipLayer := layers.IPv4{
+		Version:  10,
+		SrcIP:    srcIPv4,
+		DstIP:    dstIPv4,
+		Protocol: layers.IPProtocolICMPv4,
+		TTL:      defaultTTL,
+	}
+	icmpLayer := layers.ICMPv4{
+		TypeCode: layers.CreateICMPv4TypeCode(uint8(ipv4.ICMPTypeEcho), 0),
+		Id:       100,
+		Seq:      52819,
+	}
+	wrongIPVersion, err := createPacket(&ipLayer, &icmpLayer, nil, nil)
+	require.NoError(t, err)
+
+	tests := []struct {
+		testCase string
+		packet   []byte
+	}{
+		{
+			testCase: "unknown IP version",
+			packet:   wrongIPVersion,
+		},
+		{
+			testCase: "invalid packet",
+			packet:   []byte("not a packet"),
+		},
+		{
+			testCase: "zero length packet",
+			packet:   []byte{},
+		},
+	}
+
+	ipDecoder := NewIPDecoder()
+	icmpDecoder := NewICMPDecoder()
+	for _, test := range tests {
+		ipPacket, err := ipDecoder.Decode(RawPacket{Data: test.packet})
+		require.Error(t, err)
+		require.Nil(t, ipPacket)
+
+		icmpPacket, err := icmpDecoder.Decode(RawPacket{Data: test.packet})
+		require.Error(t, err)
+		require.Nil(t, icmpPacket)
+	}
+}
+
+func createPacket(ipLayer, secondLayer, thirdLayer gopacket.SerializableLayer, body []byte) ([]byte, error) {
+	payload := gopacket.Payload(body)
+	packet := gopacket.NewSerializeBuffer()
+	var err error
+	if thirdLayer != nil {
+		err = gopacket.SerializeLayers(packet, serializeOpts, ipLayer, secondLayer, thirdLayer, payload)
+	} else {
+		err = gopacket.SerializeLayers(packet, serializeOpts, ipLayer, secondLayer, payload)
+	}
+	if err != nil {
+		return nil, err
+	}
+	return packet.Bytes(), nil
+}
+
+func assertIPLayer(t *testing.T, expected, actual *IP) {
+	require.Equal(t, expected.Src, actual.Src)
+	require.Equal(t, expected.Dst, actual.Dst)
+	require.Equal(t, expected.Protocol, actual.Protocol)
+}
+
+type UDP struct {
+	IP
+	SrcPort, DstPort layers.UDPPort
+}
+
+func (u *UDP) EncodeLayers() ([]gopacket.SerializableLayer, error) {
+	ipLayers, err := u.IP.EncodeLayers()
+	if err != nil {
+		return nil, err
+	}
+	udpLayer := layers.UDP{
+		SrcPort: u.SrcPort,
+		DstPort: u.DstPort,
+	}
+	udpLayer.SetNetworkLayerForChecksum(ipLayers[0].(gopacket.NetworkLayer))
+	return append(ipLayers, &udpLayer), nil
+}

packet/encoder.go

@@ -0,0 +1,39 @@
+package packet
+
+import "github.com/google/gopacket"
+
+var (
+	serializeOpts = gopacket.SerializeOptions{
+		FixLengths:       true,
+		ComputeChecksums: true,
+	}
+)
+
+// RawPacket represents a raw packet or one encoded by Encoder
+type RawPacket struct {
+	Data []byte
+}
+
+type Encoder struct {
+	// buf is reusable because SerializeLayers calls the Clear method before each encoding
+	buf gopacket.SerializeBuffer
+}
+
+func NewEncoder() *Encoder {
+	return &Encoder{
+		buf: gopacket.NewSerializeBuffer(),
+	}
+}
+
+func (e Encoder) Encode(packet Packet) (RawPacket, error) {
+	encodedLayers, err := packet.EncodeLayers()
+	if err != nil {
+		return RawPacket{}, err
+	}
+	if err := gopacket.SerializeLayers(e.buf, serializeOpts, encodedLayers...); err != nil {
+		return RawPacket{}, err
+	}
+	return RawPacket{
+		Data: e.buf.Bytes(),
+	}, nil
+}

packet/flow.go

@@ -0,0 +1,94 @@
+package packet
+
+import (
+	"errors"
+	"fmt"
+	"net/netip"
+	"sync"
+)
+
+var (
+	ErrFlowNotFound = errors.New("flow not found")
+)
+
+// FlowID represents a key type that can be used by FlowTracker
+type FlowID interface {
+	// Type returns the name of the type that implements the FlowID
+	Type() string
+	fmt.Stringer
+}
+
+type Flow struct {
+	Src       netip.Addr
+	Dst       netip.Addr
+	Responder FlowResponder
+}
+
+func isSameFlow(f1, f2 *Flow) bool {
+	if f1 == nil || f2 == nil {
+		return false
+	}
+	return *f1 == *f2
+}
+
+// FlowResponder sends response packets to the flow
+type FlowResponder interface {
+	// SendPacket returns a packet to the flow. It must not modify the packet,
+	// and after return it must not read the packet
+	SendPacket(pk RawPacket) error
+}
+
+// FlowTracker tracks flow from the perspective of eyeball to origin
+type FlowTracker struct {
+	lock  sync.RWMutex
+	flows map[FlowID]*Flow
+}
+
+func NewFlowTracker() *FlowTracker {
+	return &FlowTracker{
+		flows: make(map[FlowID]*Flow),
+	}
+}
+
+func (sft *FlowTracker) Get(id FlowID) (*Flow, bool) {
+	sft.lock.RLock()
+	defer sft.lock.RUnlock()
+	flow, ok := sft.flows[id]
+	return flow, ok
+}
+
+// Registers a flow. If shouldReplace = true, replace the current flow
+func (sft *FlowTracker) Register(id FlowID, flow *Flow, shouldReplace bool) (replaced bool) {
+	sft.lock.Lock()
+	defer sft.lock.Unlock()
+	currentFlow, ok := sft.flows[id]
+	if !ok {
+		sft.flows[id] = flow
+		return false
+	}
+
+	if shouldReplace && !isSameFlow(currentFlow, flow) {
+		sft.flows[id] = flow
+		return true
+	}
+	return false
+}
+
+// Unregisters a flow. If force = true, delete it even if it maps to a different flow
+func (sft *FlowTracker) Unregister(id FlowID, flow *Flow, force bool) (forceDeleted bool) {
+	sft.lock.Lock()
+	defer sft.lock.Unlock()
+	currentFlow, ok := sft.flows[id]
+	if !ok {
+		return false
+	}
+	if isSameFlow(currentFlow, flow) {
+		delete(sft.flows, id)
+		return false
+	}
+	if force {
+		delete(sft.flows, id)
+		return true
+	}
+	return false
+}

packet/packet.go

@@ -0,0 +1,115 @@
+package packet
+
+import (
+	"fmt"
+	"net/netip"
+
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/layers"
+	"golang.org/x/net/icmp"
+)
+
+const (
+	defaultTTL    uint8 = 64
+	ipv4HeaderLen       = 20
+	ipv6HeaderLen       = 40
+)
+
+// Packet represents an IP packet or a packet that is encapsulated by IP
+type Packet interface {
+	// IPLayer returns the IP of the packet
+	IPLayer() *IP
+	// EncodeLayers returns the layers that make up this packet. They can be passed to an Encoder to serialize into RawPacket
+	EncodeLayers() ([]gopacket.SerializableLayer, error)
+}
+
+// IP represents a generic IP packet. It can be embedded in more specific IP protocols
+type IP struct {
+	Src      netip.Addr
+	Dst      netip.Addr
+	Protocol layers.IPProtocol
+}
+
+func newIPv4(ipLayer *layers.IPv4) (*IP, error) {
+	src, ok := netip.AddrFromSlice(ipLayer.SrcIP)
+	if !ok {
+		return nil, fmt.Errorf("cannot convert source IP %s to netip.Addr", ipLayer.SrcIP)
+	}
+	dst, ok := netip.AddrFromSlice(ipLayer.DstIP)
+	if !ok {
+		return nil, fmt.Errorf("cannot convert source IP %s to netip.Addr", ipLayer.DstIP)
+	}
+	return &IP{
+		Src:      src,
+		Dst:      dst,
+		Protocol: ipLayer.Protocol,
+	}, nil
+}
+
+func newIPv6(ipLayer *layers.IPv6) (*IP, error) {
+	src, ok := netip.AddrFromSlice(ipLayer.SrcIP)
+	if !ok {
+		return nil, fmt.Errorf("cannot convert source IP %s to netip.Addr", ipLayer.SrcIP)
+	}
+	dst, ok := netip.AddrFromSlice(ipLayer.DstIP)
+	if !ok {
+		return nil, fmt.Errorf("cannot convert source IP %s to netip.Addr", ipLayer.DstIP)
+	}
+	return &IP{
+		Src:      src,
+		Dst:      dst,
+		Protocol: ipLayer.NextHeader,
+	}, nil
+}
+
+func (ip *IP) IPLayer() *IP {
+	return ip
+}
+
+func (ip *IP) isIPv4() bool {
+	return ip.Src.Is4()
+}
+
+func (ip *IP) EncodeLayers() ([]gopacket.SerializableLayer, error) {
+	if ip.isIPv4() {
+		return []gopacket.SerializableLayer{
+			&layers.IPv4{
+				Version:  4,
+				SrcIP:    ip.Src.AsSlice(),
+				DstIP:    ip.Dst.AsSlice(),
+				Protocol: layers.IPProtocol(ip.Protocol),
+				TTL:      defaultTTL,
+			},
+		}, nil
+	} else {
+		return []gopacket.SerializableLayer{
+			&layers.IPv6{
+				Version:    6,
+				SrcIP:      ip.Src.AsSlice(),
+				DstIP:      ip.Dst.AsSlice(),
+				NextHeader: layers.IPProtocol(ip.Protocol),
+				HopLimit:   defaultTTL,
+			},
+		}, nil
+	}
+}
+
+// ICMP represents is an IP packet + ICMP message
+type ICMP struct {
+	*IP
+	*icmp.Message
+}
+
+func (i *ICMP) EncodeLayers() ([]gopacket.SerializableLayer, error) {
+	ipLayers, err := i.IP.EncodeLayers()
+	if err != nil {
+		return nil, err
+	}
+
+	msg, err := i.Marshal(nil)
+	if err != nil {
+		return nil, err
+	}
+	icmpLayer := gopacket.Payload(msg)
+	return append(ipLayers, icmpLayer), nil
+}

packet/session.go

@@ -0,0 +1,8 @@
+package packet
+
+import "github.com/google/uuid"
+
+type Session struct {
+	ID      uuid.UUID
+	Payload []byte
+}

proxy/proxy.go

@@ -1,7 +1,6 @@
 package proxy
 
 import (
-	"bufio"
 	"context"
 	"fmt"
 	"io"
@@ -29,6 +28,8 @@ const (
 	LogFieldRule          = "ingressRule"
 	LogFieldOriginService = "originService"
 	LogFieldFlowID        = "flowID"
+
+	trailerHeaderName = "Trailer"
 )
 
 // Proxy represents a means to Proxy between cloudflared and the origin services.
@@ -63,7 +64,7 @@ func NewOriginProxy(
 // a simple roundtrip or a tcp/websocket dial depending on ingres rule setup.
 func (p *Proxy) ProxyHTTP(
 	w connection.ResponseWriter,
-	tr *tracing.TracedRequest,
+	tr *tracing.TracedHTTPRequest,
 	isWebsocket bool,
 ) error {
 	incrementRequests()
@@ -108,7 +109,7 @@ func (p *Proxy) ProxyHTTP(
 		}
 
 		rws := connection.NewHTTPResponseReadWriterAcker(w, req)
-		if err := p.proxyStream(req.Context(), rws, dest, originProxy); err != nil {
+		if err := p.proxyStream(tr.ToTracedContext(), rws, dest, originProxy); err != nil {
 			rule, srv := ruleField(p.ingressRules, ruleNum)
 			p.logRequestError(err, cfRay, "", rule, srv)
 			return err
@@ -137,9 +138,11 @@ func (p *Proxy) ProxyTCP(
 	serveCtx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
+	tracedCtx := tracing.NewTracedContext(serveCtx, req.CfTraceID, p.log)
+
 	p.log.Debug().Str(LogFieldFlowID, req.FlowID).Msg("tcp proxy stream started")
 
-	if err := p.proxyStream(serveCtx, rwa, req.Dest, p.warpRouting.Proxy); err != nil {
+	if err := p.proxyStream(tracedCtx, rwa, req.Dest, p.warpRouting.Proxy); err != nil {
 		p.logRequestError(err, req.CFRay, req.FlowID, "", ingress.ServiceWarpRouting)
 		return err
 	}
@@ -160,7 +163,7 @@ func ruleField(ing ingress.Ingress, ruleNum int) (ruleID string, srv string) {
 // ProxyHTTPRequest proxies requests of underlying type http and websocket to the origin service.
 func (p *Proxy) proxyHTTPRequest(
 	w connection.ResponseWriter,
-	tr *tracing.TracedRequest,
+	tr *tracing.TracedHTTPRequest,
 	httpService ingress.HTTPOriginProxy,
 	isWebsocket bool,
 	disableChunkedEncoding bool,
@@ -205,15 +208,16 @@ func (p *Proxy) proxyHTTPRequest(
 	tracing.EndWithStatusCode(ttfbSpan, resp.StatusCode)
 	defer resp.Body.Close()
 
-	// resp headers can be nil
-	if resp.Header == nil {
-		resp.Header = make(http.Header)
+	headers := make(http.Header, len(resp.Header))
+	// copy headers
+	for k, v := range resp.Header {
+		headers[k] = v
 	}
 
 	// Add spans to response header (if available)
-	tr.AddSpans(resp.Header, p.log)
+	tr.AddSpans(headers)
 
-	err = w.WriteRespHeaders(resp.StatusCode, resp.Header)
+	err = w.WriteRespHeaders(resp.StatusCode, headers)
 	if err != nil {
 		return errors.Wrap(err, "Error writing response header")
 	}
@@ -234,12 +238,10 @@ func (p *Proxy) proxyHTTPRequest(
 		return nil
 	}
 
-	if connection.IsServerSentEvent(resp.Header) {
-		p.log.Debug().Msg("Detected Server-Side Events from Origin")
-		p.writeEventStream(w, resp.Body)
-	} else {
-		_, _ = cfio.Copy(w, resp.Body)
-	}
+	_, _ = cfio.Copy(w, resp.Body)
+
+	// copy trailers
+	copyTrailers(w, resp)
 
 	p.logOriginResponse(resp, fields)
 	return nil
@@ -248,17 +250,23 @@ func (p *Proxy) proxyHTTPRequest(
 // proxyStream proxies type TCP and other underlying types if the connection is defined as a stream oriented
 // ingress rule.
 func (p *Proxy) proxyStream(
-	ctx context.Context,
+	tr *tracing.TracedContext,
 	rwa connection.ReadWriteAcker,
 	dest string,
 	connectionProxy ingress.StreamBasedOriginProxy,
 ) error {
+	ctx := tr.Context
+	_, connectSpan := tr.Tracer().Start(ctx, "stream-connect")
 	originConn, err := connectionProxy.EstablishConnection(ctx, dest)
 	if err != nil {
+		tracing.EndWithErrorStatus(connectSpan, err)
 		return err
 	}
+	connectSpan.End()
 
-	if err := rwa.AckConnection(); err != nil {
+	encodedSpans := tr.GetSpans()
+
+	if err := rwa.AckConnection(encodedSpans); err != nil {
 		return err
 	}
 
@@ -288,26 +296,6 @@ func (wr *bidirectionalStream) Write(p []byte) (n int, err error) {
 	return wr.writer.Write(p)
 }
 
-func (p *Proxy) writeEventStream(w connection.ResponseWriter, respBody io.ReadCloser) {
-	reader := bufio.NewReader(respBody)
-	for {
-		line, readErr := reader.ReadBytes('\n')
-
-		// We first try to write whatever we read even if an error occurred
-		// The reason for doing it is to guarantee we really push everything to the eyeball side
-		// before returning
-		if len(line) > 0 {
-			if _, writeErr := w.Write(line); writeErr != nil {
-				return
-			}
-		}
-
-		if readErr != nil {
-			return
-		}
-	}
-}
-
 func (p *Proxy) appendTagHeaders(r *http.Request) {
 	for _, tag := range p.tags {
 		r.Header.Add(TagHeaderNamePrefix+tag.Name, tag.Value)
@@ -321,6 +309,14 @@ type logFields struct {
 	flowID  string
 }
 
+func copyTrailers(w connection.ResponseWriter, response *http.Response) {
+	for trailerHeader, trailerValues := range response.Trailer {
+		for _, trailerValue := range trailerValues {
+			w.AddTrailer(trailerHeader, trailerValue)
+		}
+	}
+}
+
 func (p *Proxy) logRequest(r *http.Request, fields logFields) {
 	if fields.cfRay != "" {
 		p.log.Debug().Msgf("CF-RAY: %s %s %s %s", fields.cfRay, r.Method, r.URL, r.Proto)

proxy/proxy_test.go

@@ -22,6 +22,8 @@ import (
 	"github.com/urfave/cli/v2"
 	"golang.org/x/sync/errgroup"
 
+	"github.com/cloudflare/cloudflared/cfio"
+
 	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/hello"
@@ -62,6 +64,10 @@ func (w *mockHTTPRespWriter) WriteRespHeaders(status int, header http.Header) er
 	return nil
 }
 
+func (w *mockHTTPRespWriter) AddTrailer(trailerName, trailerValue string) {
+	// do nothing
+}
+
 func (w *mockHTTPRespWriter) Read(data []byte) (int, error) {
 	return 0, fmt.Errorf("mockHTTPRespWriter doesn't implement io.Reader")
 }
@@ -117,7 +123,10 @@ func newMockSSERespWriter() *mockSSERespWriter {
 }
 
 func (w *mockSSERespWriter) Write(data []byte) (int, error) {
-	w.writeNotification <- data
+	newData := make([]byte, len(data))
+	copy(newData, data)
+
+	w.writeNotification <- newData
 	return len(data), nil
 }
 
@@ -157,7 +166,8 @@ func testProxyHTTP(proxy connection.OriginProxy) func(t *testing.T) {
 		req, err := http.NewRequest(http.MethodGet, "http://localhost:8080", nil)
 		require.NoError(t, err)
 
-		err = proxy.ProxyHTTP(responseWriter, tracing.NewTracedRequest(req), false)
+		log := zerolog.Nop()
+		err = proxy.ProxyHTTP(responseWriter, tracing.NewTracedHTTPRequest(req, &log), false)
 		require.NoError(t, err)
 		for _, tag := range testTags {
 			assert.Equal(t, tag.Value, req.Header.Get(TagHeaderNamePrefix+tag.Name))
@@ -184,7 +194,8 @@ func testProxyWebsocket(proxy connection.OriginProxy) func(t *testing.T) {
 
 		errGroup, ctx := errgroup.WithContext(ctx)
 		errGroup.Go(func() error {
-			err = proxy.ProxyHTTP(responseWriter, tracing.NewTracedRequest(req), true)
+			log := zerolog.Nop()
+			err = proxy.ProxyHTTP(responseWriter, tracing.NewTracedHTTPRequest(req, &log), true)
 			require.NoError(t, err)
 
 			require.Equal(t, http.StatusSwitchingProtocols, responseWriter.Code)
@@ -245,7 +256,8 @@ func testProxySSE(proxy connection.OriginProxy) func(t *testing.T) {
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
-			err = proxy.ProxyHTTP(responseWriter, tracing.NewTracedRequest(req), false)
+			log := zerolog.Nop()
+			err = proxy.ProxyHTTP(responseWriter, tracing.NewTracedHTTPRequest(req, &log), false)
 			require.NoError(t, err)
 
 			require.Equal(t, http.StatusOK, responseWriter.Code)
@@ -253,11 +265,8 @@ func testProxySSE(proxy connection.OriginProxy) func(t *testing.T) {
 
 		for i := 0; i < pushCount; i++ {
 			line := responseWriter.ReadBytes()
-			expect := fmt.Sprintf("%d\n", i)
+			expect := fmt.Sprintf("%d\n\n", i)
 			require.Equal(t, []byte(expect), line, fmt.Sprintf("Expect to read %v, got %v", expect, line))
-
-			line = responseWriter.ReadBytes()
-			require.Equal(t, []byte("\n"), line, fmt.Sprintf("Expect to read '\n', got %v", line))
 		}
 
 		cancel()
@@ -273,7 +282,7 @@ func testProxySSEAllData(proxy *Proxy) func(t *testing.T) {
 		responseWriter := newMockSSERespWriter()
 
 		// responseWriter uses an unbuffered channel, so we call in a different go-routine
-		go proxy.writeEventStream(responseWriter, eyeballReader)
+		go cfio.Copy(responseWriter, eyeballReader)
 
 		result := string(<-responseWriter.writeNotification)
 		require.Equal(t, "data\r\r", result)
@@ -357,7 +366,7 @@ func runIngressTestScenarios(t *testing.T, unvalidatedIngress []config.Unvalidat
 		req, err := http.NewRequest(http.MethodGet, test.url, nil)
 		require.NoError(t, err)
 
-		err = proxy.ProxyHTTP(responseWriter, tracing.NewTracedRequest(req), false)
+		err = proxy.ProxyHTTP(responseWriter, tracing.NewTracedHTTPRequest(req, &log), false)
 		require.NoError(t, err)
 
 		assert.Equal(t, test.expectedStatus, responseWriter.Code)
@@ -404,7 +413,7 @@ func TestProxyError(t *testing.T) {
 	req, err := http.NewRequest(http.MethodGet, "http://127.0.0.1", nil)
 	assert.NoError(t, err)
 
-	assert.Error(t, proxy.ProxyHTTP(responseWriter, tracing.NewTracedRequest(req), false))
+	assert.Error(t, proxy.ProxyHTTP(responseWriter, tracing.NewTracedHTTPRequest(req, &log), false))
 }
 
 type replayer struct {
@@ -522,6 +531,7 @@ func TestConnections(t *testing.T) {
 			},
 			want: want{
 				message: []byte("echo-test2"),
+				headers: http.Header{},
 			},
 		},
 		{
@@ -541,6 +551,7 @@ func TestConnections(t *testing.T) {
 				message: []byte("echo-test3"),
 				// We expect no headers here because they are sent back via
 				// the stream.
+				headers: http.Header{},
 			},
 		},
 		{
@@ -682,7 +693,8 @@ func TestConnections(t *testing.T) {
 				rwa := connection.NewHTTPResponseReadWriterAcker(respWriter, req)
 				err = proxy.ProxyTCP(ctx, rwa, &connection.TCPRequest{Dest: dest})
 			} else {
-				err = proxy.ProxyHTTP(respWriter, tracing.NewTracedRequest(req), test.args.connectionType == connection.TypeWebsocket)
+				log := zerolog.Nop()
+				err = proxy.ProxyHTTP(respWriter, tracing.NewTracedHTTPRequest(req, &log), test.args.connectionType == connection.TypeWebsocket)
 			}
 
 			cancel()
@@ -819,6 +831,10 @@ func (w *wsRespWriter) WriteRespHeaders(status int, header http.Header) error {
 	return nil
 }
 
+func (w *wsRespWriter) AddTrailer(trailerName, trailerValue string) {
+	// do nothing
+}
+
 // respHeaders is a test function to read respHeaders
 func (w *wsRespWriter) headers() http.Header {
 	// Removing indeterminstic header because it cannot be asserted.
@@ -846,6 +862,10 @@ func (m *mockTCPRespWriter) Write(p []byte) (n int, err error) {
 	return m.w.Write(p)
 }
 
+func (w *mockTCPRespWriter) AddTrailer(trailerName, trailerValue string) {
+	// do nothing
+}
+
 func (m *mockTCPRespWriter) WriteRespHeaders(status int, header http.Header) error {
 	m.responseHeaders = header
 	m.code = status

quic/datagram.go

@@ -1,65 +1,97 @@
 package quic
 
 import (
+	"context"
 	"fmt"
 
 	"github.com/google/uuid"
 	"github.com/lucas-clemente/quic-go"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
+
+	"github.com/cloudflare/cloudflared/packet"
 )
 
 const (
 	sessionIDLen = len(uuid.UUID{})
 )
 
+type BaseDatagramMuxer interface {
+	// SendToSession suffix the session ID to the payload so the other end of the QUIC connection can demultiplex the
+	// payload from multiple datagram sessions.
+	SendToSession(session *packet.Session) error
+	// ServeReceive starts a loop to receive datagrams from the QUIC connection
+	ServeReceive(ctx context.Context) error
+}
+
 type DatagramMuxer struct {
-	session quic.Connection
-	logger  *zerolog.Logger
+	session   quic.Connection
+	logger    *zerolog.Logger
+	demuxChan chan<- *packet.Session
 }
 
-func NewDatagramMuxer(quicSession quic.Connection, logger *zerolog.Logger) (*DatagramMuxer, error) {
+func NewDatagramMuxer(quicSession quic.Connection, log *zerolog.Logger, demuxChan chan<- *packet.Session) *DatagramMuxer {
+	logger := log.With().Uint8("datagramVersion", 1).Logger()
 	return &DatagramMuxer{
-		session: quicSession,
-		logger:  logger,
-	}, nil
+		session:   quicSession,
+		logger:    &logger,
+		demuxChan: demuxChan,
+	}
 }
 
-// SendTo suffix the session ID to the payload so the other end of the QUIC session can demultiplex
-// the payload from multiple datagram sessions
-func (dm *DatagramMuxer) SendTo(sessionID uuid.UUID, payload []byte) error {
-	if len(payload) > maxDatagramPayloadSize {
-		// TODO: TUN-5302 return ICMP packet too big message
-		return fmt.Errorf("origin UDP payload has %d bytes, which exceeds transport MTU %d", len(payload), dm.MTU())
+// Maximum application payload to send to / receive from QUIC datagram frame
+func (dm *DatagramMuxer) mtu() int {
+	return maxDatagramPayloadSize
+}
+
+func (dm *DatagramMuxer) SendToSession(session *packet.Session) error {
+	if len(session.Payload) > dm.mtu() {
+		packetTooBigDropped.Inc()
+		return fmt.Errorf("origin UDP payload has %d bytes, which exceeds transport MTU %d", len(session.Payload), dm.mtu())
 	}
-	msgWithID, err := suffixSessionID(sessionID, payload)
+	payloadWithMetadata, err := suffixSessionID(session.ID, session.Payload)
 	if err != nil {
 		return errors.Wrap(err, "Failed to suffix session ID to datagram, it will be dropped")
 	}
-	if err := dm.session.SendMessage(msgWithID); err != nil {
+	if err := dm.session.SendMessage(payloadWithMetadata); err != nil {
 		return errors.Wrap(err, "Failed to send datagram back to edge")
 	}
 	return nil
 }
 
-// ReceiveFrom extracts datagram session ID, then sends the session ID and payload to session manager
-// which determines how to proxy to the origin. It assumes the datagram session has already been
-// registered with session manager through other side channel
-func (dm *DatagramMuxer) ReceiveFrom() (uuid.UUID, []byte, error) {
-	msg, err := dm.session.ReceiveMessage()
-	if err != nil {
-		return uuid.Nil, nil, err
+func (dm *DatagramMuxer) ServeReceive(ctx context.Context) error {
+	for {
+		// Extracts datagram session ID, then sends the session ID and payload to receiver
+		// which determines how to proxy to the origin. It assumes the datagram session has already been
+		// registered with receiver through other side channel
+		msg, err := dm.session.ReceiveMessage()
+		if err != nil {
+			return err
+		}
+		if err := dm.demux(ctx, msg); err != nil {
+			dm.logger.Error().Err(err).Msg("Failed to demux datagram")
+			if err == context.Canceled {
+				return err
+			}
+		}
 	}
-	sessionID, payload, err := extractSessionID(msg)
-	if err != nil {
-		return uuid.Nil, nil, err
-	}
-	return sessionID, payload, nil
 }
 
-// Maximum application payload to send to / receive from QUIC datagram frame
-func (dm *DatagramMuxer) MTU() int {
-	return maxDatagramPayloadSize
+func (dm *DatagramMuxer) demux(ctx context.Context, msg []byte) error {
+	sessionID, payload, err := extractSessionID(msg)
+	if err != nil {
+		return err
+	}
+	sessionDatagram := packet.Session{
+		ID:      sessionID,
+		Payload: payload,
+	}
+	select {
+	case dm.demuxChan <- &sessionDatagram:
+		return nil
+	case <-ctx.Done():
+		return ctx.Err()
+	}
 }
 
 // Each QUIC datagram should be suffixed with session ID.

quic/datagram_test.go

@@ -1,22 +1,28 @@
 package quic
 
 import (
-	"bytes"
 	"context"
 	"crypto/rand"
 	"crypto/rsa"
 	"crypto/tls"
 	"crypto/x509"
 	"encoding/pem"
+	"fmt"
 	"math/big"
+	"net/netip"
 	"testing"
 	"time"
 
+	"github.com/google/gopacket/layers"
 	"github.com/google/uuid"
 	"github.com/lucas-clemente/quic-go"
 	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/require"
+	"golang.org/x/net/icmp"
+	"golang.org/x/net/ipv4"
 	"golang.org/x/sync/errgroup"
+
+	"github.com/cloudflare/cloudflared/packet"
 )
 
 var (
@@ -52,9 +58,59 @@ func TestSuffixSessionIDError(t *testing.T) {
 	require.Error(t, err)
 }
 
-func TestMaxDatagramPayload(t *testing.T) {
-	payload := make([]byte, maxDatagramPayloadSize)
+func TestDatagram(t *testing.T) {
+	maxPayload := make([]byte, maxDatagramPayloadSize)
+	noPayloadSession := uuid.New()
+	maxPayloadSession := uuid.New()
+	sessionToPayload := []*packet.Session{
+		{
+			ID:      noPayloadSession,
+			Payload: make([]byte, 0),
+		},
+		{
+			ID:      maxPayloadSession,
+			Payload: maxPayload,
+		},
+	}
 
+	packets := []packet.ICMP{
+		{
+			IP: &packet.IP{
+				Src:      netip.MustParseAddr("172.16.0.1"),
+				Dst:      netip.MustParseAddr("192.168.0.1"),
+				Protocol: layers.IPProtocolICMPv4,
+			},
+			Message: &icmp.Message{
+				Type: ipv4.ICMPTypeTimeExceeded,
+				Code: 0,
+				Body: &icmp.TimeExceeded{
+					Data: []byte("original packet"),
+				},
+			},
+		},
+		{
+			IP: &packet.IP{
+				Src:      netip.MustParseAddr("172.16.0.2"),
+				Dst:      netip.MustParseAddr("192.168.0.2"),
+				Protocol: layers.IPProtocolICMPv4,
+			},
+			Message: &icmp.Message{
+				Type: ipv4.ICMPTypeEcho,
+				Code: 0,
+				Body: &icmp.Echo{
+					ID:   6182,
+					Seq:  9151,
+					Data: []byte("Test ICMP echo"),
+				},
+			},
+		},
+	}
+
+	testDatagram(t, 1, sessionToPayload, nil)
+	testDatagram(t, 2, sessionToPayload, packets)
+}
+
+func testDatagram(t *testing.T, version uint8, sessionToPayloads []*packet.Session, packets []packet.ICMP) {
 	quicConfig := &quic.Config{
 		KeepAlivePeriod:      5 * time.Millisecond,
 		EnableDatagrams:      true,
@@ -63,6 +119,8 @@ func TestMaxDatagramPayload(t *testing.T) {
 	quicListener := newQUICListener(t, quicConfig)
 	defer quicListener.Close()
 
+	logger := zerolog.Nop()
+
 	errGroup, ctx := errgroup.WithContext(context.Background())
 	// Run edge side of datagram muxer
 	errGroup.Go(func() error {
@@ -72,22 +130,40 @@ func TestMaxDatagramPayload(t *testing.T) {
 			return err
 		}
 
-		logger := zerolog.Nop()
-		muxer, err := NewDatagramMuxer(quicSession, &logger)
-		if err != nil {
-			return err
+		sessionDemuxChan := make(chan *packet.Session, 16)
+
+		switch version {
+		case 1:
+			muxer := NewDatagramMuxer(quicSession, &logger, sessionDemuxChan)
+			muxer.ServeReceive(ctx)
+		case 2:
+			muxer := NewDatagramMuxerV2(quicSession, &logger, sessionDemuxChan)
+			muxer.ServeReceive(ctx)
+
+			icmpDecoder := packet.NewICMPDecoder()
+			for _, pk := range packets {
+				received, err := muxer.ReceivePacket(ctx)
+				require.NoError(t, err)
+
+				receivedICMP, err := icmpDecoder.Decode(received)
+				require.NoError(t, err)
+				require.Equal(t, pk.IP, receivedICMP.IP)
+				require.Equal(t, pk.Type, receivedICMP.Type)
+				require.Equal(t, pk.Code, receivedICMP.Code)
+				require.Equal(t, pk.Body, receivedICMP.Body)
+			}
+		default:
+			return fmt.Errorf("unknown datagram version %d", version)
 		}
 
-		sessionID, receivedPayload, err := muxer.ReceiveFrom()
-		if err != nil {
-			return err
+		for _, expectedPayload := range sessionToPayloads {
+			actualPayload := <-sessionDemuxChan
+			require.Equal(t, expectedPayload, actualPayload)
 		}
-		require.Equal(t, testSessionID, sessionID)
-		require.True(t, bytes.Equal(payload, receivedPayload))
-
 		return nil
 	})
 
+	largePayload := make([]byte, MaxDatagramFrameSize)
 	// Run cloudflared side of datagram muxer
 	errGroup.Go(func() error {
 		tlsClientConfig := &tls.Config{
@@ -97,24 +173,43 @@ func TestMaxDatagramPayload(t *testing.T) {
 		// Establish quic connection
 		quicSession, err := quic.DialAddrEarly(quicListener.Addr().String(), tlsClientConfig, quicConfig)
 		require.NoError(t, err)
-
-		logger := zerolog.Nop()
-		muxer, err := NewDatagramMuxer(quicSession, &logger)
-		if err != nil {
-			return err
-		}
+		defer quicSession.CloseWithError(0, "")
 
 		// Wait a few milliseconds for MTU discovery to take place
 		time.Sleep(time.Millisecond * 100)
-		err = muxer.SendTo(testSessionID, payload)
-		if err != nil {
-			return err
+
+		var muxer BaseDatagramMuxer
+		switch version {
+		case 1:
+			muxer = NewDatagramMuxer(quicSession, &logger, nil)
+		case 2:
+			muxerV2 := NewDatagramMuxerV2(quicSession, &logger, nil)
+			encoder := packet.NewEncoder()
+			for _, pk := range packets {
+				encodedPacket, err := encoder.Encode(&pk)
+				require.NoError(t, err)
+				require.NoError(t, muxerV2.SendPacket(encodedPacket))
+			}
+			// Payload larger than transport MTU, should not be sent
+			require.Error(t, muxerV2.SendPacket(packet.RawPacket{
+				Data: largePayload,
+			}))
+			muxer = muxerV2
+		default:
+			return fmt.Errorf("unknown datagram version %d", version)
 		}
 
-		// Payload larger than transport MTU, should return an error
-		largePayload := make([]byte, MaxDatagramFrameSize)
-		err = muxer.SendTo(testSessionID, largePayload)
-		require.Error(t, err)
+		for _, session := range sessionToPayloads {
+			require.NoError(t, muxer.SendToSession(session))
+		}
+		// Payload larger than transport MTU, should not be sent
+		require.Error(t, muxer.SendToSession(&packet.Session{
+			ID:      testSessionID,
+			Payload: largePayload,
+		}))
+
+		// Wait for edge to finish receiving the messages
+		time.Sleep(time.Millisecond * 100)
 
 		return nil
 	})

quic/datagramv2.go

@@ -0,0 +1,158 @@
+package quic
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/lucas-clemente/quic-go"
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog"
+
+	"github.com/cloudflare/cloudflared/packet"
+)
+
+type datagramV2Type byte
+
+const (
+	udp datagramV2Type = iota
+	ip
+	// Same as sessionDemuxChan capacity
+	packetChanCapacity = 16
+)
+
+func suffixType(b []byte, datagramType datagramV2Type) ([]byte, error) {
+	if len(b)+1 > MaxDatagramFrameSize {
+		return nil, fmt.Errorf("datagram size %d exceeds max frame size %d", len(b), MaxDatagramFrameSize)
+	}
+	b = append(b, byte(datagramType))
+	return b, nil
+}
+
+// Maximum application payload to send to / receive from QUIC datagram frame
+func (dm *DatagramMuxerV2) mtu() int {
+	return maxDatagramPayloadSize
+}
+
+type DatagramMuxerV2 struct {
+	session          quic.Connection
+	logger           *zerolog.Logger
+	sessionDemuxChan chan<- *packet.Session
+	packetDemuxChan  chan packet.RawPacket
+}
+
+func NewDatagramMuxerV2(
+	quicSession quic.Connection,
+	log *zerolog.Logger,
+	sessionDemuxChan chan<- *packet.Session,
+) *DatagramMuxerV2 {
+	logger := log.With().Uint8("datagramVersion", 2).Logger()
+	return &DatagramMuxerV2{
+		session:          quicSession,
+		logger:           &logger,
+		sessionDemuxChan: sessionDemuxChan,
+		packetDemuxChan:  make(chan packet.RawPacket, packetChanCapacity),
+	}
+}
+
+// SendToSession suffix the session ID and datagram version to the payload so the other end of the QUIC connection can
+// demultiplex the payload from multiple datagram sessions
+func (dm *DatagramMuxerV2) SendToSession(session *packet.Session) error {
+	if len(session.Payload) > dm.mtu() {
+		packetTooBigDropped.Inc()
+		return fmt.Errorf("origin UDP payload has %d bytes, which exceeds transport MTU %d", len(session.Payload), dm.mtu())
+	}
+	msgWithID, err := suffixSessionID(session.ID, session.Payload)
+	if err != nil {
+		return errors.Wrap(err, "Failed to suffix session ID to datagram, it will be dropped")
+	}
+	msgWithIDAndType, err := suffixType(msgWithID, udp)
+	if err != nil {
+		return errors.Wrap(err, "Failed to suffix datagram type, it will be dropped")
+	}
+	if err := dm.session.SendMessage(msgWithIDAndType); err != nil {
+		return errors.Wrap(err, "Failed to send datagram back to edge")
+	}
+	return nil
+}
+
+// SendPacket suffix the datagram type to the packet. The other end of the QUIC connection can demultiplex by parsing
+// the payload as IP and look at the source and destination.
+func (dm *DatagramMuxerV2) SendPacket(pk packet.RawPacket) error {
+	payloadWithVersion, err := suffixType(pk.Data, ip)
+	if err != nil {
+		return errors.Wrap(err, "Failed to suffix datagram type, it will be dropped")
+	}
+	if err := dm.session.SendMessage(payloadWithVersion); err != nil {
+		return errors.Wrap(err, "Failed to send datagram back to edge")
+	}
+	return nil
+}
+
+// Demux reads datagrams from the QUIC connection and demuxes depending on whether it's a session or packet
+func (dm *DatagramMuxerV2) ServeReceive(ctx context.Context) error {
+	for {
+		msg, err := dm.session.ReceiveMessage()
+		if err != nil {
+			return err
+		}
+		if err := dm.demux(ctx, msg); err != nil {
+			dm.logger.Error().Err(err).Msg("Failed to demux datagram")
+			if err == context.Canceled {
+				return err
+			}
+		}
+	}
+}
+
+func (dm *DatagramMuxerV2) ReceivePacket(ctx context.Context) (packet.RawPacket, error) {
+	select {
+	case <-ctx.Done():
+		return packet.RawPacket{}, ctx.Err()
+	case pk := <-dm.packetDemuxChan:
+		return pk, nil
+	}
+}
+
+func (dm *DatagramMuxerV2) demux(ctx context.Context, msgWithType []byte) error {
+	if len(msgWithType) < 1 {
+		return fmt.Errorf("QUIC datagram should have at least 1 byte")
+	}
+	msgType := datagramV2Type(msgWithType[len(msgWithType)-1])
+	msg := msgWithType[0 : len(msgWithType)-1]
+	switch msgType {
+	case udp:
+		return dm.handleSession(ctx, msg)
+	case ip:
+		return dm.handlePacket(ctx, msg)
+	default:
+		return fmt.Errorf("Unexpected datagram type %d", msgType)
+	}
+}
+
+func (dm *DatagramMuxerV2) handleSession(ctx context.Context, session []byte) error {
+	sessionID, payload, err := extractSessionID(session)
+	if err != nil {
+		return err
+	}
+	sessionDatagram := packet.Session{
+		ID:      sessionID,
+		Payload: payload,
+	}
+	select {
+	case dm.sessionDemuxChan <- &sessionDatagram:
+		return nil
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+}
+
+func (dm *DatagramMuxerV2) handlePacket(ctx context.Context, pk []byte) error {
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case dm.packetDemuxChan <- packet.RawPacket{
+		Data: pk,
+	}:
+		return nil
+	}
+}

quic/metrics.go

@@ -144,6 +144,13 @@ var (
 	}
 	registerClient = sync.Once{}
 	registerServer = sync.Once{}
+
+	packetTooBigDropped = prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: namespace,
+		Subsystem: perspectiveString(logging.PerspectiveClient),
+		Name:      "packet_too_big_dropped",
+		Help:      "Count of packets received from origin that are too big to send to the edge and are dropped as a result",
+	})
 )
 
 // MetricsCollector abstracts the difference between client and server metrics from connTracer
@@ -264,6 +271,7 @@ func newClientCollector(index uint8) MetricsCollector {
 			clientMetrics.minRTT,
 			clientMetrics.latestRTT,
 			clientMetrics.smoothedRTT,
+			packetTooBigDropped,
 		)
 	})
 	return &clientCollector{

quic/safe_stream.go

@@ -41,3 +41,14 @@ func (s *SafeStreamCloser) Close() error {
 	s.stream.CancelRead(0)
 	return s.stream.Close()
 }
+
+func (s *SafeStreamCloser) CloseWrite() error {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	// As documented by the quic-go library, this doesn't actually close the entire stream.
+	// It prevents further writes, which in turn will result in an EOF signal being sent the other side of stream when
+	// reading.
+	// We can still read from this stream.
+	return s.stream.Close()
+}

supervisor/conn_aware_logger.go

@@ -12,9 +12,9 @@ type ConnAwareLogger struct {
 	logger  *zerolog.Logger
 }
 
-func NewConnAwareLogger(logger *zerolog.Logger, observer *connection.Observer) *ConnAwareLogger {
+func NewConnAwareLogger(logger *zerolog.Logger, tracker *tunnelstate.ConnTracker, observer *connection.Observer) *ConnAwareLogger {
 	connAwareLogger := &ConnAwareLogger{
-		tracker: tunnelstate.NewConnTracker(logger),
+		tracker: tracker,
 		logger:  logger,
 	}
 

supervisor/supervisor.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"net"
 	"strings"
 	"time"
 
@@ -15,10 +16,12 @@ import (
 	"github.com/cloudflare/cloudflared/edgediscovery"
 	"github.com/cloudflare/cloudflared/edgediscovery/allregions"
 	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/ingress"
 	"github.com/cloudflare/cloudflared/orchestration"
 	"github.com/cloudflare/cloudflared/retry"
 	"github.com/cloudflare/cloudflared/signal"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/cloudflare/cloudflared/tunnelstate"
 )
 
 const (
@@ -43,7 +46,7 @@ type Supervisor struct {
 	config                  *TunnelConfig
 	orchestrator            *orchestration.Orchestrator
 	edgeIPs                 *edgediscovery.Edge
-	edgeTunnelServer        EdgeTunnelServer
+	edgeTunnelServer        *EdgeTunnelServer
 	tunnelErrors            chan tunnelError
 	tunnelsConnecting       map[int]chan struct{}
 	tunnelsProtocolFallback map[int]*protocolFallback
@@ -88,7 +91,9 @@ func NewSupervisor(config *TunnelConfig, orchestrator *orchestration.Orchestrato
 	}
 
 	reconnectCredentialManager := newReconnectCredentialManager(connection.MetricsNamespace, connection.TunnelSubsystem, config.HAConnections)
-	log := NewConnAwareLogger(config.Log, config.Observer)
+
+	tracker := tunnelstate.NewConnTracker(config.Log)
+	log := NewConnAwareLogger(config.Log, tracker, config.Observer)
 
 	var edgeAddrHandler EdgeAddrHandler
 	if isStaticEdge { // static edge addresses
@@ -106,10 +111,21 @@ func NewSupervisor(config *TunnelConfig, orchestrator *orchestration.Orchestrato
 		credentialManager: reconnectCredentialManager,
 		edgeAddrs:         edgeIPs,
 		edgeAddrHandler:   edgeAddrHandler,
+		tracker:           tracker,
 		reconnectCh:       reconnectCh,
 		gracefulShutdownC: gracefulShutdownC,
 		connAwareLogger:   log,
 	}
+	if useDatagramV2(config) {
+		// For non-privileged datagram-oriented ICMP endpoints, network must be "udp4" or "udp6"
+		// TODO: TUN-6654 listen for IPv6 and decide if it should listen on specific IP
+		icmpProxy, err := ingress.NewICMPProxy(net.IPv4zero, config.Log)
+		if err != nil {
+			log.Logger().Warn().Err(err).Msg("Failed to create icmp proxy, will continue to use datagram v1")
+		} else {
+			edgeTunnelServer.icmpProxy = icmpProxy
+		}
+	}
 
 	useReconnectToken := false
 	if config.ClassicTunnel != nil {
@@ -121,7 +137,7 @@ func NewSupervisor(config *TunnelConfig, orchestrator *orchestration.Orchestrato
 		config:                     config,
 		orchestrator:               orchestrator,
 		edgeIPs:                    edgeIPs,
-		edgeTunnelServer:           edgeTunnelServer,
+		edgeTunnelServer:           &edgeTunnelServer,
 		tunnelErrors:               make(chan tunnelError),
 		tunnelsConnecting:          map[int]chan struct{}{},
 		tunnelsProtocolFallback:    map[int]*protocolFallback{},
@@ -138,6 +154,14 @@ func (s *Supervisor) Run(
 	ctx context.Context,
 	connectedSignal *signal.Signal,
 ) error {
+	if s.edgeTunnelServer.icmpProxy != nil {
+		go func() {
+			if err := s.edgeTunnelServer.icmpProxy.ListenResponse(ctx); err != nil {
+				s.log.Logger().Err(err).Msg("icmp proxy terminated")
+			}
+		}()
+	}
+
 	if err := s.initialize(ctx, connectedSignal); err != nil {
 		if err == errEarlyShutdown {
 			return nil
@@ -245,7 +269,7 @@ func (s *Supervisor) initialize(
 		s.config.HAConnections = availableAddrs
 	}
 	s.tunnelsProtocolFallback[0] = &protocolFallback{
-		retry.BackoffHandler{MaxRetries: s.config.Retries},
+		retry.BackoffHandler{MaxRetries: s.config.Retries, RetryForever: true},
 		s.config.ProtocolSelector.Current(),
 		false,
 	}
@@ -267,7 +291,7 @@ func (s *Supervisor) initialize(
 	// At least one successful connection, so start the rest
 	for i := 1; i < s.config.HAConnections; i++ {
 		s.tunnelsProtocolFallback[i] = &protocolFallback{
-			retry.BackoffHandler{MaxRetries: s.config.Retries},
+			retry.BackoffHandler{MaxRetries: s.config.Retries, RetryForever: true},
 			s.config.ProtocolSelector.Current(),
 			false,
 		}
@@ -409,3 +433,15 @@ func (s *Supervisor) authenticate(ctx context.Context, numPreviousAttempts int)
 	registrationOptions.NumPreviousAttempts = uint8(numPreviousAttempts)
 	return rpcClient.Authenticate(ctx, s.config.ClassicTunnel, registrationOptions)
 }
+
+func useDatagramV2(config *TunnelConfig) bool {
+	if config.NamedTunnel == nil {
+		return false
+	}
+	for _, feature := range config.NamedTunnel.Client.Features {
+		if feature == FeatureDatagramV2 {
+			return true
+		}
+	}
+	return false
+}

supervisor/tunnel.go

@@ -20,12 +20,14 @@ import (
 	"github.com/cloudflare/cloudflared/edgediscovery"
 	"github.com/cloudflare/cloudflared/edgediscovery/allregions"
 	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/ingress"
 	"github.com/cloudflare/cloudflared/orchestration"
 	quicpogs "github.com/cloudflare/cloudflared/quic"
 	"github.com/cloudflare/cloudflared/retry"
 	"github.com/cloudflare/cloudflared/signal"
 	"github.com/cloudflare/cloudflared/tunnelrpc"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/cloudflare/cloudflared/tunnelstate"
 )
 
 const (
@@ -33,6 +35,7 @@ const (
 	FeatureSerializedHeaders = "serialized_headers"
 	FeatureQuickReconnects   = "quick_reconnects"
 	FeatureAllowRemoteConfig = "allow_remote_config"
+	FeatureDatagramV2        = "support_datagram_v2"
 )
 
 type TunnelConfig struct {
@@ -190,11 +193,13 @@ type EdgeTunnelServer struct {
 	edgeAddrs         *edgediscovery.Edge
 	reconnectCh       chan ReconnectSignal
 	gracefulShutdownC <-chan struct{}
+	tracker           *tunnelstate.ConnTracker
+	icmpProxy         ingress.ICMPProxy
 
 	connAwareLogger *ConnAwareLogger
 }
 
-func (e EdgeTunnelServer) Serve(ctx context.Context, connIndex uint8, protocolFallback *protocolFallback, connectedSignal *signal.Signal) error {
+func (e *EdgeTunnelServer) Serve(ctx context.Context, connIndex uint8, protocolFallback *protocolFallback, connectedSignal *signal.Signal) error {
 	haConnections.Inc()
 	defer haConnections.Dec()
 
@@ -226,20 +231,14 @@ func (e EdgeTunnelServer) Serve(ctx context.Context, connIndex uint8, protocolFa
 	// to another protocol when a particular metal doesn't support new protocol
 	// Each connection can also have it's own IP version because individual connections might fallback
 	// to another IP version.
-	err, recoverable := ServeTunnel(
+	err, recoverable := e.serveTunnel(
 		ctx,
 		connLog,
-		e.credentialManager,
-		e.config,
-		e.orchestrator,
 		addr,
 		connIndex,
 		connectedFuse,
 		protocolFallback,
-		e.cloudflaredUUID,
-		e.reconnectCh,
 		protocolFallback.protocol,
-		e.gracefulShutdownC,
 	)
 
 	// If the connection is recoverable, we want to maintain the same IP
@@ -272,6 +271,12 @@ func (e EdgeTunnelServer) Serve(ctx context.Context, connIndex uint8, protocolFa
 			return err
 		}
 
+		// If a single connection has connected with the current protocol, we know we know we don't have to fallback
+		// to a different protocol.
+		if e.tracker.HasConnectedWith(e.config.ProtocolSelector.Current()) {
+			return err
+		}
+
 		if !selectNextProtocol(
 			connLog.Logger(),
 			protocolFallback,
@@ -352,20 +357,14 @@ func selectNextProtocol(
 
 // ServeTunnel runs a single tunnel connection, returns nil on graceful shutdown,
 // on error returns a flag indicating if error can be retried
-func ServeTunnel(
+func (e *EdgeTunnelServer) serveTunnel(
 	ctx context.Context,
 	connLog *ConnAwareLogger,
-	credentialManager *reconnectCredentialManager,
-	config *TunnelConfig,
-	orchestrator *orchestration.Orchestrator,
 	addr *allregions.EdgeAddr,
 	connIndex uint8,
 	fuse *h2mux.BooleanFuse,
 	backoff *protocolFallback,
-	cloudflaredUUID uuid.UUID,
-	reconnectCh chan ReconnectSignal,
 	protocol connection.Protocol,
-	gracefulShutdownC <-chan struct{},
 ) (err error, recoverable bool) {
 	// Treat panics as recoverable errors
 	defer func() {
@@ -380,21 +379,15 @@ func ServeTunnel(
 		}
 	}()
 
-	defer config.Observer.SendDisconnect(connIndex)
-	err, recoverable = serveTunnel(
+	defer e.config.Observer.SendDisconnect(connIndex)
+	err, recoverable = e.serveConnection(
 		ctx,
 		connLog,
-		credentialManager,
-		config,
-		orchestrator,
 		addr,
 		connIndex,
 		fuse,
 		backoff,
-		cloudflaredUUID,
-		reconnectCh,
 		protocol,
-		gracefulShutdownC,
 	)
 
 	if err != nil {
@@ -407,7 +400,7 @@ func ServeTunnel(
 			connLog.ConnAwareLogger().Err(err).Msg("Register tunnel error from server side")
 			// Don't send registration error return from server to Sentry. They are
 			// logged on server side
-			if incidents := config.IncidentLookup.ActiveIncidents(); len(incidents) > 0 {
+			if incidents := e.config.IncidentLookup.ActiveIncidents(); len(incidents) > 0 {
 				connLog.ConnAwareLogger().Msg(activeIncidentsMsg(incidents))
 			}
 			return err.Cause, !err.Permanent
@@ -433,92 +426,73 @@ func ServeTunnel(
 	return nil, false
 }
 
-func serveTunnel(
+func (e *EdgeTunnelServer) serveConnection(
 	ctx context.Context,
 	connLog *ConnAwareLogger,
-	credentialManager *reconnectCredentialManager,
-	config *TunnelConfig,
-	orchestrator *orchestration.Orchestrator,
 	addr *allregions.EdgeAddr,
 	connIndex uint8,
 	fuse *h2mux.BooleanFuse,
 	backoff *protocolFallback,
-	cloudflaredUUID uuid.UUID,
-	reconnectCh chan ReconnectSignal,
 	protocol connection.Protocol,
-	gracefulShutdownC <-chan struct{},
 ) (err error, recoverable bool) {
 	connectedFuse := &connectedFuse{
 		fuse:    fuse,
 		backoff: backoff,
 	}
 	controlStream := connection.NewControlStream(
-		config.Observer,
+		e.config.Observer,
 		connectedFuse,
-		config.NamedTunnel,
+		e.config.NamedTunnel,
 		connIndex,
 		addr.UDP.IP,
 		nil,
-		gracefulShutdownC,
-		config.GracePeriod,
+		e.gracefulShutdownC,
+		e.config.GracePeriod,
+		protocol,
 	)
 
 	switch protocol {
 	case connection.QUIC, connection.QUICWarp:
-		connOptions := config.connectionOptions(addr.UDP.String(), uint8(backoff.Retries()))
-		return ServeQUIC(ctx,
+		connOptions := e.config.connectionOptions(addr.UDP.String(), uint8(backoff.Retries()))
+		return e.serveQUIC(ctx,
 			addr.UDP,
-			config,
-			orchestrator,
 			connLog,
 			connOptions,
 			controlStream,
-			connIndex,
-			reconnectCh,
-			gracefulShutdownC)
+			connIndex)
 
 	case connection.HTTP2, connection.HTTP2Warp:
-		edgeConn, err := edgediscovery.DialEdge(ctx, dialTimeout, config.EdgeTLSConfigs[protocol], addr.TCP)
+		edgeConn, err := edgediscovery.DialEdge(ctx, dialTimeout, e.config.EdgeTLSConfigs[protocol], addr.TCP)
 		if err != nil {
 			connLog.ConnAwareLogger().Err(err).Msg("Unable to establish connection with Cloudflare edge")
 			return err, true
 		}
 
-		connOptions := config.connectionOptions(edgeConn.LocalAddr().String(), uint8(backoff.Retries()))
-		if err := ServeHTTP2(
+		connOptions := e.config.connectionOptions(edgeConn.LocalAddr().String(), uint8(backoff.Retries()))
+		if err := e.serveHTTP2(
 			ctx,
 			connLog,
-			config,
-			orchestrator,
 			edgeConn,
 			connOptions,
 			controlStream,
 			connIndex,
-			gracefulShutdownC,
-			reconnectCh,
 		); err != nil {
 			return err, false
 		}
 
 	default:
-		edgeConn, err := edgediscovery.DialEdge(ctx, dialTimeout, config.EdgeTLSConfigs[protocol], addr.TCP)
+		edgeConn, err := edgediscovery.DialEdge(ctx, dialTimeout, e.config.EdgeTLSConfigs[protocol], addr.TCP)
 		if err != nil {
 			connLog.ConnAwareLogger().Err(err).Msg("Unable to establish connection with Cloudflare edge")
 			return err, true
 		}
 
-		if err := ServeH2mux(
+		if err := e.serveH2mux(
 			ctx,
 			connLog,
-			credentialManager,
-			config,
-			orchestrator,
 			edgeConn,
 			connIndex,
 			connectedFuse,
-			cloudflaredUUID,
-			reconnectCh,
-			gracefulShutdownC,
 		); err != nil {
 			return err, false
 		}
@@ -534,29 +508,24 @@ func (r unrecoverableError) Error() string {
 	return r.err.Error()
 }
 
-func ServeH2mux(
+func (e *EdgeTunnelServer) serveH2mux(
 	ctx context.Context,
 	connLog *ConnAwareLogger,
-	credentialManager *reconnectCredentialManager,
-	config *TunnelConfig,
-	orchestrator *orchestration.Orchestrator,
 	edgeConn net.Conn,
 	connIndex uint8,
 	connectedFuse *connectedFuse,
-	cloudflaredUUID uuid.UUID,
-	reconnectCh chan ReconnectSignal,
-	gracefulShutdownC <-chan struct{},
 ) error {
 	connLog.Logger().Debug().Msgf("Connecting via h2mux")
 	// Returns error from parsing the origin URL or handshake errors
 	handler, err, recoverable := connection.NewH2muxConnection(
-		orchestrator,
-		config.GracePeriod,
-		config.MuxerConfig,
+		e.orchestrator,
+		e.config.GracePeriod,
+		e.config.MuxerConfig,
 		edgeConn,
 		connIndex,
-		config.Observer,
-		gracefulShutdownC,
+		e.config.Observer,
+		e.gracefulShutdownC,
+		e.config.Log,
 	)
 	if err != nil {
 		if !recoverable {
@@ -568,42 +537,38 @@ func ServeH2mux(
 	errGroup, serveCtx := errgroup.WithContext(ctx)
 
 	errGroup.Go(func() error {
-		if config.NamedTunnel != nil {
-			connOptions := config.connectionOptions(edgeConn.LocalAddr().String(), uint8(connectedFuse.backoff.Retries()))
-			return handler.ServeNamedTunnel(serveCtx, config.NamedTunnel, connOptions, connectedFuse)
+		if e.config.NamedTunnel != nil {
+			connOptions := e.config.connectionOptions(edgeConn.LocalAddr().String(), uint8(connectedFuse.backoff.Retries()))
+			return handler.ServeNamedTunnel(serveCtx, e.config.NamedTunnel, connOptions, connectedFuse)
 		}
-		registrationOptions := config.registrationOptions(connIndex, edgeConn.LocalAddr().String(), cloudflaredUUID)
-		return handler.ServeClassicTunnel(serveCtx, config.ClassicTunnel, credentialManager, registrationOptions, connectedFuse)
+		registrationOptions := e.config.registrationOptions(connIndex, edgeConn.LocalAddr().String(), e.cloudflaredUUID)
+		return handler.ServeClassicTunnel(serveCtx, e.config.ClassicTunnel, e.credentialManager, registrationOptions, connectedFuse)
 	})
 
 	errGroup.Go(func() error {
-		return listenReconnect(serveCtx, reconnectCh, gracefulShutdownC)
+		return listenReconnect(serveCtx, e.reconnectCh, e.gracefulShutdownC)
 	})
 
 	return errGroup.Wait()
 }
 
-func ServeHTTP2(
+func (e *EdgeTunnelServer) serveHTTP2(
 	ctx context.Context,
 	connLog *ConnAwareLogger,
-	config *TunnelConfig,
-	orchestrator *orchestration.Orchestrator,
 	tlsServerConn net.Conn,
 	connOptions *tunnelpogs.ConnectionOptions,
 	controlStreamHandler connection.ControlStreamHandler,
 	connIndex uint8,
-	gracefulShutdownC <-chan struct{},
-	reconnectCh chan ReconnectSignal,
 ) error {
 	connLog.Logger().Debug().Msgf("Connecting via http2")
 	h2conn := connection.NewHTTP2Connection(
 		tlsServerConn,
-		orchestrator,
+		e.orchestrator,
 		connOptions,
-		config.Observer,
+		e.config.Observer,
 		connIndex,
 		controlStreamHandler,
-		config.Log,
+		e.config.Log,
 	)
 
 	errGroup, serveCtx := errgroup.WithContext(ctx)
@@ -612,7 +577,7 @@ func ServeHTTP2(
 	})
 
 	errGroup.Go(func() error {
-		err := listenReconnect(serveCtx, reconnectCh, gracefulShutdownC)
+		err := listenReconnect(serveCtx, e.reconnectCh, e.gracefulShutdownC)
 		if err != nil {
 			// forcefully break the connection (this is only used for testing)
 			connLog.Logger().Debug().Msg("Forcefully breaking http2 connection")
@@ -624,19 +589,15 @@ func ServeHTTP2(
 	return errGroup.Wait()
 }
 
-func ServeQUIC(
+func (e *EdgeTunnelServer) serveQUIC(
 	ctx context.Context,
 	edgeAddr *net.UDPAddr,
-	config *TunnelConfig,
-	orchestrator *orchestration.Orchestrator,
 	connLogger *ConnAwareLogger,
 	connOptions *tunnelpogs.ConnectionOptions,
 	controlStreamHandler connection.ControlStreamHandler,
 	connIndex uint8,
-	reconnectCh chan ReconnectSignal,
-	gracefulShutdownC <-chan struct{},
 ) (err error, recoverable bool) {
-	tlsConfig := config.EdgeTLSConfigs[connection.QUIC]
+	tlsConfig := e.config.EdgeTLSConfigs[connection.QUIC]
 	quicConfig := &quic.Config{
 		HandshakeIdleTimeout:  quicpogs.HandshakeIdleTimeout,
 		MaxIdleTimeout:        quicpogs.MaxIdleTimeout,
@@ -652,10 +613,11 @@ func ServeQUIC(
 		quicConfig,
 		edgeAddr,
 		tlsConfig,
-		orchestrator,
+		e.orchestrator,
 		connOptions,
 		controlStreamHandler,
-		connLogger.Logger())
+		connLogger.Logger(),
+		e.icmpProxy)
 	if err != nil {
 		connLogger.ConnAwareLogger().Err(err).Msgf("Failed to create new quic connection")
 		return err, true
@@ -671,7 +633,7 @@ func ServeQUIC(
 	})
 
 	errGroup.Go(func() error {
-		err := listenReconnect(serveCtx, reconnectCh, gracefulShutdownC)
+		err := listenReconnect(serveCtx, e.reconnectCh, e.gracefulShutdownC)
 		if err != nil {
 			// forcefully break the connection (this is only used for testing)
 			connLogger.Logger().Debug().Msg("Forcefully breaking quic connection")

tracing/tracing.go

@@ -8,6 +8,7 @@ import (
 	"net/http"
 	"os"
 	"runtime"
+	"strings"
 
 	"github.com/rs/zerolog"
 	otelContrib "go.opentelemetry.io/contrib/propagators/jaeger"
@@ -33,6 +34,9 @@ const (
 
 	MaxErrorDescriptionLen = 100
 	traceHttpStatusCodeKey = "upstreamStatusCode"
+
+	traceID128bitsWidth = 128 / 4
+	separator           = ":"
 )
 
 var (
@@ -66,22 +70,50 @@ func Init(version string) {
 	cloudflaredVersionAttribute = semconv.ProcessRuntimeVersionKey.String(version)
 }
 
-type TracedRequest struct {
+type TracedHTTPRequest struct {
 	*http.Request
-	trace.TracerProvider
-	exporter InMemoryClient
+	*cfdTracer
 }
 
-// NewTracedRequest creates a new tracer for the current request context.
-func NewTracedRequest(req *http.Request) *TracedRequest {
+// NewTracedHTTPRequest creates a new tracer for the current HTTP request context.
+func NewTracedHTTPRequest(req *http.Request, log *zerolog.Logger) *TracedHTTPRequest {
 	ctx, exists := extractTrace(req)
 	if !exists {
-		return &TracedRequest{req, trace.NewNoopTracerProvider(), &NoopOtlpClient{}}
+		return &TracedHTTPRequest{req, &cfdTracer{trace.NewNoopTracerProvider(), &NoopOtlpClient{}, log}}
 	}
+	return &TracedHTTPRequest{req.WithContext(ctx), newCfdTracer(ctx, log)}
+}
+
+func (tr *TracedHTTPRequest) ToTracedContext() *TracedContext {
+	return &TracedContext{tr.Context(), tr.cfdTracer}
+}
+
+type TracedContext struct {
+	context.Context
+	*cfdTracer
+}
+
+// NewTracedHTTPRequest creates a new tracer for the current HTTP request context.
+func NewTracedContext(ctx context.Context, traceContext string, log *zerolog.Logger) *TracedContext {
+	ctx, exists := extractTraceFromString(ctx, traceContext)
+	if !exists {
+		return &TracedContext{ctx, &cfdTracer{trace.NewNoopTracerProvider(), &NoopOtlpClient{}, log}}
+	}
+	return &TracedContext{ctx, newCfdTracer(ctx, log)}
+}
+
+type cfdTracer struct {
+	trace.TracerProvider
+	exporter InMemoryClient
+	log      *zerolog.Logger
+}
+
+// NewCfdTracer creates a new tracer for the current request context.
+func newCfdTracer(ctx context.Context, log *zerolog.Logger) *cfdTracer {
 	mc := new(InMemoryOtlpClient)
-	exp, err := otlptrace.New(req.Context(), mc)
+	exp, err := otlptrace.New(ctx, mc)
 	if err != nil {
-		return &TracedRequest{req, trace.NewNoopTracerProvider(), &NoopOtlpClient{}}
+		return &cfdTracer{trace.NewNoopTracerProvider(), &NoopOtlpClient{}, log}
 	}
 	tp := tracesdk.NewTracerProvider(
 		// We want to dump to in-memory exporter immediately
@@ -98,36 +130,41 @@ func NewTracedRequest(req *http.Request) *TracedRequest {
 		)),
 	)
 
-	return &TracedRequest{req.WithContext(ctx), tp, mc}
+	return &cfdTracer{tp, mc, log}
 }
 
-func (cft *TracedRequest) Tracer() trace.Tracer {
+func (cft *cfdTracer) Tracer() trace.Tracer {
 	return cft.TracerProvider.Tracer(tracerInstrumentName)
 }
 
-// Spans returns the spans as base64 encoded protobuf otlp traces.
-func (cft *TracedRequest) AddSpans(headers http.Header, log *zerolog.Logger) {
-	if headers == nil {
-		log.Error().Msgf("provided headers map is nil")
-		return
-	}
-
+// GetSpans returns the spans as base64 encoded string of protobuf otlp traces.
+func (cft *cfdTracer) GetSpans() (enc string) {
 	enc, err := cft.exporter.Spans()
 	switch err {
 	case nil:
 		break
 	case errNoTraces:
-		log.Error().Err(err).Msgf("expected traces to be available")
+		cft.log.Trace().Err(err).Msgf("expected traces to be available")
 		return
 	case errNoopTracer:
 		return // noop tracer has no traces
 	default:
-		log.Error().Err(err)
+		cft.log.Debug().Err(err)
 		return
 	}
+	return
+}
+
+// AddSpans assigns spans as base64 encoded protobuf otlp traces to provided
+// HTTP headers.
+func (cft *cfdTracer) AddSpans(headers http.Header) {
+	if headers == nil {
+		return
+	}
+
+	enc := cft.GetSpans()
 	// No need to add header if no traces
 	if enc == "" {
-		log.Error().Msgf("no traces provided and no error from exporter")
 		return
 	}
 
@@ -166,6 +203,33 @@ func endSpan(span trace.Span, upstreamStatusCode int, spanStatusCode codes.Code,
 	span.End()
 }
 
+// extractTraceFromString will extract the trace information from the provided
+// propagated trace string context.
+func extractTraceFromString(ctx context.Context, trace string) (context.Context, bool) {
+	if trace == "" {
+		return ctx, false
+	}
+	// Jaeger specific separator
+	parts := strings.Split(trace, separator)
+	if len(parts) != 4 {
+		return ctx, false
+	}
+	if parts[0] == "" {
+		return ctx, false
+	}
+	// Correctly left pad the trace to a length of 32
+	if len(parts[0]) < traceID128bitsWidth {
+		left := traceID128bitsWidth - len(parts[0])
+		parts[0] = strings.Repeat("0", left) + parts[0]
+		trace = strings.Join(parts, separator)
+	}
+
+	// Override the 'cf-trace-id' as 'uber-trace-id' so the jaeger propagator can extract it.
+	traceHeader := map[string]string{TracerContextNameOverride: trace}
+	remoteCtx := otel.GetTextMapPropagator().Extract(ctx, propagation.MapCarrier(traceHeader))
+	return remoteCtx, true
+}
+
 // extractTrace attempts to check for a cf-trace-id from a request and return the
 // trace context with the provided http.Request.
 func extractTrace(req *http.Request) (context.Context, bool) {

tracing/tracing_test.go

@@ -14,38 +14,42 @@ import (
 )
 
 func TestNewCfTracer(t *testing.T) {
+	log := zerolog.Nop()
 	req := httptest.NewRequest("GET", "http://localhost", nil)
 	req.Header.Add(TracerContextName, "14cb070dde8e51fc5ae8514e69ba42ca:b38f1bf5eae406f3:0:1")
-	tr := NewTracedRequest(req)
+	tr := NewTracedHTTPRequest(req, &log)
 	assert.NotNil(t, tr)
 	assert.IsType(t, tracesdk.NewTracerProvider(), tr.TracerProvider)
 	assert.IsType(t, &InMemoryOtlpClient{}, tr.exporter)
 }
 
 func TestNewCfTracerMultiple(t *testing.T) {
+	log := zerolog.Nop()
 	req := httptest.NewRequest("GET", "http://localhost", nil)
 	req.Header.Add(TracerContextName, "1241ce3ecdefc68854e8514e69ba42ca:b38f1bf5eae406f3:0:1")
 	req.Header.Add(TracerContextName, "14cb070dde8e51fc5ae8514e69ba42ca:b38f1bf5eae406f3:0:1")
-	tr := NewTracedRequest(req)
+	tr := NewTracedHTTPRequest(req, &log)
 	assert.NotNil(t, tr)
 	assert.IsType(t, tracesdk.NewTracerProvider(), tr.TracerProvider)
 	assert.IsType(t, &InMemoryOtlpClient{}, tr.exporter)
 }
 
 func TestNewCfTracerNilHeader(t *testing.T) {
+	log := zerolog.Nop()
 	req := httptest.NewRequest("GET", "http://localhost", nil)
 	req.Header[http.CanonicalHeaderKey(TracerContextName)] = nil
-	tr := NewTracedRequest(req)
+	tr := NewTracedHTTPRequest(req, &log)
 	assert.NotNil(t, tr)
 	assert.IsType(t, trace.NewNoopTracerProvider(), tr.TracerProvider)
 	assert.IsType(t, &NoopOtlpClient{}, tr.exporter)
 }
 
 func TestNewCfTracerInvalidHeaders(t *testing.T) {
+	log := zerolog.Nop()
 	req := httptest.NewRequest("GET", "http://localhost", nil)
 	for _, test := range [][]string{nil, {""}} {
 		req.Header[http.CanonicalHeaderKey(TracerContextName)] = test
-		tr := NewTracedRequest(req)
+		tr := NewTracedHTTPRequest(req, &log)
 		assert.NotNil(t, tr)
 		assert.IsType(t, trace.NewNoopTracerProvider(), tr.TracerProvider)
 		assert.IsType(t, &NoopOtlpClient{}, tr.exporter)
@@ -53,9 +57,10 @@ func TestNewCfTracerInvalidHeaders(t *testing.T) {
 }
 
 func TestAddingSpansWithNilMap(t *testing.T) {
+	log := zerolog.Nop()
 	req := httptest.NewRequest("GET", "http://localhost", nil)
 	req.Header.Add(TracerContextName, "14cb070dde8e51fc5ae8514e69ba42ca:b38f1bf5eae406f3:0:1")
-	tr := NewTracedRequest(req)
+	tr := NewTracedHTTPRequest(req, &log)
 
 	exporter := tr.exporter.(*InMemoryOtlpClient)
 
@@ -65,5 +70,5 @@ func TestAddingSpansWithNilMap(t *testing.T) {
 	assert.NoError(t, err)
 
 	// a panic shouldn't occur
-	tr.AddSpans(nil, &zerolog.Logger{})
+	tr.AddSpans(nil)
 }

tunnelstate/conntracker.go

@@ -10,20 +10,26 @@ import (
 
 type ConnTracker struct {
 	sync.RWMutex
-	isConnected map[int]bool
-	log         *zerolog.Logger
+	// int is the connection Index
+	connectionInfo map[uint8]ConnectionInfo
+	log            *zerolog.Logger
+}
+
+type ConnectionInfo struct {
+	IsConnected bool
+	Protocol    connection.Protocol
 }
 
 func NewConnTracker(log *zerolog.Logger) *ConnTracker {
 	return &ConnTracker{
-		isConnected: make(map[int]bool, 0),
-		log:         log,
+		connectionInfo: make(map[uint8]ConnectionInfo, 0),
+		log:            log,
 	}
 }
 
-func MockedConnTracker(mocked map[int]bool) *ConnTracker {
+func MockedConnTracker(mocked map[uint8]ConnectionInfo) *ConnTracker {
 	return &ConnTracker{
-		isConnected: mocked,
+		connectionInfo: mocked,
 	}
 }
 
@@ -31,11 +37,17 @@ func (ct *ConnTracker) OnTunnelEvent(c connection.Event) {
 	switch c.EventType {
 	case connection.Connected:
 		ct.Lock()
-		ct.isConnected[int(c.Index)] = true
+		ci := ConnectionInfo{
+			IsConnected: true,
+			Protocol:    c.Protocol,
+		}
+		ct.connectionInfo[c.Index] = ci
 		ct.Unlock()
 	case connection.Disconnected, connection.Reconnecting, connection.RegisteringTunnel, connection.Unregistering:
 		ct.Lock()
-		ct.isConnected[int(c.Index)] = false
+		ci := ct.connectionInfo[c.Index]
+		ci.IsConnected = false
+		ct.connectionInfo[c.Index] = ci
 		ct.Unlock()
 	default:
 		ct.log.Error().Msgf("Unknown connection event case %v", c)
@@ -46,10 +58,23 @@ func (ct *ConnTracker) CountActiveConns() uint {
 	ct.RLock()
 	defer ct.RUnlock()
 	active := uint(0)
-	for _, connected := range ct.isConnected {
-		if connected {
+	for _, ci := range ct.connectionInfo {
+		if ci.IsConnected {
 			active++
 		}
 	}
 	return active
 }
+
+// HasConnectedWith checks if we've ever had a successful connection to the edge
+// with said protocol.
+func (ct *ConnTracker) HasConnectedWith(protocol connection.Protocol) bool {
+	ct.RLock()
+	defer ct.RUnlock()
+	for _, ci := range ct.connectionInfo {
+		if ci.Protocol == protocol {
+			return true
+		}
+	}
+	return false
+}

vendor/github.com/gdamore/encoding/.appveyor.yml

@@ -1,13 +0,0 @@
-version: 1.0.{build}
-clone_folder: c:\gopath\src\github.com\gdamore\encoding
-environment:
-  GOPATH: c:\gopath
-build_script:
-- go version
-- go env
-- SET PATH=%LOCALAPPDATA%\atom\bin;%GOPATH%\bin;%PATH%
-- go get -t ./...
-- go build
-- go install ./...
-test_script:
-- go test ./...

vendor/github.com/gdamore/encoding/.travis.yml

@@ -1,7 +0,0 @@
-language: go
-
-go:
-  - 1.9.x
-  - 1.10.x
-  - 1.11.x
-  - tip

vendor/github.com/gdamore/encoding/LICENSE

@@ -1,202 +0,0 @@
-
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.

vendor/github.com/gdamore/encoding/README.md

@@ -1,19 +0,0 @@
-## encoding
-
-[![Linux Status](https://img.shields.io/travis/gdamore/encoding.svg?label=linux)](https://travis-ci.org/gdamore/encoding)
-[![Windows Status](https://img.shields.io/appveyor/ci/gdamore/encoding.svg?label=windows)](https://ci.appveyor.com/project/gdamore/encoding)
-[![Apache License](https://img.shields.io/badge/license-APACHE2-blue.svg)](https://github.com/gdamore/encoding/blob/master/LICENSE)
-[![GoDoc](https://img.shields.io/badge/godoc-reference-blue.svg)](https://godoc.org/github.com/gdamore/encoding)
-[![Go Report Card](http://goreportcard.com/badge/gdamore/encoding)](http://goreportcard.com/report/gdamore/encoding)
-
-Package encoding provides a number of encodings that are missing from the
-standard Go [encoding]("https://godoc.org/golang.org/x/text/encoding") package.
-
-We hope that we can contribute these to the standard Go library someday.  It
-turns out that some of these are useful for dealing with I/O streams coming
-from non-UTF friendly sources.
-
-The UTF8 Encoder is also useful for situations where valid UTF-8 might be
-carried in streams that contain non-valid UTF; in particular I use it for
-helping me cope with terminals that embed escape sequences in otherwise
-valid UTF-8.

vendor/github.com/gdamore/encoding/ascii.go

@@ -1,36 +0,0 @@
-// Copyright 2015 Garrett D'Amore
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use file except in compliance with the License.
-// You may obtain a copy of the license at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package encoding
-
-import (
-	"golang.org/x/text/encoding"
-)
-
-// ASCII represents the 7-bit US-ASCII scheme.  It decodes directly to
-// UTF-8 without change, as all ASCII values are legal UTF-8.
-// Unicode values less than 128 (i.e. 7 bits) map 1:1 with ASCII.
-// It encodes runes outside of that to 0x1A, the ASCII substitution character.
-var ASCII encoding.Encoding
-
-func init() {
-	amap := make(map[byte]rune)
-	for i := 128; i <= 255; i++ {
-		amap[byte(i)] = RuneError
-	}
-
-	cm := &Charmap{Map: amap}
-	cm.Init()
-	ASCII = cm
-}

vendor/github.com/gdamore/encoding/charmap.go

@@ -1,196 +0,0 @@
-// Copyright 2015 Garrett D'Amore
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use file except in compliance with the License.
-// You may obtain a copy of the license at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package encoding
-
-import (
-	"sync"
-	"unicode/utf8"
-
-	"golang.org/x/text/encoding"
-	"golang.org/x/text/transform"
-)
-
-const (
-	// RuneError is an alias for the UTF-8 replacement rune, '\uFFFD'.
-	RuneError = '\uFFFD'
-
-	// RuneSelf is the rune below which UTF-8 and the Unicode values are
-	// identical.  Its also the limit for ASCII.
-	RuneSelf = 0x80
-
-	// ASCIISub is the ASCII substitution character.
-	ASCIISub = '\x1a'
-)
-
-// Charmap is a structure for setting up encodings for 8-bit character sets,
-// for transforming between UTF8 and that other character set.  It has some
-// ideas borrowed from golang.org/x/text/encoding/charmap, but it uses a
-// different implementation.  This implementation uses maps, and supports
-// user-defined maps.
-//
-// We do assume that a character map has a reasonable substitution character,
-// and that valid encodings are stable (exactly a 1:1 map) and stateless
-// (that is there is no shift character or anything like that.)  Hence this
-// approach will not work for many East Asian character sets.
-//
-// Measurement shows little or no measurable difference in the performance of
-// the two approaches.  The difference was down to a couple of nsec/op, and
-// no consistent pattern as to which ran faster.  With the conversion to
-// UTF-8 the code takes about 25 nsec/op.  The conversion in the reverse
-// direction takes about 100 nsec/op.  (The larger cost for conversion
-// from UTF-8 is most likely due to the need to convert the UTF-8 byte stream
-// to a rune before conversion.
-//
-type Charmap struct {
-	transform.NopResetter
-	bytes map[rune]byte
-	runes [256][]byte
-	once  sync.Once
-
-	// The map between bytes and runes.  To indicate that a specific
-	// byte value is invalid for a charcter set, use the rune
-	// utf8.RuneError.  Values that are absent from this map will
-	// be assumed to have the identity mapping -- that is the default
-	// is to assume ISO8859-1, where all 8-bit characters have the same
-	// numeric value as their Unicode runes.  (Not to be confused with
-	// the UTF-8 values, which *will* be different for non-ASCII runes.)
-	//
-	// If no values less than RuneSelf are changed (or have non-identity
-	// mappings), then the character set is assumed to be an ASCII
-	// superset, and certain assumptions and optimizations become
-	// available for ASCII bytes.
-	Map map[byte]rune
-
-	// The ReplacementChar is the byte value to use for substitution.
-	// It should normally be ASCIISub for ASCII encodings.  This may be
-	// unset (left to zero) for mappings that are strictly ASCII supersets.
-	// In that case ASCIISub will be assumed instead.
-	ReplacementChar byte
-}
-
-type cmapDecoder struct {
-	transform.NopResetter
-	runes [256][]byte
-}
-
-type cmapEncoder struct {
-	transform.NopResetter
-	bytes   map[rune]byte
-	replace byte
-}
-
-// Init initializes internal values of a character map.  This should
-// be done early, to minimize the cost of allocation of transforms
-// later.  It is not strictly necessary however, as the allocation
-// functions will arrange to call it if it has not already been done.
-func (c *Charmap) Init() {
-	c.once.Do(c.initialize)
-}
-
-func (c *Charmap) initialize() {
-	c.bytes = make(map[rune]byte)
-	ascii := true
-
-	for i := 0; i < 256; i++ {
-		r, ok := c.Map[byte(i)]
-		if !ok {
-			r = rune(i)
-		}
-		if r < 128 && r != rune(i) {
-			ascii = false
-		}
-		if r != RuneError {
-			c.bytes[r] = byte(i)
-		}
-		utf := make([]byte, utf8.RuneLen(r))
-		utf8.EncodeRune(utf, r)
-		c.runes[i] = utf
-	}
-	if ascii && c.ReplacementChar == '\x00' {
-		c.ReplacementChar = ASCIISub
-	}
-}
-
-// NewDecoder returns a Decoder the converts from the 8-bit
-// character set to UTF-8.  Unknown mappings, if any, are mapped
-// to '\uFFFD'.
-func (c *Charmap) NewDecoder() *encoding.Decoder {
-	c.Init()
-	return &encoding.Decoder{Transformer: &cmapDecoder{runes: c.runes}}
-}
-
-// NewEncoder returns a Transformer that converts from UTF8 to the
-// 8-bit character set.  Unknown mappings are mapped to 0x1A.
-func (c *Charmap) NewEncoder() *encoding.Encoder {
-	c.Init()
-	return &encoding.Encoder{
-		Transformer: &cmapEncoder{
-			bytes:   c.bytes,
-			replace: c.ReplacementChar,
-		},
-	}
-}
-
-func (d *cmapDecoder) Transform(dst, src []byte, atEOF bool) (int, int, error) {
-	var e error
-	var ndst, nsrc int
-
-	for _, c := range src {
-		b := d.runes[c]
-		l := len(b)
-
-		if ndst+l > len(dst) {
-			e = transform.ErrShortDst
-			break
-		}
-		for i := 0; i < l; i++ {
-			dst[ndst] = b[i]
-			ndst++
-		}
-		nsrc++
-	}
-	return ndst, nsrc, e
-}
-
-func (d *cmapEncoder) Transform(dst, src []byte, atEOF bool) (int, int, error) {
-	var e error
-	var ndst, nsrc int
-	for nsrc < len(src) {
-		if ndst >= len(dst) {
-			e = transform.ErrShortDst
-			break
-		}
-
-		r, sz := utf8.DecodeRune(src[nsrc:])
-		if r == utf8.RuneError && sz == 1 {
-			// If its inconclusive due to insufficient data in
-			// in the source, report it
-			if !atEOF && !utf8.FullRune(src[nsrc:]) {
-				e = transform.ErrShortSrc
-				break
-			}
-		}
-
-		if c, ok := d.bytes[r]; ok {
-			dst[ndst] = c
-		} else {
-			dst[ndst] = d.replace
-		}
-		nsrc += sz
-		ndst++
-	}
-
-	return ndst, nsrc, e
-}

vendor/github.com/gdamore/encoding/doc.go

@@ -1,17 +0,0 @@
-// Copyright 2015 Garrett D'Amore
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use file except in compliance with the License.
-// You may obtain a copy of the license at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// Package encoding provides a few of the encoding structures that are
-// missing from the Go x/text/encoding tree.
-package encoding

vendor/github.com/gdamore/encoding/ebcdic.go

@@ -1,273 +0,0 @@
-// Copyright 2015 Garrett D'Amore
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use file except in compliance with the License.
-// You may obtain a copy of the license at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package encoding
-
-import (
-	"golang.org/x/text/encoding"
-)
-
-// EBCDIC represents the 8-bit EBCDIC scheme, found in some mainframe
-// environments.  If you don't know what this is, consider yourself lucky.
-var EBCDIC encoding.Encoding
-
-func init() {
-	cm := &Charmap{
-		ReplacementChar: '\x3f',
-		Map: map[byte]rune{
-			// 0x00-0x03 match
-			0x04: RuneError,
-			0x05: '\t',
-			0x06: RuneError,
-			0x07: '\x7f',
-			0x08: RuneError,
-			0x09: RuneError,
-			0x0a: RuneError,
-			// 0x0b-0x13 match
-			0x14: RuneError,
-			0x15: '\x85', // Not in any ISO code
-			0x16: '\x08',
-			0x17: RuneError,
-			// 0x18-0x19 match
-			0x1a: RuneError,
-			0x1b: RuneError,
-			// 0x1c-0x1f match
-			0x20: RuneError,
-			0x21: RuneError,
-			0x22: RuneError,
-			0x23: RuneError,
-			0x24: RuneError,
-			0x25: '\n',
-			0x26: '\x17',
-			0x27: '\x1b',
-			0x28: RuneError,
-			0x29: RuneError,
-			0x2a: RuneError,
-			0x2b: RuneError,
-			0x2c: RuneError,
-			0x2d: '\x05',
-			0x2e: '\x06',
-			0x2f: '\x07',
-			0x30: RuneError,
-			0x31: RuneError,
-			0x32: '\x16',
-			0x33: RuneError,
-			0x34: RuneError,
-			0x35: RuneError,
-			0x36: RuneError,
-			0x37: '\x04',
-			0x38: RuneError,
-			0x39: RuneError,
-			0x3a: RuneError,
-			0x3b: RuneError,
-			0x3c: '\x14',
-			0x3d: '\x15',
-			0x3e: RuneError,
-			0x3f: '\x1a', // also replacement char
-			0x40: ' ',
-			0x41: '\xa0',
-			0x42: RuneError,
-			0x43: RuneError,
-			0x44: RuneError,
-			0x45: RuneError,
-			0x46: RuneError,
-			0x47: RuneError,
-			0x48: RuneError,
-			0x49: RuneError,
-			0x4a: RuneError,
-			0x4b: '.',
-			0x4c: '<',
-			0x4d: '(',
-			0x4e: '+',
-			0x4f: '|',
-			0x50: '&',
-			0x51: RuneError,
-			0x52: RuneError,
-			0x53: RuneError,
-			0x54: RuneError,
-			0x55: RuneError,
-			0x56: RuneError,
-			0x57: RuneError,
-			0x58: RuneError,
-			0x59: RuneError,
-			0x5a: '!',
-			0x5b: '$',
-			0x5c: '*',
-			0x5d: ')',
-			0x5e: ';',
-			0x5f: '¬',
-			0x60: '-',
-			0x61: '/',
-			0x62: RuneError,
-			0x63: RuneError,
-			0x64: RuneError,
-			0x65: RuneError,
-			0x66: RuneError,
-			0x67: RuneError,
-			0x68: RuneError,
-			0x69: RuneError,
-			0x6a: '¦',
-			0x6b: ',',
-			0x6c: '%',
-			0x6d: '_',
-			0x6e: '>',
-			0x6f: '?',
-			0x70: RuneError,
-			0x71: RuneError,
-			0x72: RuneError,
-			0x73: RuneError,
-			0x74: RuneError,
-			0x75: RuneError,
-			0x76: RuneError,
-			0x77: RuneError,
-			0x78: RuneError,
-			0x79: '`',
-			0x7a: ':',
-			0x7b: '#',
-			0x7c: '@',
-			0x7d: '\'',
-			0x7e: '=',
-			0x7f: '"',
-			0x80: RuneError,
-			0x81: 'a',
-			0x82: 'b',
-			0x83: 'c',
-			0x84: 'd',
-			0x85: 'e',
-			0x86: 'f',
-			0x87: 'g',
-			0x88: 'h',
-			0x89: 'i',
-			0x8a: RuneError,
-			0x8b: RuneError,
-			0x8c: RuneError,
-			0x8d: RuneError,
-			0x8e: RuneError,
-			0x8f: '±',
-			0x90: RuneError,
-			0x91: 'j',
-			0x92: 'k',
-			0x93: 'l',
-			0x94: 'm',
-			0x95: 'n',
-			0x96: 'o',
-			0x97: 'p',
-			0x98: 'q',
-			0x99: 'r',
-			0x9a: RuneError,
-			0x9b: RuneError,
-			0x9c: RuneError,
-			0x9d: RuneError,
-			0x9e: RuneError,
-			0x9f: RuneError,
-			0xa0: RuneError,
-			0xa1: '~',
-			0xa2: 's',
-			0xa3: 't',
-			0xa4: 'u',
-			0xa5: 'v',
-			0xa6: 'w',
-			0xa7: 'x',
-			0xa8: 'y',
-			0xa9: 'z',
-			0xaa: RuneError,
-			0xab: RuneError,
-			0xac: RuneError,
-			0xad: RuneError,
-			0xae: RuneError,
-			0xaf: RuneError,
-			0xb0: '^',
-			0xb1: RuneError,
-			0xb2: RuneError,
-			0xb3: RuneError,
-			0xb4: RuneError,
-			0xb5: RuneError,
-			0xb6: RuneError,
-			0xb7: RuneError,
-			0xb8: RuneError,
-			0xb9: RuneError,
-			0xba: '[',
-			0xbb: ']',
-			0xbc: RuneError,
-			0xbd: RuneError,
-			0xbe: RuneError,
-			0xbf: RuneError,
-			0xc0: '{',
-			0xc1: 'A',
-			0xc2: 'B',
-			0xc3: 'C',
-			0xc4: 'D',
-			0xc5: 'E',
-			0xc6: 'F',
-			0xc7: 'G',
-			0xc8: 'H',
-			0xc9: 'I',
-			0xca: '\xad', // NB: soft hyphen
-			0xcb: RuneError,
-			0xcc: RuneError,
-			0xcd: RuneError,
-			0xce: RuneError,
-			0xcf: RuneError,
-			0xd0: '}',
-			0xd1: 'J',
-			0xd2: 'K',
-			0xd3: 'L',
-			0xd4: 'M',
-			0xd5: 'N',
-			0xd6: 'O',
-			0xd7: 'P',
-			0xd8: 'Q',
-			0xd9: 'R',
-			0xda: RuneError,
-			0xdb: RuneError,
-			0xdc: RuneError,
-			0xdd: RuneError,
-			0xde: RuneError,
-			0xdf: RuneError,
-			0xe0: '\\',
-			0xe1: '\u2007', // Non-breaking space
-			0xe2: 'S',
-			0xe3: 'T',
-			0xe4: 'U',
-			0xe5: 'V',
-			0xe6: 'W',
-			0xe7: 'X',
-			0xe8: 'Y',
-			0xe9: 'Z',
-			0xea: RuneError,
-			0xeb: RuneError,
-			0xec: RuneError,
-			0xed: RuneError,
-			0xee: RuneError,
-			0xef: RuneError,
-			0xf0: '0',
-			0xf1: '1',
-			0xf2: '2',
-			0xf3: '3',
-			0xf4: '4',
-			0xf5: '5',
-			0xf6: '6',
-			0xf7: '7',
-			0xf8: '8',
-			0xf9: '9',
-			0xfa: RuneError,
-			0xfb: RuneError,
-			0xfc: RuneError,
-			0xfd: RuneError,
-			0xfe: RuneError,
-			0xff: RuneError,
-		}}
-	cm.Init()
-	EBCDIC = cm
-}

vendor/github.com/gdamore/encoding/latin1.go

@@ -1,33 +0,0 @@
-// Copyright 2015 Garrett D'Amore
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use file except in compliance with the License.
-// You may obtain a copy of the license at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package encoding
-
-import (
-	"golang.org/x/text/encoding"
-)
-
-// ISO8859_1 represents the 8-bit ISO8859-1 scheme.  It decodes directly to
-// UTF-8 without change, as all ISO8859-1 values are legal UTF-8.
-// Unicode values less than 256 (i.e. 8 bits) map 1:1 with 8859-1.
-// It encodes runes outside of that to 0x1A, the ASCII substitution character.
-var ISO8859_1 encoding.Encoding
-
-func init() {
-	cm := &Charmap{}
-	cm.Init()
-
-	// 8859-1 is the 8-bit identity map for Unicode.
-	ISO8859_1 = cm
-}

vendor/github.com/gdamore/encoding/latin5.go

@@ -1,35 +0,0 @@
-// Copyright 2015 Garrett D'Amore
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use file except in compliance with the License.
-// You may obtain a copy of the license at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package encoding
-
-import (
-	"golang.org/x/text/encoding"
-)
-
-// ISO8859_9 represents the 8-bit ISO8859-9 scheme.
-var ISO8859_9 encoding.Encoding
-
-func init() {
-	cm := &Charmap{Map: map[byte]rune{
-		0xD0: 'Ğ',
-		0xDD: 'İ',
-		0xDE: 'Ş',
-		0xF0: 'ğ',
-		0xFD: 'ı',
-		0xFE: 'ş',
-	}}
-	cm.Init()
-	ISO8859_9 = cm
-}

vendor/github.com/gdamore/encoding/utf8.go

@@ -1,35 +0,0 @@
-// Copyright 2015 Garrett D'Amore
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use file except in compliance with the License.
-// You may obtain a copy of the license at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package encoding
-
-import (
-	"golang.org/x/text/encoding"
-)
-
-type validUtf8 struct{}
-
-// UTF8 is an encoding for UTF-8.  All it does is verify that the UTF-8
-// in is valid.  The main reason for its existence is that it will detect
-// and report ErrSrcShort or ErrDstShort, whereas the Nop encoding just
-// passes every byte, blithely.
-var UTF8 encoding.Encoding = validUtf8{}
-
-func (validUtf8) NewDecoder() *encoding.Decoder {
-	return &encoding.Decoder{Transformer: encoding.UTF8Validator}
-}
-
-func (validUtf8) NewEncoder() *encoding.Encoder {
-	return &encoding.Encoder{Transformer: encoding.UTF8Validator}
-}

vendor/github.com/gdamore/tcell/.appveyor.yml

@@ -1,13 +0,0 @@
-version: 1.0.{build}
-clone_folder: c:\gopath\src\github.com\gdamore\tcell
-environment:
-  GOPATH: c:\gopath
-build_script:
-- go version
-- go env
-- SET PATH=%LOCALAPPDATA%\atom\bin;%GOPATH%\bin;%PATH%
-- go get -t ./...
-- go build
-- go install ./...
-test_script:
-- go test ./...

vendor/github.com/gdamore/tcell/.gitignore

@@ -1 +0,0 @@
-coverage.txt

vendor/github.com/gdamore/tcell/.travis.yml

@@ -1,15 +0,0 @@
-language: go
-
-go:
-  - 1.10.x
-  - 1.11.x
-  - master
-
-before_install:
-  - go get -t -v ./...
-
-script:
-  - go test -race -coverprofile=coverage.txt -covermode=atomic
-
-after_success:
-  - bash <(curl -s https://codecov.io/bash)

vendor/github.com/gdamore/tcell/AUTHORS

@@ -1,4 +0,0 @@
-Garrett D'Amore <garrett@damore.org>
-Zachary Yedidia <zyedidia@gmail.com>
-Junegunn Choi <junegunn.c@gmail.com>
-Staysail Systems, Inc. <info@staysail.tech>

vendor/github.com/gdamore/tcell/LICENSE

@@ -1,202 +0,0 @@
-
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.

vendor/github.com/gdamore/tcell/README.adoc

@@ -1,272 +0,0 @@
-= tcell
-
-
-image:https://img.shields.io/travis/gdamore/tcell.svg?label=linux[Linux Status,link="https://travis-ci.org/gdamore/tcell"]
-image:https://img.shields.io/appveyor/ci/gdamore/tcell.svg?label=windows[Windows Status,link="https://ci.appveyor.com/project/gdamore/tcell"]
-image:https://img.shields.io/badge/license-APACHE2-blue.svg[Apache License,link="https://github.com/gdamore/tcell/blob/master/LICENSE"]
-image:https://img.shields.io/badge/gitter-join-brightgreen.svg[Gitter,link="https://gitter.im/gdamore/tcell"]
-image:https://img.shields.io/badge/godoc-reference-blue.svg[GoDoc,link="https://godoc.org/github.com/gdamore/tcell"]
-image:http://goreportcard.com/badge/gdamore/tcell[Go Report Card,link="http://goreportcard.com/report/gdamore/tcell"]
-image:https://codecov.io/gh/gdamore/tcell/branch/master/graph/badge.svg[codecov,link="https://codecov.io/gh/gdamore/tcell"]
-image:https://tidelift.com/badges/github/gdamore/tcell?style=flat[Dependencies]
-
-[cols="2",grid="none"]
-|===
-|_Tcell_ is a _Go_ package that provides a cell based view for text terminals, like _xterm_.
-It was inspired by _termbox_, but includes many additional improvements.
-a|[.right]
-image::logos/tcell.png[float="right"]
-|===
-
-## Examples
-
-* https://github.com/gdamore/proxima5[proxima5] - space shooter (https://youtu.be/jNxKTCmY_bQ[video])
-* https://github.com/gdamore/govisor[govisor] - service management UI (http://2.bp.blogspot.com/--OsvnfzSNow/Vf7aqMw3zXI/AAAAAAAAARo/uOMtOvw4Sbg/s1600/Screen%2BShot%2B2015-09-20%2Bat%2B9.08.41%2BAM.png[screenshot])
-* mouse demo - included mouse test (http://2.bp.blogspot.com/-fWvW5opT0es/VhIdItdKqJI/AAAAAAAAATE/7Ojc0L1SpB0/s1600/Screen%2BShot%2B2015-10-04%2Bat%2B11.47.13%2BPM.png[screenshot])
-* https://github.com/gdamore/gomatrix[gomatrix] - converted from Termbox
-* https://github.com/zyedidia/micro/[micro] - lightweight text editor with syntax-highlighting and themes
-* https://github.com/viktomas/godu[godu] - simple golang utility helping to discover large files/folders.
-* https://github.com/rivo/tview[tview] - rich interactive widgets for terminal UIs
-* https://github.com/marcusolsson/tui-go[tui-go] - UI library for terminal apps
-* https://github.com/rgm3/gomandelbrot[gomandelbrot] - Mandelbrot!
-* https://github.com/senorprogrammer/wtf[WTF]- Personal information dashboard for your terminal
-* https://github.com/browsh-org/browsh[browsh] - A fully-modern text-based browser, rendering to TTY and browsers (https://www.youtube.com/watch?v=HZq86XfBoRo[video])
-* https://github.com/sachaos/go-life[go-life] - Conway's Game of Life.
-* https://github.com/gcla/gowid[gowid] - compositional widgets for terminal UIs, inspired by urwid
-* https://termshark.io[termshark] - a terminal UI for tshark, inspired by Wireshark, built on gowid
-
-## Pure Go Terminfo Database
-
-_Tcell_ includes a full parser and expander for terminfo capability strings,
-so that it can avoid hard coding escape strings for formatting.  It also favors
-portability, and includes support for all POSIX systems.
-
-The database is also flexible & extensible, and can modified by either running
-a program to build the entire database, or an entry for just a single terminal.
-
-## More Portable
-
-_Tcell_ is portable to a wide variety of systems.
-_Tcell_ is believed
-to work with all of the systems officially supported by golang with
-the exception of nacl (which lacks any kind of a terminal interface).
-(Plan9 is not supported by _Tcell_, but it is experimental status only
-in golang.)  For all of these systems *except Solaris/illumos*, _Tcell_
-is pure Go, with no need for CGO.
-
-## No Async IO
-
-_Tcell_ is able to operate without requiring `SIGIO` signals (unlike _termbox_),
-or asynchronous I/O, and can instead use standard Go file
-objects and Go routines.
-This means it should be safe, especially for
-use with programs that use exec, or otherwise need to manipulate the
-tty streams.
-This model is also much closer to idiomatic Go, leading
-to fewer surprises.
-
-## Rich Unicode & non-Unicode support
-
-_Tcell_ includes enhanced support for Unicode, including wide characters and
-combining characters, provided your terminal can support them.
-Note that
-Windows terminals generally don't support the full Unicode repertoire.
-
-It will also convert to and from Unicode locales, so that the program
-can work with UTF-8 internally, and get reasonable output in other locales.
-_Tcell_ tries hard to convert to native characters on both input and output, and
-on output _Tcell_ even makes use of the alternate character set to facilitate
-drawing certain characters.
-
-## More Function Keys
-
-_Tcell_ also has richer support for a larger number of special keys that some terminals can send.
-
-## Better Color Handling
-
-_Tcell_ will respect your terminal's color space as specified within your terminfo
-entries, so that for example attempts to emit color sequences on VT100 terminals
-won't result in unintended consequences.
-
-In Windows mode, _Tcell_ supports 16 colors, bold, dim, and reverse,
-instead of just termbox's 8 colors with reverse.  (Note that there is some
-conflation with bold/dim and colors.)
-
-_Tcell_ maps 16 colors down to 8, for terminals that need it.
-(The upper 8 colors are just brighter versions of the lower 8.)
-
-## Better Mouse Support
-
-_Tcell_ supports enhanced mouse tracking mode, so your application can receive
-regular mouse motion events, and wheel events, if your terminal supports it.
-
-## _Termbox_ Compatibility
-
-A compatibility layer for _termbox_ is provided in the `compat` directory.
-To use it, try importing `github.com/gdamore/tcell/termbox`
-instead.  Most _termbox-go_ programs will probably work without further
-modification.
-
-## Working With Unicode
-
-Internally Tcell uses UTF-8, just like Go.
-However, Tcell understands how to
-convert to and from other character sets, using the capabilities of
-the `golang.org/x/text/encoding packages`.
-Your application must supply
-them, as the full set of the most common ones bloats the program by about 2MB.
-If you're lazy, and want them all anyway, see the `encoding` sub-directory.
-
-## Wide & Combining Characters
-
-The `SetContent()` API takes a primary rune, and an optional list of combining runes.
-If any of the runes is a wide (East Asian) rune occupying two cells,
-then the library will skip output from the following cell, but care must be
-taken in the application to avoid explicitly attempting to set content in the
-next cell, otherwise the results are undefined.  (Normally wide character
-is displayed, and the other character is not; do not depend on that behavior.)
-
-Experience has shown that the vanilla Windows 8 console application does not
-support any of these characters properly, but at least some options like
-_ConEmu_ do support Wide characters.
-
-## Colors
-
-_Tcell_ assumes the ANSI/XTerm color model, including the 256 color map that
-XTerm uses when it supports 256 colors.  The terminfo guidance will be
-honored, with respect to the number of colors supported.  Also, only
-terminals which expose ANSI style `setaf` and `setab` will support color;
-if you have a color terminal that only has `setf` and `setb`, please let me
-know; it wouldn't be hard to add that if there is need.
-
-## 24-bit Color
-
-_Tcell_ _supports true color_!  (That is, if your terminal can support it,
-_Tcell_ can accurately display 24-bit color.)
-
-To use 24-bit color, you need to use a terminal that supports it.  Modern
-xterm and similar teminal emulators can support this.  As terminfo lacks any
-way to describe this capability, we fabricate the capability for
-terminals with names ending in `*-truecolor`.  The stock distribution ships
-with a database that defines `xterm-truecolor`.
-To try it out, set your
-`TERM` variable to `xterm-truecolor`.
-
-When using TrueColor, programs will display the colors that the programmer
-intended, overriding any "`themes`" you may have set in your terminal
-emulator.  (For some cases, accurate color fidelity is more important
-than respecting themes.  For other cases, such as typical text apps that
-only use a few colors, its more desirable to respect the themes that
-the user has established.)
-
-If you find this undesirable, you can either use a `TERM` variable
-that lacks the `TRUECOLOR` setting, or set `TCELL_TRUECOLOR=disable` in your
-environment.
-
-## Performance
-
-Reasonable attempts have been made to minimize sending data to terminals,
-avoiding repeated sequences or drawing the same cell on refresh updates.
-
-## Terminfo
-
-(Not relevent for Windows users.)
-
-The Terminfo implementation operates with two forms of database.  The first
-is the built-in go database, which contains a number of real database entries
-that are compiled into the program directly.  This should minimize calling
-out to database file searches.
-
-The second is in the form of JSON files, that contain the same information,
-which can be located either by the `$TCELLDB` environment file, `$HOME/.tcelldb`,
-or is located in the Go source directory as `database.json`.
-
-These files (both the Go and the JSON files) can be generated using the
-mkinfo.go program.  If you need to regnerate the entire set for some reason,
-run the mkdatabase.sh file.  The generation uses the infocmp(1) program on
-the system to collect the necessary information.
-
-The `mkinfo.go` program can also be used to generate specific database entries
-for named terminals, in case your favorite terminal is missing.  (If you
-find that this is the case, please let me know and I'll try to add it!)
-
-_Tcell_ requires that the terminal support the `cup` mode of cursor addressing.
-Terminals without absolute cursor addressability are not supported.
-This is unlikely to be a problem; such terminals have not been mass produced
-since the early 1970s.
-
-## Mouse Support
-
-Mouse support is detected via the `kmous` terminfo variable, however,
-enablement/disablement and decoding mouse events is done using hard coded
-sequences based on the XTerm X11 model.  As of this writing all popular
-terminals with mouse tracking support this model.  (Full terminfo support
-is not possible as terminfo sequences are not defined.)
-
-On Windows, the mouse works normally.
-
-Mouse wheel buttons on various terminals are known to work, but the support
-in terminal emulators, as well as support for various buttons and
-live mouse tracking, varies widely.  Modern _xterm_, macOS _Terminal_, and _iTerm_ all work well.
-
-## Testablity
-
-There is a `SimulationScreen`, that can be used to simulate a real screen
-for automated testing.  The supplied tests do this.  The simulation contains
-event delivery, screen resizing support, and capabilities to inject events
-and examine "`physical`" screen contents.
-
-## Platforms
-
-### POSIX (Linux, FreeBSD, macOS, Solaris, etc.)
-
-For mainstream systems with a suitably well defined system call interface
-to tty settings, everything works using pure Go.
-
-For the remainder (right now means only Solaris/illumos) we use POSIX function
-calls to manage termios, which implies that CGO is required on those platforms.
-
-### Windows
-
-Windows console mode applications are supported.  Unfortunately _mintty_
-and other _cygwin_ style applications are not supported.
-
-Modern console applications like ConEmu, as well as the Windows 10
-console itself, support all the good features (resize, mouse tracking, etc.)
-
-I haven't figured out how to cleanly resolve the dichotomy between cygwin
-style termios and the Windows Console API; it seems that perhaps nobody else
-has either.  If anyone has suggestions, let me know!  Really, if you're
-using a Windows application, you should use the native Windows console or a
-fully compatible console implementation.
-
-### Plan9 and Native Client (Nacl)
-
-The nacl and plan9 platforms won't work, but compilation stubs are supplied
-for folks that want to include parts of this in software targetting those
-platforms.  The Simulation screen works, but as Tcell doesn't know how to
-allocate a real screen object on those platforms, `NewScreen()` will fail.
-
-If anyone has wisdom about how to improve support for either of these,
-please let me know.  PRs are especially welcome.
-
-### Commercial Support
-
-_Tcell_ is absolutely free, but if you want to obtain commercial, professional support, there are options.
-
-[cols="2",align="center",frame="none", grid="none"]
-|===
-^.^|
-image:logos/tidelift.png[100,100]
-a|
-https://tidelift.com/[Tidelift] subscriptions include support for _Tcell_, as well as many other open source packages.
-
-^.^|
-image:logos/staysail.png[100,100]
-a|
-mailto:info@staysail.tech[Staysail Systems, Inc.] offers direct support, and custom development around _Tcell_ on an hourly basis.
-
-^.^|
-image:logos/patreon.png[100,100]
-a|I also welcome donations at https://www.patreon.com/gedamore/[Patreon], if you just want to make a contribution.
-|===

vendor/github.com/gdamore/tcell/attr.go

@@ -1,32 +0,0 @@
-// Copyright 2015 The TCell Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use file except in compliance with the License.
-// You may obtain a copy of the license at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package tcell
-
-// AttrMask represents a mask of text attributes, apart from color.
-// Note that support for attributes may vary widely across terminals.
-type AttrMask int
-
-// Attributes are not colors, but affect the display of text.  They can
-// be combined.
-const (
-	AttrBold AttrMask = 1 << (25 + iota)
-	AttrBlink
-	AttrReverse
-	AttrUnderline
-	AttrDim
-	AttrNone AttrMask = 0 // Just normal text.
-)
-
-const attrAll = AttrBold | AttrBlink | AttrReverse | AttrUnderline | AttrDim

vendor/github.com/gdamore/tcell/cell.go

@@ -1,177 +0,0 @@
-// Copyright 2019 The TCell Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use file except in compliance with the License.
-// You may obtain a copy of the license at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package tcell
-
-import (
-	runewidth "github.com/mattn/go-runewidth"
-)
-
-type cell struct {
-	currMain  rune
-	currComb  []rune
-	currStyle Style
-	lastMain  rune
-	lastStyle Style
-	lastComb  []rune
-	width     int
-}
-
-// CellBuffer represents a two dimensional array of character cells.
-// This is primarily intended for use by Screen implementors; it
-// contains much of the common code they need.  To create one, just
-// declare a variable of its type; no explicit initialization is necessary.
-//
-// CellBuffer is not thread safe.
-type CellBuffer struct {
-	w     int
-	h     int
-	cells []cell
-}
-
-// SetContent sets the contents (primary rune, combining runes,
-// and style) for a cell at a given location.
-func (cb *CellBuffer) SetContent(x int, y int,
-	mainc rune, combc []rune, style Style) {
-
-	if x >= 0 && y >= 0 && x < cb.w && y < cb.h {
-		c := &cb.cells[(y*cb.w)+x]
-
-		c.currComb = append([]rune{}, combc...)
-
-		if c.currMain != mainc {
-			c.width = runewidth.RuneWidth(mainc)
-		}
-		c.currMain = mainc
-		c.currStyle = style
-	}
-}
-
-// GetContent returns the contents of a character cell, including the
-// primary rune, any combining character runes (which will usually be
-// nil), the style, and the display width in cells.  (The width can be
-// either 1, normally, or 2 for East Asian full-width characters.)
-func (cb *CellBuffer) GetContent(x, y int) (rune, []rune, Style, int) {
-	var mainc rune
-	var combc []rune
-	var style Style
-	var width int
-	if x >= 0 && y >= 0 && x < cb.w && y < cb.h {
-		c := &cb.cells[(y*cb.w)+x]
-		mainc, combc, style = c.currMain, c.currComb, c.currStyle
-		if width = c.width; width == 0 || mainc < ' ' {
-			width = 1
-			mainc = ' '
-		}
-	}
-	return mainc, combc, style, width
-}
-
-// Size returns the (width, height) in cells of the buffer.
-func (cb *CellBuffer) Size() (int, int) {
-	return cb.w, cb.h
-}
-
-// Invalidate marks all characters within the buffer as dirty.
-func (cb *CellBuffer) Invalidate() {
-	for i := range cb.cells {
-		cb.cells[i].lastMain = rune(0)
-	}
-}
-
-// Dirty checks if a character at the given location needs an
-// to be refreshed on the physical display.  This returns true
-// if the cell content is different since the last time it was
-// marked clean.
-func (cb *CellBuffer) Dirty(x, y int) bool {
-	if x >= 0 && y >= 0 && x < cb.w && y < cb.h {
-		c := &cb.cells[(y*cb.w)+x]
-		if c.lastMain == rune(0) {
-			return true
-		}
-		if c.lastMain != c.currMain {
-			return true
-		}
-		if c.lastStyle != c.currStyle {
-			return true
-		}
-		if len(c.lastComb) != len(c.currComb) {
-			return true
-		}
-		for i := range c.lastComb {
-			if c.lastComb[i] != c.currComb[i] {
-				return true
-			}
-		}
-	}
-	return false
-}
-
-// SetDirty is normally used to indicate that a cell has
-// been displayed (in which case dirty is false), or to manually
-// force a cell to be marked dirty.
-func (cb *CellBuffer) SetDirty(x, y int, dirty bool) {
-	if x >= 0 && y >= 0 && x < cb.w && y < cb.h {
-		c := &cb.cells[(y*cb.w)+x]
-		if dirty {
-			c.lastMain = rune(0)
-		} else {
-			if c.currMain == rune(0) {
-				c.currMain = ' '
-			}
-			c.lastMain = c.currMain
-			c.lastComb = c.currComb
-			c.lastStyle = c.currStyle
-		}
-	}
-}
-
-// Resize is used to resize the cells array, with different dimensions,
-// while preserving the original contents.  The cells will be invalidated
-// so that they can be redrawn.
-func (cb *CellBuffer) Resize(w, h int) {
-
-	if cb.h == h && cb.w == w {
-		return
-	}
-
-	newc := make([]cell, w*h)
-	for y := 0; y < h && y < cb.h; y++ {
-		for x := 0; x < w && x < cb.w; x++ {
-			oc := &cb.cells[(y*cb.w)+x]
-			nc := &newc[(y*w)+x]
-			nc.currMain = oc.currMain
-			nc.currComb = oc.currComb
-			nc.currStyle = oc.currStyle
-			nc.width = oc.width
-			nc.lastMain = rune(0)
-		}
-	}
-	cb.cells = newc
-	cb.h = h
-	cb.w = w
-}
-
-// Fill fills the entire cell buffer array with the specified character
-// and style.  Normally choose ' ' to clear the screen.  This API doesn't
-// support combining characters, or characters with a width larger than one.
-func (cb *CellBuffer) Fill(r rune, style Style) {
-	for i := range cb.cells {
-		c := &cb.cells[i]
-		c.currMain = r
-		c.currComb = nil
-		c.currStyle = style
-		c.width = 1
-	}
-}

vendor/github.com/gdamore/tcell/charset_stub.go

@@ -1,21 +0,0 @@
-// +build plan9 nacl
-
-// Copyright 2015 The TCell Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use file except in compliance with the License.
-// You may obtain a copy of the license at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package tcell
-
-func getCharset() string {
-	return ""
-}

vendor/github.com/gdamore/tcell/charset_unix.go

@@ -1,49 +0,0 @@
-// +build !windows,!nacl,!plan9
-
-// Copyright 2016 The TCell Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use file except in compliance with the License.
-// You may obtain a copy of the license at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package tcell
-
-import (
-	"os"
-	"strings"
-)
-
-func getCharset() string {
-	// Determine the character set.  This can help us later.
-	// Per POSIX, we search for LC_ALL first, then LC_CTYPE, and
-	// finally LANG.  First one set wins.
-	locale := ""
-	if locale = os.Getenv("LC_ALL"); locale == "" {
-		if locale = os.Getenv("LC_CTYPE"); locale == "" {
-			locale = os.Getenv("LANG")
-		}
-	}
-	if locale == "POSIX" || locale == "C" {
-		return "US-ASCII"
-	}
-	if i := strings.IndexRune(locale, '@'); i >= 0 {
-		locale = locale[:i]
-	}
-	if i := strings.IndexRune(locale, '.'); i >= 0 {
-		locale = locale[i+1:]
-	} else {
-		// Default assumption, and on Linux we can see LC_ALL
-		// without a character set, which we assume implies UTF-8.
-		return "UTF-8"
-	}
-	// XXX: add support for aliases
-	return locale
-}

vendor/github.com/gdamore/tcell/charset_windows.go

@@ -1,21 +0,0 @@
-// +build windows
-
-// Copyright 2015 The TCell Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use file except in compliance with the License.
-// You may obtain a copy of the license at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package tcell
-
-func getCharset() string {
-	return "UTF-16"
-}

vendor/github.com/gdamore/tcell/colorfit.go

@@ -1,52 +0,0 @@
-// Copyright 2016 The TCell Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use file except in compliance with the License.
-// You may obtain a copy of the license at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package tcell
-
-import (
-	"github.com/lucasb-eyer/go-colorful"
-	"math"
-)
-
-// FindColor attempts to find a given color, or the best match possible for it,
-// from the palette given.  This is an expensive operation, so results should
-// be cached by the caller.
-func FindColor(c Color, palette []Color) Color {
-	match := ColorDefault
-	dist := float64(0)
-	r, g, b := c.RGB()
-	c1 := colorful.Color{
-		R: float64(r) / 255.0,
-		G: float64(g) / 255.0,
-		B: float64(b) / 255.0,
-	}
-	for _, d := range palette {
-		r, g, b = d.RGB()
-		c2 := colorful.Color{
-			R: float64(r) / 255.0,
-			G: float64(g) / 255.0,
-			B: float64(b) / 255.0,
-		}
-		// CIE94 is more accurate, but really really expensive.
-		nd := c1.DistanceCIE76(c2)
-		if math.IsNaN(nd) {
-			nd = math.Inf(1)
-		}
-		if match == ColorDefault || nd < dist {
-			match = d
-			dist = nd
-		}
-	}
-	return match
-}

vendor/github.com/gdamore/tcell/console_stub.go

@@ -1,23 +0,0 @@
-// +build !windows
-
-// Copyright 2015 The TCell Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use file except in compliance with the License.
-// You may obtain a copy of the license at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package tcell
-
-// NewConsoleScreen returns a console based screen.  This platform
-// doesn't have support for any, so it returns nil and a suitable error.
-func NewConsoleScreen() (Screen, error) {
-	return nil, ErrNoScreen
-}

vendor/github.com/gdamore/tcell/doc.go

@@ -1,48 +0,0 @@
-// Copyright 2018 The TCell Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use file except in compliance with the License.
-// You may obtain a copy of the license at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// Package tcell provides a lower-level, portable API for building
-// programs that interact with terminals or consoles.  It works with
-// both common (and many uncommon!) terminals or terminal emulators,
-// and Windows console implementations.
-//
-// It provides support for up to 256 colors, text attributes, and box drawing
-// elements.  A database of terminals built from a real terminfo database
-// is provided, along with code to generate new database entries.
-//
-// Tcell offers very rich support for mice, dependent upon the terminal
-// of course.  (Windows, XTerm, and iTerm 2 are known to work very well.)
-//
-// If the environment is not Unicode by default, such as an ISO8859 based
-// locale or GB18030, Tcell can convert input and output, so that your
-// terminal can operate in whatever locale is most convenient, while the
-// application program can just assume "everything is UTF-8".  Reasonable
-// defaults are used for updating characters to something suitable for
-// display.  Unicode box drawing characters will be converted to use the
-// alternate character set of your terminal, if native conversions are
-// not available.  If no ACS is available, then some ASCII fallbacks will
-// be used.
-//
-// Note that support for non-UTF-8 locales (other than C)  must be enabled
-// by the application using RegisterEncoding() -- we don't have them all
-// enabled by default to avoid bloating the application unneccessarily.
-// (These days UTF-8 is good enough for almost everyone, and nobody should
-// be using legacy locales anymore.)  Also, actual glyphs for various code
-// point will only be displayed if your terminal or emulator (or the font
-// the emulator is using) supports them.
-//
-// A rich set of keycodes is supported, with support for up to 65 function
-// keys, and various other special keys.
-//
-package tcell

vendor/github.com/gdamore/tcell/encoding.go

@@ -1,139 +0,0 @@
-// Copyright 2015 The TCell Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use file except in compliance with the License.
-// You may obtain a copy of the license at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package tcell
-
-import (
-	"strings"
-	"sync"
-
-	"golang.org/x/text/encoding"
-
-	gencoding "github.com/gdamore/encoding"
-)
-
-var encodings map[string]encoding.Encoding
-var encodingLk sync.Mutex
-var encodingFallback EncodingFallback = EncodingFallbackFail
-
-// RegisterEncoding may be called by the application to register an encoding.
-// The presence of additional encodings will facilitate application usage with
-// terminal environments where the I/O subsystem does not support Unicode.
-//
-// Windows systems use Unicode natively, and do not need any of the encoding
-// subsystem when using Windows Console screens.
-//
-// Please see the Go documentation for golang.org/x/text/encoding -- most of
-// the common ones exist already as stock variables.  For example, ISO8859-15
-// can be registered using the following code:
-//
-//   import "golang.org/x/text/encoding/charmap"
-//
-//     ...
-//     RegisterEncoding("ISO8859-15", charmap.ISO8859_15)
-//
-// Aliases can be registered as well, for example "8859-15" could be an alias
-// for "ISO8859-15".
-//
-// For POSIX systems, the tcell package will check the environment variables
-// LC_ALL, LC_CTYPE,  and LANG (in that order) to determine the character set.
-// These are expected to have the following pattern:
-//
-//	 $language[.$codeset[@$variant]
-//
-// We extract only the $codeset part, which will usually be something like
-// UTF-8 or ISO8859-15 or KOI8-R.  Note that if the locale is either "POSIX"
-// or "C", then we assume US-ASCII (the POSIX 'portable character set'
-// and assume all other characters are somehow invalid.)
-//
-// Modern POSIX systems and terminal emulators may use UTF-8, and for those
-// systems, this API is also unnecessary.  For example, Darwin (MacOS X) and
-// modern Linux running modern xterm generally will out of the box without
-// any of this.  Use of UTF-8 is recommended when possible, as it saves
-// quite a lot processing overhead.
-//
-// Note that some encodings are quite large (for example GB18030 which is a
-// superset of Unicode) and so the application size can be expected ot
-// increase quite a bit as each encoding is added.  The East Asian encodings
-// have been seen to add 100-200K per encoding to the application size.
-//
-func RegisterEncoding(charset string, enc encoding.Encoding) {
-	encodingLk.Lock()
-	charset = strings.ToLower(charset)
-	encodings[charset] = enc
-	encodingLk.Unlock()
-}
-
-// EncodingFallback describes how the system behavees when the locale
-// requires a character set that we do not support.  The system always
-// supports UTF-8 and US-ASCII. On Windows consoles, UTF-16LE is also
-// supported automatically.  Other character sets must be added using the
-// RegisterEncoding API.  (A large group of nearly all of them can be
-// added using the RegisterAll function in the encoding sub package.)
-type EncodingFallback int
-
-const (
-	// EncodingFallbackFail behavior causes GetEncoding to fail
-	// when it cannot find an encoding.
-	EncodingFallbackFail = iota
-
-	// EncodingFallbackASCII behaviore causes GetEncoding to fall back
-	// to a 7-bit ASCII encoding, if no other encoding can be found.
-	EncodingFallbackASCII
-
-	// EncodingFallbackUTF8 behavior causes GetEncoding to assume
-	// UTF8 can pass unmodified upon failure.  Note that this behavior
-	// is not recommended, unless you are sure your terminal can cope
-	// with real UTF8 sequences.
-	EncodingFallbackUTF8
-)
-
-// SetEncodingFallback changes the behavior of GetEncoding when a suitable
-// encoding is not found.  The default is EncodingFallbackFail, which
-// causes GetEncoding to simply return nil.
-func SetEncodingFallback(fb EncodingFallback) {
-	encodingLk.Lock()
-	encodingFallback = fb
-	encodingLk.Unlock()
-}
-
-// GetEncoding is used by Screen implementors who want to locate an encoding
-// for the given character set name.  Note that this will return nil for
-// either the Unicode (UTF-8) or ASCII encodings, since we don't use
-// encodings for them but instead have our own native methods.
-func GetEncoding(charset string) encoding.Encoding {
-	charset = strings.ToLower(charset)
-	encodingLk.Lock()
-	defer encodingLk.Unlock()
-	if enc, ok := encodings[charset]; ok {
-		return enc
-	}
-	switch encodingFallback {
-	case EncodingFallbackASCII:
-		return gencoding.ASCII
-	case EncodingFallbackUTF8:
-		return encoding.Nop
-	}
-	return nil
-}
-
-func init() {
-	// We always support UTF-8 and ASCII.
-	encodings = make(map[string]encoding.Encoding)
-	encodings["utf-8"] = gencoding.UTF8
-	encodings["utf8"] = gencoding.UTF8
-	encodings["us-ascii"] = gencoding.ASCII
-	encodings["ascii"] = gencoding.ASCII
-	encodings["iso646"] = gencoding.ASCII
-}

vendor/github.com/gdamore/tcell/errors.go

@@ -1,73 +0,0 @@
-// Copyright 2015 The TCell Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use file except in compliance with the License.
-// You may obtain a copy of the license at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package tcell
-
-import (
-	"errors"
-	"time"
-
-	"github.com/gdamore/tcell/terminfo"
-)
-
-var (
-	// ErrTermNotFound indicates that a suitable terminal entry could
-	// not be found.  This can result from either not having TERM set,
-	// or from the TERM failing to support certain minimal functionality,
-	// in particular absolute cursor addressability (the cup capability)
-	// is required.  For example, legacy "adm3" lacks this capability,
-	// whereas the slightly newer "adm3a" supports it.  This failure
-	// occurs most often with "dumb".
-	ErrTermNotFound = terminfo.ErrTermNotFound
-
-	// ErrNoScreen indicates that no suitable screen could be found.
-	// This may result from attempting to run on a platform where there
-	// is no support for either termios or console I/O (such as nacl),
-	// or from running in an environment where there is no access to
-	// a suitable console/terminal device.  (For example, running on
-	// without a controlling TTY or with no /dev/tty on POSIX platforms.)
-	ErrNoScreen = errors.New("no suitable screen available")
-
-	// ErrNoCharset indicates that the locale environment the
-	// program is not supported by the program, because no suitable
-	// encoding was found for it.  This problem never occurs if
-	// the environment is UTF-8 or UTF-16.
-	ErrNoCharset = errors.New("character set not supported")
-
-	// ErrEventQFull indicates that the event queue is full, and
-	// cannot accept more events.
-	ErrEventQFull = errors.New("event queue full")
-)
-
-// An EventError is an event representing some sort of error, and carries
-// an error payload.
-type EventError struct {
-	t   time.Time
-	err error
-}
-
-// When returns the time when the event was created.
-func (ev *EventError) When() time.Time {
-	return ev.t
-}
-
-// Error implements the error.
-func (ev *EventError) Error() string {
-	return ev.err.Error()
-}
-
-// NewEventError creates an ErrorEvent with the given error payload.
-func NewEventError(err error) *EventError {
-	return &EventError{t: time.Now(), err: err}
-}

vendor/github.com/gdamore/tcell/event.go

@@ -1,53 +0,0 @@
-// Copyright 2015 The TCell Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use file except in compliance with the License.
-// You may obtain a copy of the license at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package tcell
-
-import (
-	"time"
-)
-
-// Event is a generic interface used for passing around Events.
-// Concrete types follow.
-type Event interface {
-	// When reports the time when the event was generated.
-	When() time.Time
-}
-
-// EventTime is a simple base event class, suitable for easy reuse.
-// It can be used to deliver actual timer events as well.
-type EventTime struct {
-	when time.Time
-}
-
-// When returns the time stamp when the event occurred.
-func (e *EventTime) When() time.Time {
-	return e.when
-}
-
-// SetEventTime sets the time of occurrence for the event.
-func (e *EventTime) SetEventTime(t time.Time) {
-	e.when = t
-}
-
-// SetEventNow sets the time of occurrence for the event to the current time.
-func (e *EventTime) SetEventNow() {
-	e.SetEventTime(time.Now())
-}
-
-// EventHandler is anything that handles events.  If the handler has
-// consumed the event, it should return true.  False otherwise.
-type EventHandler interface {
-	HandleEvent(Event) bool
-}

vendor/github.com/gdamore/tcell/interrupt.go

@@ -1,41 +0,0 @@
-// Copyright 2015 The TCell Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use file except in compliance with the License.
-// You may obtain a copy of the license at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package tcell
-
-import (
-	"time"
-)
-
-// EventInterrupt is a generic wakeup event.  Its can be used to
-// to request a redraw.  It can carry an arbitrary payload, as well.
-type EventInterrupt struct {
-	t time.Time
-	v interface{}
-}
-
-// When returns the time when this event was created.
-func (ev *EventInterrupt) When() time.Time {
-	return ev.t
-}
-
-// Data is used to obtain the opaque event payload.
-func (ev *EventInterrupt) Data() interface{} {
-	return ev.v
-}
-
-// NewEventInterrupt creates an EventInterrupt with the given payload.
-func NewEventInterrupt(data interface{}) *EventInterrupt {
-	return &EventInterrupt{t: time.Now(), v: data}
-}

vendor/github.com/gdamore/tcell/key.go

@@ -1,464 +0,0 @@
-// Copyright 2016 The TCell Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use file except in compliance with the License.
-// You may obtain a copy of the license at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package tcell
-
-import (
-	"fmt"
-	"strings"
-	"time"
-)
-
-// EventKey represents a key press.  Usually this is a key press followed
-// by a key release, but since terminal programs don't have a way to report
-// key release events, we usually get just one event.  If a key is held down
-// then the terminal may synthesize repeated key presses at some predefined
-// rate.  We have no control over that, nor visibility into it.
-//
-// In some cases, we can have a modifier key, such as ModAlt, that can be
-// generated with a key press.  (This usually is represented by having the
-// high bit set, or in some cases, by sending an ESC prior to the rune.)
-//
-// If the value of Key() is KeyRune, then the actual key value will be
-// available with the Rune() method.  This will be the case for most keys.
-// In most situations, the modifiers will not be set.  For example, if the
-// rune is 'A', this will be reported without the ModShift bit set, since
-// really can't tell if the Shift key was pressed (it might have been CAPSLOCK,
-// or a terminal that only can send capitals, or keyboard with separate
-// capital letters from lower case letters).
-//
-// Generally, terminal applications have far less visibility into keyboard
-// activity than graphical applications.  Hence, they should avoid depending
-// overly much on availability of modifiers, or the availability of any
-// specific keys.
-type EventKey struct {
-	t   time.Time
-	mod ModMask
-	key Key
-	ch  rune
-}
-
-// When returns the time when this Event was created, which should closely
-// match the time when the key was pressed.
-func (ev *EventKey) When() time.Time {
-	return ev.t
-}
-
-// Rune returns the rune corresponding to the key press, if it makes sense.
-// The result is only defined if the value of Key() is KeyRune.
-func (ev *EventKey) Rune() rune {
-	return ev.ch
-}
-
-// Key returns a virtual key code.  We use this to identify specific key
-// codes, such as KeyEnter, etc.  Most control and function keys are reported
-// with unique Key values.  Normal alphanumeric and punctuation keys will
-// generally return KeyRune here; the specific key can be further decoded
-// using the Rune() function.
-func (ev *EventKey) Key() Key {
-	return ev.key
-}
-
-// Modifiers returns the modifiers that were present with the key press.  Note
-// that not all platforms and terminals support this equally well, and some
-// cases we will not not know for sure.  Hence, applications should avoid
-// using this in most circumstances.
-func (ev *EventKey) Modifiers() ModMask {
-	return ev.mod
-}
-
-// KeyNames holds the written names of special keys. Useful to echo back a key
-// name, or to look up a key from a string value.
-var KeyNames = map[Key]string{
-	KeyEnter:          "Enter",
-	KeyBackspace:      "Backspace",
-	KeyTab:            "Tab",
-	KeyBacktab:        "Backtab",
-	KeyEsc:            "Esc",
-	KeyBackspace2:     "Backspace2",
-	KeyDelete:         "Delete",
-	KeyInsert:         "Insert",
-	KeyUp:             "Up",
-	KeyDown:           "Down",
-	KeyLeft:           "Left",
-	KeyRight:          "Right",
-	KeyHome:           "Home",
-	KeyEnd:            "End",
-	KeyUpLeft:         "UpLeft",
-	KeyUpRight:        "UpRight",
-	KeyDownLeft:       "DownLeft",
-	KeyDownRight:      "DownRight",
-	KeyCenter:         "Center",
-	KeyPgDn:           "PgDn",
-	KeyPgUp:           "PgUp",
-	KeyClear:          "Clear",
-	KeyExit:           "Exit",
-	KeyCancel:         "Cancel",
-	KeyPause:          "Pause",
-	KeyPrint:          "Print",
-	KeyF1:             "F1",
-	KeyF2:             "F2",
-	KeyF3:             "F3",
-	KeyF4:             "F4",
-	KeyF5:             "F5",
-	KeyF6:             "F6",
-	KeyF7:             "F7",
-	KeyF8:             "F8",
-	KeyF9:             "F9",
-	KeyF10:            "F10",
-	KeyF11:            "F11",
-	KeyF12:            "F12",
-	KeyF13:            "F13",
-	KeyF14:            "F14",
-	KeyF15:            "F15",
-	KeyF16:            "F16",
-	KeyF17:            "F17",
-	KeyF18:            "F18",
-	KeyF19:            "F19",
-	KeyF20:            "F20",
-	KeyF21:            "F21",
-	KeyF22:            "F22",
-	KeyF23:            "F23",
-	KeyF24:            "F24",
-	KeyF25:            "F25",
-	KeyF26:            "F26",
-	KeyF27:            "F27",
-	KeyF28:            "F28",
-	KeyF29:            "F29",
-	KeyF30:            "F30",
-	KeyF31:            "F31",
-	KeyF32:            "F32",
-	KeyF33:            "F33",
-	KeyF34:            "F34",
-	KeyF35:            "F35",
-	KeyF36:            "F36",
-	KeyF37:            "F37",
-	KeyF38:            "F38",
-	KeyF39:            "F39",
-	KeyF40:            "F40",
-	KeyF41:            "F41",
-	KeyF42:            "F42",
-	KeyF43:            "F43",
-	KeyF44:            "F44",
-	KeyF45:            "F45",
-	KeyF46:            "F46",
-	KeyF47:            "F47",
-	KeyF48:            "F48",
-	KeyF49:            "F49",
-	KeyF50:            "F50",
-	KeyF51:            "F51",
-	KeyF52:            "F52",
-	KeyF53:            "F53",
-	KeyF54:            "F54",
-	KeyF55:            "F55",
-	KeyF56:            "F56",
-	KeyF57:            "F57",
-	KeyF58:            "F58",
-	KeyF59:            "F59",
-	KeyF60:            "F60",
-	KeyF61:            "F61",
-	KeyF62:            "F62",
-	KeyF63:            "F63",
-	KeyF64:            "F64",
-	KeyCtrlA:          "Ctrl-A",
-	KeyCtrlB:          "Ctrl-B",
-	KeyCtrlC:          "Ctrl-C",
-	KeyCtrlD:          "Ctrl-D",
-	KeyCtrlE:          "Ctrl-E",
-	KeyCtrlF:          "Ctrl-F",
-	KeyCtrlG:          "Ctrl-G",
-	KeyCtrlJ:          "Ctrl-J",
-	KeyCtrlK:          "Ctrl-K",
-	KeyCtrlL:          "Ctrl-L",
-	KeyCtrlN:          "Ctrl-N",
-	KeyCtrlO:          "Ctrl-O",
-	KeyCtrlP:          "Ctrl-P",
-	KeyCtrlQ:          "Ctrl-Q",
-	KeyCtrlR:          "Ctrl-R",
-	KeyCtrlS:          "Ctrl-S",
-	KeyCtrlT:          "Ctrl-T",
-	KeyCtrlU:          "Ctrl-U",
-	KeyCtrlV:          "Ctrl-V",
-	KeyCtrlW:          "Ctrl-W",
-	KeyCtrlX:          "Ctrl-X",
-	KeyCtrlY:          "Ctrl-Y",
-	KeyCtrlZ:          "Ctrl-Z",
-	KeyCtrlSpace:      "Ctrl-Space",
-	KeyCtrlUnderscore: "Ctrl-_",
-	KeyCtrlRightSq:    "Ctrl-]",
-	KeyCtrlBackslash:  "Ctrl-\\",
-	KeyCtrlCarat:      "Ctrl-^",
-}
-
-// Name returns a printable value or the key stroke.  This can be used
-// when printing the event, for example.
-func (ev *EventKey) Name() string {
-	s := ""
-	m := []string{}
-	if ev.mod&ModShift != 0 {
-		m = append(m, "Shift")
-	}
-	if ev.mod&ModAlt != 0 {
-		m = append(m, "Alt")
-	}
-	if ev.mod&ModMeta != 0 {
-		m = append(m, "Meta")
-	}
-	if ev.mod&ModCtrl != 0 {
-		m = append(m, "Ctrl")
-	}
-
-	ok := false
-	if s, ok = KeyNames[ev.key]; !ok {
-		if ev.key == KeyRune {
-			s = "Rune[" + string(ev.ch) + "]"
-		} else {
-			s = fmt.Sprintf("Key[%d,%d]", ev.key, int(ev.ch))
-		}
-	}
-	if len(m) != 0 {
-		if ev.mod&ModCtrl != 0 && strings.HasPrefix(s, "Ctrl-") {
-			s = s[5:]
-		}
-		return fmt.Sprintf("%s+%s", strings.Join(m, "+"), s)
-	}
-	return s
-}
-
-// NewEventKey attempts to create a suitable event.  It parses the various
-// ASCII control sequences if KeyRune is passed for Key, but if the caller
-// has more precise information it should set that specifically.  Callers
-// that aren't sure about modifier state (most) should just pass ModNone.
-func NewEventKey(k Key, ch rune, mod ModMask) *EventKey {
-	if k == KeyRune && (ch < ' ' || ch == 0x7f) {
-		// Turn specials into proper key codes.  This is for
-		// control characters and the DEL.
-		k = Key(ch)
-		if mod == ModNone && ch < ' ' {
-			switch Key(ch) {
-			case KeyBackspace, KeyTab, KeyEsc, KeyEnter:
-				// these keys are directly typeable without CTRL
-			default:
-				// most likely entered with a CTRL keypress
-				mod = ModCtrl
-			}
-		}
-	}
-	return &EventKey{t: time.Now(), key: k, ch: ch, mod: mod}
-}
-
-// ModMask is a mask of modifier keys.  Note that it will not always be
-// possible to report modifier keys.
-type ModMask int16
-
-// These are the modifiers keys that can be sent either with a key press,
-// or a mouse event.  Note that as of now, due to the confusion associated
-// with Meta, and the lack of support for it on many/most platforms, the
-// current implementations never use it.  Instead, they use ModAlt, even for
-// events that could possibly have been distinguished from ModAlt.
-const (
-	ModShift ModMask = 1 << iota
-	ModCtrl
-	ModAlt
-	ModMeta
-	ModNone ModMask = 0
-)
-
-// Key is a generic value for representing keys, and especially special
-// keys (function keys, cursor movement keys, etc.)  For normal keys, like
-// ASCII letters, we use KeyRune, and then expect the application to
-// inspect the Rune() member of the EventKey.
-type Key int16
-
-// This is the list of named keys.  KeyRune is special however, in that it is
-// a place holder key indicating that a printable character was sent.  The
-// actual value of the rune will be transported in the Rune of the associated
-// EventKey.
-const (
-	KeyRune Key = iota + 256
-	KeyUp
-	KeyDown
-	KeyRight
-	KeyLeft
-	KeyUpLeft
-	KeyUpRight
-	KeyDownLeft
-	KeyDownRight
-	KeyCenter
-	KeyPgUp
-	KeyPgDn
-	KeyHome
-	KeyEnd
-	KeyInsert
-	KeyDelete
-	KeyHelp
-	KeyExit
-	KeyClear
-	KeyCancel
-	KeyPrint
-	KeyPause
-	KeyBacktab
-	KeyF1
-	KeyF2
-	KeyF3
-	KeyF4
-	KeyF5
-	KeyF6
-	KeyF7
-	KeyF8
-	KeyF9
-	KeyF10
-	KeyF11
-	KeyF12
-	KeyF13
-	KeyF14
-	KeyF15
-	KeyF16
-	KeyF17
-	KeyF18
-	KeyF19
-	KeyF20
-	KeyF21
-	KeyF22
-	KeyF23
-	KeyF24
-	KeyF25
-	KeyF26
-	KeyF27
-	KeyF28
-	KeyF29
-	KeyF30
-	KeyF31
-	KeyF32
-	KeyF33
-	KeyF34
-	KeyF35
-	KeyF36
-	KeyF37
-	KeyF38
-	KeyF39
-	KeyF40
-	KeyF41
-	KeyF42
-	KeyF43
-	KeyF44
-	KeyF45
-	KeyF46
-	KeyF47
-	KeyF48
-	KeyF49
-	KeyF50
-	KeyF51
-	KeyF52
-	KeyF53
-	KeyF54
-	KeyF55
-	KeyF56
-	KeyF57
-	KeyF58
-	KeyF59
-	KeyF60
-	KeyF61
-	KeyF62
-	KeyF63
-	KeyF64
-)
-
-// These are the control keys.  Note that they overlap with other keys,
-// perhaps.  For example, KeyCtrlH is the same as KeyBackspace.
-const (
-	KeyCtrlSpace Key = iota
-	KeyCtrlA
-	KeyCtrlB
-	KeyCtrlC
-	KeyCtrlD
-	KeyCtrlE
-	KeyCtrlF
-	KeyCtrlG
-	KeyCtrlH
-	KeyCtrlI
-	KeyCtrlJ
-	KeyCtrlK
-	KeyCtrlL
-	KeyCtrlM
-	KeyCtrlN
-	KeyCtrlO
-	KeyCtrlP
-	KeyCtrlQ
-	KeyCtrlR
-	KeyCtrlS
-	KeyCtrlT
-	KeyCtrlU
-	KeyCtrlV
-	KeyCtrlW
-	KeyCtrlX
-	KeyCtrlY
-	KeyCtrlZ
-	KeyCtrlLeftSq // Escape
-	KeyCtrlBackslash
-	KeyCtrlRightSq
-	KeyCtrlCarat
-	KeyCtrlUnderscore
-)
-
-// Special values - these are fixed in an attempt to make it more likely
-// that aliases will encode the same way.
-
-// These are the defined ASCII values for key codes.  They generally match
-// with KeyCtrl values.
-const (
-	KeyNUL Key = iota
-	KeySOH
-	KeySTX
-	KeyETX
-	KeyEOT
-	KeyENQ
-	KeyACK
-	KeyBEL
-	KeyBS
-	KeyTAB
-	KeyLF
-	KeyVT
-	KeyFF
-	KeyCR
-	KeySO
-	KeySI
-	KeyDLE
-	KeyDC1
-	KeyDC2
-	KeyDC3
-	KeyDC4
-	KeyNAK
-	KeySYN
-	KeyETB
-	KeyCAN
-	KeyEM
-	KeySUB
-	KeyESC
-	KeyFS
-	KeyGS
-	KeyRS
-	KeyUS
-	KeyDEL Key = 0x7F
-)
-
-// These keys are aliases for other names.
-const (
-	KeyBackspace  = KeyBS
-	KeyTab        = KeyTAB
-	KeyEsc        = KeyESC
-	KeyEscape     = KeyESC
-	KeyEnter      = KeyCR
-	KeyBackspace2 = KeyDEL
-)

vendor/github.com/gdamore/tcell/mouse.go

@@ -1,97 +0,0 @@
-// Copyright 2015 The TCell Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use file except in compliance with the License.
-// You may obtain a copy of the license at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package tcell
-
-import (
-	"time"
-)
-
-// EventMouse is a mouse event.  It is sent on either mouse up or mouse down
-// events.  It is also sent on mouse motion events - if the terminal supports
-// it.  We make every effort to ensure that mouse release events are delivered.
-// Hence, click drag can be identified by a motion event with the mouse down,
-// without any intervening button release.  On some terminals only the initiating
-// press and terminating release event will be delivered.
-//
-// Mouse wheel events, when reported, may appear on their own as individual
-// impulses; that is, there will normally not be a release event delivered
-// for mouse wheel movements.
-//
-// Most terminals cannot report the state of more than one button at a time --
-// and some cannot report motion events unless a button is pressed.
-//
-// Applications can inspect the time between events to resolve double or
-// triple clicks.
-type EventMouse struct {
-	t   time.Time
-	btn ButtonMask
-	mod ModMask
-	x   int
-	y   int
-}
-
-// When returns the time when this EventMouse was created.
-func (ev *EventMouse) When() time.Time {
-	return ev.t
-}
-
-// Buttons returns the list of buttons that were pressed or wheel motions.
-func (ev *EventMouse) Buttons() ButtonMask {
-	return ev.btn
-}
-
-// Modifiers returns a list of keyboard modifiers that were pressed
-// with the mouse button(s).
-func (ev *EventMouse) Modifiers() ModMask {
-	return ev.mod
-}
-
-// Position returns the mouse position in character cells.  The origin
-// 0, 0 is at the upper left corner.
-func (ev *EventMouse) Position() (int, int) {
-	return ev.x, ev.y
-}
-
-// NewEventMouse is used to create a new mouse event.  Applications
-// shouldn't need to use this; its mostly for screen implementors.
-func NewEventMouse(x, y int, btn ButtonMask, mod ModMask) *EventMouse {
-	return &EventMouse{t: time.Now(), x: x, y: y, btn: btn, mod: mod}
-}
-
-// ButtonMask is a mask of mouse buttons and wheel events.  Mouse button presses
-// are normally delivered as both press and release events.  Mouse wheel events
-// are normally just single impulse events.  Windows supports up to eight
-// separate buttons plus all four wheel directions, but XTerm can only support
-// mouse buttons 1-3 and wheel up/down.  Its not unheard of for terminals
-// to support only one or two buttons (think Macs).  Old terminals, and true
-// emulations (such as vt100) won't support mice at all, of course.
-type ButtonMask int16
-
-// These are the actual button values.
-const (
-	Button1 ButtonMask = 1 << iota // Usually left mouse button.
-	Button2                        // Usually the middle mouse button.
-	Button3                        // Usually the right mouse button.
-	Button4                        // Often a side button (thumb/next).
-	Button5                        // Often a side button (thumb/prev).
-	Button6
-	Button7
-	Button8
-	WheelUp                   // Wheel motion up/away from user.
-	WheelDown                 // Wheel motion down/towards user.
-	WheelLeft                 // Wheel motion to left.
-	WheelRight                // Wheel motion to right.
-	ButtonNone ButtonMask = 0 // No button or wheel events.
-)