Release 2022.5.3

Allow for cloudflared to be built with debug mode to be used with dlv exec

.teamcity/update-homebrew-core.sh

@@ -23,4 +23,4 @@ git config --global user.name "cloudflare-warp-bot"
 git config --global user.email "warp-bot@cloudflare.com"
 
 # bump formula pr
-brew bump-formula-pr cloudflared --version="$VERSION"
+brew bump-formula-pr cloudflared --version="$VERSION" --no-browse

Makefile

@@ -40,6 +40,10 @@ ifneq ($(GO_BUILD_TAGS),)
 	GO_BUILD_TAGS := -tags "$(GO_BUILD_TAGS)"
 endif
 
+ifeq ($(debug), 1)
+	GO_BUILD_TAGS += -gcflags="all=-N -l"
+endif
+
 IMPORT_PATH    := github.com/cloudflare/cloudflared
 PACKAGE_DIR    := $(CURDIR)/packaging
 PREFIX         := /usr
@@ -261,6 +265,10 @@ github-release: cloudflared
 github-release-built-pkgs:
 	python3 github_release.py --path $(PWD)/built_artifacts --release-version $(VERSION)
 
+.PHONY: release-pkgs-linux
+release-pkgs-linux:
+	python3 ./release_pkgs.py
+
 .PHONY: github-message
 github-message:
 	python3 github_message.py --release-version $(VERSION)

RELEASE_NOTES

@@ -1,3 +1,21 @@
+2022.5.3
+- 2022-05-30 TUN-6308: Add debug logs to see if packets are sent/received from edge
+- 2022-05-30 TUN-6301: Allow to update logger used by UDP session manager
+
+2022.5.2
+- 2022-05-23 TUN-6270: Import gpg keys from environment variables
+- 2022-05-24 TUN-6209: Improve feedback process if release_pkgs to deb and rpm fail
+- 2022-05-24 TUN-6280: Don't wrap qlog connection tracer for gatethering QUIC metrics since we're not writing qlog files.
+- 2022-05-25 TUN-6209: Sign RPM packages
+- 2022-05-25 TUN-6285: Upload pkg assets to repos when cloudflared is released.
+- 2022-05-24 TUN-6282: Upgrade golang to 1.17.10, go-boring to 1.17.9
+- 2022-05-26 TUN-6292: Debug builds for cloudflared
+- 2022-05-28 TUN-6304: Fixed some file permission issues
+- 2022-05-11 TUN-6197: Publish to brew core should not try to open the browser
+- 2022-05-12 TUN-5943: Add RPM support
+- 2022-05-18 TUN-6248: Fix panic in cloudflared during tracing when origin doesn't provide header map
+- 2022-05-18 TUN-6250: Add upstream response status code to tracing span attributes
+
 2022.5.1
 - 2022-05-06 TUN-6146: Release_pkgs is now a generic command line script
 - 2022-05-06 TUN-6185: Fix tcpOverWSOriginService not using original scheme for String representation

cfsetup.yaml

@@ -1,5 +1,5 @@
-pinned_go: &pinned_go go=1.17.5-1
-pinned_go_fips: &pinned_go_fips go-boring=1.17.5-1
+pinned_go: &pinned_go go=1.17.10-1
+pinned_go_fips: &pinned_go_fips go-boring=1.17.9-1
 
 build_dir: &build_dir /cfsetup_build
 default-flavor: buster
@@ -40,16 +40,22 @@ stretch: &stretch
       - libffi-dev
       - python3-setuptools
       - python3-pip
+      - reprepro
+      - createrepo
     pre-cache: &github_release_pkgs_pre_cache
       - wget https://github.com/sudarshan-reddy/msitools/releases/download/v0.101b/wixl -P /usr/local/bin
       - chmod a+x /usr/local/bin/wixl
       - pip3 install pynacl==1.4.0
       - pip3 install pygithub==1.55
+      - pip3 install boto3==1.22.9
+      - pip3 install python-gnupg==0.4.9
     post-cache:
       # build all packages (except macos and FIPS) and move them to /cfsetup/built_artifacts
       - ./build-packages.sh
       # release the packages built and moved to /cfsetup/built_artifacts
       - make github-release-built-pkgs
+      # publish packages to linux repos
+      - make release-pkgs-linux
   # handle FIPS separately so that we built with gofips compiler
   github-fips-release-pkgs:
     build_dir: *build_dir
@@ -249,10 +255,10 @@ centos-7:
     pre-cache:
       - yum install -y fakeroot
       - yum upgrade -y binutils-2.27-44.base.el7.x86_64
-      - wget https://go.dev/dl/go1.17.5.linux-amd64.tar.gz -P /tmp/
-      - tar -C /usr/local -xzf /tmp/go1.17.5.linux-amd64.tar.gz
+      - wget https://go.dev/dl/go1.17.10.linux-amd64.tar.gz -P /tmp/
+      - tar -C /usr/local -xzf /tmp/go1.17.10.linux-amd64.tar.gz
     post-cache:
       - export PATH=$PATH:/usr/local/go/bin
       - export GOOS=linux
       - export GOARCH=amd64
-      - make publish-rpm
+      - make publish-rpm

connection/quic.go

@@ -58,7 +58,7 @@ func NewQUICConnection(
 		return nil, fmt.Errorf("failed to dial to edge: %w", err)
 	}
 
-	datagramMuxer, err := quicpogs.NewDatagramMuxer(session)
+	datagramMuxer, err := quicpogs.NewDatagramMuxer(session, logger)
 	if err != nil {
 		return nil, err
 	}
@@ -217,7 +217,7 @@ func (q *QUICConnection) RegisterUdpSession(ctx context.Context, sessionID uuid.
 
 	go q.serveUDPSession(session, closeAfterIdleHint)
 
-	q.logger.Debug().Msgf("Registered session %v, %v, %v", sessionID, dstIP, dstPort)
+	q.logger.Debug().Str("sessionID", sessionID.String()).Str("src", originProxy.LocalAddr().String()).Str("dst", fmt.Sprintf("%s:%d", dstIP, dstPort)).Msgf("Registered session")
 	return nil
 }
 

datagramsession/manager.go

@@ -29,6 +29,8 @@ type Manager interface {
 	RegisterSession(ctx context.Context, sessionID uuid.UUID, dstConn io.ReadWriteCloser) (*Session, error)
 	// UnregisterSession stops tracking the session and terminates it
 	UnregisterSession(ctx context.Context, sessionID uuid.UUID, message string, byRemote bool) error
+	// UpdateLogger updates the logger used by the Manager
+	UpdateLogger(log *zerolog.Logger)
 }
 
 type manager struct {
@@ -57,6 +59,11 @@ func NewManager(transport transport, log *zerolog.Logger) *manager {
 	}
 }
 
+func (m *manager) UpdateLogger(log *zerolog.Logger) {
+	// Benign data race, no problem if the old pointer is read or not concurrently.
+	m.log = log
+}
+
 func (m *manager) Serve(ctx context.Context) error {
 	errGroup, ctx := errgroup.WithContext(ctx)
 	errGroup.Go(func() error {

dev.Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.17.5 as builder
+FROM golang:1.17.10 as builder
 ENV GO111MODULE=on \
     CGO_ENABLED=0
 WORKDIR /go/src/github.com/cloudflare/cloudflared/
@@ -6,4 +6,4 @@ RUN apt-get update
 COPY . .
 # compile cloudflared
 RUN make cloudflared
-RUN cp /go/src/github.com/cloudflare/cloudflared/cloudflared /usr/local/bin/
+RUN cp /go/src/github.com/cloudflare/cloudflared/cloudflared /usr/local/bin/

ingress/origin_proxy.go

@@ -1,6 +1,7 @@
 package ingress
 
 import (
+	"fmt"
 	"net"
 	"net/http"
 
@@ -49,7 +50,13 @@ func (o *httpService) RoundTrip(req *http.Request) (*http.Response, error) {
 }
 
 func (o *statusCode) RoundTrip(_ *http.Request) (*http.Response, error) {
-	return o.resp, nil
+	resp := &http.Response{
+		StatusCode: o.code,
+		Status:     fmt.Sprintf("%d %s", o.code, http.StatusText(o.code)),
+		Body:       new(NopReadCloser),
+	}
+
+	return resp, nil
 }
 
 func (o *rawTCPService) EstablishConnection(dest string) (OriginConnection, error) {

ingress/origin_service.go

@@ -238,20 +238,15 @@ func (o helloWorld) MarshalJSON() ([]byte, error) {
 // statusCode is an OriginService that just responds with a given HTTP status.
 // Typical use-case is "user wants the catch-all rule to just respond 404".
 type statusCode struct {
-	resp *http.Response
+	code int
 }
 
 func newStatusCode(status int) statusCode {
-	resp := &http.Response{
-		StatusCode: status,
-		Status:     fmt.Sprintf("%d %s", status, http.StatusText(status)),
-		Body:       new(NopReadCloser),
-	}
-	return statusCode{resp: resp}
+	return statusCode{code: status}
 }
 
 func (o *statusCode) String() string {
-	return fmt.Sprintf("http_status:%d", o.resp.StatusCode)
+	return fmt.Sprintf("http_status:%d", o.code)
 }
 
 func (o *statusCode) start(

ingress/origin_udp_proxy.go

@@ -6,11 +6,16 @@ import (
 	"net"
 )
 
-type UDPProxy struct {
+type UDPProxy interface {
 	io.ReadWriteCloser
+	LocalAddr() net.Addr
 }
 
-func DialUDP(dstIP net.IP, dstPort uint16) (*UDPProxy, error) {
+type udpProxy struct {
+	*net.UDPConn
+}
+
+func DialUDP(dstIP net.IP, dstPort uint16) (UDPProxy, error) {
 	dstAddr := &net.UDPAddr{
 		IP:   dstIP,
 		Port: int(dstPort),
@@ -23,5 +28,5 @@ func DialUDP(dstIP net.IP, dstPort uint16) (*UDPProxy, error) {
 		return nil, fmt.Errorf("unable to create UDP proxy to origin (%v:%v): %w", dstIP, dstPort, err)
 	}
 
-	return &UDPProxy{udpConn}, nil
+	return &udpProxy{udpConn}, nil
 }

proxy/proxy.go

@@ -11,7 +11,6 @@ import (
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
 	"go.opentelemetry.io/otel/attribute"
-	"go.opentelemetry.io/otel/codes"
 	"go.opentelemetry.io/otel/trace"
 
 	"github.com/cloudflare/cloudflared/carrier"
@@ -197,12 +196,18 @@ func (p *Proxy) proxyHTTPRequest(
 	_, ttfbSpan := tr.Tracer().Start(tr.Context(), "ttfb_origin")
 	resp, err := httpService.RoundTrip(roundTripReq)
 	if err != nil {
-		tracing.EndWithStatus(ttfbSpan, codes.Error, "")
+		tracing.EndWithErrorStatus(ttfbSpan, err)
 		return errors.Wrap(err, "Unable to reach the origin service. The service may be down or it may not be responding to traffic from cloudflared")
 	}
-	tracing.EndWithStatus(ttfbSpan, codes.Ok, resp.Status)
+
+	tracing.EndWithStatusCode(ttfbSpan, resp.StatusCode)
 	defer resp.Body.Close()
 
+	// resp headers can be nil
+	if resp.Header == nil {
+		resp.Header = make(http.Header)
+	}
+
 	// Add spans to response header (if available)
 	tr.AddSpans(resp.Header, p.log)
 

quic/datagram.go

@@ -6,6 +6,7 @@ import (
 	"github.com/google/uuid"
 	"github.com/lucas-clemente/quic-go"
 	"github.com/pkg/errors"
+	"github.com/rs/zerolog"
 )
 
 const (
@@ -13,18 +14,14 @@ const (
 )
 
 type DatagramMuxer struct {
-	ID      uuid.UUID
 	session quic.Session
+	logger  *zerolog.Logger
 }
 
-func NewDatagramMuxer(quicSession quic.Session) (*DatagramMuxer, error) {
-	muxerID, err := uuid.NewRandom()
-	if err != nil {
-		return nil, err
-	}
+func NewDatagramMuxer(quicSession quic.Session, logger *zerolog.Logger) (*DatagramMuxer, error) {
 	return &DatagramMuxer{
-		ID:      muxerID,
 		session: quicSession,
+		logger:  logger,
 	}, nil
 }
 
@@ -42,6 +39,7 @@ func (dm *DatagramMuxer) SendTo(sessionID uuid.UUID, payload []byte) error {
 	if err := dm.session.SendMessage(msgWithID); err != nil {
 		return errors.Wrap(err, "Failed to send datagram back to edge")
 	}
+	dm.logger.Debug().Str("sessionID", sessionID.String()).Int("bytes", len(payload)).Msg("Send datagram back to edge")
 	return nil
 }
 
@@ -53,7 +51,12 @@ func (dm *DatagramMuxer) ReceiveFrom() (uuid.UUID, []byte, error) {
 	if err != nil {
 		return uuid.Nil, nil, err
 	}
-	return extractSessionID(msg)
+	sessionID, payload, err := extractSessionID(msg)
+	if err != nil {
+		return uuid.Nil, nil, err
+	}
+	dm.logger.Debug().Str("sessionID", sessionID.String()).Int("bytes", len(payload)).Msg("Received datagram from edge")
+	return sessionID, payload, nil
 }
 
 // Maximum application payload to send to / receive from QUIC datagram frame

quic/datagram_test.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/google/uuid"
 	"github.com/lucas-clemente/quic-go"
+	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/require"
 	"golang.org/x/sync/errgroup"
 )
@@ -71,7 +72,8 @@ func TestMaxDatagramPayload(t *testing.T) {
 			return err
 		}
 
-		muxer, err := NewDatagramMuxer(quicSession)
+		logger := zerolog.Nop()
+		muxer, err := NewDatagramMuxer(quicSession, &logger)
 		if err != nil {
 			return err
 		}
@@ -96,7 +98,8 @@ func TestMaxDatagramPayload(t *testing.T) {
 		quicSession, err := quic.DialAddrEarly(quicListener.Addr().String(), tlsClientConfig, quicConfig)
 		require.NoError(t, err)
 
-		muxer, err := NewDatagramMuxer(quicSession)
+		logger := zerolog.Nop()
+		muxer, err := NewDatagramMuxer(quicSession, &logger)
 		if err != nil {
 			return err
 		}

quic/tracing.go

@@ -3,9 +3,9 @@ package quic
 import (
 	"context"
 	"net"
+	"time"
 
 	"github.com/lucas-clemente/quic-go/logging"
-	"github.com/lucas-clemente/quic-go/qlog"
 	"github.com/rs/zerolog"
 )
 
@@ -40,75 +40,116 @@ func NewServerTracer(logger *zerolog.Logger) logging.Tracer {
 	}
 }
 
-func (t *tracer) TracerForConnection(_ context.Context, p logging.Perspective, odcid logging.ConnectionID) logging.ConnectionTracer {
-	connID := logging.ConnectionID(odcid).String()
-	ql := &quicLogger{
-		logger:       t.logger,
-		connectionID: connID,
-	}
+func (t *tracer) TracerForConnection(_ctx context.Context, _p logging.Perspective, _odcid logging.ConnectionID) logging.ConnectionTracer {
 	if t.config.isClient {
-		return newConnTracer(ql, p, odcid, newClientCollector(t.config.index))
+		return newConnTracer(newClientCollector(t.config.index))
 	}
-	return newConnTracer(ql, p, odcid, newServiceCollector())
+	return newConnTracer(newServiceCollector())
 }
 
 func (*tracer) SentPacket(net.Addr, *logging.Header, logging.ByteCount, []logging.Frame) {}
 func (*tracer) DroppedPacket(net.Addr, logging.PacketType, logging.ByteCount, logging.PacketDropReason) {
 }
 
-// connTracer is a wrapper around https://pkg.go.dev/github.com/lucas-clemente/quic-go@v0.23.0/qlog#NewConnectionTracer to collect metrics
+var _ logging.Tracer = (*tracer)(nil)
+
+// connTracer collects connection level metrics
 type connTracer struct {
-	logging.ConnectionTracer
 	metricsCollector MetricsCollector
-	connectionID     string
 }
 
-func newConnTracer(ql *quicLogger, p logging.Perspective, odcid logging.ConnectionID, metricsCollector MetricsCollector) logging.ConnectionTracer {
+var _ logging.ConnectionTracer = (*connTracer)(nil)
+
+func newConnTracer(metricsCollector MetricsCollector) logging.ConnectionTracer {
 	return &connTracer{
-		qlog.NewConnectionTracer(ql, p, odcid),
-		metricsCollector,
-		logging.ConnectionID(odcid).String(),
+		metricsCollector: metricsCollector,
 	}
 }
 
 func (ct *connTracer) StartedConnection(local, remote net.Addr, srcConnID, destConnID logging.ConnectionID) {
 	ct.metricsCollector.startedConnection()
-	ct.ConnectionTracer.StartedConnection(local, remote, srcConnID, destConnID)
 }
 
 func (ct *connTracer) ClosedConnection(err error) {
 	ct.metricsCollector.closedConnection(err)
-	ct.ConnectionTracer.ClosedConnection(err)
 }
 
 func (ct *connTracer) SentPacket(hdr *logging.ExtendedHeader, packetSize logging.ByteCount, ack *logging.AckFrame, frames []logging.Frame) {
 	ct.metricsCollector.sentPackets(packetSize)
-	ct.ConnectionTracer.SentPacket(hdr, packetSize, ack, frames)
 }
 
 func (ct *connTracer) ReceivedPacket(hdr *logging.ExtendedHeader, size logging.ByteCount, frames []logging.Frame) {
 	ct.metricsCollector.receivedPackets(size)
-	ct.ConnectionTracer.ReceivedPacket(hdr, size, frames)
 }
 
 func (ct *connTracer) BufferedPacket(pt logging.PacketType) {
 	ct.metricsCollector.bufferedPackets(pt)
-	ct.ConnectionTracer.BufferedPacket(pt)
 }
 
 func (ct *connTracer) DroppedPacket(pt logging.PacketType, size logging.ByteCount, reason logging.PacketDropReason) {
 	ct.metricsCollector.droppedPackets(pt, size, reason)
-	ct.ConnectionTracer.DroppedPacket(pt, size, reason)
 }
 
 func (ct *connTracer) LostPacket(level logging.EncryptionLevel, number logging.PacketNumber, reason logging.PacketLossReason) {
 	ct.metricsCollector.lostPackets(reason)
-	ct.ConnectionTracer.LostPacket(level, number, reason)
 }
 
 func (ct *connTracer) UpdatedMetrics(rttStats *logging.RTTStats, cwnd, bytesInFlight logging.ByteCount, packetsInFlight int) {
 	ct.metricsCollector.updatedRTT(rttStats)
-	ct.ConnectionTracer.UpdatedMetrics(rttStats, cwnd, bytesInFlight, packetsInFlight)
+}
+
+func (ct *connTracer) NegotiatedVersion(chosen logging.VersionNumber, clientVersions, serverVersions []logging.VersionNumber) {
+}
+
+func (ct *connTracer) SentTransportParameters(parameters *logging.TransportParameters) {
+}
+
+func (ct *connTracer) ReceivedTransportParameters(parameters *logging.TransportParameters) {
+}
+
+func (ct *connTracer) RestoredTransportParameters(parameters *logging.TransportParameters) {
+}
+
+func (ct *connTracer) ReceivedVersionNegotiationPacket(header *logging.Header, numbers []logging.VersionNumber) {
+}
+
+func (ct *connTracer) ReceivedRetry(header *logging.Header) {
+}
+
+func (ct *connTracer) AcknowledgedPacket(level logging.EncryptionLevel, number logging.PacketNumber) {
+}
+
+func (ct *connTracer) UpdatedCongestionState(state logging.CongestionState) {
+}
+
+func (ct *connTracer) UpdatedPTOCount(value uint32) {
+}
+
+func (ct *connTracer) UpdatedKeyFromTLS(level logging.EncryptionLevel, perspective logging.Perspective) {
+}
+
+func (ct *connTracer) UpdatedKey(generation logging.KeyPhase, remote bool) {
+}
+
+func (ct *connTracer) DroppedEncryptionLevel(level logging.EncryptionLevel) {
+}
+
+func (ct *connTracer) DroppedKey(generation logging.KeyPhase) {
+}
+
+func (ct *connTracer) SetLossTimer(timerType logging.TimerType, level logging.EncryptionLevel, time time.Time) {
+}
+
+func (ct *connTracer) LossTimerExpired(timerType logging.TimerType, level logging.EncryptionLevel) {
+}
+
+func (ct *connTracer) LossTimerCanceled() {
+}
+
+func (ct *connTracer) Close() {
+}
+
+func (ct *connTracer) Debug(name, msg string) {
 }
 
 type quicLogger struct {

release_pkgs.py

@@ -8,12 +8,16 @@
        them to be in an uploadable state.
     2. Upload these packages to a storage in a format that apt and yum expect.
 """
-import subprocess
+from subprocess import Popen, PIPE
 import os
 import argparse
+import base64
+from pathlib import Path
 import logging
+import shutil
 from hashlib import sha256
 
+import gnupg
 import boto3
 from botocore.client import Config
 from botocore.exceptions import ClientError
@@ -47,7 +51,7 @@ class PkgUploader:
             config=config,
         )
 
-        print(f"uploading asset: {filename} to {upload_file_path}...")
+        print(f"uploading asset: {filename} to {upload_file_path} in bucket{self.bucket_name}...")
         try:
             r2.upload_file(filename, self.bucket_name, upload_file_path)
         except ClientError as e:
@@ -77,7 +81,7 @@ class PkgCreator:
             components, 
             description,
             gpg_key_id ):
-        with open(file_path, "w") as distributions_file:
+        with open(file_path, "w+") as distributions_file:
             for release in releases:
                 distributions_file.write(f"Origin: {origin}\n")
                 distributions_file.write(f"Label: {label}\n")
@@ -99,14 +103,87 @@ class PkgCreator:
         dist: contains all the pkgs and signed releases that are necessary for an apt download.
     """
     def create_deb_pkgs(self, release, deb_file):
-        self._clean_build_resources()
-        subprocess.call(("reprepro", "includedeb", release, deb_file))
+        print(f"creating deb pkgs: {release} : {deb_file}")
+        p = Popen(["reprepro", "includedeb", release, deb_file], stdout=PIPE, stderr=PIPE)
+        out, err = p.communicate()
+        if p.returncode != 0:
+            print(f"create deb_pkgs result => {out}, {err}")
+            raise
+
+    def create_rpm_pkgs(self, artifacts_path, gpg_key_name):
+        self._setup_rpm_pkg_directories(artifacts_path, gpg_key_name)
+        p = Popen(["createrepo", "./rpm"], stdout=PIPE, stderr=PIPE)
+        out, err = p.communicate()
+        if p.returncode != 0:
+            print(f"create rpm_pkgs result => {out}, {err}")
+            raise
+
+        self._sign_repomd()
+
+    def _sign_rpms(self, file_path):
+        p = Popen(["rpm" , "--define", f"_gpg_name {gpg_key_name}", "--addsign", file_path], stdout=PIPE, stderr=PIPE)
+        out, err = p.communicate()
+        if p.returncode != 0:
+            print(f"rpm sign result result => {out}, {err}")
+            raise
+
+    def _sign_repomd(self):
+        p = Popen(["gpg", "--batch", "--detach-sign", "--armor", "./rpm/repodata/repomd.xml"], stdout=PIPE, stderr=PIPE) 
+        out, err = p.communicate()
+        if p.returncode != 0:
+            print(f"sign repomd result => {out}, {err}")
+            raise
+        
+    """
+        sets up and signs the RPM directories in the following format:
+        - rpm 
+           - aarch64
+           - x86_64
+           - 386
+
+        this assumes the assets are in the format <prefix>-<aarch64/x86_64/386>.rpm
+    """
+    def _setup_rpm_pkg_directories(self, artifacts_path, gpg_key_name, archs=["aarch64", "x86_64", "386"]):
+        for arch in archs:
+            for root, _ , files in os.walk(artifacts_path):
+                for file in files:
+                    if file.endswith(f"{arch}.rpm"):
+                        new_dir = f"./rpm/{arch}"
+                        os.makedirs(new_dir, exist_ok=True)
+                        old_path = os.path.join(root, file)
+                        new_path = os.path.join(new_dir, file)
+                        shutil.copyfile(old_path, new_path)
+                        self._sign_rpms(new_path)
+    
+    """
+        imports gpg keys into the system so reprepro and createrepo can use it to sign packages.
+        it returns the GPG ID after a successful import
+    """
+    def import_gpg_keys(self, private_key, public_key):
+        gpg = gnupg.GPG()
+        private_key = base64.b64decode(private_key)
+        gpg.import_keys(private_key)
+        public_key = base64.b64decode(public_key)
+        gpg.import_keys(public_key)
+        data = gpg.list_keys(secret=True)
+        return (data[0]["fingerprint"], data[0]["uids"][0])
 
     """
-        This is mostly useful to clear previously built db, dist and pool resources.
+        basically rpm --import <key_file>
+        This enables us to sign rpms.
     """
-    def _clean_build_resources(self):
-        subprocess.call(("reprepro", "clearvanished"))
+    def import_rpm_key(self, public_key):
+        file_name = "pb.key"
+        with open(file_name, "wb") as f:
+            public_key = base64.b64decode(public_key)
+            f.write(public_key)
+        
+        p = Popen(["rpm", "--import", file_name], stdout=PIPE, stderr=PIPE)
+        out, err = p.communicate()
+        if p.returncode != 0:
+            print(f"create rpm import result => {out}, {err}")
+            raise
+
 
 """
     Walks through a directory and uploads it's assets to R2.
@@ -138,11 +215,12 @@ def upload_from_directories(pkg_uploader, directory, release, binary):
     gpg_key_id: is an id indicating the key the package should be signed with. The public key of this id will be 
     uploaded to R2 so it can be presented to apt downloaders.
 
-    release_version: is the cloudflared release version.
+    release_version: is the cloudflared release version. Only provide this if you want a permanent backup.
 """
 def create_deb_packaging(pkg_creator, pkg_uploader, releases, gpg_key_id, binary_name, archs, package_component, release_version):
     # set configuration for package creation.
     print(f"initialising configuration for {binary_name} , {archs}")
+    Path("./conf").mkdir(parents=True, exist_ok=True)
     pkg_creator.create_distribution_conf(
     "./conf/distributions",
     binary_name,
@@ -163,9 +241,22 @@ def create_deb_packaging(pkg_creator, pkg_uploader, releases, gpg_key_id, binary
     upload_from_directories(pkg_uploader, "dists", None, binary_name)
     upload_from_directories(pkg_uploader, "pool", None, binary_name)
 
-    print(f"uploading versioned release {release_version} to r2...")
-    upload_from_directories(pkg_uploader, "dists", release_version, binary_name)
-    upload_from_directories(pkg_uploader, "pool", release_version, binary_name)
+    if release_version:
+        print(f"uploading versioned release {release_version} to r2...")
+        upload_from_directories(pkg_uploader, "dists", release_version, binary_name)
+        upload_from_directories(pkg_uploader, "pool", release_version, binary_name)
+
+def create_rpm_packaging(pkg_creator, pkg_uploader, artifacts_path, release_version, binary_name, gpg_key_name):
+    print(f"creating rpm pkgs...")
+    pkg_creator.create_rpm_pkgs(artifacts_path, gpg_key_name)
+
+    print("uploading latest to r2...")
+    upload_from_directories(pkg_uploader, "rpm", None, binary_name)
+
+    if release_version:
+        print(f"uploading versioned release {release_version} to r2...")
+        upload_from_directories(pkg_uploader, "rpm", release_version, binary_name)
+
 
 def parse_args():
     parser = argparse.ArgumentParser(
@@ -173,7 +264,7 @@ def parse_args():
     )
 
     parser.add_argument(
-            "--bucket", default=os.environ.get("R2_BUCKET_NAME"), help="R2 Bucket name"
+            "--bucket", default=os.environ.get("R2_BUCKET"), help="R2 Bucket name"
     )
     parser.add_argument(
             "--id", default=os.environ.get("R2_CLIENT_ID"), help="R2 Client ID"
@@ -186,7 +277,7 @@ def parse_args():
     )
     parser.add_argument(
             "--release-tag", default=os.environ.get("RELEASE_VERSION"), help="Release version you want your pkgs to be\
-            prefixed with"
+            prefixed with. Leave empty if you don't want tagged release versions backed up to R2."
     )
 
     parser.add_argument(
@@ -194,8 +285,13 @@ def parse_args():
     )
 
     parser.add_argument(
-            "--gpg-key-id", default=os.environ.get("GPG_KEY_ID"), help="gpg key ID that's being used to sign release\
-            packages."
+            "--gpg-private-key", default=os.environ.get("LINUX_SIGNING_PRIVATE_KEY"), help="GPG private key to sign the\
+            packages"
+    )
+
+    parser.add_argument(
+            "--gpg-public-key", default=os.environ.get("LINUX_SIGNING_PUBLIC_KEY"), help="GPG public key used for\
+            signing packages"
     )
 
     parser.add_argument(
@@ -220,6 +316,12 @@ if __name__ == "__main__":
         exit(1)
 
     pkg_creator = PkgCreator()
+    (gpg_key_id, gpg_key_name) = pkg_creator.import_gpg_keys(args.gpg_private_key, args.gpg_public_key)
+    pkg_creator.import_rpm_key(args.gpg_public_key)
+
     pkg_uploader = PkgUploader(args.account, args.bucket, args.id, args.secret)
-    create_deb_packaging(pkg_creator, pkg_uploader, args.deb_based_releases, args.gpg_key_id, args.binary, 
-            args.archs, "main", args.release_tag)
+    print(f"signing with gpg_key: {gpg_key_id}")
+    create_deb_packaging(pkg_creator, pkg_uploader, args.deb_based_releases, gpg_key_id, args.binary, args.archs,
+            "main", args.release_tag)
+    
+    create_rpm_packaging(pkg_creator, pkg_uploader, "./built_artifacts", args.release_tag, args.binary, gpg_key_name)

tracing/tracing.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"math"
 	"net/http"
 	"os"
 	"runtime"
@@ -29,6 +30,9 @@ const (
 	TracerContextNameOverride = "uber-trace-id"
 
 	IntCloudflaredTracingHeader = "cf-int-cloudflared-tracing"
+
+	MaxErrorDescriptionLen = 100
+	traceHttpStatusCodeKey = "upstreamStatusCode"
 )
 
 var (
@@ -103,6 +107,11 @@ func (cft *TracedRequest) Tracer() trace.Tracer {
 
 // Spans returns the spans as base64 encoded protobuf otlp traces.
 func (cft *TracedRequest) AddSpans(headers http.Header, log *zerolog.Logger) {
+	if headers == nil {
+		log.Error().Msgf("provided headers map is nil")
+		return
+	}
+
 	enc, err := cft.exporter.Spans()
 	switch err {
 	case nil:
@@ -121,15 +130,39 @@ func (cft *TracedRequest) AddSpans(headers http.Header, log *zerolog.Logger) {
 		log.Error().Msgf("no traces provided and no error from exporter")
 		return
 	}
+
 	headers[CanonicalCloudflaredTracingHeader] = []string{enc}
 }
 
-// EndWithStatus will set a status for the span and then end it.
-func EndWithStatus(span trace.Span, code codes.Code, status string) {
+// EndWithErrorStatus will set a status for the span and then end it.
+func EndWithErrorStatus(span trace.Span, err error) {
+	endSpan(span, -1, codes.Error, err)
+}
+
+// EndWithStatusCode will set a status for the span and then end it.
+func EndWithStatusCode(span trace.Span, statusCode int) {
+	endSpan(span, statusCode, codes.Ok, nil)
+}
+
+// EndWithErrorStatus will set a status for the span and then end it.
+func endSpan(span trace.Span, upstreamStatusCode int, spanStatusCode codes.Code, err error) {
 	if span == nil {
 		return
 	}
-	span.SetStatus(code, status)
+
+	if upstreamStatusCode > 0 {
+		span.SetAttributes(attribute.Int(traceHttpStatusCodeKey, upstreamStatusCode))
+	}
+
+	// add error to status buf cap description
+	errDescription := ""
+	if err != nil {
+		errDescription = err.Error()
+		l := int(math.Min(float64(len(errDescription)), MaxErrorDescriptionLen))
+		errDescription = errDescription[:l]
+	}
+
+	span.SetStatus(spanStatusCode, errDescription)
 	span.End()
 }
 

tracing/tracing_test.go

@@ -1,13 +1,16 @@
 package tracing
 
 import (
+	"context"
 	"net/http"
 	"net/http/httptest"
 	"testing"
 
+	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/assert"
 	tracesdk "go.opentelemetry.io/otel/sdk/trace"
 	"go.opentelemetry.io/otel/trace"
+	tracepb "go.opentelemetry.io/proto/otlp/trace/v1"
 )
 
 func TestNewCfTracer(t *testing.T) {
@@ -48,3 +51,19 @@ func TestNewCfTracerInvalidHeaders(t *testing.T) {
 		assert.IsType(t, &NoopOtlpClient{}, tr.exporter)
 	}
 }
+
+func TestAddingSpansWithNilMap(t *testing.T) {
+	req := httptest.NewRequest("GET", "http://localhost", nil)
+	req.Header.Add(TracerContextName, "14cb070dde8e51fc5ae8514e69ba42ca:b38f1bf5eae406f3:0:1")
+	tr := NewTracedRequest(req)
+
+	exporter := tr.exporter.(*InMemoryOtlpClient)
+
+	// add fake spans
+	spans := createResourceSpans([]*tracepb.Span{createOtlpSpan(traceId)})
+	err := exporter.UploadTraces(context.Background(), spans)
+	assert.NoError(t, err)
+
+	// a panic shouldn't occur
+	tr.AddSpans(nil, &zerolog.Logger{})
+}