Release 2025.4.2

## Summary
If we define pipelines to trigger on merge requests,
they will take precedence over branch pipelines,
which is currently the way our old pipelines are still
triggered. This means that we can have a merge request
with green pipelines, but actually the external pipelines failed.
Therefore, we need to only rely on branch pipelines,
to ensure that we don't ignore the results from
external pipelines.

More information here:
- https://forum.gitlab.com/t/merge-request-considering-merge-request-pipelines-instead-of-branch-pipelines/111248/2
- https://docs.gitlab.com/17.6/ci/jobs/job_rules/#run-jobs-only-in-specific-pipeline-types

.gitlab-ci.yml

@@ -0,0 +1,131 @@
+stages: [build, release]
+
+default:
+  id_tokens:
+    VAULT_ID_TOKEN:
+      aud: https://vault.cfdata.org
+
+# This before_script is injected into every job that runs on master meaning that if there is no tag the step
+# will succeed but only write "No tag present - Skipping" to the console.
+.check_tag:
+  before_script:
+    - |
+      # Check if there is a Git tag pointing to HEAD
+      echo "Tag found: $(git tag --points-at HEAD | grep .)"
+      if git tag --points-at HEAD | grep .; then
+        echo "Tag found: $(git tag --points-at HEAD | grep .)"
+        export "VERSION=$(git tag --points-at HEAD | grep .)"
+      else
+        echo "No tag present — skipping."
+        exit 0
+      fi
+
+## A set of predefined rules to use on the different jobs
+.default_rules:
+  # Rules to run the job only on the master branch
+  run_on_master:
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+      when: always
+    - when: never
+  # Rules to run the job only on branches that are not master. This is needed because for now
+  # we need to keep a similar behavior due to the integration with teamcity, which requires us
+  # to not trigger pipelines on tags and/or merge requests.
+  run_on_branch:
+    - if: $CI_COMMIT_TAG
+      when: never
+    - if: $CI_PIPELINE_SOURCE != "merge_request_event" && $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH
+      when: always
+    - when: never
+
+# -----------------------------------------------
+# Stage 1: Build on every PR
+# -----------------------------------------------
+build_cloudflared_macos: &build
+  stage: build
+  rules:
+    - !reference [.default_rules, run_on_branch]
+  tags:
+    - "macstadium-${RUNNER_ARCH}"
+  parallel:
+    matrix:
+      - RUNNER_ARCH: [arm, intel]
+  artifacts:
+    paths:
+      - artifacts/*
+  script:
+    - '[ "${RUNNER_ARCH}" = "arm" ] && export TARGET_ARCH=arm64'
+    - '[ "${RUNNER_ARCH}" = "intel" ] && export TARGET_ARCH=amd64'
+    - ARCH=$(uname -m)
+    - echo ARCH=$ARCH - TARGET_ARCH=$TARGET_ARCH
+    - ./.teamcity/mac/install-cloudflare-go.sh
+    - export PATH="/tmp/go/bin:$PATH"
+    - BUILD_SCRIPT=.teamcity/mac/build.sh
+    - if [[ ! -x ${BUILD_SCRIPT} ]] ; then exit ; fi
+    - set -euo pipefail
+    - echo "Executing ${BUILD_SCRIPT}"
+    - exec ${BUILD_SCRIPT}
+
+# -----------------------------------------------
+# Stage 1: Build and sign only on releases
+# -----------------------------------------------
+build_and_sign_cloudflared_macos:
+  <<: *build
+  rules:
+    - !reference [.default_rules, run_on_master]
+  secrets:
+    APPLE_DEV_CA_CERT:
+      vault: gitlab/cloudflare/tun/cloudflared/_branch/master/apple_dev_ca_cert_v2/data@kv
+      file: false
+    CFD_CODE_SIGN_CERT:
+      vault: gitlab/cloudflare/tun/cloudflared/_branch/master/cfd_code_sign_cert_v2/data@kv
+      file: false
+    CFD_CODE_SIGN_KEY:
+      vault: gitlab/cloudflare/tun/cloudflared/_branch/master/cfd_code_sign_key_v2/data@kv
+      file: false
+    CFD_CODE_SIGN_PASS:
+      vault: gitlab/cloudflare/tun/cloudflared/_branch/master/cfd_code_sign_pass_v2/data@kv
+      file: false
+    CFD_INSTALLER_CERT:
+      vault: gitlab/cloudflare/tun/cloudflared/_branch/master/cfd_installer_cert_v2/data@kv
+      file: false
+    CFD_INSTALLER_KEY:
+      vault: gitlab/cloudflare/tun/cloudflared/_branch/master/cfd_installer_key_v2/data@kv
+      file: false
+    CFD_INSTALLER_PASS:
+      vault: gitlab/cloudflare/tun/cloudflared/_branch/master/cfd_installer_pass_v2/data@kv
+      file: false
+
+# -----------------------------------------------
+# Stage 2: Release to Github after building and signing
+# -----------------------------------------------
+release_cloudflared_macos_to_github:
+  stage: release
+  image: docker-registry.cfdata.org/stash/tun/docker-images/cloudflared-ci/main:6-8616fe631b76-amd64@sha256:96f4fd05e66cec03e0864c1bcf09324c130d4728eef45ee994716da499183614
+  extends: .check_tag
+  dependencies:
+    - build_and_sign_cloudflared_macos
+  rules:
+    - !reference [.default_rules, run_on_master]
+  cache:
+    paths:
+      - .cache/pip
+  variables:
+    PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
+    KV_NAMESPACE: 380e19aa04314648949b6ad841417ebe
+    KV_ACCOUNT: 5ab4e9dfbd435d24068829fda0077963
+  secrets:
+    KV_API_TOKEN:
+      vault: gitlab/cloudflare/tun/cloudflared/_dev/cfd_kv_api_token/data@kv
+      file: false
+    API_KEY:
+      vault: gitlab/cloudflare/tun/cloudflared/_dev/cfd_github_api_key/data@kv
+      file: false
+  script:
+    - python3 --version ; pip --version  # For debugging
+    - python3 -m venv venv
+    - source venv/bin/activate
+    - pip install pynacl==1.4.0 pygithub==1.55
+    - echo $VERSION
+    - echo $TAG_EXISTS
+    - echo "Running release because tag exists."
+    - make macos-release

.teamcity/mac/build.sh

@@ -49,7 +49,7 @@ import_certificate() {
       echo -n -e ${CERTIFICATE_ENV_VAR} | base64 -D > ${CERTIFICATE_FILE_NAME}
       # we set || true  here and for every `security import invoke` because the  "duplicate SecKeychainItemImport" error
       # will cause set -e to exit 1. It is okay we do this because we deliberately handle this error in the lines below.
-      local out=$(security import ${CERTIFICATE_FILE_NAME} -A 2>&1) || true
+      local out=$(security import ${CERTIFICATE_FILE_NAME} -T /usr/bin/pkgbuild -A 2>&1) || true
       local exitcode=$?
       # delete the certificate from disk
       rm -rf ${CERTIFICATE_FILE_NAME}
@@ -68,6 +68,28 @@ import_certificate() {
     fi
 }
 
+create_cloudflared_build_keychain() {
+  # Reusing the private key password as the keychain key
+  local PRIVATE_KEY_PASS=$1
+
+  # Create keychain only if it doesn't already exist
+  if [ ! -f "$HOME/Library/Keychains/cloudflared_build_keychain.keychain-db" ]; then
+    security create-keychain -p "$PRIVATE_KEY_PASS" cloudflared_build_keychain
+  else
+    echo "Keychain already exists: cloudflared_build_keychain"
+  fi
+
+  # Append temp keychain to the user domain
+  security list-keychains -d user -s cloudflared_build_keychain $(security list-keychains -d user | sed s/\"//g)
+
+  # Remove relock timeout
+  security set-keychain-settings cloudflared_build_keychain
+
+  # Unlock keychain so it doesn't require password
+  security unlock-keychain -p "$PRIVATE_KEY_PASS" cloudflared_build_keychain
+
+}
+
 # Imports private keys to the Apple KeyChain
 import_private_keys() {
     local PRIVATE_KEY_NAME=$1
@@ -83,7 +105,7 @@ import_private_keys() {
         echo -n -e ${PRIVATE_KEY_ENV_VAR} | base64 -D > ${PRIVATE_KEY_FILE_NAME}
         # we set || true  here and for every `security import invoke` because the  "duplicate SecKeychainItemImport" error
         # will cause set -e to exit 1. It is okay we do this because we deliberately handle this error in the lines below.
-        local out=$(security import ${PRIVATE_KEY_FILE_NAME} -A -P "${PRIVATE_KEY_PASS}" 2>&1) || true
+        local out=$(security import ${PRIVATE_KEY_FILE_NAME} -k cloudflared_build_keychain -P "$PRIVATE_KEY_PASS" -T /usr/bin/pkgbuild -A -P "${PRIVATE_KEY_PASS}" 2>&1) || true
         local exitcode=$?
         rm -rf ${PRIVATE_KEY_FILE_NAME}
         if [ -n "$out" ]; then
@@ -100,6 +122,9 @@ import_private_keys() {
     fi
 }
 
+# Create temp keychain only for this build
+create_cloudflared_build_keychain "${CFD_CODE_SIGN_PASS}"
+
 # Add Apple Root Developer certificate to the key chain
 import_certificate "Apple Developer CA" "${APPLE_DEV_CA_CERT}" "${APPLE_CA_CERT}"
 
@@ -119,8 +144,8 @@ import_certificate "Developer ID Installer" "${CFD_INSTALLER_CERT}" "${INSTALLER
 if [[ ! -z "$CFD_CODE_SIGN_NAME" ]]; then
   CODE_SIGN_NAME="${CFD_CODE_SIGN_NAME}"
 else
-  if [[ -n "$(security find-certificate -c "Developer ID Application" | cut -d'"' -f 4 -s | grep "Developer ID Application:" | head -1)" ]]; then
-    CODE_SIGN_NAME=$(security find-certificate -c "Developer ID Application" | cut -d'"' -f 4 -s | grep "Developer ID Application:" | head -1)
+  if [[ -n "$(security find-certificate -c "Developer ID Application" cloudflared_build_keychain | cut -d'"' -f 4 -s | grep "Developer ID Application:" | head -1)" ]]; then
+    CODE_SIGN_NAME=$(security find-certificate -c "Developer ID Application" cloudflared_build_keychain | cut -d'"' -f 4 -s | grep "Developer ID Application:" | head -1)
   else
     CODE_SIGN_NAME=""
   fi
@@ -130,8 +155,8 @@ fi
 if [[ ! -z "$CFD_INSTALLER_NAME" ]]; then
   PKG_SIGN_NAME="${CFD_INSTALLER_NAME}"
 else
-  if [[ -n "$(security find-certificate -c "Developer ID Installer" | cut -d'"' -f 4 -s | grep "Developer ID Installer:" | head -1)" ]]; then
-    PKG_SIGN_NAME=$(security find-certificate -c "Developer ID Installer" | cut -d'"' -f 4 -s | grep "Developer ID Installer:" | head -1)
+  if [[ -n "$(security find-certificate -c "Developer ID Installer" cloudflared_build_keychain | cut -d'"' -f 4 -s | grep "Developer ID Installer:" | head -1)" ]]; then
+    PKG_SIGN_NAME=$(security find-certificate -c "Developer ID Installer" cloudflared_build_keychain | cut -d'"' -f 4 -s | grep "Developer ID Installer:" | head -1)
   else
     PKG_SIGN_NAME=""
   fi
@@ -142,9 +167,16 @@ rm -rf "${TARGET_DIRECTORY}"
 export TARGET_OS="darwin"
 GOCACHE="$PWD/../../../../" GOPATH="$PWD/../../../../" CGO_ENABLED=1 make cloudflared
 
+
+# This allows apple tools to use the certificates in the keychain without requiring password input.
+# This command always needs to run after the certificates have been loaded into the keychain
+if [[ ! -z "$CFD_CODE_SIGN_PASS" ]]; then
+  security set-key-partition-list -S apple-tool:,apple: -s -k "${CFD_CODE_SIGN_PASS}" cloudflared_build_keychain
+fi
+
 # sign the cloudflared binary
 if [[ ! -z "$CODE_SIGN_NAME" ]]; then
-  codesign -s "${CODE_SIGN_NAME}" -f -v --timestamp --options runtime ${BINARY_NAME}
+  codesign --keychain $HOME/Library/Keychains/cloudflared_build_keychain.keychain-db -s "${CODE_SIGN_NAME}" -fv --options runtime --timestamp ${BINARY_NAME}
 
  # notarize the binary
  # TODO: TUN-5789
@@ -165,11 +197,13 @@ tar czf "$FILENAME" "${BINARY_NAME}"
 
 # build the installer package
 if [[ ! -z "$PKG_SIGN_NAME" ]]; then
+
   pkgbuild --identifier com.cloudflare.${PRODUCT} \
       --version ${VERSION} \
       --scripts ${ARCH_TARGET_DIRECTORY}/scripts \
       --root ${ARCH_TARGET_DIRECTORY}/contents \
       --install-location /usr/local/bin \
+      --keychain cloudflared_build_keychain \
       --sign "${PKG_SIGN_NAME}" \
       ${PKGNAME}
 
@@ -187,3 +221,8 @@ fi
 # cleanup build directory because this script is not ran within containers,
 # which might lead to future issues in subsequent runs.
 rm -rf "${TARGET_DIRECTORY}"
+
+# cleanup the keychain
+security default-keychain -d user -s login.keychain-db
+security list-keychains -d user -s login.keychain-db
+security delete-keychain cloudflared_build_keychain

Makefile

@@ -237,6 +237,10 @@ github-release:
 	python3 github_release.py --path $(PWD)/built_artifacts --release-version $(VERSION)
 	python3 github_message.py --release-version $(VERSION)
 
+.PHONY: macos-release
+macos-release:
+	python3 github_release.py --path $(PWD)/artifacts/ --release-version $(VERSION)
+
 .PHONY: r2-linux-release
 r2-linux-release:
 	python3 ./release_pkgs.py

RELEASE_NOTES

@@ -1,3 +1,9 @@
+2025.4.2
+- 2025-04-30 chore: Do not use gitlab merge request pipelines
+- 2025-04-30 DEVTOOLS-16383: Create GitlabCI pipeline to release Mac builds
+- 2025-04-24 TUN-9255: Improve flush on write conditions in http2 tunnel type to match what is done on the edge
+- 2025-04-10 SDLC-3727 - Adding FIPS status to backstage
+
 2025.4.0
 - 2025-04-02 Fix broken links in `cmd/cloudflared/*.go` related to running tunnel as a service
 - 2025-04-02 chore: remove repetitive words

catalog-info.yaml

@@ -13,3 +13,5 @@ spec:
   type: "service"
   lifecycle: "Active"
   owner: "teams/tunnel-teams-routing"
+  cf:
+    FIPS: "required"

connection/connection.go

@@ -26,14 +26,20 @@ const (
 	MaxGracePeriod         = time.Minute * 3
 	MaxConcurrentStreams   = math.MaxUint32
 
-	contentTypeHeader = "content-type"
-	sseContentType    = "text/event-stream"
-	grpcContentType   = "application/grpc"
+	contentTypeHeader      = "content-type"
+	contentLengthHeader    = "content-length"
+	transferEncodingHeader = "transfer-encoding"
+
+	sseContentType     = "text/event-stream"
+	grpcContentType    = "application/grpc"
+	sseJsonContentType = "application/x-ndjson"
+
+	chunkTransferEncoding = "chunked"
 )
 
 var (
 	switchingProtocolText = fmt.Sprintf("%d %s", http.StatusSwitchingProtocols, http.StatusText(http.StatusSwitchingProtocols))
-	flushableContentTypes = []string{sseContentType, grpcContentType}
+	flushableContentTypes = []string{sseContentType, grpcContentType, sseJsonContentType}
 )
 
 // TunnelConnection represents the connection to the edge.
@@ -274,6 +280,22 @@ type ConnectedFuse interface {
 // Helper method to let the caller know what content-types should require a flush on every
 // write to a ResponseWriter.
 func shouldFlush(headers http.Header) bool {
+	// When doing Server Side Events (SSE), some frameworks don't respect the `Content-Type` header.
+	// Therefore, we need to rely on other ways to know whether we should flush on write or not. A good
+	// approach is to assume that responses without `Content-Length` or with `Transfer-Encoding: chunked`
+	// are streams, and therefore, should be flushed right away to the eyeball.
+	// References:
+	// - https://datatracker.ietf.org/doc/html/rfc7230#section-4.1
+	// - https://datatracker.ietf.org/doc/html/rfc9112#section-6.1
+	if contentLength := headers.Get(contentLengthHeader); contentLength == "" {
+		return true
+	}
+	if transferEncoding := headers.Get(transferEncodingHeader); transferEncoding != "" {
+		transferEncoding = strings.ToLower(transferEncoding)
+		if strings.Contains(transferEncoding, chunkTransferEncoding) {
+			return true
+		}
+	}
 	if contentType := headers.Get(contentTypeHeader); contentType != "" {
 		contentType = strings.ToLower(contentType)
 		for _, c := range flushableContentTypes {
@@ -282,7 +304,6 @@ func shouldFlush(headers http.Header) bool {
 			}
 		}
 	}
-
 	return false
 }
 

connection/connection_test.go

@@ -7,10 +7,12 @@ import (
 	"io"
 	"math/big"
 	"net/http"
+	"testing"
 	"time"
 
 	pkgerrors "github.com/pkg/errors"
 	"github.com/rs/zerolog"
+	"github.com/stretchr/testify/require"
 
 	cfdflow "github.com/cloudflare/cloudflared/flow"
 
@@ -209,3 +211,48 @@ func (mcf mockConnectedFuse) Connected() {}
 func (mcf mockConnectedFuse) IsConnected() bool {
 	return true
 }
+
+func TestShouldFlushHeaders(t *testing.T) {
+	tests := []struct {
+		headers     map[string]string
+		shouldFlush bool
+	}{
+		{
+			headers:     map[string]string{contentTypeHeader: "application/json", contentLengthHeader: "1"},
+			shouldFlush: false,
+		},
+		{
+			headers:     map[string]string{contentTypeHeader: "text/html", contentLengthHeader: "1"},
+			shouldFlush: false,
+		},
+		{
+			headers:     map[string]string{contentTypeHeader: "text/event-stream", contentLengthHeader: "1"},
+			shouldFlush: true,
+		},
+		{
+			headers:     map[string]string{contentTypeHeader: "application/grpc", contentLengthHeader: "1"},
+			shouldFlush: true,
+		},
+		{
+			headers:     map[string]string{contentTypeHeader: "application/x-ndjson", contentLengthHeader: "1"},
+			shouldFlush: true,
+		},
+		{
+			headers:     map[string]string{contentTypeHeader: "application/json"},
+			shouldFlush: true,
+		},
+		{
+			headers:     map[string]string{contentTypeHeader: "application/json", contentLengthHeader: "-1", transferEncodingHeader: "chunked"},
+			shouldFlush: true,
+		},
+	}
+
+	for _, test := range tests {
+		headers := http.Header{}
+		for k, v := range test.headers {
+			headers.Add(k, v)
+		}
+
+		require.Equal(t, test.shouldFlush, shouldFlush(headers))
+	}
+}

github_release.py

@@ -61,7 +61,7 @@ def assert_tag_exists(repo, version):
         raise Exception("Tag {} not found".format(version))
 
 
-def get_or_create_release(repo, version, dry_run=False):
+def get_or_create_release(repo, version, dry_run=False, is_draft=False):
     """
     Get a Github Release matching the version tag or create a new one.
     If a conflict occurs on creation, attempt to fetch the Release on last time
@@ -81,8 +81,11 @@ def get_or_create_release(repo, version, dry_run=False):
         return
 
     try:
-        logging.info("Creating release %s", version)
-        return repo.create_git_release(version, version, "")
+        if is_draft:
+            logging.info("Drafting release %s", version)
+        else:
+            logging.info("Creating release %s", version)
+        return repo.create_git_release(version, version, "", is_draft)
     except GithubException as e:
         errors = e.data.get("errors", [])
         if e.status == 422 and any(
@@ -129,6 +132,10 @@ def parse_args():
         "--dry-run", action="store_true", help="Do not create release or upload asset"
     )
 
+    parser.add_argument(
+        "--draft", action="store_true", help="Create a draft release"
+    )
+
     args = parser.parse_args()
     is_valid = True
     if not args.release_version:
@@ -292,7 +299,7 @@ def main():
                 for filename in onlyfiles:
                     binary_path = os.path.join(args.path, filename)
                     assert_asset_version(binary_path, args.release_version)
-                release = get_or_create_release(repo, args.release_version, args.dry_run)
+                release = get_or_create_release(repo, args.release_version, args.dry_run, args.draft)
                 for filename in onlyfiles:
                     binary_path = os.path.join(args.path, filename)
                     upload_asset(release, binary_path, filename, args.release_version, args.kv_account_id, args.namespace_id,