Compare commits

...

15 Commits

Author SHA1 Message Date
Sudarsan Reddy 6a6ba704f1 Release 2022.6.3 2022-06-20 13:52:31 +01:00
Sudarsan Reddy 135c8e6d13 TUN-6362: Add armhf support to cloudflare packaging
We now will have `armhf` based debs on our github pages

This will also sync to our R2 Release process allowing legacy rpi users to
eventually be able to apt-get install cloudflared.
2022-06-20 12:05:03 +01:00
Sudarsan Reddy 420e80ea50 Release 2022.6.2 2022-06-17 10:07:49 +01:00
Sudarsan Reddy 337591b2bb TUN-6414: Remove go-sumtype from cloudflared build process
This PR removes go-sumtype from cloudflared's build processes.

The value we see from analysing exhaustive match patterns in go is minimal (we can do this in code reviews) compared to using a tool that is unmaintained and (now broken) in Go 1.18.

We'd already been using the patched version here: https://github.com/sudarshan-reddy/go-sumtype because the original is broken for go tools > 1.16
2022-06-17 09:23:44 +01:00
Silver fa6bcdad04 Merge pull request #655 from legonigel/patch-1
Add image source label to docker container.
2022-06-16 12:23:32 -05:00
Silver ee87c43eb9 Merge pull request #656 from nikr-canva/http2-origins
Add Http2Origin option to force HTTP/2 origin connections
2022-06-16 12:23:07 -05:00
Silver bccc58b54d Merge pull request #641 from Albonycal/patch-1
Warp Private Network link updated
2022-06-16 12:21:15 -05:00
Igor Postelnik 3da1c25471 TUN-6381: Write error data on QUIC stream when we fail to talk to the origin; separate logging for protocol errors vs. origin errors. 2022-06-15 13:06:45 +00:00
Sudarsan Reddy 7d0a271000 Release 2022.6.1 2022-06-15 10:00:02 +01:00
Igor Postelnik 102631d98d TUN-6395: Fix writing RPM repo data 2022-06-14 22:35:01 -05:00
Niklas Rehfeld 5ed3d4e29a Update remaining OriginRequestConfig functions for Http2Origins 2022-06-09 16:44:40 +12:00
Niklas Rehfeld 2345720b2b fix ingress rules unit test 2022-06-02 09:49:06 +12:00
Niklas Rehfeld 7d4afd4ae0 Add Http2Origin option to force HTTP/2 origin connections
If `http2Origin` is set, it will set `ForceAttemptHTTP2` in the transport config of the `OriginService`.
2022-06-01 12:57:29 +12:00
Nigel Armstrong 056693c814 Add image source label to docker container.
This label allows tools like [whitesource renovate](https://docs.renovatebot.com/modules/datasource/#docker-datasource) to find changelogs.
2022-05-31 10:29:51 -07:00
Albony Cal f758361730 Warp Private Network link updated
The old one was returning 404
2022-05-10 19:40:31 +05:30
16 changed files with 188 additions and 89 deletions
-2
View File
@@ -12,8 +12,6 @@ jobs:
uses: actions/setup-go@v2
with:
go-version: ${{ matrix.go-version }}
- name: Install go-sumtype
run: go get github.com/sudarshan-reddy/go-sumtype
- name: Checkout code
uses: actions/checkout@v2
- name: Test
+2
View File
@@ -6,6 +6,8 @@ ENV GO111MODULE=on \
CGO_ENABLED=0 \
TARGET_GOOS=${TARGET_GOOS} \
TARGET_GOARCH=${TARGET_GOARCH}
LABEL org.opencontainers.image.source="https://github.com/cloudflare/cloudflared"
WORKDIR /go/src/github.com/cloudflare/cloudflared/
+12 -7
View File
@@ -98,6 +98,16 @@ else
TARGET_PUBLIC_REPO ?= $(FLAVOR)
endif
ifneq ($(TARGET_ARM), )
ARM_COMMAND := GOARM=$(TARGET_ARM)
endif
ifeq ($(TARGET_ARM), 7)
PACKAGE_ARCH := armhf
else
PACKAGE_ARCH := $(TARGET_ARCH)
endif
.PHONY: all
all: cloudflared test
@@ -111,7 +121,7 @@ ifeq ($(FIPS), true)
$(info Building cloudflared with go-fips)
cp -f fips/fips.go.linux-amd64 cmd/cloudflared/fips.go
endif
GOOS=$(TARGET_OS) GOARCH=$(TARGET_ARCH) go build -v -mod=vendor $(GO_BUILD_TAGS) $(LDFLAGS) $(IMPORT_PATH)/cmd/cloudflared
GOOS=$(TARGET_OS) GOARCH=$(TARGET_ARCH) $(ARM_COMMAND) go build -v -mod=vendor $(GO_BUILD_TAGS) $(LDFLAGS) $(IMPORT_PATH)/cmd/cloudflared
ifeq ($(FIPS), true)
rm -f cmd/cloudflared/fips.go
./check-fips.sh cloudflared
@@ -171,7 +181,7 @@ define build_package
--license 'Apache License Version 2.0' \
--url 'https://github.com/cloudflare/cloudflared' \
-m 'Cloudflare <support@cloudflare.com>' \
-a $(TARGET_ARCH) -v $(VERSION) -n $(DEB_PACKAGE_NAME) $(NIGHTLY_FLAGS) --after-install postinst.sh --after-remove postrm.sh \
-a $(PACKAGE_ARCH) -v $(VERSION) -n $(DEB_PACKAGE_NAME) $(NIGHTLY_FLAGS) --after-install postinst.sh --after-remove postrm.sh \
cloudflared=$(INSTALL_BINDIR) cloudflared.1=$(INSTALL_MANDIR)
endef
@@ -293,11 +303,6 @@ quic-deps:
.PHONY: vet
vet:
go vet -mod=vendor ./...
# go get github.com/sudarshan-reddy/go-sumtype (don't do this in build directory or this will cause vendor issues)
# Note: If you have github.com/BurntSushi/go-sumtype then you might have to use the repo above instead
# for now because it uses an older version of golang.org/x/tools.
which go-sumtype
go-sumtype $$(go list -mod=vendor ./...)
.PHONY: goimports
goimports:
+1 -1
View File
@@ -43,7 +43,7 @@ Once installed, you can authenticate `cloudflared` into your Cloudflare account
* Route traffic to that Tunnel:
* Via public [DNS records in Cloudflare](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/routing-to-tunnel/dns)
* Or via a public hostname guided by a [Cloudflare Load Balancer](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/routing-to-tunnel/lb)
* Or from [WARP client private traffic](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/configuration/private-networks)
* Or from [WARP client private traffic](https://developers.cloudflare.com/cloudflare-one/connections/connect-networks/private-net/)
## TryCloudflare
+15
View File
@@ -1,3 +1,18 @@
2022.6.3
- 2022-06-20 TUN-6362: Add armhf support to cloudflare packaging
2022.6.2
- 2022-06-13 TUN-6381: Write error data on QUIC stream when we fail to talk to the origin; separate logging for protocol errors vs. origin errors.
- 2022-06-17 TUN-6414: Remove go-sumtype from cloudflared build process
- 2022-06-01 Add Http2Origin option to force HTTP/2 origin connections
- 2022-06-02 fix ingress rules unit test
- 2022-06-09 Update remaining OriginRequestConfig functions for Http2Origins
- 2022-05-31 Add image source label to docker container.
- 2022-05-10 Warp Private Network link updated
2022.6.1
- 2022-06-14 TUN-6395: Fix writing RPM repo data
2022.6.0
- 2022-06-14 Revert "TUN-6010: Add component tests for --edge-ip-version"
- 2022-06-14 Revert "TUN-6373: Add edge-ip-version to remotely pushed configuration"
+9 -1
View File
@@ -17,10 +17,18 @@ for arch in ${windowsArchs[@]}; do
done
linuxArchs=("386" "amd64" "arm" "arm64")
linuxArchs=("386" "amd64" "arm" "armhf" "arm64")
export TARGET_OS=linux
for arch in ${linuxArchs[@]}; do
unset TARGET_ARM
export TARGET_ARCH=$arch
## Support for armhf builds
if [[ $arch == armhf ]] ; then
export TARGET_ARCH=arm
export TARGET_ARM=7
fi
make cloudflared-deb
mv cloudflared\_$VERSION\_$arch.deb $ARTIFACT_DIR/cloudflared-linux-$arch.deb
-1
View File
@@ -153,7 +153,6 @@ stretch: &stretch
- gotest-to-teamcity
pre-cache: &test_pre_cache
- go get golang.org/x/tools/cmd/goimports
- go get github.com/sudarshan-reddy/go-sumtype@v0.0.0-20210827105221-82eca7e5abb1
post-cache:
- export GOOS=linux
- export GOARCH=amd64
+7
View File
@@ -837,6 +837,13 @@ func configureProxyFlags(shouldHide bool) []cli.Flag {
EnvVars: []string{"TUNNEL_NO_CHUNKED_ENCODING"},
Hidden: shouldHide,
}),
altsrc.NewBoolFlag(&cli.BoolFlag{
Name: ingress.Http2OriginFlag,
Usage: "Enables HTTP/2 origin servers.",
EnvVars: []string{"TUNNEL_ORIGIN_ENABLE_HTTP2"},
Hidden: shouldHide,
Value: false,
}),
}
return append(flags, sshFlags(shouldHide)...)
}
+2
View File
@@ -227,6 +227,8 @@ type OriginRequestConfig struct {
ProxyType *string `yaml:"proxyType" json:"proxyType,omitempty"`
// IP rules for the proxy service
IPRules []IngressIPRule `yaml:"ipRules" json:"ipRules,omitempty"`
// Attempt to connect to origin with HTTP/2
Http2Origin *bool `yaml:"http2Origin" json:"http2Origin,omitempty"`
}
type IngressIPRule struct {
+3 -1
View File
@@ -144,7 +144,8 @@ var rawJsonConfig = []byte(`
"ports": [443, 4443],
"allow": true
}
]
],
"http2Origin": true
}
`)
@@ -191,6 +192,7 @@ func assertConfig(
assert.Equal(t, true, *config.NoTLSVerify)
assert.Equal(t, uint(9000), *config.ProxyPort)
assert.Equal(t, "socks", *config.ProxyType)
assert.Equal(t, true, *config.Http2Origin)
privateV4 := "10.0.0.0/8"
privateV6 := "fc00::/7"
+20 -9
View File
@@ -160,7 +160,7 @@ func (q *QUICConnection) handleStream(stream io.ReadWriteCloser) error {
case quicpogs.DataStreamProtocolSignature:
reqServerStream, err := quicpogs.NewRequestServerStream(stream, signature)
if err != nil {
return nil
return err
}
return q.handleDataStream(reqServerStream)
case quicpogs.RPCStreamProtocolSignature:
@@ -175,30 +175,41 @@ func (q *QUICConnection) handleStream(stream io.ReadWriteCloser) error {
}
func (q *QUICConnection) handleDataStream(stream *quicpogs.RequestServerStream) error {
connectRequest, err := stream.ReadConnectRequestData()
request, err := stream.ReadConnectRequestData()
if err != nil {
return err
}
if err := q.dispatchRequest(stream, err, request); err != nil {
_ = stream.WriteConnectResponseData(err)
q.logger.Err(err).Str("type", request.Type.String()).Str("dest", request.Dest).Msg("Request failed")
}
return nil
}
func (q *QUICConnection) dispatchRequest(stream *quicpogs.RequestServerStream, err error, request *quicpogs.ConnectRequest) error {
originProxy, err := q.orchestrator.GetOriginProxy()
if err != nil {
return err
}
switch connectRequest.Type {
switch request.Type {
case quicpogs.ConnectionTypeHTTP, quicpogs.ConnectionTypeWebsocket:
tracedReq, err := buildHTTPRequest(connectRequest, stream)
tracedReq, err := buildHTTPRequest(request, stream)
if err != nil {
return err
}
w := newHTTPResponseAdapter(stream)
return originProxy.ProxyHTTP(w, tracedReq, connectRequest.Type == quicpogs.ConnectionTypeWebsocket)
return originProxy.ProxyHTTP(w, tracedReq, request.Type == quicpogs.ConnectionTypeWebsocket)
case quicpogs.ConnectionTypeTCP:
rwa := &streamReadWriteAcker{stream}
metadata := connectRequest.MetadataMap()
return originProxy.ProxyTCP(context.Background(), rwa, &TCPRequest{Dest: connectRequest.Dest,
FlowID: metadata[QUICMetadataFlowID]})
metadata := request.MetadataMap()
return originProxy.ProxyTCP(context.Background(), rwa, &TCPRequest{
Dest: request.Dest,
FlowID: metadata[QUICMetadataFlowID],
})
}
return nil
}
+20
View File
@@ -36,6 +36,7 @@ const (
NoChunkedEncodingFlag = "no-chunked-encoding"
ProxyAddressFlag = "proxy-address"
ProxyPortFlag = "proxy-port"
Http2OriginFlag = "http2-origin"
)
const (
@@ -128,6 +129,7 @@ func originRequestFromSingeRule(c *cli.Context) OriginRequestConfig {
var proxyAddress = defaultProxyAddress
var proxyPort uint
var proxyType string
var http2Origin bool
if flag := ProxyConnectTimeoutFlag; c.IsSet(flag) {
connectTimeout = config.CustomDuration{Duration: c.Duration(flag)}
}
@@ -171,9 +173,13 @@ func originRequestFromSingeRule(c *cli.Context) OriginRequestConfig {
// Note TUN-3758 , we use Int because UInt is not supported with altsrc
proxyPort = uint(c.Int(flag))
}
if flag := Http2OriginFlag; c.IsSet(flag) {
http2Origin = c.Bool(flag)
}
if c.IsSet(Socks5Flag) {
proxyType = socksProxy
}
return OriginRequestConfig{
ConnectTimeout: connectTimeout,
TLSTimeout: tlsTimeout,
@@ -190,6 +196,7 @@ func originRequestFromSingeRule(c *cli.Context) OriginRequestConfig {
ProxyAddress: proxyAddress,
ProxyPort: proxyPort,
ProxyType: proxyType,
Http2Origin: http2Origin,
}
}
@@ -255,6 +262,9 @@ func originRequestFromConfig(c config.OriginRequestConfig) OriginRequestConfig {
}
}
}
if c.Http2Origin != nil {
out.Http2Origin = *c.Http2Origin
}
return out
}
@@ -298,6 +308,8 @@ type OriginRequestConfig struct {
ProxyType string `yaml:"proxyType" json:"proxyType"`
// IP rules for the proxy service
IPRules []ipaccess.Rule `yaml:"ipRules" json:"ipRules"`
// Attempt to connect to origin with HTTP/2
Http2Origin bool `yaml:"http2Origin" json:"http2Origin"`
}
func (defaults *OriginRequestConfig) setConnectTimeout(overrides config.OriginRequestConfig) {
@@ -403,6 +415,12 @@ func (defaults *OriginRequestConfig) setIPRules(overrides config.OriginRequestCo
}
}
func (defaults *OriginRequestConfig) setHttp2Origin(overrides config.OriginRequestConfig) {
if val := overrides.Http2Origin; val != nil {
defaults.Http2Origin = *val
}
}
// SetConfig gets config for the requests that cloudflared sends to origins.
// Each field has a setter method which sets a value for the field by trying to find:
// 1. The user config for this rule
@@ -428,6 +446,7 @@ func setConfig(defaults OriginRequestConfig, overrides config.OriginRequestConfi
cfg.setProxyAddress(overrides)
cfg.setProxyType(overrides)
cfg.setIPRules(overrides)
cfg.setHttp2Origin(overrides)
return cfg
}
@@ -475,6 +494,7 @@ func ConvertToRawOriginConfig(c OriginRequestConfig) config.OriginRequestConfig
ProxyPort: zeroUIntToNil(c.ProxyPort),
ProxyType: emptyStringToNil(c.ProxyType),
IPRules: convertToRawIPRules(c.IPRules),
Http2Origin: defaultBoolToNil(c.Http2Origin),
}
}
+1
View File
@@ -290,6 +290,7 @@ func newHTTPTransport(service OriginService, cfg OriginRequestConfig, log *zerol
TLSHandshakeTimeout: cfg.TLSTimeout.Duration,
ExpectContinueTimeout: 1 * time.Second,
TLSClientConfig: &tls.Config{RootCAs: originCertPool, InsecureSkipVerify: cfg.NoTLSVerify},
ForceAttemptHTTP2: cfg.Http2Origin,
}
if _, isHelloWorld := service.(*helloWorld); !isHelloWorld && cfg.OriginServerName != "" {
httpTransport.TLSClientConfig.ServerName = cfg.OriginServerName
+4 -4
View File
@@ -182,25 +182,25 @@ func TestMarshalJSON(t *testing.T) {
{
name: "Nil",
path: nil,
expected: `{"hostname":"example.com","path":null,"service":"https://localhost:8000","originRequest":{"connectTimeout":30,"tlsTimeout":10,"tcpKeepAlive":30,"noHappyEyeballs":false,"keepAliveTimeout":90,"keepAliveConnections":100,"httpHostHeader":"","originServerName":"","caPool":"","noTLSVerify":false,"disableChunkedEncoding":false,"bastionMode":false,"proxyAddress":"127.0.0.1","proxyPort":0,"proxyType":"","ipRules":null}}`,
expected: `{"hostname":"example.com","path":null,"service":"https://localhost:8000","originRequest":{"connectTimeout":30,"tlsTimeout":10,"tcpKeepAlive":30,"noHappyEyeballs":false,"keepAliveTimeout":90,"keepAliveConnections":100,"httpHostHeader":"","originServerName":"","caPool":"","noTLSVerify":false,"disableChunkedEncoding":false,"bastionMode":false,"proxyAddress":"127.0.0.1","proxyPort":0,"proxyType":"","ipRules":null,"http2Origin":false}}`,
want: true,
},
{
name: "Nil regex",
path: &Regexp{Regexp: nil},
expected: `{"hostname":"example.com","path":null,"service":"https://localhost:8000","originRequest":{"connectTimeout":30,"tlsTimeout":10,"tcpKeepAlive":30,"noHappyEyeballs":false,"keepAliveTimeout":90,"keepAliveConnections":100,"httpHostHeader":"","originServerName":"","caPool":"","noTLSVerify":false,"disableChunkedEncoding":false,"bastionMode":false,"proxyAddress":"127.0.0.1","proxyPort":0,"proxyType":"","ipRules":null}}`,
expected: `{"hostname":"example.com","path":null,"service":"https://localhost:8000","originRequest":{"connectTimeout":30,"tlsTimeout":10,"tcpKeepAlive":30,"noHappyEyeballs":false,"keepAliveTimeout":90,"keepAliveConnections":100,"httpHostHeader":"","originServerName":"","caPool":"","noTLSVerify":false,"disableChunkedEncoding":false,"bastionMode":false,"proxyAddress":"127.0.0.1","proxyPort":0,"proxyType":"","ipRules":null,"http2Origin":false}}`,
want: true,
},
{
name: "Empty",
path: &Regexp{Regexp: regexp.MustCompile("")},
expected: `{"hostname":"example.com","path":"","service":"https://localhost:8000","originRequest":{"connectTimeout":30,"tlsTimeout":10,"tcpKeepAlive":30,"noHappyEyeballs":false,"keepAliveTimeout":90,"keepAliveConnections":100,"httpHostHeader":"","originServerName":"","caPool":"","noTLSVerify":false,"disableChunkedEncoding":false,"bastionMode":false,"proxyAddress":"127.0.0.1","proxyPort":0,"proxyType":"","ipRules":null}}`,
expected: `{"hostname":"example.com","path":"","service":"https://localhost:8000","originRequest":{"connectTimeout":30,"tlsTimeout":10,"tcpKeepAlive":30,"noHappyEyeballs":false,"keepAliveTimeout":90,"keepAliveConnections":100,"httpHostHeader":"","originServerName":"","caPool":"","noTLSVerify":false,"disableChunkedEncoding":false,"bastionMode":false,"proxyAddress":"127.0.0.1","proxyPort":0,"proxyType":"","ipRules":null,"http2Origin":false}}`,
want: true,
},
{
name: "Basic",
path: &Regexp{Regexp: regexp.MustCompile("/echo")},
expected: `{"hostname":"example.com","path":"/echo","service":"https://localhost:8000","originRequest":{"connectTimeout":30,"tlsTimeout":10,"tcpKeepAlive":30,"noHappyEyeballs":false,"keepAliveTimeout":90,"keepAliveConnections":100,"httpHostHeader":"","originServerName":"","caPool":"","noTLSVerify":false,"disableChunkedEncoding":false,"bastionMode":false,"proxyAddress":"127.0.0.1","proxyPort":0,"proxyType":"","ipRules":null}}`,
expected: `{"hostname":"example.com","path":"/echo","service":"https://localhost:8000","originRequest":{"connectTimeout":30,"tlsTimeout":10,"tcpKeepAlive":30,"noHappyEyeballs":false,"keepAliveTimeout":90,"keepAliveConnections":100,"httpHostHeader":"","originServerName":"","caPool":"","noTLSVerify":false,"disableChunkedEncoding":false,"bastionMode":false,"proxyAddress":"127.0.0.1","proxyPort":0,"proxyType":"","ipRules":null,"http2Origin":false}}`,
want: true,
},
}
+14
View File
@@ -1,6 +1,8 @@
package quic
import (
"fmt"
capnp "zombiezen.com/go/capnproto2"
"zombiezen.com/go/capnproto2/pogs"
@@ -16,6 +18,18 @@ const (
ConnectionTypeTCP
)
func (c ConnectionType) String() string {
switch c {
case ConnectionTypeHTTP:
return "http"
case ConnectionTypeWebsocket:
return "ws"
case ConnectionTypeTCP:
return "tcp"
}
panic(fmt.Sprintf("invalid ConnectionType: %d", c))
}
// ConnectRequest is the representation of metadata sent at the start of a QUIC application handshake.
type ConnectRequest struct {
Dest string `capnp:"dest"`
+78 -63
View File
@@ -8,23 +8,24 @@
them to be in an uploadable state.
2. Upload these packages to a storage in a format that apt and yum expect.
"""
from subprocess import Popen, PIPE
import os
import argparse
import base64
from pathlib import Path
import logging
import os
import shutil
from hashlib import sha256
from pathlib import Path
from subprocess import Popen, PIPE
import gnupg
import boto3
import gnupg
from botocore.client import Config
from botocore.exceptions import ClientError
# The front facing R2 URL to access assets from.
R2_ASSET_URL = 'https://demo-r2-worker.cloudflare-tunnel.workers.dev/'
class PkgUploader:
def __init__(self, account_id, bucket_name, client_id, client_secret):
self.account_id = account_id
@@ -35,14 +36,14 @@ class PkgUploader:
def upload_pkg_to_r2(self, filename, upload_file_path):
endpoint_url = f"https://{self.account_id}.r2.cloudflarestorage.com"
token_secret_hash = sha256(self.client_secret.encode()).hexdigest()
config = Config(
region_name = 'auto',
region_name='auto',
s3={
"addressing_style": "path",
}
)
r2 = boto3.client(
"s3",
endpoint_url=endpoint_url,
@@ -57,7 +58,7 @@ class PkgUploader:
except ClientError as e:
raise e
class PkgCreator:
"""
The distribution conf is what dictates to reprepro, the debian packaging building
@@ -72,15 +73,16 @@ class PkgCreator:
description - (String)
gpg_key_id - gpg key id of what you want to use to sign the packages.(String)
"""
def create_distribution_conf(self,
file_path,
origin,
label,
releases,
archs,
components,
description,
gpg_key_id ):
def create_distribution_conf(self,
file_path,
origin,
label,
releases,
archs,
components,
description,
gpg_key_id):
with open(file_path, "w+") as distributions_file:
for release in releases:
distributions_file.write(f"Origin: {origin}\n")
@@ -102,6 +104,7 @@ class PkgCreator:
db and pool contain information and metadata about builds. We can ignore these.
dist: contains all the pkgs and signed releases that are necessary for an apt download.
"""
def create_deb_pkgs(self, release, deb_file):
print(f"creating deb pkgs: {release} : {deb_file}")
p = Popen(["reprepro", "includedeb", release, deb_file], stdout=PIPE, stderr=PIPE)
@@ -130,8 +133,9 @@ class PkgCreator:
gpgcheck=1
gpgkey=https://pkg.cloudflare.com/cloudflare-main.gpg
"""
def create_repo_file(self, file_path, binary_name, baseurl, gpgkey_url):
with open(file_path, "w+") as repo_file:
with open(os.path.join(file_path, binary_name + '.repo'), "w+") as repo_file:
repo_file.write(f"[{binary_name}-stable]")
repo_file.write(f"{binary_name}-stable")
repo_file.write(f"baseurl={baseurl}/rpm")
@@ -141,19 +145,19 @@ class PkgCreator:
repo_file.write(f"gpgkey={gpgkey_url}")
def _sign_rpms(self, file_path):
p = Popen(["rpm" , "--define", f"_gpg_name {gpg_key_name}", "--addsign", file_path], stdout=PIPE, stderr=PIPE)
p = Popen(["rpm", "--define", f"_gpg_name {gpg_key_name}", "--addsign", file_path], stdout=PIPE, stderr=PIPE)
out, err = p.communicate()
if p.returncode != 0:
print(f"rpm sign result result => {out}, {err}")
raise
def _sign_repomd(self):
p = Popen(["gpg", "--batch", "--detach-sign", "--armor", "./rpm/repodata/repomd.xml"], stdout=PIPE, stderr=PIPE)
p = Popen(["gpg", "--batch", "--detach-sign", "--armor", "./rpm/repodata/repomd.xml"], stdout=PIPE, stderr=PIPE)
out, err = p.communicate()
if p.returncode != 0:
print(f"sign repomd result => {out}, {err}")
raise
"""
sets up and signs the RPM directories in the following format:
- rpm
@@ -163,9 +167,10 @@ class PkgCreator:
this assumes the assets are in the format <prefix>-<aarch64/x86_64/386>.rpm
"""
def _setup_rpm_pkg_directories(self, artifacts_path, gpg_key_name, archs=["aarch64", "x86_64", "386"]):
for arch in archs:
for root, _ , files in os.walk(artifacts_path):
for root, _, files in os.walk(artifacts_path):
for file in files:
if file.endswith(f"{arch}.rpm"):
new_dir = f"./rpm/{arch}"
@@ -174,11 +179,12 @@ class PkgCreator:
new_path = os.path.join(new_dir, file)
shutil.copyfile(old_path, new_path)
self._sign_rpms(new_path)
"""
imports gpg keys into the system so reprepro and createrepo can use it to sign packages.
it returns the GPG ID after a successful import
"""
def import_gpg_keys(self, private_key, public_key):
gpg = gnupg.GPG()
private_key = base64.b64decode(private_key)
@@ -192,12 +198,13 @@ class PkgCreator:
basically rpm --import <key_file>
This enables us to sign rpms.
"""
def import_rpm_key(self, public_key):
file_name = "pb.key"
with open(file_name, "wb") as f:
public_key = base64.b64decode(public_key)
f.write(public_key)
p = Popen(["rpm", "--import", file_name], stdout=PIPE, stderr=PIPE)
out, err = p.communicate()
if p.returncode != 0:
@@ -212,18 +219,21 @@ class PkgCreator:
and the release will be uploaded to the default path.
binary: name of the binary to upload
"""
def upload_from_directories(pkg_uploader, directory, release, binary):
for root, _ , files in os.walk(directory):
for root, _, files in os.walk(directory):
for file in files:
upload_file_name = os.path.join(binary, root, file)
if release:
upload_file_name = os.path.join(release, upload_file_name)
filename = os.path.join(root,file)
try:
filename = os.path.join(root, file)
try:
pkg_uploader.upload_pkg_to_r2(filename, upload_file_name)
except ClientError as e:
logging.error(e)
return
return
"""
1. looks into a built_artifacts folder for cloudflared debs
@@ -237,19 +247,22 @@ def upload_from_directories(pkg_uploader, directory, release, binary):
release_version: is the cloudflared release version. Only provide this if you want a permanent backup.
"""
def create_deb_packaging(pkg_creator, pkg_uploader, releases, gpg_key_id, binary_name, archs, package_component, release_version):
def create_deb_packaging(pkg_creator, pkg_uploader, releases, gpg_key_id, binary_name, archs, package_component,
release_version):
# set configuration for package creation.
print(f"initialising configuration for {binary_name} , {archs}")
Path("./conf").mkdir(parents=True, exist_ok=True)
pkg_creator.create_distribution_conf(
"./conf/distributions",
binary_name,
binary_name,
releases,
archs,
package_component,
f"apt repository for {binary_name}",
gpg_key_id)
"./conf/distributions",
binary_name,
binary_name,
releases,
archs,
package_component,
f"apt repository for {binary_name}",
gpg_key_id)
# create deb pkgs
for release in releases:
@@ -266,6 +279,7 @@ def create_deb_packaging(pkg_creator, pkg_uploader, releases, gpg_key_id, binary
upload_from_directories(pkg_uploader, "dists", release_version, binary_name)
upload_from_directories(pkg_uploader, "pool", release_version, binary_name)
def create_rpm_packaging(
pkg_creator,
pkg_uploader,
@@ -294,60 +308,61 @@ def parse_args():
)
parser.add_argument(
"--bucket", default=os.environ.get("R2_BUCKET"), help="R2 Bucket name"
"--bucket", default=os.environ.get("R2_BUCKET"), help="R2 Bucket name"
)
parser.add_argument(
"--id", default=os.environ.get("R2_CLIENT_ID"), help="R2 Client ID"
"--id", default=os.environ.get("R2_CLIENT_ID"), help="R2 Client ID"
)
parser.add_argument(
"--secret", default=os.environ.get("R2_CLIENT_SECRET"), help="R2 Client Secret"
"--secret", default=os.environ.get("R2_CLIENT_SECRET"), help="R2 Client Secret"
)
parser.add_argument(
"--account", default=os.environ.get("R2_ACCOUNT_ID"), help="R2 Account Tag"
"--account", default=os.environ.get("R2_ACCOUNT_ID"), help="R2 Account Tag"
)
parser.add_argument(
"--release-tag", default=os.environ.get("RELEASE_VERSION"), help="Release version you want your pkgs to be\
"--release-tag", default=os.environ.get("RELEASE_VERSION"), help="Release version you want your pkgs to be\
prefixed with. Leave empty if you don't want tagged release versions backed up to R2."
)
parser.add_argument(
"--binary", default=os.environ.get("BINARY_NAME"), help="The name of the binary the packages are for"
"--binary", default=os.environ.get("BINARY_NAME"), help="The name of the binary the packages are for"
)
parser.add_argument(
"--gpg-private-key", default=os.environ.get("LINUX_SIGNING_PRIVATE_KEY"), help="GPG private key to sign the\
"--gpg-private-key", default=os.environ.get("LINUX_SIGNING_PRIVATE_KEY"), help="GPG private key to sign the\
packages"
)
parser.add_argument(
"--gpg-public-key", default=os.environ.get("LINUX_SIGNING_PUBLIC_KEY"), help="GPG public key used for\
"--gpg-public-key", default=os.environ.get("LINUX_SIGNING_PUBLIC_KEY"), help="GPG public key used for\
signing packages"
)
parser.add_argument(
"--gpg-public-key-url", default=os.environ.get("GPG_PUBLIC_KEY_URL"), help="GPG public key url that\
"--gpg-public-key-url", default=os.environ.get("GPG_PUBLIC_KEY_URL"), help="GPG public key url that\
downloaders can use to verify signing"
)
parser.add_argument(
"--pkg-upload-url", default=os.environ.get("PKG_URL"), help="URL to be used by downloaders"
"--pkg-upload-url", default=os.environ.get("PKG_URL"), help="URL to be used by downloaders"
)
parser.add_argument(
"--deb-based-releases", default=["bookworm", "bullseye", "buster", "jammy", "impish", "focal", "bionic",
"xenial", "trusty"],
help="list of debian based releases that need to be packaged for"
"--deb-based-releases", default=["bookworm", "bullseye", "buster", "jammy", "impish", "focal", "bionic",
"xenial", "trusty"],
help="list of debian based releases that need to be packaged for"
)
parser.add_argument(
"--archs", default=["amd64", "386", "arm64", "arm"], help="list of architectures we want to package for. Note that\
"--archs", default=["amd64", "386", "arm64", "arm", "armhf"], help="list of architectures we want to package for. Note that\
it is the caller's responsiblity to ensure that these debs are already present in a directory. This script\
will not build binaries or create their debs."
)
)
args = parser.parse_args()
return args
if __name__ == "__main__":
try:
args = parse_args()
@@ -362,15 +377,15 @@ if __name__ == "__main__":
pkg_uploader = PkgUploader(args.account, args.bucket, args.id, args.secret)
print(f"signing with gpg_key: {gpg_key_id}")
create_deb_packaging(pkg_creator, pkg_uploader, args.deb_based_releases, gpg_key_id, args.binary, args.archs,
"main", args.release_tag)
"main", args.release_tag)
create_rpm_packaging(
pkg_creator,
pkg_uploader,
"./built_artifacts",
args.release_tag,
args.binary,
gpg_key_name,
args.gpg_public_key_url,
args.pkg_upload_url,
)
pkg_creator,
pkg_uploader,
"./built_artifacts",
args.release_tag,
args.binary,
gpg_key_name,
args.gpg_public_key_url,
args.pkg_upload_url,
)