kernelCTF: add submission PR verification workflow

.github/workflows/kernelctf-submission-verification.yaml

@@ -0,0 +1,205 @@
+name: kernelCTF PR check
+on:
+  pull_request_target:
+    types: [opened, synchronize, reopened, labeled]
+    paths: [pocs/linux/kernelctf/**]
+  workflow_dispatch:
+    inputs:
+      prNumber:
+        description: 'PR number'
+        type: number
+        required: true
+permissions: {}
+env:
+  PR_REF: ${{ github.event_name == 'workflow_dispatch' && format('refs/pull/{0}/merge', github.event.inputs.prNumber) || github.event.pull_request.head.sha }}
+jobs:
+  structure_check:
+    # if labeling triggered the job then only run in case of the "recheck" label
+    if: github.event.action != 'labeled' || github.event.label.name == 'recheck'
+    runs-on: ubuntu-latest
+    outputs:
+      targets: ${{ steps.check_submission.outputs.targets }}
+      submission_dir: ${{ steps.check_submission.outputs.submission_dir }}
+    steps:
+      - run: pip install -U jsonschema
+
+      - name: Checkout repo content
+        uses: actions/checkout@v3
+
+      - name: Checkout PR content
+        uses: actions/checkout@v3
+        with:
+          path: pr
+          ref: ${{ env.PR_REF }}
+          fetch-depth: 0
+
+      - id: check_submission
+        name: Check submission
+        working-directory: pr
+        run: |
+          echo "::stop-commands::$(uuidgen)"
+          ../kernelctf/check-submission.py ${{ github.event.pull_request.base.sha }}
+
+  exploit_build:
+    runs-on: ubuntu-latest
+    needs: structure_check
+    strategy:
+      matrix:
+        target: ${{ fromJSON(needs.structure_check.outputs.targets) }}
+      fail-fast: false # do not cancel other targets
+    env:
+      RELEASE_ID: ${{ matrix.target }}
+      EXPLOIT_DIR: pr/pocs/linux/kernelctf/${{ needs.structure_check.outputs.submission_dir }}/exploit/${{ matrix.target }}
+    steps:
+      - name: Checkout PR content
+        uses: actions/checkout@v3
+        with:
+          path: pr
+          ref: ${{ env.PR_REF }}
+          fetch-depth: 0
+
+      - name: List files
+        run: find .
+
+      - name: Backup original exploit
+        run: mv $EXPLOIT_DIR/exploit ./
+
+      - name: Build exploit
+        id: build_exploit
+        working-directory: ${{ env.EXPLOIT_DIR }}
+        run: |
+          if make -n prerequisites; then
+            make prerequisites
+          fi
+          make exploit
+
+      - name: Upload exploit (newly compiled)
+        if: success()
+        uses: actions/upload-artifact@v3
+        with:
+          name: exploit_${{ env.RELEASE_ID }}
+          path: ${{ env.EXPLOIT_DIR }}/exploit
+          if-no-files-found: error
+
+      - name: Upload exploit (original, build failed)
+        if: failure() && steps.build_exploit.outcome == 'failure'
+        uses: actions/upload-artifact@v3
+        with:
+          name: exploit_${{ env.RELEASE_ID }}
+          path: ./exploit
+          if-no-files-found: error
+
+      - name: Summarize result (success)
+        if: success()
+        run: printf '✅ Exploit was built successfully.\n\nIt can be found under the artifacts (`exploit_${{ env.RELEASE_ID }}`).\n' >> $GITHUB_STEP_SUMMARY
+
+      - name: Summarize result (failure)
+        if: failure() && steps.build_exploit.outcome == 'failure'
+        run: printf '❌ The exploit compilation failed.\n\nPlease fix it.\n\nYou can see the build logs by clicking on `...` here and then on "View job logs". Or by selecting `exploit_build (${{ env.RELEASE_ID }})` under Jobs in the left menubar.\n' >> $GITHUB_STEP_SUMMARY
+
+  exploit_repro:
+    runs-on: ubuntu-22.04-4core
+    timeout-minutes: 30
+    needs: [structure_check, exploit_build]
+    strategy:
+      matrix:
+        target: ${{ fromJSON(needs.structure_check.outputs.targets) }}
+      fail-fast: false
+    if: always() && needs.structure_check.result == 'success'
+    env:
+      RELEASE_ID: ${{ matrix.target }}
+      SUBMISSION_DIR: ${{ needs.structure_check.outputs.submission_dir }}
+    steps:
+      - name: Checkout repo content
+        uses: actions/checkout@v3
+
+      - name: Install tools (QEMU, inotify, expect)
+        run: sudo apt-get update && sudo apt-get install -y qemu-system-x86 inotify-tools expect
+
+      - name: Enable KVM group perms
+        run: |
+          echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules
+          sudo udevadm control --reload-rules
+          sudo udevadm trigger --name-match=kvm
+
+      - name: Download exploit
+        uses: actions/download-artifact@v3
+        with:
+          name: exploit_${{ env.RELEASE_ID }}
+          path: exp/
+
+      - name: Fetch rootfs
+        run: |
+          wget https://storage.googleapis.com/kernelctf-build/files/rootfs_repro_v1.img.gz
+          mv rootfs_repro_v1.img.gz rootfs.img.gz
+          gzip -d rootfs.img.gz
+
+      - name: Download bzImage
+        run: |
+          if [ "$RELEASE_ID" == "mitigation-6.1" ]; then RELEASE_ID="mitigation-6.1-v2"; fi
+          wget https://storage.googleapis.com/kernelctf-build/releases/$RELEASE_ID/bzImage
+
+      # ugly hack to make Github Actions UI to show repro logs separately in somewhat readable fashion
+      - id: repro1
+        name: Reproduction (1 / 10)
+        continue-on-error: true
+        run: ./kernelctf/repro.sh 1
+
+      - id: repro2
+        name: Reproduction (2 / 10)
+        continue-on-error: true
+        run: ./kernelctf/repro.sh 2
+
+      - id: repro3
+        name: Reproduction (3 / 10)
+        continue-on-error: true
+        run: ./kernelctf/repro.sh 3
+
+      - id: repro4
+        name: Reproduction (4 / 10)
+        continue-on-error: true
+        run: ./kernelctf/repro.sh 4
+
+      - id: repro5
+        name: Reproduction (5 / 10)
+        continue-on-error: true
+        run: ./kernelctf/repro.sh 5
+
+      - id: repro6
+        name: Reproduction (6 / 10)
+        continue-on-error: true
+        run: ./kernelctf/repro.sh 6
+
+      - id: repro7
+        name: Reproduction (7 / 10)
+        continue-on-error: true
+        run: ./kernelctf/repro.sh 7
+
+      - id: repro8
+        name: Reproduction (8 / 10)
+        continue-on-error: true
+        run: ./kernelctf/repro.sh 8
+
+      - id: repro9
+        name: Reproduction (9 / 10)
+        continue-on-error: true
+        run: ./kernelctf/repro.sh 9
+
+      - id: repro10
+        name: Reproduction (10 / 10)
+        continue-on-error: true
+        run: ./kernelctf/repro.sh 10
+
+      - name: Upload repro QEMU logs as an artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: repro_logs_${{ env.RELEASE_ID }}
+          path: repro_log_*.txt
+
+      - name: Reproduction // Summary
+        env:
+          STEPS: ${{ toJSON(steps) }}
+        run: |
+          env
+          echo $STEPS >> steps.json
+          ./kernelctf/repro_summary.py ${{ github.run_id }}

kernelctf/check-submission.py

@@ -0,0 +1,220 @@
+#!/usr/bin/env -S python3 -u
+import os
+import re
+import subprocess
+import sys
+import json
+import jsonschema
+import requests
+import csv
+import io
+import hashlib
+
+PUBLIC_CSV_URL = "https://docs.google.com/spreadsheets/d/e/2PACX-1vS1REdTA29OJftst8xN5B5x8iIUcxuK6bXdzF8G1UXCmRtoNsoQ9MbebdRdFnj6qZ0Yd7LwQfvYC2oF/pub?output=csv"
+POC_FOLDER = "pocs/linux/kernelctf/"
+EXPLOIT_DIR = "exploit/"
+DEBUG = "--debug" in sys.argv
+
+errors = []
+warnings = []
+
+def error(msg):
+    global errors
+    errors.append(msg)
+    print("\n[!] [ERROR] " + msg.replace('\n', '\n    '))
+
+def warning(msg):
+    global warnings
+    warnings.append(msg)
+    print("\n[!] [WARN] " + msg.replace('\n', '\n    '))
+
+def fail(msg):
+    print("\n[!] [FAIL] " + msg.replace('\n', '\n    '))
+    os._exit(1)
+
+def run(cmd):
+    try:
+        result = subprocess.check_output(cmd, shell=True).decode('utf-8').split('\n')
+        return result if result[-1] != "" else result[0:-1]
+    except subprocess.CalledProcessError as e:
+        fail(f"executing '{cmd}' failed with exit code {e.returncode}")
+
+def subdirEntries(files, subdir):
+    return list(set([f[len(subdir):].split('/')[0] for f in files if f.startswith(subdir)]))
+
+def formatList(items):
+    return ''.join([f"\n - {item}" for item in items])
+
+def printList(title, items):
+    print(f"\n{title}:" + formatList(items))
+
+def errorList(errorMsg, items, warningOnly=False):
+    itemsStr = ", ".join(f"`{x}`" for x in items)
+    errorMsg = errorMsg.replace("<LIST>", itemsStr) if "<LIST>" in errorMsg else f"{errorMsg}: {itemsStr}"
+    if warningOnly:
+        warning(errorMsg)
+    else:
+        error(errorMsg)
+
+def checkOnlyOne(list, errorMsg):
+    if len(list) > 1:
+        errorList(errorMsg, list)
+    return list[0]
+
+def checkList(items, isAllowedFunc, errorMsg, warningOnly=False):
+    disallowedItems = [item for item in items if not isAllowedFunc(item)]
+    if len(disallowedItems) > 0:
+        errorList(errorMsg, disallowedItems, warningOnly)
+    return list(sorted(set(items) - set(disallowedItems)))
+
+def checkAtLeastOne(list, errorMsg):
+    if len(list) == 0:
+        fail(errorMsg)
+
+def checkRegex(text, pattern, errorMsg):
+    m = re.match(pattern, text)
+    if not m:
+        error(f"{errorMsg}. Must match regex `{pattern}`")
+    return m
+
+def fetch(url):
+    response = requests.get(url)
+    if response.status_code != 200:
+        fail(f"expected 200 OK for request: {url}")
+    return response.content.decode('utf-8')
+
+def parseCsv(csvContent):
+    columns, *rows = list(csv.reader(io.StringIO(csvContent), strict=True))
+    return [{ columns[i]: row[i] for i in range(len(columns)) } for row in rows]
+
+argv = [arg for arg in sys.argv if not arg.startswith("--")]
+print(f"[-] Argv: {argv}")
+
+mergeInto = argv[1] if len(argv) >= 2 else "origin/main"
+print(f"[-] Params: mergeInto = {mergeInto}")
+
+mergeBase = run(f"git merge-base HEAD {mergeInto}")[0]
+print(f"[-] mergeBase = {mergeBase}")
+
+prFiles = run(f"git diff --name-only {mergeBase}")
+
+checkAtLeastOne(prFiles, "There are no files in the submission")
+prFiles = checkList(prFiles, lambda f: f.startswith(POC_FOLDER), f"The following files are outside of the `{POC_FOLDER}` folder which is not allowed")
+
+subDirName = checkOnlyOne(subdirEntries(prFiles, POC_FOLDER), "Only one submission is allowed per PR. Found multiple submissions")
+checkRegex(subDirName, r"^CVE-\d+-\d+(_lts|_cos|_mitigation)+$", f"The submission folder name is invalid (`{subDirName}`)")
+
+print(f"[-] Processing submission... Folder = {subDirName}")
+cve, *targets = subDirName.split('_')
+submissionFolder = f"{POC_FOLDER}{subDirName}/"
+files = [f[len(submissionFolder):] for f in prFiles]
+printList("Submission files", files)
+
+exploitFolders = subdirEntries(files, EXPLOIT_DIR)
+printList("Exploit folders", exploitFolders)
+
+validExploitFolderPrefixes = [f"{t}-" for t in targets] + ["extra-"]
+checkList(exploitFolders, lambda f: any(f.startswith(p) for p in validExploitFolderPrefixes),
+    f"The submission folder name (`{subDirName}`) is not consistent with the exploits in the `{EXPLOIT_DIR}` folder. " +
+    f"I expected the subfolders to be prefixed with one of these: [{', '.join(f'`{x}`' for x in validExploitFolderPrefixes)}], " +
+    "but this is not true for the following entries: <LIST>. You can put the extra files into a folder prefixed with `extra-`, " + 
+    "but try to make it clear what's the difference between this exploit and the others.")
+
+reqFilesPerExploit = ["Makefile", "exploit.c", "exploit"]
+
+checkList(["metadata.json", "docs/vulnerability.md"], lambda f: f in files, "The following files are missing")
+if "docs/exploit.md" not in files:
+    warning("docs/exploit.md was not found, expecting per-exploit exploit.md")
+    reqFilesPerExploit.append("exploit.md")
+
+for exploitFolder in exploitFolders:
+    checkList(reqFilesPerExploit, lambda file: f"{EXPLOIT_DIR}{exploitFolder}/{file}",
+        f"The following files are missing from exploit ({exploitFolder})")
+
+with open(f"{submissionFolder}metadata.json", "rt") as f: metadata = json.load(f)
+print("\nMetadata:\n" + json.dumps(metadata, indent=4) + "\n")
+
+schemaUrl = checkRegex(metadata["$schema"], r"^https://google.github.io/security-research/kernelctf/metadata.schema.v\d+.json$",
+                       "The `$schema` field of the `metadata.yaml` file is invalid").group(0)
+if schemaUrl:
+    if DEBUG:
+        with open("metadata.schema.v1.json", "rt") as f: schema = json.load(f)
+    else:
+        schema = json.loads(fetch(schemaUrl))
+
+    metadataErrors = list(jsonschema.Draft202012Validator(schema).iter_errors(metadata))
+    if len(metadataErrors) > 0:
+        for err in metadataErrors:
+            error(f"Schema validation of `metadata.json` failed with the following errors: {err}")
+
+submissionIds = metadata.get("submission_ids", None) or metadata["submission_id"]
+if isinstance(submissionIds, str):
+    submissionIds = [submissionIds]
+print(f"[-] Submission IDs = {submissionIds}")
+
+if DEBUG:
+    with open("public.csv", "rt") as f: publicCsv = f.read()
+else:
+    publicCsv = fetch(PUBLIC_CSV_URL)
+
+publicSheet = { x["ID"]: x for x in parseCsv(publicCsv) }
+# print(json.dumps(publicSheet, indent=4))
+
+for submissionId in set(submissionIds).difference(publicSheet.keys()):
+    fail(f"submission ID ({submissionId}) was not found on public spreadsheet")
+
+submissionIds = list(set(submissionIds).intersection(publicSheet.keys()))
+
+flags = []
+for submissionId in submissionIds:
+    publicData = publicSheet[submissionId]
+    is0Day = publicData["0-day / 1-day"] == "0-day"
+    exploitHash = publicData["Exploit hash"]
+    archiveFn = "original.tar.gz" if len(submissionIds) == 1 else f"original_{submissionId}.tar.gz"
+
+    if exploitHash != "":
+        if archiveFn not in files:
+            if not is0Day:
+                warning(f"The file `{archiveFn}` is missing, but submission is not a 0-day submission, so skipping exploit hash verification.")
+            else:
+                error(f"The file `{archiveFn}` is missing. Expected file with SHA256 hash of `{exploitHash}`.")
+        else:
+            with open(f"{submissionFolder}{archiveFn}", "rb") as f: originalTarGz = f.read()
+            calculated = hashlib.sha256(originalTarGz).hexdigest()
+
+            if exploitHash != calculated:
+                error(f"Expected `{archiveFn}` with SHA256 hash of `{exploitHash}`, but the file's checksum is `{calculated}`.")
+
+    flags.extend(publicData["Flags"].strip().split('\n'))
+
+    if cve != publicData["CVE"]:
+        error(f"The CVE on the public spreadsheet for submission `{submissionId}` is `{publicData['CVE']}` but the PR is for `{cve}`.")
+
+flagTargets = set([checkRegex(flag, r"kernelCTF\{v1:([^:]+):\d+\}", f"The flag (`{flag}`) is invalid").group(1) for flag in flags])
+if "mitigation-6.1-v2" in flagTargets:
+    flagTargets = flagTargets - set(["mitigation-6.1-v2"]) | set(["mitigation-6.1"])
+print(f"[-] Got flags for the following targets: {', '.join(flagTargets)}")
+checkList(flagTargets, lambda t: t in exploitFolders, f"Missing exploit for target(s)")
+checkList(exploitFolders, lambda t: t in flagTargets, f"Found extra exploit(s) without flag submission", True)
+
+def ghSet(varName, content):
+    varName = f"GITHUB_{varName}"
+    print(f"[+] Writing {json.dumps(content)} to ${varName}")
+    if varName in os.environ:
+        with open(os.environ[varName], 'at') as f: f.write(content + "\n")
+
+def summary(success, text):
+    if warnings:
+        text += "\n\n**Warnings:**\n" + '\n'.join(f" - ⚠️ {x}" for x in warnings)
+
+    ghSet("STEP_SUMMARY", text)
+    print(f"\n[+] {text}") if success else fail(text)
+
+if len(errors) > 0:
+    summary(False, f"The file structure verification of the PR failed with the following errors:{formatList(f'❌ {e}' for e in errors)}")
+
+ghSet("OUTPUT", "targets=" + json.dumps([f for f in exploitFolders if not f.startswith("extra-")]))
+ghSet("OUTPUT", f"submission_dir={subDirName}")
+
+summary(True, f"✅ The file structure verification of the PR was successful!")
+

kernelctf/repro.sh

@@ -0,0 +1,78 @@
+#!/bin/bash
+set -e
+
+TRY_ID="$1"
+STDOUT_TIMEOUT=60
+QEMU_TXT=qemu.txt
+
+FLAG="kernelCTF{$(uuidgen)}"
+#echo "::add-mask::$FLAG"
+echo $FLAG > flag
+
+STOP_MARKER=$(uuidgen)
+echo "::stop-commands::$STOP_MARKER"
+
+touch $QEMU_TXT
+
+START_TIME=$(date +%s)
+
+expect -c '
+    set timeout -1
+    set stty_init raw
+
+    spawn qemu-system-x86_64 -m 3.5G -nographic \
+    -monitor none \
+    -enable-kvm -cpu host -smp cores=2 \
+    -kernel bzImage \
+    -nic user,model=virtio-net-pci \
+    -drive file=rootfs.img,if=virtio,cache=none,aio=native,format=raw,discard=on,readonly=on \
+    -drive file=flag,if=virtio,format=raw,readonly=on \
+    -virtfs local,path=exp,mount_tag=exp,security_model=none \
+    -append "console=ttyS0 root=/dev/vda1 rootfstype=ext4 rootflags=discard ro init=/init hostname=repro" \
+    -nographic -no-reboot
+
+    expect "# "
+    send "id\n"
+
+    expect "# "
+    send "cat /flag\n"
+
+    expect "# "
+    send "exit\n"
+
+    expect eof
+' | tee $QEMU_TXT | sed $'s/\r//' &
+QEMU_PID="$!"
+
+while true; do
+    # check if qemu.txt modified within $STDOUT_TIMEOUT seconds
+    inotifywait -qq -t $STDOUT_TIMEOUT -e modify $QEMU_TXT &
+
+    # wait for either QEMU or inotifywait to exit
+    if ! wait -n $QEMU_PID $!; then break; fi
+
+    # exit loop if QEMU exited already
+    if ! ps -p $QEMU_PID > /dev/null; then break; fi
+done
+
+if ps -p $QEMU_PID > /dev/null; then
+    echo "Repro error: no stdout response within the expected timeout of $STDOUT_TIMEOUT seconds"
+    echo "Killing QEMU..."
+    kill -9 $QEMU_PID
+else
+    echo "QEMU exited cleanly"
+fi
+
+echo "::$STOP_MARKER::"
+
+cp $QEMU_TXT repro_log_$TRY_ID.txt
+# echo "QEMU_OUTPUT_B64=$(cat $QEMU_TXT|base64 -w0)" >> "$GITHUB_OUTPUT"
+echo "RUN_TIME=$(expr $(date +%s) - $START_TIME)" >> "$GITHUB_OUTPUT"
+
+if grep -q $FLAG $QEMU_TXT; then
+    echo "Got the flag! Congrats!"
+    exit 0
+else
+    echo "Failed, did not get the flag."
+    exit 1
+fi

kernelctf/repro_summary.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env -S python3 -u
+import base64
+import json
+import os
+import re
+
+with open("steps.json", "rt") as f: steps = json.loads(f.read())
+
+repros = [{ "idx": id[len("repro"):], **step } for (id,step) in steps.items() if id.startswith("repro")]
+print(repros)
+
+success_count = 0
+for repro in repros:
+    success = repro["outcome"] == "success"
+    repro["icon"] = "✅" if success else "❌"
+    success_count += 1 if success else 0
+
+result = f"""
+# Reproduction summary
+
+Reliability: {'%d' % (success_count / len(repros) * 100)}%
+
+Runs: {' '.join(x['icon'] for x in repros)}"""
+
+for repro in repros:
+    result += f"\n\n## Reproduction {repro['idx']} / {len(repros)} - {repro['icon']}\n\n"
+
+    run_time = repro["outputs"].get("RUN_TIME")
+    if run_time:
+        result += f"Time: {run_time}s\n\n"
+
+    repro_log_fn = f"repro_log_{repro['idx']}.txt"
+    if os.path.isfile(repro_log_fn):
+        with open(repro_log_fn, 'rb') as f: repro_log = f.read()
+
+        repro_log = repro_log.replace(b'\r\r\n', b'\n').replace(b'\r\n', b'\n').decode('utf-8')
+
+        def split(pattern, last=False):
+            arr = repro_log.rsplit(pattern, 1) if last else repro_log.split(pattern, 1)
+            return arr[1].strip() if len(arr) == 2 else ""
+
+        def getLastLine(pattern):
+            return split(pattern, True).split('\n')[0].strip()
+
+        panic = getLastLine('Kernel panic - ')
+        if "Attempted to kill init!" in panic:
+            result += f"The kernel did not panic (init exited).\n\n"
+        elif panic:
+            result += f"Kernel panic: `{panic}`\n\nRIP: `{getLastLine('RIP: ')}`\n\n"
+
+        repro_error = getLastLine('Repro error: ')
+        if repro_error:
+            result += f"Error during reproduction: `{repro_error}`.\n\n"
+
+        expl_out = split('su user -c /tmp/exp/exploit')
+
+        m = re.search(r"exploit.*?: (segfault at.*)", expl_out)
+        if m:
+            result += f"The exploit crashed: `{m.groups()[0]}`.\n\n"
+
+        if expl_out:
+            result += f"""
+<details>
+    <summary>Exploit / QEMU output</summary>
+
+```
+{expl_out.replace('`', '')}
+```
+
+</details>
+"""
+
+print(result)
+
+if "GITHUB_STEP_SUMMARY" in os.environ:
+    with open(os.environ["GITHUB_STEP_SUMMARY"], 'at') as f: f.write(result.strip() + "\n")
+
+os._exit(1 if success_count == 0 else 0)