diff --git a/.github/workflows/validate-and-build.yml b/.github/workflows/validate-and-build.yml deleted file mode 100644 index a21d963981..0000000000 --- a/.github/workflows/validate-and-build.yml +++ /dev/null @@ -1,619 +0,0 @@ -#This file makes use of a number of useful, external Github Actions. -#Check the links below for additional documentation on each of these: -#https://github.com/actions/setup-python -#https://github.com/actions/setup-node -#https://github.com/actions/checkout -#https://github.com/actions/upload-artifact - -#The mechanism for persisting data between jobs in a workflow is the same as for persisting it -#permanently: -#https://docs.github.com/en/actions/guides/storing-workflow-data-as-artifacts -#In CircleCI, this was different (store_artifacts vs persist_to_workspace) - - - -name: validate-and-build -on: [push, pull_request] -jobs: - validate-tag-if-present: - runs-on: ubuntu-latest - - steps: - - name: TAGGED, Validate that the tag is in the correct format - - run: | - echo "The GITHUB_REF: $GITHUB_REF" - #First check to see if the release is a tag - if [[ $GITHUB_REF =~ refs/tags/* ]]; then - #Yes, this is a tag, so we need to test to make sure that the tag - #is in the correct format (like v1.10.20) - if [[ $GITHUB_REF =~ refs/tags/v[0-9]+.[0-9]+.[0-9]+ ]]; then - echo "PASS: Tagged release with good format" - exit 0 - else - echo "FAIL: Tagged release with bad format" - exit 1 - fi - else - echo "PASS: Not a tagged release" - exit 0 - fi - - validate-content: - #Note that the CircleCI job used a Container. The way to do this with Github Actions - #is to first start up a Virtual Machine, then we can by following: - # https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#jobsjob_idcontainer - runs-on: ubuntu-latest - needs: [validate-tag-if-present] - steps: - #Previous config chose which branch/tag to operate on. - #I think Github is smart enough to choose based on whether it's a pull request or push + other info? - - name: Check out the repository code - uses: actions/checkout@v2 - #with: - # repository: splunk/security-content #check out https://github.com/mitre/cti.git, defaults to HEAD - # path: "security-content" - - - - uses: actions/setup-python@v2 - with: - python-version: '3.9' #Available versions here - https://github.com/actions/python-versions/releases easy to change/make a matrix/use pypy - architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified - - - #TODO: CircleCI restore_cache equivalent - - #don't need to install python3 or python3-dev since it was handled by the action above? - #Also, no support for YAML anchors/aliases in Github Actions... - - name: Install System Packages - run: | - sudo apt update -qq - sudo apt install jq -qq - #TODO: CircleCI save_cache equivalent - - - name: Install Python Dependencies - run: | - #Get the virtualenv set up - rm -rf venv - python3 -m venv --clear venv - source venv/bin/activate - python3 -m pip install -q -r requirements.txt - - - - name: run validate - run: | - source venv/bin/activate - python3 contentctl.py --path . --verbose validate - - - name: Get CTI Repo for Mitre context - uses: actions/checkout@v2 - with: - repository: mitre/cti #check out https://github.com/mitre/cti.git, defaults to HEAD - path: "cti/" - - - #Now generate the documentation (uses Node) - - uses: actions/setup-node@v2 - with: - node-version: '14' #can easily be changed to a different version - - name: Generate documentation - run: | - ls -lah - - #Enter the virtualenv and run the docgen - source venv/bin/activate - python3 bin/doc_gen.py --path . --output docs -v - - #Now generate the spec docs - npm install -g @adobe/jsonschema2md - jsonschema2md -d spec -o docs/spec -f yaml -e spec.json -x - - - #Clean up extra properties on docs - rm -rf docs/spec/*-*.md - - echo "****** BRANCH INFORMATION ******" - git branch - git branch --show-current - - build-sources: - runs-on: ubuntu-latest - needs: validate-content - steps: - - name: Checkout Repo - uses: actions/checkout@v2 - - - name: Install System Packages - run: | - sudo apt update -qq - sudo apt install jq -qq - - - name: Install Python Dependencies - run: | - #Get the virtualenv set up - rm -rf venv - python3 -m venv --clear venv - source venv/bin/activate - python3 -m pip install -q -r requirements.txt - - - name: Run Generate - run: | - source venv/bin/activate - python3 contentctl.py --path . --verbose generate --product ESCU --output dist/escu - python3 contentctl.py --path . --verbose generate --product SAAWS --output dist/saaws - python3 contentctl.py --path . --verbose generate --product DevSecOps --output dist/devsecops - #make a copy of use_case_lib in order to have ES work :-( - cp dist/escu/default/use_case_library.conf dist/escu/default/analyticstories.conf - cp dist/saaws/default/use_case_library.conf dist/saaws/default/analyticstories.conf - cp dist/devsecops/default/use_case_library.conf dist/devsecops/default/analyticstories.conf - - - name: Copy lookups .csv files - run: | - # clean up current lookups - rm -rf dist/escu/lookups - rm -rf dist/saaws/lookups - rm -rf dist/devsecops/lookups - mkdir dist/escu/lookups - mkdir dist/saaws/lookups - mkdir dist/devsecops/lookups - #copy over lookups - cd lookups - cp -rv *.csv ../dist/escu/lookups - cp -rv *.csv ../dist/saaws/lookups - cp -rv *.csv ../dist/devsecops/lookups - - #Tag is '' for non-tagged push and the tag name for a tagged release - - name: Set tag - id: vars - run: | - if [ echo ${GITHUB_REF} | grep "^refs/tags/*" ]; then - #failed to find the refs/tags/ beginning, grab and set the tag - echo "::set-output name=tag::${GITHUB_REF#refs/tags/}" - else - #Not a tagged relese - echo "::set-output name=tag::" - fi - - - name: Update Version and Build number - run : | - # check if tag is set, get build number from the tag if set - if [ -z "${{ steps.vars.outputs.tag }}" ]; then - CONTENT_VERSION=$(grep -oP "(\d+.\d+.\d+$)" dist/escu/default/content-version.conf) - echo "detected content version: $CONTENT_VERSION" - else - CONTENT_VERSION=$(echo ${{ steps.vars.outputs.tag }} | grep -oP "\d+.\d+.\d+") - echo "content version: $CONTENT_VERSION, set by tag: ${{ steps.vars.outputs.tag }}" - fi - # update build number and version for ESCU - sed -i "s/build = .*$/build = ${{ github.run_number }}/g" dist/escu/default/app.conf - sed -i "s/^version = .*$/version = $CONTENT_VERSION/g" dist/escu/default/app.conf - sed -i "s/\"version\": .*$/\"version\": \"$CONTENT_VERSION\"/g" dist/escu/app.manifest - sed -i "s/version = .*$/version = $CONTENT_VERSION/g" dist/escu/default/content-version.conf - tar -czf content-pack-build-escu.tar.gz dist/escu/* - # update build number and version for saaws - sed -i "s/build = .*$/build = ${{ github.run_number }}/g" dist/saaws/default/app.conf - sed -i "s/^version = .*$/version = $CONTENT_VERSION/g" dist/saaws/default/app.conf - sed -i "s/\"version\": .*$/\"version\": \"$CONTENT_VERSION\"/g" dist/saaws/app.manifest - sed -i "s/version = .*$/version = $CONTENT_VERSION/g" dist/saaws/default/content-version.conf - tar -czf content-pack-build-saaws.tar.gz dist/saaws/* - - # update build number and version for devsecops - tar -czf content-pack-build-devsecops.tar.gz dist/devsecops/* - - - name: Persist to Workspace - uses: actions/upload-artifact@v2 - with: - name: content-pack-build - path: | - content-pack-build-escu.tar.gz - content-pack-build-saaws.tar.gz - content-pack-build-devsecops.tar.gz - - - build-package: - runs-on: ubuntu-latest - needs: [validate-content, build-sources] - - steps: - - uses: actions/download-artifact@v2 - with: - name: content-pack-build - path: build/ - - #This explicitly uses a different version of python (2.7) - - uses: actions/setup-python@v2 - with: - python-version: '2.7' #Available versions here - https://github.com/actions/python-versions/releases easy to change/make a matrix/use pypy - architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified - - - name: Get virtualenv for Python 2.7 - run: | - sudo apt install virtualenv - - - name: Grab Splunk Packaging Toolkit - run : | - curl -Ls https://download.splunk.com/misc/packaging-toolkit/splunk-packaging-toolkit-0.9.0.tar.gz -o splunk-packaging-toolkit-latest.tar.gz - mkdir slim-latest - tar -zxf splunk-packaging-toolkit-latest.tar.gz -C slim-latest --strip-components=1 - - - name: Install Splunk Packaging Toolkit (slim) - run: | - cd slim-latest - virtualenv --python=/usr/bin/python2.7 --clear venv - source venv/bin/activate - python -m pip install semantic_version - python -m pip install . - - name: Create a .spl for this Build Using Slim - run: | - source slim-latest/venv/bin/activate - cd build - tar -zxf content-pack-build-escu.tar.gz - tar -zxf content-pack-build-saaws.tar.gz - tar -zxf content-pack-build-devsecops.tar.gz - mv dist/escu DA-ESS-ContentUpdate - mv dist/saaws DA-ESS_AmazonWebServices_Content - mv dist/devsecops devsecops - slim package -o upload DA-ESS-ContentUpdate - slim package -o upload DA-ESS_AmazonWebServices_Content - slim package -o upload devsecops - - cp upload/DA-ESS-ContentUpdate-*.tar.gz DA-ESS-ContentUpdate-latest.tar.gz - sha256sum DA-ESS-ContentUpdate-latest.tar.gz > checksum.txt - - cp upload/DA-ESS_AmazonWebServices_Content-*tar.gz DA-ESS_AmazonWebServices_Content-latest.tar.gz - sha256sum DA-ESS_AmazonWebServices_Content-latest.tar.gz >> checksum.txt - - cp upload/devsecops-*tar.gz devsecops.tar.gz - - - name: store_artifacts - uses: actions/upload-artifact@v2 - with: - name: package - path: | - build/upload - - name: store_artifacts_two - uses: actions/upload-artifact@v2 - with: - name: content-latest - path: | - build/DA-ESS-ContentUpdate-latest.tar.gz - build/DA-ESS_AmazonWebServices_Content-latest.tar.gz - build/devsecops.tar.gz - build/checksum.txt - - - run-appinspect: - runs-on: ubuntu-latest - needs: [validate-content, build-sources, build-package] - #Only run when tagged - if: startsWith(github.ref, 'refs/tags/v') - steps: - - - name: Checkout Repo - uses: actions/checkout@v2 - with: - ref: 'develop' - - #Download the artifacts we want to check - - name: Restore Content-Pack Artifacts for AppInspect testing - uses: actions/download-artifact@v2 - with: - name: content-latest - path: build/ - - - - - name: Install System Packages - run: | - sudo apt update -qq - sudo apt install jq -qq - - - - - name: Submit ESCU Package to AppInspect API - env: - APPINSPECT_USERNAME: ${{ secrets.AppInspectUsername }} - APPINSPECT_PASSWORD: ${{ secrets.AppInspectPassword }} - run: | - cd bin - #Enclose in quotes in case there are any special characters in the username/password - #Better not to pass these arguments on the command line, if possible - ./appinspect.sh ../ DA-ESS-ContentUpdate-latest.tar.gz "$APPINSPECT_USERNAME" "$APPINSPECT_PASSWORD" - - - name: Submit SAAWS Package to AppInspect API - env: - APPINSPECT_USERNAME: ${{ secrets.AppInspectUsername }} - APPINSPECT_PASSWORD: ${{ secrets.AppInspectPassword }} - run: | - cd bin - ./appinspect.sh ../ DA-ESS_AmazonWebServices_Content-latest.tar.gz "$APPINSPECT_USERNAME" "$APPINSPECT_PASSWORD" - - - name: Create report artifact - if: always() - run: | - #Always create this, regardless of whether success or failure above - tar -cvzf report.tar.gz report/ - - - name: store_artifacts - uses: actions/upload-artifact@v2 - with: - name: appinspect_reports - path: | - report.tar.gz - - #Still store the report, even if we have failed (otherwise we don't know why/how we failed) - - name: store_artifacts_on_failure - uses: actions/upload-artifact@v2 - if: failure() - with: - name: appinspect_reports_failure - path: | - report.tar.gz - - create-report: - runs-on: ubuntu-latest - needs: [validate-content, build-sources, build-package, run-appinspect] - #Only run when tagged - if: startsWith(github.ref, 'refs/tags/v') - steps: - - name: Checkout Repo - uses: actions/checkout@v2 - with: - ref: 'develop' - - - - name: Install System Packages - run: | - sudo apt update -qq - sudo apt install jq -qq - - - uses: actions/setup-python@v2 - with: - python-version: '3.9' #Available versions here - https://github.com/actions/python-versions/releases easy to change/make a matrix/use pypy - architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified - - - name: Install Python Dependencies - run: | - #Get the virtualenv set up - rm -rf venv - python3 -m venv --clear venv - source venv/bin/activate - python3 -m pip install -q -r requirements.txt - - - name: run reporting - run: | - source venv/bin/activate - python3 bin/reporting.py - - #Official, Verified Amazon-AWS Github Account Provided Action - - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - # aws-session-token: ${{ secrets.AWS_SESSION_TOKEN }} # if you have/need it - aws-region: us-west-1 #assume we will always use this, could make this an environment variable... - - - name: Upload Reporting - run: | - aws s3 cp bin/reporting s3://security-content/reporting --recursive --exclude "*" --include "*.svg" - - update-sources-github: - runs-on: ubuntu-latest - needs: [validate-content, build-sources, build-package, run-appinspect, create-report] - #Only run when tagged - if: startsWith(github.ref, 'refs/tags/v') - steps: - - - name: Checkout Repo - uses: actions/checkout@v2 - with: - ref: 'develop' - - - uses: actions/setup-python@v2 - with: - python-version: '3.9' #Available versions here - https://github.com/actions/python-versions/releases easy to change/make a matrix/use pypy - architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified - - - - uses: actions/download-artifact@v2 - with: - name: content-latest - - - name: Stage artifacts in proper directories - run: | - mkdir latest-escu - tar -zxf DA-ESS-ContentUpdate-latest.tar.gz -C latest-escu --strip-components=1 - mkdir latest-saaws - tar -zxf DA-ESS_AmazonWebServices_Content-latest.tar.gz -C latest-saaws --strip-components=1 - - name: Install Python Dependencies - run: | - #Get the virtualenv set up - rm -rf venv - python3 -m venv --clear venv - source venv/bin/activate - python3 -m pip install -q -r requirements.txt - - - name: Get CTI Repo for Mitre context - uses: actions/checkout@v2 - with: - repository: mitre/cti #check out https://github.com/mitre/cti.git, defaults to HEAD - path: "cti/" - - - name: Get branch and PR required for detection testing main.py - id: vars - run: | - echo "::set-output name=branch::${GITHUB_REF#refs/heads/}" - - - name: Run doc-gen - run: | - source venv/bin/activate - python3 bin/doc_gen.py --path . --output docs -v - - - name: Make YAMLs Pretty - run: | - source venv/bin/activate - python3 bin/pretty_yaml.py --path . -v - - - name: Run generate-actors-map - run: | - source venv/bin/activate - python3 bin/generate-actors-map.py --projects_path . --output docs/mitre-map/ - - - name: Run generate-coverage-map - run: | - source venv/bin/activate - python3 bin/generate-coverage-map.py --projects_path . --output docs/mitre-map - - - name: Update github with new docs and package bits - run: | - rm -rf dist - mkdir dist - echo "Directory layout 3" - pwd - ls -lah - mv latest-escu dist/escu - mv latest-saaws dist/saaws - # configure git to prep for commit - #git config credential.helper 'cache --timeout=120' - git config user.email "research@splunk.com" - git config user.name "research bot" - git config --global push.default simple - git add dist/* - git add docs/* - git add detections/* - git commit --allow-empty -m "updating docs and package bits [ci skip]" - # Push quietly to prevent showing the token in log - #No need to provide any credentials - git push - - - publish-github-release: - #Github-maintained release action is in archived state: https://github.com/actions/create-release - #They recommend several and we use the following with the most stars: https://github.com/softprops/action-gh-release - runs-on: ubuntu-latest - needs: [validate-content, build-sources, build-package, run-appinspect, create-report, update-sources-github] - #Only run when tagged - if: startsWith(github.ref, 'refs/tags/v') - steps: - - #Get the artifacts that we need - - uses: actions/download-artifact@v2 - with: - name: content-latest - - uses: actions/download-artifact@v2 - with: - name: appinspect_reports - - #Rename those artifacts appropriately - - name: Set tag - id: vars - run: echo "::set-output name=tag::${GITHUB_REF#refs/*/}" - - - name: Rename the content-update appropriately - run: | - cp DA-ESS-ContentUpdate-latest.tar.gz DA-ESS-ContentUpdate-${{ steps.vars.outputs.tag }}.tar.gz - cp DA-ESS_AmazonWebServices_Content-latest.tar.gz DA-ESS_AmazonWebServices_Content-${{ steps.vars.outputs.tag }}.tar.gz - - #No checksum on the reports - cp report.tar.gz report-${{ steps.vars.outputs.tag }}.tar.gz - - cp checksum.txt checksum-${{ steps.vars.outputs.tag }}.txt - - #Upload all of the artifacts that we have created using the third party - #action recommended bu Github - - name: Upload Release Artifacts - uses: softprops/action-gh-release@v1 - with: - files: | - DA-ESS-ContentUpdate-${{ steps.vars.outputs.tag }}.tar.gz - DA-ESS_AmazonWebServices_Content-${{ steps.vars.outputs.tag }}.tar.gz - report-${{ steps.vars.outputs.tag }}.tar.gz - checksum-${{ steps.vars.outputs.tag }}.txt - - - - attack-range-update: - runs-on: ubuntu-latest - needs: [validate-content, build-sources, build-package, run-appinspect, create-report, update-sources-github, publish-github-release] - #Only run when tagged - if: startsWith(github.ref, 'refs/tags/v') - steps: - - - uses: actions/download-artifact@v2 - with: - name: content-latest - - #Official, Verified Amazon-AWS Github Account Provided Action - - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - # aws-session-token: ${{ secrets.AWS_SESSION_TOKEN }} # if you have/need it - aws-region: us-west-1 #assume we will always use this, could make this an environment variable... - - - name: Sync latest ESCU to the Attack Range S3 bucket for apps - run: | - aws s3 cp DA-ESS-ContentUpdate-latest.tar.gz s3://attack-range-appbinaries/ - # make the file public since it is not by default - aws s3api put-object-acl --bucket attack-range-appbinaries --key DA-ESS-ContentUpdate-latest.tar.gz --acl public-read - - master-api-update: - runs-on: ubuntu-latest - needs: [validate-content, build-sources, build-package, run-appinspect, create-report, update-sources-github, publish-github-release, attack-range-update] - #Only run when tagged - if: startsWith(github.ref, 'refs/tags/v') - steps: - - name: Checkout Repo - uses: actions/checkout@v2 - with: - ref: 'develop' - - - - uses: actions/setup-python@v2 - with: - python-version: '3.9' #Available versions here - https://github.com/actions/python-versions/releases easy to change/make a matrix/use pypy - architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified - - - name: Install Python Dependencies - run: | - #Get the virtualenv set up - rm -rf venv - python3 -m venv --clear venv - source venv/bin/activate - python3 -m pip install -q -r requirements.txt - - - name: Create Baseline Folder - run: | - source venv/bin/activate - python3 bin/create_baseline_folder.py - - #Official, Verified Amazon-AWS Github Account Provided Action - - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - # aws-session-token: ${{ secrets.AWS_SESSION_TOKEN }} # if you have/need it - aws-region: us-west-1 #assume we will always use this, could make this an environment variable... - - - name: Update API sources - run: | - aws s3 rm s3://security-content --recursive --exclude "*" --include "*.yml" - aws s3 cp stories s3://security-content/stories --recursive --exclude "*" --include "*.yml" - aws s3 cp baselines s3://security-content/baselines --recursive --exclude "*" --include "*.yml" - aws s3 cp detections s3://security-content/detections --recursive --exclude "*" --include "*.yml" - aws s3 cp response_tasks s3://security-content/response_tasks --recursive --exclude "*" --include "*.yml" - aws s3 cp responses s3://security-content/responses --recursive --exclude "*" --include "*.yml" - aws s3 cp lookups s3://security-content/lookups --recursive --exclude "*" --include "*.yml" - aws s3 cp lookups s3://security-content/lookups --recursive --exclude "*" --include "*.csv" - aws s3 cp macros s3://security-content/macros --recursive --exclude "*" --include "*.yml" - aws s3 cp deployments s3://security-content/deployments --recursive --exclude "*" --include "*.yml" - - name: Security Content API Smoke Test - run: | - API_URL='https://content.splunkresearch.com/detections' - API_STATUS=$(curl -s -o /dev/null -w "%{http_code}" $API_URL) - echo "Security Content API Status: $API_STATUS" - if [ "$API_STATUS" != "200" ]; then - echo "Error [Security Content API status: $API_STATUS]" - exit 1 - fi - -