admin/splunk-security_content
1
0
Fork 0
mirror of https://github.com/splunk/security_content synced 2026-06-08 17:32:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

removed old ci

Browse Source
This commit is contained in:
P4T12ICK
2021-08-27 14:32:04 +02:00
parent ea1e6fa207
commit dbe2edc2eb
1 changed files with 0 additions and 619 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/validate-and-build.yml
-619
View File
@@ -1,619 +0,0 @@
#This file makes use of a number of useful, external Github Actions.
#Check the links below for additional documentation on each of these:
#https://github.com/actions/setup-python
#https://github.com/actions/setup-node
#https://github.com/actions/checkout
#https://github.com/actions/upload-artifact
#The mechanism for persisting data between jobs in a workflow is the same as for persisting it
#permanently:
#https://docs.github.com/en/actions/guides/storing-workflow-data-as-artifacts
#In CircleCI, this was different (store_artifacts vs persist_to_workspace)
name: validate-and-build
on: [push, pull_request]
jobs:
validate-tag-if-present:
runs-on: ubuntu-latest
steps:
- name: TAGGED, Validate that the tag is in the correct format
run: |
echo "The GITHUB_REF: $GITHUB_REF"
#First check to see if the release is a tag
if [[ $GITHUB_REF =~ refs/tags/* ]]; then
#Yes, this is a tag, so we need to test to make sure that the tag
#is in the correct format (like v1.10.20)
if [[ $GITHUB_REF =~ refs/tags/v[0-9]+.[0-9]+.[0-9]+ ]]; then
echo "PASS: Tagged release with good format"
exit 0
else
echo "FAIL: Tagged release with bad format"
exit 1
fi
else
echo "PASS: Not a tagged release"
exit 0
fi
validate-content:
#Note that the CircleCI job used a Container. The way to do this with Github Actions
#is to first start up a Virtual Machine, then we can by following:
# https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#jobsjob_idcontainer
runs-on: ubuntu-latest
needs: [validate-tag-if-present]
steps:
#Previous config chose which branch/tag to operate on.
#I think Github is smart enough to choose based on whether it's a pull request or push + other info?
- name: Check out the repository code
uses: actions/checkout@v2
#with:
# repository: splunk/security-content #check out https://github.com/mitre/cti.git, defaults to HEAD
# path: "security-content"
- uses: actions/setup-python@v2
with:
python-version: '3.9' #Available versions here - https://github.com/actions/python-versions/releases easy to change/make a matrix/use pypy
architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified
#TODO: CircleCI restore_cache equivalent
#don't need to install python3 or python3-dev since it was handled by the action above?
#Also, no support for YAML anchors/aliases in Github Actions...
- name: Install System Packages
run: |
sudo apt update -qq
sudo apt install jq -qq
#TODO: CircleCI save_cache equivalent
- name: Install Python Dependencies
run: |
#Get the virtualenv set up
rm -rf venv
python3 -m venv --clear venv
source venv/bin/activate
python3 -m pip install -q -r requirements.txt
- name: run validate
run: |
source venv/bin/activate
python3 contentctl.py --path . --verbose validate
- name: Get CTI Repo for Mitre context
uses: actions/checkout@v2
with:
repository: mitre/cti #check out https://github.com/mitre/cti.git, defaults to HEAD
path: "cti/"
#Now generate the documentation (uses Node)
- uses: actions/setup-node@v2
with:
node-version: '14' #can easily be changed to a different version
- name: Generate documentation
run: |
ls -lah
#Enter the virtualenv and run the docgen
source venv/bin/activate
python3 bin/doc_gen.py --path . --output docs -v
#Now generate the spec docs
npm install -g @adobe/jsonschema2md
jsonschema2md -d spec -o docs/spec -f yaml -e spec.json -x -
#Clean up extra properties on docs
rm -rf docs/spec/*-*.md
echo "****** BRANCH INFORMATION ******"
git branch
git branch --show-current
build-sources:
runs-on: ubuntu-latest
needs: validate-content
steps:
- name: Checkout Repo
uses: actions/checkout@v2
- name: Install System Packages
run: |
sudo apt update -qq
sudo apt install jq -qq
- name: Install Python Dependencies
run: |
#Get the virtualenv set up
rm -rf venv
python3 -m venv --clear venv
source venv/bin/activate
python3 -m pip install -q -r requirements.txt
- name: Run Generate
run: |
source venv/bin/activate
python3 contentctl.py --path . --verbose generate --product ESCU --output dist/escu
python3 contentctl.py --path . --verbose generate --product SAAWS --output dist/saaws
python3 contentctl.py --path . --verbose generate --product DevSecOps --output dist/devsecops
#make a copy of use_case_lib in order to have ES work :-(
cp dist/escu/default/use_case_library.conf dist/escu/default/analyticstories.conf
cp dist/saaws/default/use_case_library.conf dist/saaws/default/analyticstories.conf
cp dist/devsecops/default/use_case_library.conf dist/devsecops/default/analyticstories.conf
- name: Copy lookups .csv files
run: |
# clean up current lookups
rm -rf dist/escu/lookups
rm -rf dist/saaws/lookups
rm -rf dist/devsecops/lookups
mkdir dist/escu/lookups
mkdir dist/saaws/lookups
mkdir dist/devsecops/lookups
#copy over lookups
cd lookups
cp -rv *.csv ../dist/escu/lookups
cp -rv *.csv ../dist/saaws/lookups
cp -rv *.csv ../dist/devsecops/lookups
#Tag is '' for non-tagged push and the tag name for a tagged release
- name: Set tag
id: vars
run: |
if [ echo ${GITHUB_REF} | grep "^refs/tags/*" ]; then
#failed to find the refs/tags/ beginning, grab and set the tag
echo "::set-output name=tag::${GITHUB_REF#refs/tags/}"
else
#Not a tagged relese
echo "::set-output name=tag::"
fi
- name: Update Version and Build number
run : |
# check if tag is set, get build number from the tag if set
if [ -z "${{ steps.vars.outputs.tag }}" ]; then
CONTENT_VERSION=$(grep -oP "(\d+.\d+.\d+$)" dist/escu/default/content-version.conf)
echo "detected content version: $CONTENT_VERSION"
else
CONTENT_VERSION=$(echo ${{ steps.vars.outputs.tag }} | grep -oP "\d+.\d+.\d+")
echo "content version: $CONTENT_VERSION, set by tag: ${{ steps.vars.outputs.tag }}"
fi
# update build number and version for ESCU
sed -i "s/build = .*$/build = ${{ github.run_number }}/g" dist/escu/default/app.conf
sed -i "s/^version = .*$/version = $CONTENT_VERSION/g" dist/escu/default/app.conf
sed -i "s/\"version\": .*$/\"version\": \"$CONTENT_VERSION\"/g" dist/escu/app.manifest
sed -i "s/version = .*$/version = $CONTENT_VERSION/g" dist/escu/default/content-version.conf
tar -czf content-pack-build-escu.tar.gz dist/escu/*
# update build number and version for saaws
sed -i "s/build = .*$/build = ${{ github.run_number }}/g" dist/saaws/default/app.conf
sed -i "s/^version = .*$/version = $CONTENT_VERSION/g" dist/saaws/default/app.conf
sed -i "s/\"version\": .*$/\"version\": \"$CONTENT_VERSION\"/g" dist/saaws/app.manifest
sed -i "s/version = .*$/version = $CONTENT_VERSION/g" dist/saaws/default/content-version.conf
tar -czf content-pack-build-saaws.tar.gz dist/saaws/*
# update build number and version for devsecops
tar -czf content-pack-build-devsecops.tar.gz dist/devsecops/*
- name: Persist to Workspace
uses: actions/upload-artifact@v2
with:
name: content-pack-build
path: |
content-pack-build-escu.tar.gz
content-pack-build-saaws.tar.gz
content-pack-build-devsecops.tar.gz
build-package:
runs-on: ubuntu-latest
needs: [validate-content, build-sources]
steps:
- uses: actions/download-artifact@v2
with:
name: content-pack-build
path: build/
#This explicitly uses a different version of python (2.7)
- uses: actions/setup-python@v2
with:
python-version: '2.7' #Available versions here - https://github.com/actions/python-versions/releases easy to change/make a matrix/use pypy
architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified
- name: Get virtualenv for Python 2.7
run: |
sudo apt install virtualenv
- name: Grab Splunk Packaging Toolkit
run : |
curl -Ls https://download.splunk.com/misc/packaging-toolkit/splunk-packaging-toolkit-0.9.0.tar.gz -o splunk-packaging-toolkit-latest.tar.gz
mkdir slim-latest
tar -zxf splunk-packaging-toolkit-latest.tar.gz -C slim-latest --strip-components=1
- name: Install Splunk Packaging Toolkit (slim)
run: |
cd slim-latest
virtualenv --python=/usr/bin/python2.7 --clear venv
source venv/bin/activate
python -m pip install semantic_version
python -m pip install .
- name: Create a .spl for this Build Using Slim
run: |
source slim-latest/venv/bin/activate
cd build
tar -zxf content-pack-build-escu.tar.gz
tar -zxf content-pack-build-saaws.tar.gz
tar -zxf content-pack-build-devsecops.tar.gz
mv dist/escu DA-ESS-ContentUpdate
mv dist/saaws DA-ESS_AmazonWebServices_Content
mv dist/devsecops devsecops
slim package -o upload DA-ESS-ContentUpdate
slim package -o upload DA-ESS_AmazonWebServices_Content
slim package -o upload devsecops
cp upload/DA-ESS-ContentUpdate-*.tar.gz DA-ESS-ContentUpdate-latest.tar.gz
sha256sum DA-ESS-ContentUpdate-latest.tar.gz > checksum.txt
cp upload/DA-ESS_AmazonWebServices_Content-*tar.gz DA-ESS_AmazonWebServices_Content-latest.tar.gz
sha256sum DA-ESS_AmazonWebServices_Content-latest.tar.gz >> checksum.txt
cp upload/devsecops-*tar.gz devsecops.tar.gz
- name: store_artifacts
uses: actions/upload-artifact@v2
with:
name: package
path: |
build/upload
- name: store_artifacts_two
uses: actions/upload-artifact@v2
with:
name: content-latest
path: |
build/DA-ESS-ContentUpdate-latest.tar.gz
build/DA-ESS_AmazonWebServices_Content-latest.tar.gz
build/devsecops.tar.gz
build/checksum.txt
run-appinspect:
runs-on: ubuntu-latest
needs: [validate-content, build-sources, build-package]
#Only run when tagged
if: startsWith(github.ref, 'refs/tags/v')
steps:
- name: Checkout Repo
uses: actions/checkout@v2
with:
ref: 'develop'
#Download the artifacts we want to check
- name: Restore Content-Pack Artifacts for AppInspect testing
uses: actions/download-artifact@v2
with:
name: content-latest
path: build/
- name: Install System Packages
run: |
sudo apt update -qq
sudo apt install jq -qq
- name: Submit ESCU Package to AppInspect API
env:
APPINSPECT_USERNAME: ${{ secrets.AppInspectUsername }}
APPINSPECT_PASSWORD: ${{ secrets.AppInspectPassword }}
run: |
cd bin
#Enclose in quotes in case there are any special characters in the username/password
#Better not to pass these arguments on the command line, if possible
./appinspect.sh ../ DA-ESS-ContentUpdate-latest.tar.gz "$APPINSPECT_USERNAME" "$APPINSPECT_PASSWORD"
- name: Submit SAAWS Package to AppInspect API
env:
APPINSPECT_USERNAME: ${{ secrets.AppInspectUsername }}
APPINSPECT_PASSWORD: ${{ secrets.AppInspectPassword }}
run: |
cd bin
./appinspect.sh ../ DA-ESS_AmazonWebServices_Content-latest.tar.gz "$APPINSPECT_USERNAME" "$APPINSPECT_PASSWORD"
- name: Create report artifact
if: always()
run: |
#Always create this, regardless of whether success or failure above
tar -cvzf report.tar.gz report/
- name: store_artifacts
uses: actions/upload-artifact@v2
with:
name: appinspect_reports
path: |
report.tar.gz
#Still store the report, even if we have failed (otherwise we don't know why/how we failed)
- name: store_artifacts_on_failure
uses: actions/upload-artifact@v2
if: failure()
with:
name: appinspect_reports_failure
path: |
report.tar.gz
create-report:
runs-on: ubuntu-latest
needs: [validate-content, build-sources, build-package, run-appinspect]
#Only run when tagged
if: startsWith(github.ref, 'refs/tags/v')
steps:
- name: Checkout Repo
uses: actions/checkout@v2
with:
ref: 'develop'
- name: Install System Packages
run: |
sudo apt update -qq
sudo apt install jq -qq
- uses: actions/setup-python@v2
with:
python-version: '3.9' #Available versions here - https://github.com/actions/python-versions/releases easy to change/make a matrix/use pypy
architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified
- name: Install Python Dependencies
run: |
#Get the virtualenv set up
rm -rf venv
python3 -m venv --clear venv
source venv/bin/activate
python3 -m pip install -q -r requirements.txt
- name: run reporting
run: |
source venv/bin/activate
python3 bin/reporting.py
#Official, Verified Amazon-AWS Github Account Provided Action
- uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
# aws-session-token: ${{ secrets.AWS_SESSION_TOKEN }} # if you have/need it
aws-region: us-west-1 #assume we will always use this, could make this an environment variable...
- name: Upload Reporting
run: |
aws s3 cp bin/reporting s3://security-content/reporting --recursive --exclude "*" --include "*.svg"
update-sources-github:
runs-on: ubuntu-latest
needs: [validate-content, build-sources, build-package, run-appinspect, create-report]
#Only run when tagged
if: startsWith(github.ref, 'refs/tags/v')
steps:
- name: Checkout Repo
uses: actions/checkout@v2
with:
ref: 'develop'
- uses: actions/setup-python@v2
with:
python-version: '3.9' #Available versions here - https://github.com/actions/python-versions/releases easy to change/make a matrix/use pypy
architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified
- uses: actions/download-artifact@v2
with:
name: content-latest
- name: Stage artifacts in proper directories
run: |
mkdir latest-escu
tar -zxf DA-ESS-ContentUpdate-latest.tar.gz -C latest-escu --strip-components=1
mkdir latest-saaws
tar -zxf DA-ESS_AmazonWebServices_Content-latest.tar.gz -C latest-saaws --strip-components=1
- name: Install Python Dependencies
run: |
#Get the virtualenv set up
rm -rf venv
python3 -m venv --clear venv
source venv/bin/activate
python3 -m pip install -q -r requirements.txt
- name: Get CTI Repo for Mitre context
uses: actions/checkout@v2
with:
repository: mitre/cti #check out https://github.com/mitre/cti.git, defaults to HEAD
path: "cti/"
- name: Get branch and PR required for detection testing main.py
id: vars
run: |
echo "::set-output name=branch::${GITHUB_REF#refs/heads/}"
- name: Run doc-gen
run: |
source venv/bin/activate
python3 bin/doc_gen.py --path . --output docs -v
- name: Make YAMLs Pretty
run: |
source venv/bin/activate
python3 bin/pretty_yaml.py --path . -v
- name: Run generate-actors-map
run: |
source venv/bin/activate
python3 bin/generate-actors-map.py --projects_path . --output docs/mitre-map/
- name: Run generate-coverage-map
run: |
source venv/bin/activate
python3 bin/generate-coverage-map.py --projects_path . --output docs/mitre-map
- name: Update github with new docs and package bits
run: |
rm -rf dist
mkdir dist
echo "Directory layout 3"
pwd
ls -lah
mv latest-escu dist/escu
mv latest-saaws dist/saaws
# configure git to prep for commit
#git config credential.helper 'cache --timeout=120'
git config user.email "research@splunk.com"
git config user.name "research bot"
git config --global push.default simple
git add dist/*
git add docs/*
git add detections/*
git commit --allow-empty -m "updating docs and package bits [ci skip]"
# Push quietly to prevent showing the token in log
#No need to provide any credentials
git push
publish-github-release:
#Github-maintained release action is in archived state: https://github.com/actions/create-release
#They recommend several and we use the following with the most stars: https://github.com/softprops/action-gh-release
runs-on: ubuntu-latest
needs: [validate-content, build-sources, build-package, run-appinspect, create-report, update-sources-github]
#Only run when tagged
if: startsWith(github.ref, 'refs/tags/v')
steps:
#Get the artifacts that we need
- uses: actions/download-artifact@v2
with:
name: content-latest
- uses: actions/download-artifact@v2
with:
name: appinspect_reports
#Rename those artifacts appropriately
- name: Set tag
id: vars
run: echo "::set-output name=tag::${GITHUB_REF#refs/*/}"
- name: Rename the content-update appropriately
run: |
cp DA-ESS-ContentUpdate-latest.tar.gz DA-ESS-ContentUpdate-${{ steps.vars.outputs.tag }}.tar.gz
cp DA-ESS_AmazonWebServices_Content-latest.tar.gz DA-ESS_AmazonWebServices_Content-${{ steps.vars.outputs.tag }}.tar.gz
#No checksum on the reports
cp report.tar.gz report-${{ steps.vars.outputs.tag }}.tar.gz
cp checksum.txt checksum-${{ steps.vars.outputs.tag }}.txt
#Upload all of the artifacts that we have created using the third party
#action recommended bu Github
- name: Upload Release Artifacts
uses: softprops/action-gh-release@v1
with:
files: |
DA-ESS-ContentUpdate-${{ steps.vars.outputs.tag }}.tar.gz
DA-ESS_AmazonWebServices_Content-${{ steps.vars.outputs.tag }}.tar.gz
report-${{ steps.vars.outputs.tag }}.tar.gz
checksum-${{ steps.vars.outputs.tag }}.txt
attack-range-update:
runs-on: ubuntu-latest
needs: [validate-content, build-sources, build-package, run-appinspect, create-report, update-sources-github, publish-github-release]
#Only run when tagged
if: startsWith(github.ref, 'refs/tags/v')
steps:
- uses: actions/download-artifact@v2
with:
name: content-latest
#Official, Verified Amazon-AWS Github Account Provided Action
- uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
# aws-session-token: ${{ secrets.AWS_SESSION_TOKEN }} # if you have/need it
aws-region: us-west-1 #assume we will always use this, could make this an environment variable...
- name: Sync latest ESCU to the Attack Range S3 bucket for apps
run: |
aws s3 cp DA-ESS-ContentUpdate-latest.tar.gz s3://attack-range-appbinaries/
# make the file public since it is not by default
aws s3api put-object-acl --bucket attack-range-appbinaries --key DA-ESS-ContentUpdate-latest.tar.gz --acl public-read
master-api-update:
runs-on: ubuntu-latest
needs: [validate-content, build-sources, build-package, run-appinspect, create-report, update-sources-github, publish-github-release, attack-range-update]
#Only run when tagged
if: startsWith(github.ref, 'refs/tags/v')
steps:
- name: Checkout Repo
uses: actions/checkout@v2
with:
ref: 'develop'
- uses: actions/setup-python@v2
with:
python-version: '3.9' #Available versions here - https://github.com/actions/python-versions/releases easy to change/make a matrix/use pypy
architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified
- name: Install Python Dependencies
run: |
#Get the virtualenv set up
rm -rf venv
python3 -m venv --clear venv
source venv/bin/activate
python3 -m pip install -q -r requirements.txt
- name: Create Baseline Folder
run: |
source venv/bin/activate
python3 bin/create_baseline_folder.py
#Official, Verified Amazon-AWS Github Account Provided Action
- uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
# aws-session-token: ${{ secrets.AWS_SESSION_TOKEN }} # if you have/need it
aws-region: us-west-1 #assume we will always use this, could make this an environment variable...
- name: Update API sources
run: |
aws s3 rm s3://security-content --recursive --exclude "*" --include "*.yml"
aws s3 cp stories s3://security-content/stories --recursive --exclude "*" --include "*.yml"
aws s3 cp baselines s3://security-content/baselines --recursive --exclude "*" --include "*.yml"
aws s3 cp detections s3://security-content/detections --recursive --exclude "*" --include "*.yml"
aws s3 cp response_tasks s3://security-content/response_tasks --recursive --exclude "*" --include "*.yml"
aws s3 cp responses s3://security-content/responses --recursive --exclude "*" --include "*.yml"
aws s3 cp lookups s3://security-content/lookups --recursive --exclude "*" --include "*.yml"
aws s3 cp lookups s3://security-content/lookups --recursive --exclude "*" --include "*.csv"
aws s3 cp macros s3://security-content/macros --recursive --exclude "*" --include "*.yml"
aws s3 cp deployments s3://security-content/deployments --recursive --exclude "*" --include "*.yml"
- name: Security Content API Smoke Test
run: |
API_URL='https://content.splunkresearch.com/detections'
API_STATUS=$(curl -s -o /dev/null -w "%{http_code}" $API_URL)
echo "Security Content API Status: $API_STATUS"
if [ "$API_STATUS" != "200" ]; then
echo "Error [Security Content API status: $API_STATUS]"
exit 1
fi