admin/splunk-security_content
1
0
Fork 0
mirror of https://github.com/splunk/security_content synced 2026-06-08 17:32:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Branch was auto-updated.

Browse Source
This commit is contained in:
github-actions[bot]
2021-08-27 19:49:27 +00:00
committed by GitHub
parent f381f02c8b f8b75c6c69
commit e0e2aaf5cd
8 changed files with 367 additions and 429 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/build-and-validate.yml
+341 -5
View File
@@ -250,7 +250,6 @@ jobs:
cp upload/DA-ESS_AmazonWebServices_Content-*tar.gz DA-ESS_AmazonWebServices_Content-latest.tar.gz
sha256sum DA-ESS_AmazonWebServices_Content-latest.tar.gz >> checksum.txt
touch tag-canary.txt
- name: store_artifacts
uses: actions/upload-artifact@v2
with:
@@ -266,10 +265,347 @@ jobs:
build/DA-ESS_AmazonWebServices_Content-latest.tar.gz
build/checksum.txt
#Store the tag to indicate that this was a tagged build
- name: store_artifacts_three
#Everything below this line should ONLY run on a tag and nothing else
#We still want all of the above checks to run and pass before running these
run-appinspect:
runs-on: ubuntu-latest
needs: [validate-content, build-sources, build-package]
#Only run when tagged
if: startsWith(github.ref, 'refs/tags/')
steps:
- name: Checkout Repo
uses: actions/checkout@v2
with:
ref: 'develop'
#Download the artifacts we want to check
- uses: actions/download-artifact@v2
with:
name: content-latest
path: build/
- name: Install System Packages
run: |
sudo apt update -qq
sudo apt install jq -qq
- name: Submit ESCU Package to AppInspect API
env:
APPINSPECT_USERNAME: ${{ secrets.AppInspectUsername }}
APPINSPECT_PASSWORD: ${{ secrets.AppInspectPassword }}
run: |
cd bin
#Enclose in quotes in case there are any special characters in the username/password
#Better not to pass these arguments on the command line, if possible
./appinspect.sh ../ DA-ESS-ContentUpdate-latest.tar.gz "$APPINSPECT_USERNAME" "$APPINSPECT_PASSWORD"
- name: Submit SAAWS Package to AppInspect API
env:
APPINSPECT_USERNAME: ${{ secrets.AppInspectUsername }}
APPINSPECT_PASSWORD: ${{ secrets.AppInspectPassword }}
run: |
cd bin
./appinspect.sh ../ DA-ESS_AmazonWebServices_Content-latest.tar.gz "$APPINSPECT_USERNAME" "$APPINSPECT_PASSWORD"
- name: Create report artifact
if: always()
run: |
#Always create this, regardless of whether success or failure above
tar -cvzf report.tar.gz report/
- name: store_artifacts
uses: actions/upload-artifact@v2
with:
name: tag-canary
name: appinspect_reports
path: |
build/tag-canary.txt
report.tar.gz
#Still store the report, even if we have failed (otherwise we don't know why/how we failed)
- name: store_artifacts_on_failure
uses: actions/upload-artifact@v2
if: failure()
with:
name: appinspect_reports_failure
path: |
report.tar.gz
create-report:
runs-on: ubuntu-latest
needs: [validate-content, build-sources, build-package, run-appinspect]
#Only run when tagged
if: startsWith(github.ref, 'refs/tags/')
steps:
- name: Checkout Repo
uses: actions/checkout@v2
with:
ref: 'develop'
- name: Install System Packages
run: |
sudo apt update -qq
sudo apt install jq -qq
- uses: actions/setup-python@v2
with:
python-version: '3.9' #Available versions here - https://github.com/actions/python-versions/releases easy to change/make a matrix/use pypy
architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified
- name: Install Python Dependencies
run: |
#Get the virtualenv set up
rm -rf venv
python3 -m venv --clear venv
source venv/bin/activate
python3 -m pip install -q -r requirements.txt
- name: run reporting
run: |
source venv/bin/activate
python3 bin/reporting.py
#Official, Verified Amazon-AWS Github Account Provided Action
- uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
# aws-session-token: ${{ secrets.AWS_SESSION_TOKEN }} # if you have/need it
aws-region: us-west-1 #assume we will always use this, could make this an environment variable...
- name: Upload Reporting
run: |
aws s3 cp bin/reporting s3://security-content-testing/reporting --recursive --exclude "*" --include "*.svg"
update-sources-github:
runs-on: ubuntu-latest
needs: [validate-content, build-sources, build-package, run-appinspect, create-report]
#Only run when tagged
if: startsWith(github.ref, 'refs/tags/')
steps:
- name: Checkout Repo
uses: actions/checkout@v2
with:
ref: 'develop'
- uses: actions/setup-python@v2
with:
python-version: '3.9' #Available versions here - https://github.com/actions/python-versions/releases easy to change/make a matrix/use pypy
architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified
- uses: actions/download-artifact@v2
with:
name: content-latest
- name: Stage artifacts in proper directories
run: |
mkdir latest-escu
tar -zxf DA-ESS-ContentUpdate-latest.tar.gz -C latest-escu --strip-components=1
mkdir latest-saaws
tar -zxf DA-ESS_AmazonWebServices_Content-latest.tar.gz -C latest-saaws --strip-components=1
- name: Install Python Dependencies
run: |
#Get the virtualenv set up
rm -rf venv
python3 -m venv --clear venv
source venv/bin/activate
python3 -m pip install -q -r requirements.txt
- name: Get CTI Repo for Mitre context
uses: actions/checkout@v2
with:
repository: mitre/cti #check out https://github.com/mitre/cti.git, defaults to HEAD
path: "cti/"
- name: Get branch and PR required for detection testing main.py
id: vars
run: |
echo "::set-output name=branch::${GITHUB_REF#refs/heads/}"
- name: Run doc-gen
run: |
source venv/bin/activate
python3 bin/doc_gen.py --path . --output docs -v
- name: Make YAMLs Pretty
run: |
source venv/bin/activate
python3 bin/pretty_yaml.py --path . -v
- name: Run generate-actors-map
run: |
source venv/bin/activate
python3 bin/generate-actors-map.py --projects_path . --output docs/mitre-map/
- name: Run generate-coverage-map
run: |
source venv/bin/activate
python3 bin/generate-coverage-map.py --projects_path . --output docs/mitre-map
- name: Update github with new docs and package bits
run: |
rm -rf dist
mkdir dist
echo "Directory layout 3"
pwd
ls -lah
mv latest-escu dist/escu
mv latest-saaws dist/saaws
# configure git to prep for commit
#git config credential.helper 'cache --timeout=120'
git config user.email "research@splunk.com"
git config user.name "research bot"
git config --global push.default simple
git add dist/*
git add docs/*
git add detections/*
git commit --allow-empty -m "updating docs and package bits [ci skip]"
# Push quietly to prevent showing the token in log
#No need to provide any credentials
git push
publish-github-release:
#Github-maintained release action is in archived state: https://github.com/actions/create-release
#They recommend several and we use the following with the most stars: https://github.com/softprops/action-gh-release
runs-on: ubuntu-latest
needs: [validate-content, build-sources, build-package, run-appinspect, create-report, update-sources-github]
#Only run when tagged
if: startsWith(github.ref, 'refs/tags/')
steps:
#Get the artifacts that we need
- uses: actions/download-artifact@v2
with:
name: content-latest
- uses: actions/download-artifact@v2
with:
name: appinspect_reports
#Rename those artifacts appropriately
- name: Set tag
id: vars
run: echo "::set-output name=tag::${GITHUB_REF#refs/*/}"
- name: Rename the content-update appropriately
run: |
cp DA-ESS-ContentUpdate-latest.tar.gz DA-ESS-ContentUpdate-${{ steps.vars.outputs.tag }}.tar.gz
cp DA-ESS_AmazonWebServices_Content-latest.tar.gz DA-ESS_AmazonWebServices_Content-${{ steps.vars.outputs.tag }}.tar.gz
#No checksum on the reports
cp report.tar.gz report-${{ steps.vars.outputs.tag }}.tar.gz
cp checksum.txt checksum-${{ steps.vars.outputs.tag }}.txt
#Upload all of the artifacts that we have created using the third party
#action recommended bu Github
- name: Upload Release Artifacts
uses: softprops/action-gh-release@v1
with:
files: |
DA-ESS-ContentUpdate-${{ steps.vars.outputs.tag }}.tar.gz
DA-ESS_AmazonWebServices_Content-${{ steps.vars.outputs.tag }}.tar.gz
report-${{ steps.vars.outputs.tag }}.tar.gz
checksum-${{ steps.vars.outputs.tag }}.txt
attack-range-update:
runs-on: ubuntu-latest
needs: [validate-content, build-sources, build-package, run-appinspect, create-report, update-sources-github, publish-github-release]
#Only run when tagged
if: startsWith(github.ref, 'refs/tags/')
steps:
#Get the artifacts that we need
- uses: actions/download-artifact@v2
with:
name: content-latest
#Official, Verified Amazon-AWS Github Account Provided Action
- uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
# aws-session-token: ${{ secrets.AWS_SESSION_TOKEN }} # if you have/need it
aws-region: us-west-1 #assume we will always use this, could make this an environment variable...
- name: Sync latest ESCU to the Attack Range S3 bucket for apps
run: |
aws s3 cp DA-ESS-ContentUpdate-latest.tar.gz s3://attack-range-appbinaries-testing/
# make the file public since it is not by default
aws s3api put-object-acl --bucket attack-range-appbinaries-testing --key DA-ESS-ContentUpdate-latest.tar.gz --acl public-read
master-api-update:
runs-on: ubuntu-latest
needs: [validate-content, build-sources, build-package, run-appinspect, create-report, update-sources-github, publish-github-release, attack-range-update]
#Only run when tagged
if: startsWith(github.ref, 'refs/tags/')
steps:
- name: Checkout Repo
uses: actions/checkout@v2
with:
ref: 'develop'
- uses: actions/setup-python@v2
with:
python-version: '3.9' #Available versions here - https://github.com/actions/python-versions/releases easy to change/make a matrix/use pypy
architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified
- name: Install Python Dependencies
run: |
#Get the virtualenv set up
rm -rf venv
python3 -m venv --clear venv
source venv/bin/activate
python3 -m pip install -q -r requirements.txt
- name: Create Baseline Folder
run: |
source venv/bin/activate
python3 bin/create_baseline_folder.py
#Official, Verified Amazon-AWS Github Account Provided Action
- uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
# aws-session-token: ${{ secrets.AWS_SESSION_TOKEN }} # if you have/need it
aws-region: us-west-1 #assume we will always use this, could make this an environment variable...
- name: Update API sources
run: |
aws s3 rm s3://security-content-testing --recursive --exclude "*" --include "*.yml"
aws s3 cp stories s3://security-content-testing/stories --recursive --exclude "*" --include "*.yml"
aws s3 cp baselines s3://security-content-testing/baselines --recursive --exclude "*" --include "*.yml"
aws s3 cp detections s3://security-content-testing/detections --recursive --exclude "*" --include "*.yml"
aws s3 cp response_tasks s3://security-content-testing/response_tasks --recursive --exclude "*" --include "*.yml"
aws s3 cp responses s3://security-content-testing/responses --recursive --exclude "*" --include "*.yml"
aws s3 cp lookups s3://security-content-testing/lookups --recursive --exclude "*" --include "*.yml"
aws s3 cp lookups s3://security-content-testing/lookups --recursive --exclude "*" --include "*.csv"
aws s3 cp macros s3://security-content-testing/macros --recursive --exclude "*" --include "*.yml"
aws s3 cp deployments s3://security-content-testing/deployments --recursive --exclude "*" --include "*.yml"
- name: Security Content API Smoke Test
run: |
API_URL='https://content.splunkresearch.com/detections'
API_STATUS=$(curl -s -o /dev/null -w "%{http_code}" $API_URL)
echo "Security Content API Status: $API_STATUS"
if [ "$API_STATUS" != "200" ]; then
echo "Error [Security Content API status: $API_STATUS]"
exit 1
fi
.github/workflows/detection-testing.yml
+3 -2
View File
@@ -30,8 +30,9 @@ jobs:
runs-on: ubuntu-latest
environment: Detection-Testing-Approval
needs: [validate-tag-if-present]
#Only run when tagged
if: startsWith(github.ref, 'refs/heads/')
#Only run when we are pushing to the head of a branch or on a PR.
#This implicitly means that we do not run on a tag
if: startsWith(github.ref, 'refs/heads/') || startsWith(github.ref, 'refs/pull/')
steps:
- name: Checkout Repo
.github/workflows/release-checks.yml
-398
View File
@@ -1,398 +0,0 @@
name: release-checks
on:
workflow_run:
workflows: ["validate-and-build"]
types:
- completed
jobs:
#Check that the validate-and-build workflow succeeded
check-validate-and-build-success:
runs-on: ubuntu-latest
steps:
- if: github.event.workflow_run.conclusion != 'success'
name: Abort if failed
run: |
echo "FAIL: validate-and-build.yml DID NOT run successfully. Terminating..."
exit 1
- name: Print Success
run: |
echo "SUCCESS: validate-and-build.yml ran successfully. Continue"
exit 0
#Enusre that we are running on a tag. There is no good way to see if this was
#triggered from a tag/release, so we use the creation of an aritifact in the
#validate-and-build workflow to represent it
verify-tag:
runs-on: ubuntu-latest
needs: [check-validate-and-build-success]
steps:
- name: Try to get the canary
uses: dawidd6/action-download-artifact@v2
with:
github_token: "${{ secrets.GITHUB_TOKEN }}"
workflow: ${{ github.event.workflow_run.workflow_id }}
#workflow: validate-and-build.yml
#run_id: ${{ github.event.workflow_run.id }}
name: tag-canary
path: canary
- name: Check for existence of canary
run: |
#If this file does not exist, then cat will return a nonzero status (failure)
#and the entire workflow will fail
cat canary/tag-canary.txt
run-appinspect:
runs-on: ubuntu-latest
needs: [check-validate-and-build-success, verify-tag]
#Only run when tagged
steps:
- name: Checkout Repo
uses: actions/checkout@v2
with:
ref: 'develop'
#Download the artifacts we want to check
- name: Restore Content-Pack Artifacts for AppInspect testing
uses: dawidd6/action-download-artifact@v2
with:
workflow: validate-and-build.yml
workflow_conclusion: success
run_id: ${{ github.event.workflow_run.id }}
name: content-latest
path: build/
- name: Install System Packages
run: |
sudo apt update -qq
sudo apt install jq -qq
- name: Submit ESCU Package to AppInspect API
env:
APPINSPECT_USERNAME: ${{ secrets.AppInspectUsername }}
APPINSPECT_PASSWORD: ${{ secrets.AppInspectPassword }}
run: |
cd bin
#Enclose in quotes in case there are any special characters in the username/password
#Better not to pass these arguments on the command line, if possible
./appinspect.sh ../ DA-ESS-ContentUpdate-latest.tar.gz "$APPINSPECT_USERNAME" "$APPINSPECT_PASSWORD"
- name: Submit SAAWS Package to AppInspect API
env:
APPINSPECT_USERNAME: ${{ secrets.AppInspectUsername }}
APPINSPECT_PASSWORD: ${{ secrets.AppInspectPassword }}
run: |
cd bin
./appinspect.sh ../ DA-ESS_AmazonWebServices_Content-latest.tar.gz "$APPINSPECT_USERNAME" "$APPINSPECT_PASSWORD"
- name: Create report artifact
if: always()
run: |
#Always create this, regardless of whether success or failure above
tar -cvzf report.tar.gz report/
- name: store_artifacts
uses: actions/upload-artifact@v2
with:
name: appinspect_reports
path: |
report.tar.gz
#Still store the report, even if we have failed (otherwise we don't know why/how we failed)
- name: store_artifacts_on_failure
uses: actions/upload-artifact@v2
if: failure()
with:
name: appinspect_reports_failure
path: |
report.tar.gz
create-report:
runs-on: ubuntu-latest
needs: [check-validate-and-build-success, verify-tag, run-appinspect]
#Only run when tagged
steps:
- name: Checkout Repo
uses: actions/checkout@v2
with:
ref: 'develop'
- name: Install System Packages
run: |
sudo apt update -qq
sudo apt install jq -qq
- uses: actions/setup-python@v2
with:
python-version: '3.9' #Available versions here - https://github.com/actions/python-versions/releases easy to change/make a matrix/use pypy
architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified
- name: Install Python Dependencies
run: |
#Get the virtualenv set up
rm -rf venv
python3 -m venv --clear venv
source venv/bin/activate
python3 -m pip install -q -r requirements.txt
- name: run reporting
run: |
source venv/bin/activate
python3 bin/reporting.py
#Official, Verified Amazon-AWS Github Account Provided Action
- uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
# aws-session-token: ${{ secrets.AWS_SESSION_TOKEN }} # if you have/need it
aws-region: us-west-1 #assume we will always use this, could make this an environment variable...
- name: Upload Reporting
run: |
aws s3 cp bin/reporting s3://security-content-testing/reporting --recursive --exclude "*" --include "*.svg"
update-sources-github:
runs-on: ubuntu-latest
needs: [check-validate-and-build-success, verify-tag, run-appinspect, create-report]
#Only run when tagged
steps:
- name: Checkout Repo
uses: actions/checkout@v2
with:
ref: 'develop'
- uses: actions/setup-python@v2
with:
python-version: '3.9' #Available versions here - https://github.com/actions/python-versions/releases easy to change/make a matrix/use pypy
architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified
- uses: dawidd6/action-download-artifact@v2
with:
workflow: validate-and-build.yml
workflow_conclusion: success
run_id: ${{ github.event.workflow_run.id }}
path: .
name: content-latest
- name: Stage artifacts in proper directories
run: |
mkdir latest-escu
tar -zxf DA-ESS-ContentUpdate-latest.tar.gz -C latest-escu --strip-components=1
mkdir latest-saaws
tar -zxf DA-ESS_AmazonWebServices_Content-latest.tar.gz -C latest-saaws --strip-components=1
- name: Install Python Dependencies
run: |
#Get the virtualenv set up
rm -rf venv
python3 -m venv --clear venv
source venv/bin/activate
python3 -m pip install -q -r requirements.txt
- name: Get CTI Repo for Mitre context
uses: actions/checkout@v2
with:
repository: mitre/cti #check out https://github.com/mitre/cti.git, defaults to HEAD
path: "cti/"
- name: Get branch and PR required for detection testing main.py
id: vars
run: |
echo "::set-output name=branch::${GITHUB_REF#refs/heads/}"
- name: Run doc-gen
run: |
source venv/bin/activate
python3 bin/doc_gen.py --path . --output docs -v
- name: Make YAMLs Pretty
run: |
source venv/bin/activate
python3 bin/pretty_yaml.py --path . -v
- name: Run generate-actors-map
run: |
source venv/bin/activate
python3 bin/generate-actors-map.py --projects_path . --output docs/mitre-map/
- name: Run generate-coverage-map
run: |
source venv/bin/activate
python3 bin/generate-coverage-map.py --projects_path . --output docs/mitre-map
- name: Update github with new docs and package bits
run: |
rm -rf dist
mkdir dist
echo "Directory layout 3"
pwd
ls -lah
mv latest-escu dist/escu
mv latest-saaws dist/saaws
# configure git to prep for commit
#git config credential.helper 'cache --timeout=120'
git config user.email "research@splunk.com"
git config user.name "research bot"
git config --global push.default simple
git add dist/*
git add docs/*
git add detections/*
git commit --allow-empty -m "updating docs and package bits [ci skip]"
# Push quietly to prevent showing the token in log
#No need to provide any credentials
git push
publish-github-release:
#Github-maintained release action is in archived state: https://github.com/actions/create-release
#They recommend several and we use the following with the most stars: https://github.com/softprops/action-gh-release
runs-on: ubuntu-latest
needs: [check-validate-and-build-success, verify-tag, run-appinspect, create-report, update-sources-github]
#Only run when tagged
steps:
#Get the artifacts that we need
- uses: dawidd6/action-download-artifact@v2
with:
workflow: validate-and-build.yml
workflow_conclusion: success
run_id: ${{ github.event.workflow_run.id }}
path: .
name: content-latest
- uses: dawidd6/action-download-artifact@v2
with:
workflow: validate-and-build.yml
workflow_conclusion: success
run_id: ${{ github.event.workflow_run.id }}
path: .
name: appinspect_reports
#Rename those artifacts appropriately
- name: Set tag
id: vars
run: echo "::set-output name=tag::${GITHUB_REF#refs/*/}"
- name: Rename the content-update appropriately
run: |
cp DA-ESS-ContentUpdate-latest.tar.gz DA-ESS-ContentUpdate-${{ steps.vars.outputs.tag }}.tar.gz
cp DA-ESS_AmazonWebServices_Content-latest.tar.gz DA-ESS_AmazonWebServices_Content-${{ steps.vars.outputs.tag }}.tar.gz
#No checksum on the reports
cp report.tar.gz report-${{ steps.vars.outputs.tag }}.tar.gz
cp checksum.txt checksum-${{ steps.vars.outputs.tag }}.txt
#Upload all of the artifacts that we have created using the third party
#action recommended bu Github
- name: Upload Release Artifacts
uses: softprops/action-gh-release@v1
with:
files: |
DA-ESS-ContentUpdate-${{ steps.vars.outputs.tag }}.tar.gz
DA-ESS_AmazonWebServices_Content-${{ steps.vars.outputs.tag }}.tar.gz
report-${{ steps.vars.outputs.tag }}.tar.gz
checksum-${{ steps.vars.outputs.tag }}.txt
attack-range-update:
runs-on: ubuntu-latest
needs: [check-validate-and-build-success, verify-tag, run-appinspect, create-report, update-sources-github, publish-github-release]
#Only run when tagged
steps:
- uses: dawidd6/action-download-artifact@v2
with:
workflow: validate-and-build.yml
workflow_conclusion: success
run_id: ${{ github.event.workflow_run.id }}
path: .
name: content-latest
#Official, Verified Amazon-AWS Github Account Provided Action
- uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
# aws-session-token: ${{ secrets.AWS_SESSION_TOKEN }} # if you have/need it
aws-region: us-west-1 #assume we will always use this, could make this an environment variable...
- name: Sync latest ESCU to the Attack Range S3 bucket for apps
run: |
aws s3 cp DA-ESS-ContentUpdate-latest.tar.gz s3://attack-range-appbinaries-testing/
# make the file public since it is not by default
aws s3api put-object-acl --bucket attack-range-appbinaries-testing --key DA-ESS-ContentUpdate-latest.tar.gz --acl public-read
master-api-update:
runs-on: ubuntu-latest
needs: [check-validate-and-build-success, verify-tag, run-appinspect, create-report, update-sources-github, publish-github-release, attack-range-update]
#Only run when tagged
steps:
- name: Checkout Repo
uses: actions/checkout@v2
with:
ref: 'develop'
- uses: actions/setup-python@v2
with:
python-version: '3.9' #Available versions here - https://github.com/actions/python-versions/releases easy to change/make a matrix/use pypy
architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified
- name: Install Python Dependencies
run: |
#Get the virtualenv set up
rm -rf venv
python3 -m venv --clear venv
source venv/bin/activate
python3 -m pip install -q -r requirements.txt
- name: Create Baseline Folder
run: |
source venv/bin/activate
python3 bin/create_baseline_folder.py
#Official, Verified Amazon-AWS Github Account Provided Action
- uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
# aws-session-token: ${{ secrets.AWS_SESSION_TOKEN }} # if you have/need it
aws-region: us-west-1 #assume we will always use this, could make this an environment variable...
- name: Update API sources
run: |
aws s3 rm s3://security-content-testing --recursive --exclude "*" --include "*.yml"
aws s3 cp stories s3://security-content-testing/stories --recursive --exclude "*" --include "*.yml"
aws s3 cp baselines s3://security-content-testing/baselines --recursive --exclude "*" --include "*.yml"
aws s3 cp detections s3://security-content-testing/detections --recursive --exclude "*" --include "*.yml"
aws s3 cp response_tasks s3://security-content-testing/response_tasks --recursive --exclude "*" --include "*.yml"
aws s3 cp responses s3://security-content-testing/responses --recursive --exclude "*" --include "*.yml"
aws s3 cp lookups s3://security-content-testing/lookups --recursive --exclude "*" --include "*.yml"
aws s3 cp lookups s3://security-content-testing/lookups --recursive --exclude "*" --include "*.csv"
aws s3 cp macros s3://security-content-testing/macros --recursive --exclude "*" --include "*.yml"
aws s3 cp deployments s3://security-content-testing/deployments --recursive --exclude "*" --include "*.yml"
- name: Security Content API Smoke Test
run: |
API_URL='https://content.splunkresearch.com/detections'
API_STATUS=$(curl -s -o /dev/null -w "%{http_code}" $API_URL)
echo "Security Content API Status: $API_STATUS"
if [ "$API_STATUS" != "200" ]; then
echo "Error [Security Content API status: $API_STATUS]"
exit 1
fi
automated_detection_testing/detection_service.py
+1 -1
View File
@@ -156,7 +156,7 @@ def main(args):
counter = counter + 1
j2_env = Environment(loader=FileSystemLoader('templates'),trim_blocks=True, autoescape=True)
j2_env = Environment(loader=FileSystemLoader('templates'),trim_blocks=True) # nosemgrep
template = j2_env.get_template('PR_template.j2')
body = template.render(results=results)
bin/doc_gen.py
+4 -4
View File
@@ -150,8 +150,8 @@ def generate_doc_stories(REPO_PATH, OUTPUT_DIR, TEMPLATE_PATH, attack, sorted_de
if category['name'] == story['tags']['category'][0]:
category['stories'].append(story)
j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH),
trim_blocks=False, autoescape=True)
j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH), # nosemgrep
trim_blocks=False)
# write markdown
template = j2_env.get_template('doc_stories_markdown.j2')
output_path = path.join(OUTPUT_DIR + '/stories.md')
@@ -207,8 +207,8 @@ def generate_doc_detections(REPO_PATH, OUTPUT_DIR, TEMPLATE_PATH, attack, messag
sorted_detections = sorted(detections, key=lambda i: i['name'])
j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH),
trim_blocks=False, autoescape=True)
j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH), # nosemgrep
trim_blocks=False)
# write markdown
template = j2_env.get_template('doc_detections_markdown.j2')
bin/generate.py
+11 -12
View File
@@ -10,7 +10,7 @@ import argparse
from os import path
import sys
import datetime
from jinja2 import Environment, FileSystemLoader, select_autoescape
from jinja2 import Environment, FileSystemLoader
import re
from attackcti import attack_client
import csv
@@ -62,8 +62,8 @@ def generate_transforms_conf(lookups, TEMPLATE_PATH, OUTPUT_PATH):
utc_time = datetime.datetime.utcnow().replace(microsecond=0).isoformat()
j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH),
trim_blocks=True)
j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH), # nosemgrep
trim_blocks=True)
template = j2_env.get_template('transforms.j2')
output_path = path.join(OUTPUT_PATH, 'default/transforms.conf')
output = template.render(lookups=sorted_lookups, time=utc_time)
@@ -78,7 +78,7 @@ def generate_collections_conf(lookups, TEMPLATE_PATH, OUTPUT_PATH):
utc_time = datetime.datetime.utcnow().replace(microsecond=0).isoformat()
j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH),
j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH), # nosemgrep
trim_blocks=True)
template = j2_env.get_template('collections.j2')
output_path = path.join(OUTPUT_PATH, 'default/collections.conf')
@@ -97,7 +97,7 @@ def generate_savedsearches_conf(detections, deployments, TEMPLATE_PATH, OUTPUT_P
utc_time = datetime.datetime.utcnow().replace(microsecond=0).isoformat()
j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH),
j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH), # nosemgrep
trim_blocks=True)
j2_env.filters['custom_jinja2_enrichment_filter'] = custom_jinja2_enrichment_filter
template = j2_env.get_template('savedsearches.j2')
@@ -112,7 +112,7 @@ def generate_savedsearches_conf(detections, deployments, TEMPLATE_PATH, OUTPUT_P
def generate_analytic_story_conf(stories, detections, TEMPLATE_PATH, OUTPUT_PATH):
utc_time = datetime.datetime.utcnow().replace(microsecond=0).isoformat()
j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH),
j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH), # nosemgrep
trim_blocks=True)
template = j2_env.get_template('analytic_stories.j2')
output_path = path.join(OUTPUT_PATH, 'default/analytic_stories.conf')
@@ -125,7 +125,7 @@ def generate_analytic_story_conf(stories, detections, TEMPLATE_PATH, OUTPUT_PATH
def generate_use_case_library_conf(stories, detections, TEMPLATE_PATH, OUTPUT_PATH):
utc_time = datetime.datetime.utcnow().replace(microsecond=0).isoformat()
j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH),
j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH), # nosemgrep
trim_blocks=True)
template = j2_env.get_template('use_case_library.j2')
output_path = path.join(OUTPUT_PATH, 'default/use_case_library.conf')
@@ -150,7 +150,7 @@ def generate_macros_conf(macros, detections, TEMPLATE_PATH, OUTPUT_PATH):
utc_time = datetime.datetime.utcnow().replace(microsecond=0).isoformat()
j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH),
j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH), # nosemgrep
trim_blocks=True)
template = j2_env.get_template('macros.j2')
output_path = path.join(OUTPUT_PATH, 'default/macros.conf')
@@ -170,7 +170,7 @@ def generate_workbench_panels(response_tasks, stories, TEMPLATE_PATH, OUTPUT_PAT
response_file_name_xml = response_file_name + "___response_task.xml"
response_task['lowercase_name'] = response_file_name
workbench_panel_objects.append(response_task)
j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH),
j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH), # nosemgrep
trim_blocks=True)
template = j2_env.get_template('panel.j2')
file_path = "default/data/ui/panels/workbench_panel_" + response_file_name_xml
@@ -182,14 +182,14 @@ def generate_workbench_panels(response_tasks, stories, TEMPLATE_PATH, OUTPUT_PAT
with open(output_path, 'w') as f:
f.write(output)
j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH),
j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH), # nosemgrep
trim_blocks=True)
template = j2_env.get_template('es_investigations.j2')
output_path = path.join(OUTPUT_PATH, 'default/es_investigations.conf')
output = template.render(response_tasks=workbench_panel_objects, stories=stories)
with open(output_path, 'w', encoding="utf-8") as f:
f.write(output)
j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH),
j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH), # nosemgrep
trim_blocks=True)
template = j2_env.get_template('workflow_actions.j2')
output_path = path.join(OUTPUT_PATH, 'default/workflow_actions.conf')
@@ -590,7 +590,6 @@ def get_objects(REPO_PATH, OUTPUT_PATH, PRODUCT, VERBOSE):
def main(REPO_PATH, OUTPUT_PATH, PRODUCT, VERBOSE):
#select_autoescape(default_for_string=True, default=True)
TEMPLATE_PATH = path.join(REPO_PATH, 'bin/jinja2_templates')
objects = get_objects(REPO_PATH, OUTPUT_PATH, PRODUCT, VERBOSE)
bin/newcontent.py
+6 -6
View File
@@ -211,8 +211,8 @@ def detection_wizard(security_content_path,type,TEMPLATE_PATH):
print(mitre_attack_id)
j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH),
trim_blocks=True, autoescape=True)
j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH), # nosemgrep
trim_blocks=True)
answers['products'] = ['Splunk Enterprise','Splunk Enterprise Security','Splunk Cloud']
answers['references'] = []
@@ -374,8 +374,8 @@ def story_wizard(security_content_path,type, TEMPLATE_PATH):
},
]
answers = prompt(questions)
j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH),
trim_blocks=True, autoescape=True)
j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH), # nosemgrep
trim_blocks=True)
template = j2_env.get_template('story.j2')
@@ -394,8 +394,8 @@ def story_wizard(security_content_path,type, TEMPLATE_PATH):
def create_example(security_content_path,type, TEMPLATE_PATH):
getpass.getuser()
j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH),
trim_blocks=True, autoescape=True)
j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH), # nosemgrep
trim_blocks=True)
if type == 'detection':
bin/reporting.py
+1 -1
View File
@@ -72,7 +72,7 @@ def main(args):
TEMPLATE_PATH = os.path.join(os.path.dirname(__file__), 'jinja2_templates')
OUTPUT_PATH = os.path.join(os.path.dirname(__file__), 'reporting')
j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH), trim_blocks=True, autoescape=True)
j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH), trim_blocks=True) # nosemgrep
template = j2_env.get_template('detection_coverage.j2')
output_path = path.join(OUTPUT_PATH, 'detection_coverage.svg')
output = template.render(detection_coverage=detection_coverage)