diff --git a/.github/workflows/build-and-validate.yml b/.github/workflows/build-and-validate.yml index 8fbbf86819..ed1bf32c20 100644 --- a/.github/workflows/build-and-validate.yml +++ b/.github/workflows/build-and-validate.yml @@ -250,7 +250,6 @@ jobs: cp upload/DA-ESS_AmazonWebServices_Content-*tar.gz DA-ESS_AmazonWebServices_Content-latest.tar.gz sha256sum DA-ESS_AmazonWebServices_Content-latest.tar.gz >> checksum.txt - touch tag-canary.txt - name: store_artifacts uses: actions/upload-artifact@v2 with: @@ -266,10 +265,347 @@ jobs: build/DA-ESS_AmazonWebServices_Content-latest.tar.gz build/checksum.txt - #Store the tag to indicate that this was a tagged build - - name: store_artifacts_three + + #Everything below this line should ONLY run on a tag and nothing else + #We still want all of the above checks to run and pass before running these + run-appinspect: + runs-on: ubuntu-latest + needs: [validate-content, build-sources, build-package] + #Only run when tagged + if: startsWith(github.ref, 'refs/tags/') + steps: + + - name: Checkout Repo + uses: actions/checkout@v2 + with: + ref: 'develop' + + #Download the artifacts we want to check + - uses: actions/download-artifact@v2 + with: + name: content-latest + path: build/ + + + + + - name: Install System Packages + run: | + sudo apt update -qq + sudo apt install jq -qq + + + + - name: Submit ESCU Package to AppInspect API + env: + APPINSPECT_USERNAME: ${{ secrets.AppInspectUsername }} + APPINSPECT_PASSWORD: ${{ secrets.AppInspectPassword }} + run: | + cd bin + #Enclose in quotes in case there are any special characters in the username/password + #Better not to pass these arguments on the command line, if possible + ./appinspect.sh ../ DA-ESS-ContentUpdate-latest.tar.gz "$APPINSPECT_USERNAME" "$APPINSPECT_PASSWORD" + + - name: Submit SAAWS Package to AppInspect API + env: + APPINSPECT_USERNAME: ${{ secrets.AppInspectUsername }} + APPINSPECT_PASSWORD: ${{ secrets.AppInspectPassword }} + run: | + cd bin + ./appinspect.sh ../ DA-ESS_AmazonWebServices_Content-latest.tar.gz "$APPINSPECT_USERNAME" "$APPINSPECT_PASSWORD" + + - name: Create report artifact + if: always() + run: | + #Always create this, regardless of whether success or failure above + tar -cvzf report.tar.gz report/ + + - name: store_artifacts uses: actions/upload-artifact@v2 with: - name: tag-canary + name: appinspect_reports path: | - build/tag-canary.txt \ No newline at end of file + report.tar.gz + + #Still store the report, even if we have failed (otherwise we don't know why/how we failed) + - name: store_artifacts_on_failure + uses: actions/upload-artifact@v2 + if: failure() + with: + name: appinspect_reports_failure + path: | + report.tar.gz + + create-report: + runs-on: ubuntu-latest + needs: [validate-content, build-sources, build-package, run-appinspect] + #Only run when tagged + if: startsWith(github.ref, 'refs/tags/') + steps: + - name: Checkout Repo + uses: actions/checkout@v2 + with: + ref: 'develop' + + + - name: Install System Packages + run: | + sudo apt update -qq + sudo apt install jq -qq + + - uses: actions/setup-python@v2 + with: + python-version: '3.9' #Available versions here - https://github.com/actions/python-versions/releases easy to change/make a matrix/use pypy + architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified + + - name: Install Python Dependencies + run: | + #Get the virtualenv set up + rm -rf venv + python3 -m venv --clear venv + source venv/bin/activate + python3 -m pip install -q -r requirements.txt + + - name: run reporting + run: | + source venv/bin/activate + python3 bin/reporting.py + + #Official, Verified Amazon-AWS Github Account Provided Action + - uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + # aws-session-token: ${{ secrets.AWS_SESSION_TOKEN }} # if you have/need it + aws-region: us-west-1 #assume we will always use this, could make this an environment variable... + + - name: Upload Reporting + run: | + aws s3 cp bin/reporting s3://security-content-testing/reporting --recursive --exclude "*" --include "*.svg" + + update-sources-github: + runs-on: ubuntu-latest + needs: [validate-content, build-sources, build-package, run-appinspect, create-report] + #Only run when tagged + if: startsWith(github.ref, 'refs/tags/') + steps: + + - name: Checkout Repo + uses: actions/checkout@v2 + with: + ref: 'develop' + + - uses: actions/setup-python@v2 + with: + python-version: '3.9' #Available versions here - https://github.com/actions/python-versions/releases easy to change/make a matrix/use pypy + architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified + + - uses: actions/download-artifact@v2 + with: + name: content-latest + + + - name: Stage artifacts in proper directories + run: | + mkdir latest-escu + tar -zxf DA-ESS-ContentUpdate-latest.tar.gz -C latest-escu --strip-components=1 + mkdir latest-saaws + tar -zxf DA-ESS_AmazonWebServices_Content-latest.tar.gz -C latest-saaws --strip-components=1 + - name: Install Python Dependencies + run: | + #Get the virtualenv set up + rm -rf venv + python3 -m venv --clear venv + source venv/bin/activate + python3 -m pip install -q -r requirements.txt + + - name: Get CTI Repo for Mitre context + uses: actions/checkout@v2 + with: + repository: mitre/cti #check out https://github.com/mitre/cti.git, defaults to HEAD + path: "cti/" + + - name: Get branch and PR required for detection testing main.py + id: vars + run: | + echo "::set-output name=branch::${GITHUB_REF#refs/heads/}" + + - name: Run doc-gen + run: | + source venv/bin/activate + python3 bin/doc_gen.py --path . --output docs -v + + - name: Make YAMLs Pretty + run: | + source venv/bin/activate + python3 bin/pretty_yaml.py --path . -v + + - name: Run generate-actors-map + run: | + source venv/bin/activate + python3 bin/generate-actors-map.py --projects_path . --output docs/mitre-map/ + + - name: Run generate-coverage-map + run: | + source venv/bin/activate + python3 bin/generate-coverage-map.py --projects_path . --output docs/mitre-map + + - name: Update github with new docs and package bits + run: | + rm -rf dist + mkdir dist + echo "Directory layout 3" + pwd + ls -lah + mv latest-escu dist/escu + mv latest-saaws dist/saaws + # configure git to prep for commit + #git config credential.helper 'cache --timeout=120' + git config user.email "research@splunk.com" + git config user.name "research bot" + git config --global push.default simple + git add dist/* + git add docs/* + git add detections/* + git commit --allow-empty -m "updating docs and package bits [ci skip]" + # Push quietly to prevent showing the token in log + #No need to provide any credentials + git push + + + publish-github-release: + #Github-maintained release action is in archived state: https://github.com/actions/create-release + #They recommend several and we use the following with the most stars: https://github.com/softprops/action-gh-release + runs-on: ubuntu-latest + needs: [validate-content, build-sources, build-package, run-appinspect, create-report, update-sources-github] + #Only run when tagged + if: startsWith(github.ref, 'refs/tags/') + steps: + + #Get the artifacts that we need + - uses: actions/download-artifact@v2 + with: + name: content-latest + + - uses: actions/download-artifact@v2 + with: + name: appinspect_reports + + + + #Rename those artifacts appropriately + - name: Set tag + id: vars + run: echo "::set-output name=tag::${GITHUB_REF#refs/*/}" + + - name: Rename the content-update appropriately + run: | + cp DA-ESS-ContentUpdate-latest.tar.gz DA-ESS-ContentUpdate-${{ steps.vars.outputs.tag }}.tar.gz + cp DA-ESS_AmazonWebServices_Content-latest.tar.gz DA-ESS_AmazonWebServices_Content-${{ steps.vars.outputs.tag }}.tar.gz + + #No checksum on the reports + cp report.tar.gz report-${{ steps.vars.outputs.tag }}.tar.gz + + cp checksum.txt checksum-${{ steps.vars.outputs.tag }}.txt + + #Upload all of the artifacts that we have created using the third party + #action recommended bu Github + - name: Upload Release Artifacts + uses: softprops/action-gh-release@v1 + with: + files: | + DA-ESS-ContentUpdate-${{ steps.vars.outputs.tag }}.tar.gz + DA-ESS_AmazonWebServices_Content-${{ steps.vars.outputs.tag }}.tar.gz + report-${{ steps.vars.outputs.tag }}.tar.gz + checksum-${{ steps.vars.outputs.tag }}.txt + + + + attack-range-update: + runs-on: ubuntu-latest + needs: [validate-content, build-sources, build-package, run-appinspect, create-report, update-sources-github, publish-github-release] + #Only run when tagged + if: startsWith(github.ref, 'refs/tags/') + steps: + + #Get the artifacts that we need + - uses: actions/download-artifact@v2 + with: + name: content-latest + + + #Official, Verified Amazon-AWS Github Account Provided Action + - uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + # aws-session-token: ${{ secrets.AWS_SESSION_TOKEN }} # if you have/need it + aws-region: us-west-1 #assume we will always use this, could make this an environment variable... + + - name: Sync latest ESCU to the Attack Range S3 bucket for apps + run: | + aws s3 cp DA-ESS-ContentUpdate-latest.tar.gz s3://attack-range-appbinaries-testing/ + # make the file public since it is not by default + aws s3api put-object-acl --bucket attack-range-appbinaries-testing --key DA-ESS-ContentUpdate-latest.tar.gz --acl public-read + + master-api-update: + runs-on: ubuntu-latest + needs: [validate-content, build-sources, build-package, run-appinspect, create-report, update-sources-github, publish-github-release, attack-range-update] + #Only run when tagged + if: startsWith(github.ref, 'refs/tags/') + steps: + - name: Checkout Repo + uses: actions/checkout@v2 + with: + ref: 'develop' + + + - uses: actions/setup-python@v2 + with: + python-version: '3.9' #Available versions here - https://github.com/actions/python-versions/releases easy to change/make a matrix/use pypy + architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified + + - name: Install Python Dependencies + run: | + #Get the virtualenv set up + rm -rf venv + python3 -m venv --clear venv + source venv/bin/activate + python3 -m pip install -q -r requirements.txt + + - name: Create Baseline Folder + run: | + source venv/bin/activate + python3 bin/create_baseline_folder.py + + #Official, Verified Amazon-AWS Github Account Provided Action + - uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + # aws-session-token: ${{ secrets.AWS_SESSION_TOKEN }} # if you have/need it + aws-region: us-west-1 #assume we will always use this, could make this an environment variable... + + - name: Update API sources + run: | + aws s3 rm s3://security-content-testing --recursive --exclude "*" --include "*.yml" + aws s3 cp stories s3://security-content-testing/stories --recursive --exclude "*" --include "*.yml" + aws s3 cp baselines s3://security-content-testing/baselines --recursive --exclude "*" --include "*.yml" + aws s3 cp detections s3://security-content-testing/detections --recursive --exclude "*" --include "*.yml" + aws s3 cp response_tasks s3://security-content-testing/response_tasks --recursive --exclude "*" --include "*.yml" + aws s3 cp responses s3://security-content-testing/responses --recursive --exclude "*" --include "*.yml" + aws s3 cp lookups s3://security-content-testing/lookups --recursive --exclude "*" --include "*.yml" + aws s3 cp lookups s3://security-content-testing/lookups --recursive --exclude "*" --include "*.csv" + aws s3 cp macros s3://security-content-testing/macros --recursive --exclude "*" --include "*.yml" + aws s3 cp deployments s3://security-content-testing/deployments --recursive --exclude "*" --include "*.yml" + - name: Security Content API Smoke Test + run: | + API_URL='https://content.splunkresearch.com/detections' + API_STATUS=$(curl -s -o /dev/null -w "%{http_code}" $API_URL) + echo "Security Content API Status: $API_STATUS" + if [ "$API_STATUS" != "200" ]; then + echo "Error [Security Content API status: $API_STATUS]" + exit 1 + fi + + diff --git a/.github/workflows/detection-testing.yml b/.github/workflows/detection-testing.yml index c7dd81d847..241c21d3ec 100644 --- a/.github/workflows/detection-testing.yml +++ b/.github/workflows/detection-testing.yml @@ -30,8 +30,9 @@ jobs: runs-on: ubuntu-latest environment: Detection-Testing-Approval needs: [validate-tag-if-present] - #Only run when tagged - if: startsWith(github.ref, 'refs/heads/') + #Only run when we are pushing to the head of a branch or on a PR. + #This implicitly means that we do not run on a tag + if: startsWith(github.ref, 'refs/heads/') || startsWith(github.ref, 'refs/pull/') steps: - name: Checkout Repo diff --git a/.github/workflows/release-checks.yml b/.github/workflows/release-checks.yml deleted file mode 100644 index ac27b9fe4e..0000000000 --- a/.github/workflows/release-checks.yml +++ /dev/null @@ -1,398 +0,0 @@ -name: release-checks -on: - workflow_run: - workflows: ["validate-and-build"] - types: - - completed - - - -jobs: - - #Check that the validate-and-build workflow succeeded - check-validate-and-build-success: - runs-on: ubuntu-latest - steps: - - if: github.event.workflow_run.conclusion != 'success' - name: Abort if failed - run: | - echo "FAIL: validate-and-build.yml DID NOT run successfully. Terminating..." - exit 1 - - name: Print Success - run: | - echo "SUCCESS: validate-and-build.yml ran successfully. Continue" - exit 0 - - - #Enusre that we are running on a tag. There is no good way to see if this was - #triggered from a tag/release, so we use the creation of an aritifact in the - #validate-and-build workflow to represent it - verify-tag: - runs-on: ubuntu-latest - needs: [check-validate-and-build-success] - steps: - - name: Try to get the canary - uses: dawidd6/action-download-artifact@v2 - with: - github_token: "${{ secrets.GITHUB_TOKEN }}" - workflow: ${{ github.event.workflow_run.workflow_id }} - #workflow: validate-and-build.yml - #run_id: ${{ github.event.workflow_run.id }} - name: tag-canary - path: canary - - name: Check for existence of canary - run: | - #If this file does not exist, then cat will return a nonzero status (failure) - #and the entire workflow will fail - cat canary/tag-canary.txt - - run-appinspect: - runs-on: ubuntu-latest - needs: [check-validate-and-build-success, verify-tag] - #Only run when tagged - steps: - - - name: Checkout Repo - uses: actions/checkout@v2 - with: - ref: 'develop' - - #Download the artifacts we want to check - - name: Restore Content-Pack Artifacts for AppInspect testing - uses: dawidd6/action-download-artifact@v2 - with: - workflow: validate-and-build.yml - workflow_conclusion: success - run_id: ${{ github.event.workflow_run.id }} - name: content-latest - path: build/ - - - - - name: Install System Packages - run: | - sudo apt update -qq - sudo apt install jq -qq - - - - - name: Submit ESCU Package to AppInspect API - env: - APPINSPECT_USERNAME: ${{ secrets.AppInspectUsername }} - APPINSPECT_PASSWORD: ${{ secrets.AppInspectPassword }} - run: | - cd bin - #Enclose in quotes in case there are any special characters in the username/password - #Better not to pass these arguments on the command line, if possible - ./appinspect.sh ../ DA-ESS-ContentUpdate-latest.tar.gz "$APPINSPECT_USERNAME" "$APPINSPECT_PASSWORD" - - - name: Submit SAAWS Package to AppInspect API - env: - APPINSPECT_USERNAME: ${{ secrets.AppInspectUsername }} - APPINSPECT_PASSWORD: ${{ secrets.AppInspectPassword }} - run: | - cd bin - ./appinspect.sh ../ DA-ESS_AmazonWebServices_Content-latest.tar.gz "$APPINSPECT_USERNAME" "$APPINSPECT_PASSWORD" - - - name: Create report artifact - if: always() - run: | - #Always create this, regardless of whether success or failure above - tar -cvzf report.tar.gz report/ - - - name: store_artifacts - uses: actions/upload-artifact@v2 - with: - name: appinspect_reports - path: | - report.tar.gz - - #Still store the report, even if we have failed (otherwise we don't know why/how we failed) - - name: store_artifacts_on_failure - uses: actions/upload-artifact@v2 - if: failure() - with: - name: appinspect_reports_failure - path: | - report.tar.gz - - create-report: - runs-on: ubuntu-latest - needs: [check-validate-and-build-success, verify-tag, run-appinspect] - #Only run when tagged - steps: - - name: Checkout Repo - uses: actions/checkout@v2 - with: - ref: 'develop' - - - - name: Install System Packages - run: | - sudo apt update -qq - sudo apt install jq -qq - - - uses: actions/setup-python@v2 - with: - python-version: '3.9' #Available versions here - https://github.com/actions/python-versions/releases easy to change/make a matrix/use pypy - architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified - - - name: Install Python Dependencies - run: | - #Get the virtualenv set up - rm -rf venv - python3 -m venv --clear venv - source venv/bin/activate - python3 -m pip install -q -r requirements.txt - - - name: run reporting - run: | - source venv/bin/activate - python3 bin/reporting.py - - #Official, Verified Amazon-AWS Github Account Provided Action - - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - # aws-session-token: ${{ secrets.AWS_SESSION_TOKEN }} # if you have/need it - aws-region: us-west-1 #assume we will always use this, could make this an environment variable... - - - name: Upload Reporting - run: | - aws s3 cp bin/reporting s3://security-content-testing/reporting --recursive --exclude "*" --include "*.svg" - - update-sources-github: - runs-on: ubuntu-latest - needs: [check-validate-and-build-success, verify-tag, run-appinspect, create-report] - #Only run when tagged - steps: - - - name: Checkout Repo - uses: actions/checkout@v2 - with: - ref: 'develop' - - - uses: actions/setup-python@v2 - with: - python-version: '3.9' #Available versions here - https://github.com/actions/python-versions/releases easy to change/make a matrix/use pypy - architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified - - - - uses: dawidd6/action-download-artifact@v2 - with: - workflow: validate-and-build.yml - workflow_conclusion: success - run_id: ${{ github.event.workflow_run.id }} - path: . - name: content-latest - - - name: Stage artifacts in proper directories - run: | - mkdir latest-escu - tar -zxf DA-ESS-ContentUpdate-latest.tar.gz -C latest-escu --strip-components=1 - mkdir latest-saaws - tar -zxf DA-ESS_AmazonWebServices_Content-latest.tar.gz -C latest-saaws --strip-components=1 - - name: Install Python Dependencies - run: | - #Get the virtualenv set up - rm -rf venv - python3 -m venv --clear venv - source venv/bin/activate - python3 -m pip install -q -r requirements.txt - - - name: Get CTI Repo for Mitre context - uses: actions/checkout@v2 - with: - repository: mitre/cti #check out https://github.com/mitre/cti.git, defaults to HEAD - path: "cti/" - - - name: Get branch and PR required for detection testing main.py - id: vars - run: | - echo "::set-output name=branch::${GITHUB_REF#refs/heads/}" - - - name: Run doc-gen - run: | - source venv/bin/activate - python3 bin/doc_gen.py --path . --output docs -v - - - name: Make YAMLs Pretty - run: | - source venv/bin/activate - python3 bin/pretty_yaml.py --path . -v - - - name: Run generate-actors-map - run: | - source venv/bin/activate - python3 bin/generate-actors-map.py --projects_path . --output docs/mitre-map/ - - - name: Run generate-coverage-map - run: | - source venv/bin/activate - python3 bin/generate-coverage-map.py --projects_path . --output docs/mitre-map - - - name: Update github with new docs and package bits - run: | - rm -rf dist - mkdir dist - echo "Directory layout 3" - pwd - ls -lah - mv latest-escu dist/escu - mv latest-saaws dist/saaws - # configure git to prep for commit - #git config credential.helper 'cache --timeout=120' - git config user.email "research@splunk.com" - git config user.name "research bot" - git config --global push.default simple - git add dist/* - git add docs/* - git add detections/* - git commit --allow-empty -m "updating docs and package bits [ci skip]" - # Push quietly to prevent showing the token in log - #No need to provide any credentials - git push - - - publish-github-release: - #Github-maintained release action is in archived state: https://github.com/actions/create-release - #They recommend several and we use the following with the most stars: https://github.com/softprops/action-gh-release - runs-on: ubuntu-latest - needs: [check-validate-and-build-success, verify-tag, run-appinspect, create-report, update-sources-github] - #Only run when tagged - - steps: - - #Get the artifacts that we need - - uses: dawidd6/action-download-artifact@v2 - with: - workflow: validate-and-build.yml - workflow_conclusion: success - run_id: ${{ github.event.workflow_run.id }} - path: . - name: content-latest - - uses: dawidd6/action-download-artifact@v2 - with: - workflow: validate-and-build.yml - workflow_conclusion: success - run_id: ${{ github.event.workflow_run.id }} - path: . - name: appinspect_reports - - #Rename those artifacts appropriately - - name: Set tag - id: vars - run: echo "::set-output name=tag::${GITHUB_REF#refs/*/}" - - - name: Rename the content-update appropriately - run: | - cp DA-ESS-ContentUpdate-latest.tar.gz DA-ESS-ContentUpdate-${{ steps.vars.outputs.tag }}.tar.gz - cp DA-ESS_AmazonWebServices_Content-latest.tar.gz DA-ESS_AmazonWebServices_Content-${{ steps.vars.outputs.tag }}.tar.gz - - #No checksum on the reports - cp report.tar.gz report-${{ steps.vars.outputs.tag }}.tar.gz - - cp checksum.txt checksum-${{ steps.vars.outputs.tag }}.txt - - #Upload all of the artifacts that we have created using the third party - #action recommended bu Github - - name: Upload Release Artifacts - uses: softprops/action-gh-release@v1 - with: - files: | - DA-ESS-ContentUpdate-${{ steps.vars.outputs.tag }}.tar.gz - DA-ESS_AmazonWebServices_Content-${{ steps.vars.outputs.tag }}.tar.gz - report-${{ steps.vars.outputs.tag }}.tar.gz - checksum-${{ steps.vars.outputs.tag }}.txt - - - - attack-range-update: - runs-on: ubuntu-latest - needs: [check-validate-and-build-success, verify-tag, run-appinspect, create-report, update-sources-github, publish-github-release] - #Only run when tagged - steps: - - - uses: dawidd6/action-download-artifact@v2 - with: - workflow: validate-and-build.yml - workflow_conclusion: success - run_id: ${{ github.event.workflow_run.id }} - path: . - name: content-latest - - #Official, Verified Amazon-AWS Github Account Provided Action - - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - # aws-session-token: ${{ secrets.AWS_SESSION_TOKEN }} # if you have/need it - aws-region: us-west-1 #assume we will always use this, could make this an environment variable... - - - name: Sync latest ESCU to the Attack Range S3 bucket for apps - run: | - aws s3 cp DA-ESS-ContentUpdate-latest.tar.gz s3://attack-range-appbinaries-testing/ - # make the file public since it is not by default - aws s3api put-object-acl --bucket attack-range-appbinaries-testing --key DA-ESS-ContentUpdate-latest.tar.gz --acl public-read - - master-api-update: - runs-on: ubuntu-latest - needs: [check-validate-and-build-success, verify-tag, run-appinspect, create-report, update-sources-github, publish-github-release, attack-range-update] - #Only run when tagged - steps: - - name: Checkout Repo - uses: actions/checkout@v2 - with: - ref: 'develop' - - - - uses: actions/setup-python@v2 - with: - python-version: '3.9' #Available versions here - https://github.com/actions/python-versions/releases easy to change/make a matrix/use pypy - architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified - - - name: Install Python Dependencies - run: | - #Get the virtualenv set up - rm -rf venv - python3 -m venv --clear venv - source venv/bin/activate - python3 -m pip install -q -r requirements.txt - - - name: Create Baseline Folder - run: | - source venv/bin/activate - python3 bin/create_baseline_folder.py - - #Official, Verified Amazon-AWS Github Account Provided Action - - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - # aws-session-token: ${{ secrets.AWS_SESSION_TOKEN }} # if you have/need it - aws-region: us-west-1 #assume we will always use this, could make this an environment variable... - - - name: Update API sources - run: | - aws s3 rm s3://security-content-testing --recursive --exclude "*" --include "*.yml" - aws s3 cp stories s3://security-content-testing/stories --recursive --exclude "*" --include "*.yml" - aws s3 cp baselines s3://security-content-testing/baselines --recursive --exclude "*" --include "*.yml" - aws s3 cp detections s3://security-content-testing/detections --recursive --exclude "*" --include "*.yml" - aws s3 cp response_tasks s3://security-content-testing/response_tasks --recursive --exclude "*" --include "*.yml" - aws s3 cp responses s3://security-content-testing/responses --recursive --exclude "*" --include "*.yml" - aws s3 cp lookups s3://security-content-testing/lookups --recursive --exclude "*" --include "*.yml" - aws s3 cp lookups s3://security-content-testing/lookups --recursive --exclude "*" --include "*.csv" - aws s3 cp macros s3://security-content-testing/macros --recursive --exclude "*" --include "*.yml" - aws s3 cp deployments s3://security-content-testing/deployments --recursive --exclude "*" --include "*.yml" - - name: Security Content API Smoke Test - run: | - API_URL='https://content.splunkresearch.com/detections' - API_STATUS=$(curl -s -o /dev/null -w "%{http_code}" $API_URL) - echo "Security Content API Status: $API_STATUS" - if [ "$API_STATUS" != "200" ]; then - echo "Error [Security Content API status: $API_STATUS]" - exit 1 - fi - - diff --git a/automated_detection_testing/detection_service.py b/automated_detection_testing/detection_service.py index 022c2dadcd..d7cd1d841d 100644 --- a/automated_detection_testing/detection_service.py +++ b/automated_detection_testing/detection_service.py @@ -156,7 +156,7 @@ def main(args): counter = counter + 1 - j2_env = Environment(loader=FileSystemLoader('templates'),trim_blocks=True, autoescape=True) + j2_env = Environment(loader=FileSystemLoader('templates'),trim_blocks=True) # nosemgrep template = j2_env.get_template('PR_template.j2') body = template.render(results=results) diff --git a/bin/doc_gen.py b/bin/doc_gen.py index dbb2e4d84f..48d9bcb338 100644 --- a/bin/doc_gen.py +++ b/bin/doc_gen.py @@ -150,8 +150,8 @@ def generate_doc_stories(REPO_PATH, OUTPUT_DIR, TEMPLATE_PATH, attack, sorted_de if category['name'] == story['tags']['category'][0]: category['stories'].append(story) - j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH), - trim_blocks=False, autoescape=True) + j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH), # nosemgrep + trim_blocks=False) # write markdown template = j2_env.get_template('doc_stories_markdown.j2') output_path = path.join(OUTPUT_DIR + '/stories.md') @@ -207,8 +207,8 @@ def generate_doc_detections(REPO_PATH, OUTPUT_DIR, TEMPLATE_PATH, attack, messag sorted_detections = sorted(detections, key=lambda i: i['name']) - j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH), - trim_blocks=False, autoescape=True) + j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH), # nosemgrep + trim_blocks=False) # write markdown template = j2_env.get_template('doc_detections_markdown.j2') diff --git a/bin/generate.py b/bin/generate.py index 3991bf7d64..86abe3e838 100644 --- a/bin/generate.py +++ b/bin/generate.py @@ -10,7 +10,7 @@ import argparse from os import path import sys import datetime -from jinja2 import Environment, FileSystemLoader, select_autoescape +from jinja2 import Environment, FileSystemLoader import re from attackcti import attack_client import csv @@ -62,8 +62,8 @@ def generate_transforms_conf(lookups, TEMPLATE_PATH, OUTPUT_PATH): utc_time = datetime.datetime.utcnow().replace(microsecond=0).isoformat() - j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH), - trim_blocks=True) + j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH), # nosemgrep + trim_blocks=True) template = j2_env.get_template('transforms.j2') output_path = path.join(OUTPUT_PATH, 'default/transforms.conf') output = template.render(lookups=sorted_lookups, time=utc_time) @@ -78,7 +78,7 @@ def generate_collections_conf(lookups, TEMPLATE_PATH, OUTPUT_PATH): utc_time = datetime.datetime.utcnow().replace(microsecond=0).isoformat() - j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH), + j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH), # nosemgrep trim_blocks=True) template = j2_env.get_template('collections.j2') output_path = path.join(OUTPUT_PATH, 'default/collections.conf') @@ -97,7 +97,7 @@ def generate_savedsearches_conf(detections, deployments, TEMPLATE_PATH, OUTPUT_P utc_time = datetime.datetime.utcnow().replace(microsecond=0).isoformat() - j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH), + j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH), # nosemgrep trim_blocks=True) j2_env.filters['custom_jinja2_enrichment_filter'] = custom_jinja2_enrichment_filter template = j2_env.get_template('savedsearches.j2') @@ -112,7 +112,7 @@ def generate_savedsearches_conf(detections, deployments, TEMPLATE_PATH, OUTPUT_P def generate_analytic_story_conf(stories, detections, TEMPLATE_PATH, OUTPUT_PATH): utc_time = datetime.datetime.utcnow().replace(microsecond=0).isoformat() - j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH), + j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH), # nosemgrep trim_blocks=True) template = j2_env.get_template('analytic_stories.j2') output_path = path.join(OUTPUT_PATH, 'default/analytic_stories.conf') @@ -125,7 +125,7 @@ def generate_analytic_story_conf(stories, detections, TEMPLATE_PATH, OUTPUT_PATH def generate_use_case_library_conf(stories, detections, TEMPLATE_PATH, OUTPUT_PATH): utc_time = datetime.datetime.utcnow().replace(microsecond=0).isoformat() - j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH), + j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH), # nosemgrep trim_blocks=True) template = j2_env.get_template('use_case_library.j2') output_path = path.join(OUTPUT_PATH, 'default/use_case_library.conf') @@ -150,7 +150,7 @@ def generate_macros_conf(macros, detections, TEMPLATE_PATH, OUTPUT_PATH): utc_time = datetime.datetime.utcnow().replace(microsecond=0).isoformat() - j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH), + j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH), # nosemgrep trim_blocks=True) template = j2_env.get_template('macros.j2') output_path = path.join(OUTPUT_PATH, 'default/macros.conf') @@ -170,7 +170,7 @@ def generate_workbench_panels(response_tasks, stories, TEMPLATE_PATH, OUTPUT_PAT response_file_name_xml = response_file_name + "___response_task.xml" response_task['lowercase_name'] = response_file_name workbench_panel_objects.append(response_task) - j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH), + j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH), # nosemgrep trim_blocks=True) template = j2_env.get_template('panel.j2') file_path = "default/data/ui/panels/workbench_panel_" + response_file_name_xml @@ -182,14 +182,14 @@ def generate_workbench_panels(response_tasks, stories, TEMPLATE_PATH, OUTPUT_PAT with open(output_path, 'w') as f: f.write(output) - j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH), + j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH), # nosemgrep trim_blocks=True) template = j2_env.get_template('es_investigations.j2') output_path = path.join(OUTPUT_PATH, 'default/es_investigations.conf') output = template.render(response_tasks=workbench_panel_objects, stories=stories) with open(output_path, 'w', encoding="utf-8") as f: f.write(output) - j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH), + j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH), # nosemgrep trim_blocks=True) template = j2_env.get_template('workflow_actions.j2') output_path = path.join(OUTPUT_PATH, 'default/workflow_actions.conf') @@ -590,7 +590,6 @@ def get_objects(REPO_PATH, OUTPUT_PATH, PRODUCT, VERBOSE): def main(REPO_PATH, OUTPUT_PATH, PRODUCT, VERBOSE): - #select_autoescape(default_for_string=True, default=True) TEMPLATE_PATH = path.join(REPO_PATH, 'bin/jinja2_templates') objects = get_objects(REPO_PATH, OUTPUT_PATH, PRODUCT, VERBOSE) diff --git a/bin/newcontent.py b/bin/newcontent.py index 736e0ada0b..ad88c90bbb 100644 --- a/bin/newcontent.py +++ b/bin/newcontent.py @@ -211,8 +211,8 @@ def detection_wizard(security_content_path,type,TEMPLATE_PATH): print(mitre_attack_id) - j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH), - trim_blocks=True, autoescape=True) + j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH), # nosemgrep + trim_blocks=True) answers['products'] = ['Splunk Enterprise','Splunk Enterprise Security','Splunk Cloud'] answers['references'] = [] @@ -374,8 +374,8 @@ def story_wizard(security_content_path,type, TEMPLATE_PATH): }, ] answers = prompt(questions) - j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH), - trim_blocks=True, autoescape=True) + j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH), # nosemgrep + trim_blocks=True) template = j2_env.get_template('story.j2') @@ -394,8 +394,8 @@ def story_wizard(security_content_path,type, TEMPLATE_PATH): def create_example(security_content_path,type, TEMPLATE_PATH): getpass.getuser() - j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH), - trim_blocks=True, autoescape=True) + j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH), # nosemgrep + trim_blocks=True) if type == 'detection': diff --git a/bin/reporting.py b/bin/reporting.py index c70cb54fce..84432a7fb9 100644 --- a/bin/reporting.py +++ b/bin/reporting.py @@ -72,7 +72,7 @@ def main(args): TEMPLATE_PATH = os.path.join(os.path.dirname(__file__), 'jinja2_templates') OUTPUT_PATH = os.path.join(os.path.dirname(__file__), 'reporting') - j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH), trim_blocks=True, autoescape=True) + j2_env = Environment(loader=FileSystemLoader(TEMPLATE_PATH), trim_blocks=True) # nosemgrep template = j2_env.get_template('detection_coverage.j2') output_path = path.join(OUTPUT_PATH, 'detection_coverage.svg') output = template.render(detection_coverage=detection_coverage)