# Python CircleCI 2.0 configuration file
#
# Check https://circleci.com/docs/2.0/language-python/ for more details
#
#############
# Automatically generated by ci-generator.py in splunk/security-content
# On Date: 2020-07-30T03:07:52 UTC
# Author: Splunk Security Research
# Contact: research@splunk.com
#############

version: 2.1
orbs:
  aws-cli: circleci/aws-cli@0.1.19

dependencies:
  cache_directories:
    - "~/.apt-cache"
  pre:
    - sudo rm -rf /var/cache/apt/archives && sudo ln -s ~/.apt-cache /var/cache/apt/archives && mkdir -p ~/.apt-cache/partial

apt-run: &apt-install
  name: install system packages
  command: |
    sudo apt update -qq
    sudo apt install -y python-dev python3-dev jq -qq
    # install go for other testing tools
    # but first lets clean up the env
    if  [ -f goinstall.sh ]; then
        rm goinstall.sh
    fi
    if [ -d ~/.go ]; then
        rm -rf ~/.go
    fi
    wget https://raw.githubusercontent.com/canha/golang-tools-install-script/master/goinstall.sh
    sudo chown circleci goinstall.sh
    chmod +x goinstall.sh
    ./goinstall.sh

executors:
  content-executor:
    docker:
      - image: circleci/python:latest
    working_directory: ~/repo

jobs:
  validate-content:
    executor: content-executor
    steps:
      - run:
          name: checkout repo
          command: |
            if [[ ! -z "${CIRCLE_PULL_REQUEST}" &&  ! -z "${CIRCLE_PR_NUMBER}" ]]; then
                git clone https://${GITHUB_TOKEN}@github.com/splunk/security-content.git
                cd security-content
                git fetch origin pull/${CIRCLE_PR_NUMBER}/head:${CIRCLE_BRANCH}
                git checkout ${CIRCLE_BRANCH}
            elif [ "${CIRCLE_BRANCH}" == "" ]; then
                git clone https://${GITHUB_TOKEN}@github.com/splunk/security-content.git
            else
                git clone --branch ${CIRCLE_BRANCH} https://${GITHUB_TOKEN}@github.com/splunk/security-content.git
            fi
      - restore_cache:
          key: virtualenv
      - run: *apt-install
      - run:
          name: install python dependencies
          command: |
            cd security-content
            rm -rf venv
            virtualenv --python=/usr/bin/python3 --clear venv
            source venv/bin/activate
            pip install -q -r requirements.txt
      - save_cache:
          key: virtualenv
          paths:
            - "/security-content/venv"
      - run:
          name: run validate
          command: |
            cd security-content
            source venv/bin/activate
            python contentctl.py --path . --verbose  validate
      - run:
          name: run doc-gen
          command: |
            cd security-content
            source venv/bin/activate
            python bin/doc-gen.py --path . --output docs -v

  build-sources:
    executor: content-executor
    steps:
      - run:
          name: checkout repo
          command: |
            if [[ ! -z "${CIRCLE_PULL_REQUEST}" &&  ! -z "${CIRCLE_PR_NUMBER}" ]]; then
                git clone https://${GITHUB_TOKEN}@github.com/splunk/security-content.git
                cd security-content
                git fetch origin pull/${CIRCLE_PR_NUMBER}/head:${CIRCLE_BRANCH}
                git checkout ${CIRCLE_BRANCH}
            elif [ "${CIRCLE_BRANCH}" == "" ]; then
                git clone https://${GITHUB_TOKEN}@github.com/splunk/security-content.git
            else
                git clone --branch ${CIRCLE_BRANCH} https://${GITHUB_TOKEN}@github.com/splunk/security-content.git
            fi
      - run: *apt-install
      - run:
          name: install python dependencies
          command: |
            cd security-content
            rm -rf venv
            virtualenv --python=/usr/bin/python3 --clear venv
            source venv/bin/activate
            pip install -q -r requirements.txt
      - run:
          name: run generate
          command: |
            cd security-content
            source venv/bin/activate
            python contentctl.py --path . --verbose generate --output package
            # make a copy of use_case_lib in order to have ES work :-(
            cp package/default/use_case_library.conf package/default/analyticstories.conf
      - run:
          name: copy lookups .csv files
          command: |
            cd security-content
            # clean up current lookups
            rm -rf package/lookups
            mkdir package/lookups
            # copy over lookups
            cd lookups
            cp -rv *.csv ../package/lookups
      - run:
          name: update version and build number
          command: |
            cd security-content
            # check if tag is set, get build number from the tag if set
            if [ -z "${CIRCLE_TAG}" ]; then
                CONTENT_VERSION=$(grep -oP "(\d+.\d+.\d+$)" package/default/content-version.conf)
                echo "detected content version: $CONTENT_VERSION"
            else
                CONTENT_VERSION=$(echo $CIRCLE_TAG | grep -oP "\d+.\d+.\d+")
                echo "content version: $CONTENT_VERSION, set by tag: $CIRCLE_TAG"
            fi
            # update build number and version
            sed -i "s/build = .*$/build = $CIRCLE_BUILD_NUM/g" package/default/app.conf
            sed -i "s/^version = .*$/version = $CONTENT_VERSION/g" package/default/app.conf
            sed -i "s/\"version\": .*$/\"version\": \"$CONTENT_VERSION\"/g" package/app.manifest
            sed -i "s/version = .*$/version = $CONTENT_VERSION/g" package/default/content-version.conf
            tar -czf content-pack-build.tar.gz package/*
      - persist_to_workspace:
          root: security-content/
          paths:
              - content-pack-build.tar.gz

  build-package:
    executor: content-executor
    steps:
      - attach_workspace:
          at: ~/dist
      - run:
          name: grab splunk packaging toolkit
          command: |
            curl -Ls https://download.splunk.com/misc/packaging-toolkit/splunk-packaging-toolkit-0.9.0.tar.gz -o ~/splunk-packaging-toolkit-latest.tar.gz
            mkdir ~/slim-latest
            tar -zxf ~/splunk-packaging-toolkit-latest.tar.gz -C ~/slim-latest --strip-components=1
      - run:
          name: install splunk packaging toolkit (slim)
          command: |
              cd ~/slim-latest
              sudo pip install --upgrade pip setuptools
              sudo pip install virtualenv
              virtualenv --python=/usr/bin/python2.7 --clear venv
              source venv/bin/activate
              pip install semantic_version
              pip install .
      - run:
          name: create a .spl for this build using slim
          command: |
              source ~/slim-latest/venv/bin/activate
              cd ~/dist
              tar -zxf content-pack-build.tar.gz
              mv package DA-ESS-ContentUpdate
              slim package -o upload DA-ESS-ContentUpdate
              cp upload/*.tar.gz DA-ESS-ContentUpdate-latest.tar.gz
      - store_artifacts:
          path: ~/dist/upload
          destination: package/
      - persist_to_workspace:
          root: ~/dist
          paths:
             - DA-ESS-ContentUpdate-latest.tar.gz

  run-appinspect:
    executor: content-executor
    steps:
      - attach_workspace:
          at: ~/
      - run: *apt-install
      - run:
          name: checkout repo
          command: |
            if [[ ! -z "${CIRCLE_PULL_REQUEST}" &&  ! -z "${CIRCLE_PR_NUMBER}" ]]; then
                git clone https://${GITHUB_TOKEN}@github.com/splunk/security-content.git
                cd security-content
                git fetch origin pull/${CIRCLE_PR_NUMBER}/head:${CIRCLE_BRANCH}
                git checkout ${CIRCLE_BRANCH}
            elif [ "${CIRCLE_BRANCH}" == "" ]; then
                git clone https://${GITHUB_TOKEN}@github.com/splunk/security-content.git
            else
                git clone --branch ${CIRCLE_BRANCH} https://${GITHUB_TOKEN}@github.com/splunk/security-content.git
            fi
      - run:
          name: submit package to appinspect API
          command: |
            cd security-content/bin
            ./appinspect.sh ~/ $APPINSPECT_USERNAME $APPINSPECT_PASSWORD
      - store_artifacts:
          path: ~/report
          destination: report/
      - persist_to_workspace:
          root: ~/
          paths:
             - DA-ESS-ContentUpdate-latest.tar.gz
  update-sources-github:
    executor: content-executor
    steps:
      - attach_workspace:
          at: ~/
      - run: *apt-install
      - run:
          name: checkout repo
          command: |
            mkdir ~/latest
            tar -zxf ~/DA-ESS-ContentUpdate-latest.tar.gz -C ~/latest --strip-components=1
            if [ "${CIRCLE_BRANCH}" == "" ]; then
                git clone https://${GITHUB_TOKEN}@github.com/splunk/security-content.git
            else
                git clone --branch ${CIRCLE_BRANCH} https://${GITHUB_TOKEN}@github.com/splunk/security-content.git
            fi
      - run:
          name: install python dependencies
          command: |
            cd security-content
            rm -rf venv
            virtualenv --python=/usr/bin/python3 --clear venv
            source venv/bin/activate
            pip install -q -r requirements.txt
      - run:
          name: run doc-gen
          command: |
            cd security-content
            source venv/bin/activate
            python bin/doc-gen.py --path . --output docs -v
      - run:
          name: make yamls pretty
          command: |
            cd security-content
            source venv/bin/activate
            python bin/pretty_yaml.py --path . -v
      - run:
          name: get cti repo for mitre-maps
          command: |
            cd security-content
            git clone https://github.com/mitre/cti.git
      - run:
          name: run generate-actors-map
          command: |
            cd security-content
            source venv/bin/activate
            python bin/generate-actors-map.py --projects_path . --output docs/mitre-map/
      - run:
          name: run generate-coverage-map
          command: |
            cd security-content
            source venv/bin/activate
            python bin/generate-coverage-map.py --projects_path . --output docs/mitre-map
      - run:
          name: update github with new docs and package bits
          command: |
            cd security-content
            rm -rf package
            mv ~/latest package
            # configure git to prep for commit
            git config credential.helper 'cache --timeout=120'
            git config user.email "research@splunk.com"
            git config user.name "research bot"
            git config --global push.default simple
            git add package/*
            git add docs/*
            git add detections/*
            git commit --allow-empty -m "updating docs and package bits [ci skip]"
            # Push quietly to prevent showing the token in log
            git push https://${GITHUB_TOKEN}@github.com/splunk/security-content.git ${CIRCLE_BRANCH}
      - persist_to_workspace:
          root: ~/
          paths:
              - DA-ESS-ContentUpdate-latest.tar.gz

  publish-github-release:
    docker:
      - image: cibuilds/github:0.10
    steps:
      - attach_workspace:
          at: ~/
      - run:
          name: publish release on github
          command: |
            cp ~/DA-ESS-ContentUpdate-latest.tar.gz ~/DA-ESS-ContentUpdate-${CIRCLE_TAG}.tar.gz
            ghr -t ${GITHUB_TOKEN} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${CIRCLE_SHA1} -delete ${CIRCLE_TAG} ~/DA-ESS-ContentUpdate-${CIRCLE_TAG}.tar.gz
      - persist_to_workspace:
          root: ~/
          paths:
              - DA-ESS-ContentUpdate-latest.tar.gz

  attack-range-update:
    executor: aws-cli/default
    steps:
      - attach_workspace:
          at: ~/
      - aws-cli/setup:
          profile-name: default
      - run:
          name: sync latest ESCU to the Attack Range S3 bucket for apps
          command: |
            aws s3 cp ~/DA-ESS-ContentUpdate-latest.tar.gz s3://attack-range-appbinaries/
            # make the file public since it is not by default
            aws s3api put-object-acl --bucket attack-range-appbinaries --key DA-ESS-ContentUpdate-latest.tar.gz --acl public-read

  master-api-update:
    executor: aws-cli/default
    steps:
      - run:
          name: checkout repo
          command: |
            if [ "${CIRCLE_BRANCH}" == "" ]; then
                git clone https://${GITHUB_TOKEN}@github.com/splunk/security-content.git
            else
                git clone --branch ${CIRCLE_BRANCH} https://${GITHUB_TOKEN}@github.com/splunk/security-content.git
            fi
      - run: *apt-install
      - aws-cli/setup:
          profile-name: default
      - run:
          name: update api sources
          command: |
            cd security-content
            aws s3 rm s3://security-content --recursive --exclude "*" --include "*.yml"
            aws s3 cp stories s3://security-content/stories --recursive --exclude "*" --include "*.yml"
            aws s3 cp baselines s3://security-content/baselines --recursive --exclude "*" --include "*.yml"
            aws s3 cp detections s3://security-content/detections --recursive --exclude "*" --include "*.yml"
            aws s3 cp response_tasks s3://security-content/response_tasks --recursive --exclude "*" --include "*.yml"
            aws s3 cp responses s3://security-content/responses --recursive --exclude "*" --include "*.yml"
            aws s3 cp lookups s3://security-content/lookups --recursive --exclude "*" --include "*.yml"
            aws s3 cp lookups s3://security-content/lookups --recursive --exclude "*" --include "*.csv"
            aws s3 cp macros s3://security-content/macros --recursive --exclude "*" --include "*.yml"
            aws s3 cp deployments s3://security-content/deployments --recursive --exclude "*" --include "*.yml"
      - run:
          name: security content api smoke test
          command: |
            API_URL='https://content.splunkresearch.com/detections'
            API_STATUS=$(curl -s -o /dev/null -w "%{http_code}" $API_URL)
            echo "Security Content API Status: $API_STATUS"
            if [ "$API_STATUS" != "200" ]; then
              echo "Error [Security Content API status: $API_STATUS]"
              exit 1
            fi

  create-report:
    executor: content-executor
    steps:
      - run:
          name: checkout repo
          command: |
            if [ "${CIRCLE_BRANCH}" == "" ]; then
                git clone https://${GITHUB_TOKEN}@github.com/splunk/security-content.git
            else
                git clone --branch ${CIRCLE_BRANCH} https://${GITHUB_TOKEN}@github.com/splunk/security-content.git
            fi
      - run: *apt-install
      - run:
          name: install python dependencies
          command: |
            cd security-content
            rm -rf venv
            virtualenv --python=/usr/bin/python3 --clear venv
            source venv/bin/activate
            pip install -q -r requirements.txt
      - run:
          name: run reporting
          command: |
            cd security-content
            source venv/bin/activate
            python bin/reporting.py
      - aws-cli/setup:
          profile-name: default
      - run:
          name: upload reporting
          command: |
            cd security-content
            aws s3 cp bin/reporting s3://security-content/reporting --recursive --exclude "*" --include "*.svg"


workflows:
  version: 2.1
  validate-and-build:
    jobs:
      - validate-content:
        # build always
          filters:
            tags:
              only: /.*/
      - build-sources:
        # build always
          requires:
            - validate-content
          filters:
            tags:
              only: /.*/
      - build-package:
        # build always
          requires:
            - validate-content
            - build-sources
          filters:
            tags:
              only: /.*/
      - run-appinspect:
        # build always
          requires:
            - validate-content
            - build-sources
            - build-package
          filters:
            tags:
              only: /.*/
      - create-report:
          requires:
            - validate-content
            - build-sources
            - build-package
            - run-appinspect
          filters:
            tags:
              only: /^v.*/
      - update-sources-github:
        # update package and docs in github if is a tag
          requires:
            - validate-content
            - build-sources
            - build-package
            - run-appinspect
            - create-report
          filters:
            tags:
              only: /^v.*/
            branches:
              ignore: /.*/
      - publish-github-release:
        # publish release in github if is a tag
          requires:
            - validate-content
            - build-sources
            - build-package
            - run-appinspect
            - create-report
            - update-sources-github
          filters:
            tags:
              only: /^v.*/
            branches:
              ignore: /.*/
      - attack-range-update:
        # update the attack range with the latest build
          requires:
            - validate-content
            - build-sources
            - build-package
            - run-appinspect
            - create-report
            - update-sources-github
            - publish-github-release
          filters:
            tags:
              only: /^v.*/
            branches:
              ignore: /.*/
      - master-api-update:
        # update master API from develop if is a tag
          requires:
            - validate-content
            - build-sources
            - build-package
            - run-appinspect
            - create-report
            - update-sources-github
            - publish-github-release
            - attack-range-update
          filters:
            tags:
              only: /^v.*/
            branches:
              ignore: /.*/
