diff --git a/.circleci/config.yml b/.circleci/config.yml index 110f50b502..023b54b0d9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -124,47 +124,60 @@ jobs: command: | cd security-content source venv/bin/activate - python contentctl.py --path . --verbose generate --output package + python contentctl.py --path . --verbose generate --product ESCU --output dist/escu + python contentctl.py --path . --verbose generate --product MUSTANG --output dist/mustang # make a copy of use_case_lib in order to have ES work :-( - cp package/default/use_case_library.conf package/default/analyticstories.conf + cp dist/escu/default/use_case_library.conf dist/escu/default/analyticstories.conf + cp dist/mustang/default/use_case_library.conf dist/mustang/default/analyticstories.conf - run: name: copy lookups .csv files command: | cd security-content # clean up current lookups - rm -rf package/lookups - mkdir package/lookups + rm -rf dist/escu/lookups + rm -rf dist/mustang/lookups + mkdir dist/escu/lookups + mkdir dist/mustang/lookups # copy over lookups cd lookups - cp -rv *.csv ../package/lookups + cp -rv *.csv ../dist/escu/lookups + cp -rv *.csv ../dist/mustang/lookups - run: name: update version and build number command: | cd security-content # check if tag is set, get build number from the tag if set if [ -z "${CIRCLE_TAG}" ]; then - CONTENT_VERSION=$(grep -oP "(\d+.\d+.\d+$)" package/default/content-version.conf) + CONTENT_VERSION=$(grep -oP "(\d+.\d+.\d+$)" dist/escu/default/content-version.conf) echo "detected content version: $CONTENT_VERSION" else CONTENT_VERSION=$(echo $CIRCLE_TAG | grep -oP "\d+.\d+.\d+") echo "content version: $CONTENT_VERSION, set by tag: $CIRCLE_TAG" fi - # update build number and version - sed -i "s/build = .*$/build = $CIRCLE_BUILD_NUM/g" package/default/app.conf - sed -i "s/^version = .*$/version = $CONTENT_VERSION/g" package/default/app.conf - sed -i "s/\"version\": .*$/\"version\": \"$CONTENT_VERSION\"/g" package/app.manifest - sed -i "s/version = .*$/version = $CONTENT_VERSION/g" package/default/content-version.conf - tar -czf content-pack-build.tar.gz package/* + # update build number and version for ESCU + sed -i "s/build = .*$/build = $CIRCLE_BUILD_NUM/g" dist/escu/default/app.conf + sed -i "s/^version = .*$/version = $CONTENT_VERSION/g" dist/escu/default/app.conf + sed -i "s/\"version\": .*$/\"version\": \"$CONTENT_VERSION\"/g" dist/escu/app.manifest + sed -i "s/version = .*$/version = $CONTENT_VERSION/g" dist/escu/default/content-version.conf + tar -czf content-pack-build-escu.tar.gz dist/escu/* + # update build number and version for MUSTANG + sed -i "s/build = .*$/build = $CIRCLE_BUILD_NUM/g" dist/mustang/default/app.conf + sed -i "s/^version = .*$/version = $CONTENT_VERSION/g" dist/mustang/default/app.conf + sed -i "s/\"version\": .*$/\"version\": \"$CONTENT_VERSION\"/g" dist/mustang/app.manifest + sed -i "s/version = .*$/version = $CONTENT_VERSION/g" dist/mustang/default/content-version.conf + tar -czf content-pack-build-mustang.tar.gz dist/mustang/* + - persist_to_workspace: root: security-content/ paths: - - content-pack-build.tar.gz + - content-pack-build-escu.tar.gz + - content-pack-build-mustang.tar.gz build-package: executor: content-executor steps: - attach_workspace: - at: ~/dist + at: ~/build - run: name: grab splunk packaging toolkit command: | @@ -185,18 +198,23 @@ jobs: name: create a .spl for this build using slim command: | source ~/slim-latest/venv/bin/activate - cd ~/dist - tar -zxf content-pack-build.tar.gz - mv package DA-ESS-ContentUpdate + cd ~/build + tar -zxf content-pack-build-escu.tar.gz + tar -zxf content-pack-build-mustang.tar.gz + mv dist/escu DA-ESS-ContentUpdate + mv dist/mustang DA-ESS-ContentUpdateMustang slim package -o upload DA-ESS-ContentUpdate - cp upload/*.tar.gz DA-ESS-ContentUpdate-latest.tar.gz + slim package -o upload DA-ESS-ContentUpdateMustang + cp upload/DA-ESS-ContentUpdate-*.tar.gz DA-ESS-ContentUpdate-latest.tar.gz + cp upload/DA-ESS-ContentUpdateMustang-*tar.gz DA-ESS-ContentUpdateMustang-latest.tar.gz - store_artifacts: - path: ~/dist/upload + path: ~/build/upload destination: package/ - persist_to_workspace: - root: ~/dist + root: ~/build paths: - DA-ESS-ContentUpdate-latest.tar.gz + - DA-ESS-ContentUpdateMustang-latest.tar.gz run-appinspect: executor: content-executor @@ -218,10 +236,15 @@ jobs: git clone --branch ${CIRCLE_BRANCH} https://${GITHUB_TOKEN}@github.com/splunk/security-content.git fi - run: - name: submit package to appinspect API + name: submit ESCU package to appinspect API command: | cd security-content/bin - ./appinspect.sh ~/ $APPINSPECT_USERNAME $APPINSPECT_PASSWORD + ./appinspect.sh ~/ DA-ESS-ContentUpdate-latest.tar.gz $APPINSPECT_USERNAME $APPINSPECT_PASSWORD + - run: + name: submit MUSTANG package to appinspect API + command: | + cd security-content/bin + ./appinspect.sh ~/ DA-ESS-ContentUpdateMustang-latest.tar.gz $APPINSPECT_USERNAME $APPINSPECT_PASSWORD - store_artifacts: path: ~/report destination: report/ @@ -229,6 +252,7 @@ jobs: root: ~/ paths: - DA-ESS-ContentUpdate-latest.tar.gz + - DA-ESS-ContentUpdateMustang-latest.tar.gz update-sources-github: executor: content-executor steps: diff --git a/.gitignore b/.gitignore index 0ac3d08ff8..26dc097f54 100644 --- a/.gitignore +++ b/.gitignore @@ -20,7 +20,6 @@ __pycache__/ .Python build/ develop-eggs/ -dist/ downloads/ eggs/ .eggs/ diff --git a/bin/appinspect.sh b/bin/appinspect.sh index 7c9769ffbe..00b0f22ce4 100755 --- a/bin/appinspect.sh +++ b/bin/appinspect.sh @@ -1,12 +1,12 @@ #!/bin/bash # simple script to run an appinspect API check -EXPECTED_ARGS=3 +EXPECTED_ARGS=4 E_BADARGS=65 -if [ $# -lt 3 ] +if [ $# -lt 4 ] then -echo "Usage: `basename $0` " -echo "Example `basename $0` ~/ doomguy R1p&T3ar" +echo "Usage: `basename $0` " +echo "Example `basename $0` ~/ DA-ESS-ContentUpdate-latest.tar.gz doomguy R1p&T3ar" exit $E_BADARGS fi @@ -17,8 +17,13 @@ echo "Too many arguments" fi APP_PATH=$1 -USERNAME=$2 -PASSWORD=$3 +PACKAGE_NAME=$2 +USERNAME=$3 +PASSWORD=$4 + +PACKAGE_PATH="/home/circleci/"$PACKAGE_NAME + + cd $APP_PATH mkdir report # get a JWT token @@ -26,7 +31,8 @@ AUTH_TOKEN=$(echo -n "$USERNAME:$PASSWORD" | base64) APPINSPECT_TOKEN=$(curl -s --location --request GET 'https://api.splunk.com/2.0/rest/login/splunk' --header "Authorization: Basic $AUTH_TOKEN" | jq -r '.data | .token') sleep 1 # submit a inspection job EXPECTS app on same directory -REQUEST_ID=$(curl -s --location --request POST 'https://appinspect.splunk.com/v1/app/validate' --header "Authorization: bearer $APPINSPECT_TOKEN" --form 'app_package=@"/home/circleci/DA-ESS-ContentUpdate-latest.tar.gz"' | jq -r '.request_id') +#REQUEST_ID=$(curl -s --location --request POST 'https://appinspect.splunk.com/v1/app/validate' --header "Authorization: bearer $APPINSPECT_TOKEN" --form 'app_package=@"/home/circleci/DA-ESS-ContentUpdate-latest.tar.gz"' | jq -r '.request_id') +REQUEST_ID=$(curl -s --location --request POST 'https://appinspect.splunk.com/v1/app/validate' --header "Authorization: bearer $APPINSPECT_TOKEN" --form 'app_package=@'$PACKAGE_PATH | jq -r '.request_id') echo "app inspect request: $REQUEST_ID" sleep 5 STATUS=$(curl -s --location --request GET https://appinspect.splunk.com/v1/app/validate/status/$REQUEST_ID --header "Authorization: bearer $APPINSPECT_TOKEN" | jq -r '.status') @@ -40,7 +46,7 @@ do # REPORT FINISHED CHECK RESULTS then echo "appinspect completed inspection" - curl -s --location --request GET https://appinspect.splunk.com/v1/app/report/$REQUEST_ID --header "Authorization: bearer $APPINSPECT_TOKEN" --header 'Content-Type: text/html' -o report/appinspect_report.html + curl -s --location --request GET https://appinspect.splunk.com/v1/app/report/$REQUEST_ID --header "Authorization: bearer $APPINSPECT_TOKEN" --header 'Content-Type: text/html' -o report/appinspect_report_$PACKAGE_NAME.html FAILS=$(curl -s --location --request GET https://appinspect.splunk.com/v1/app/report/$REQUEST_ID --header "Authorization: bearer $APPINSPECT_TOKEN" --header 'Content-Type: application/json' | jq -r '.summary | .failure') ERRORS=$(curl -s --location --request GET https://appinspect.splunk.com/v1/app/report/$REQUEST_ID --header "Authorization: bearer $APPINSPECT_TOKEN" --header 'Content-Type: application/json' | jq -r '.summary | .error') if [ $FAILS -gt 0 -o $ERRORS -gt 0 ] diff --git a/bin/generate.py b/bin/generate.py index 2b0e12eaaf..3c2763e8e1 100644 --- a/bin/generate.py +++ b/bin/generate.py @@ -602,7 +602,7 @@ def generate_mitre_lookup(OUTPUT_PATH): -def main(REPO_PATH, OUTPUT_PATH, VERBOSE): +def main(REPO_PATH, OUTPUT_PATH, PRODUCT, VERBOSE): TEMPLATE_PATH = path.join(REPO_PATH, 'bin/jinja2_templates') @@ -619,6 +619,12 @@ def main(REPO_PATH, OUTPUT_PATH, VERBOSE): detections = load_objects("detections/*/*.yml", VERBOSE, REPO_PATH) detections.extend(load_objects("detections/*/*/*.yml", VERBOSE, REPO_PATH)) + if PRODUCT == "MUSTANG": + detections = [object for object in detections if 'Splunk Security Analytics for AWS' in object['tags']['product']] + stories = [object for object in stories if 'Splunk Security Analytics for AWS' in object['tags']['product']] + baselines = [object for object in baselines if 'Splunk Security Analytics for AWS' in object['tags']['product']] + response_tasks = [object for object in response_tasks if 'Splunk Security Analytics for AWS' in object['tags']['product']] + try: if VERBOSE: print("generating Mitre lookups") @@ -641,6 +647,9 @@ def main(REPO_PATH, OUTPUT_PATH, VERBOSE): # only use ESCU stories to the configuration stories = sorted(filter(lambda s: s['type'].lower() == 'batch', stories), key=lambda s: s['name']) + + + story_path = generate_analytic_story_conf(stories, detections, response_tasks, baselines, TEMPLATE_PATH, OUTPUT_PATH) use_case_lib_path = generate_use_case_library_conf(stories, detections, response_tasks, baselines, TEMPLATE_PATH, OUTPUT_PATH) @@ -669,11 +678,13 @@ if __name__ == "__main__": parser.add_argument("-p", "--path", required=True, help="path to security_content repo") parser.add_argument("-o", "--output", required=True, help="path to the output directory") parser.add_argument("-v", "--verbose", required=False, default=False, action='store_true', help="prints verbose output") + parser.add_argument("--product", required=True, default="ESCU", help="package type") # parse them args = parser.parse_args() REPO_PATH = args.path OUTPUT_PATH = args.output VERBOSE = args.verbose + PRODUCT = args.product - main(REPO_PATH, OUTPUT_PATH, VERBOSE) \ No newline at end of file + main(REPO_PATH, OUTPUT_PATH, PRODUCT, VERBOSE) \ No newline at end of file diff --git a/contentctl.py b/contentctl.py index 72c115cedd..736bf605a0 100644 --- a/contentctl.py +++ b/contentctl.py @@ -74,7 +74,7 @@ def generate(args): sys.exit(1) print("contentctl is generating a new splunk_app under ".format(output)) - generator.main(security_content_path, args.output, args.verbose) + generator.main(security_content_path, args.output, args.product, args.verbose) def main(args): @@ -111,6 +111,8 @@ def main(args): help="Format of our deployment package, defaults to `splunk_app`.\n The deployment `splunk_app` runs on product Splunk Enterprise Security and Splunk Enterprise.") generate_parser.add_argument("-o", "--output", required=False, type=str, default="package", help="Path where to store the deployment package, defaults to `package`") + generate_parser.add_argument("--product", required=False, type=str, default="ESCU", + help="Type of package to create, choose between `ESCU`, or `MUSTANG`. Defaults to `ESCU`") generate_parser.set_defaults(func=generate) # # parse them diff --git a/package/README.md b/dist/escu/README.md similarity index 100% rename from package/README.md rename to dist/escu/README.md diff --git a/package/README/essoc_story_detail.txt b/dist/escu/README/essoc_story_detail.txt similarity index 100% rename from package/README/essoc_story_detail.txt rename to dist/escu/README/essoc_story_detail.txt diff --git a/package/README/essoc_summary.txt b/dist/escu/README/essoc_summary.txt similarity index 100% rename from package/README/essoc_summary.txt rename to dist/escu/README/essoc_summary.txt diff --git a/package/README/essoc_usage_dashboard.txt b/dist/escu/README/essoc_usage_dashboard.txt similarity index 100% rename from package/README/essoc_usage_dashboard.txt rename to dist/escu/README/essoc_usage_dashboard.txt diff --git a/package/app.manifest b/dist/escu/app.manifest similarity index 100% rename from package/app.manifest rename to dist/escu/app.manifest diff --git a/package/bin/dnstwist.py b/dist/escu/bin/dnstwist.py similarity index 100% rename from package/bin/dnstwist.py rename to dist/escu/bin/dnstwist.py diff --git a/package/bin/escu_contextualize.py b/dist/escu/bin/escu_contextualize.py similarity index 100% rename from package/bin/escu_contextualize.py rename to dist/escu/bin/escu_contextualize.py diff --git a/package/bin/escu_investigate.py b/dist/escu/bin/escu_investigate.py similarity index 100% rename from package/bin/escu_investigate.py rename to dist/escu/bin/escu_investigate.py diff --git a/package/bin/runstory.py b/dist/escu/bin/runstory.py similarity index 100% rename from package/bin/runstory.py rename to dist/escu/bin/runstory.py diff --git a/package/bin/splunklib/__init__.py b/dist/escu/bin/splunklib/__init__.py similarity index 100% rename from package/bin/splunklib/__init__.py rename to dist/escu/bin/splunklib/__init__.py diff --git a/package/bin/splunklib/binding.py b/dist/escu/bin/splunklib/binding.py similarity index 100% rename from package/bin/splunklib/binding.py rename to dist/escu/bin/splunklib/binding.py diff --git a/package/bin/splunklib/client.py b/dist/escu/bin/splunklib/client.py similarity index 100% rename from package/bin/splunklib/client.py rename to dist/escu/bin/splunklib/client.py diff --git a/package/bin/splunklib/data.py b/dist/escu/bin/splunklib/data.py similarity index 100% rename from package/bin/splunklib/data.py rename to dist/escu/bin/splunklib/data.py diff --git a/package/bin/splunklib/modularinput/__init__.py b/dist/escu/bin/splunklib/modularinput/__init__.py similarity index 100% rename from package/bin/splunklib/modularinput/__init__.py rename to dist/escu/bin/splunklib/modularinput/__init__.py diff --git a/package/bin/splunklib/modularinput/argument.py b/dist/escu/bin/splunklib/modularinput/argument.py similarity index 100% rename from package/bin/splunklib/modularinput/argument.py rename to dist/escu/bin/splunklib/modularinput/argument.py diff --git a/package/bin/splunklib/modularinput/event.py b/dist/escu/bin/splunklib/modularinput/event.py similarity index 100% rename from package/bin/splunklib/modularinput/event.py rename to dist/escu/bin/splunklib/modularinput/event.py diff --git a/package/bin/splunklib/modularinput/event_writer.py b/dist/escu/bin/splunklib/modularinput/event_writer.py similarity index 100% rename from package/bin/splunklib/modularinput/event_writer.py rename to dist/escu/bin/splunklib/modularinput/event_writer.py diff --git a/package/bin/splunklib/modularinput/input_definition.py b/dist/escu/bin/splunklib/modularinput/input_definition.py similarity index 100% rename from package/bin/splunklib/modularinput/input_definition.py rename to dist/escu/bin/splunklib/modularinput/input_definition.py diff --git a/package/bin/splunklib/modularinput/scheme.py b/dist/escu/bin/splunklib/modularinput/scheme.py similarity index 100% rename from package/bin/splunklib/modularinput/scheme.py rename to dist/escu/bin/splunklib/modularinput/scheme.py diff --git a/package/bin/splunklib/modularinput/script.py b/dist/escu/bin/splunklib/modularinput/script.py similarity index 100% rename from package/bin/splunklib/modularinput/script.py rename to dist/escu/bin/splunklib/modularinput/script.py diff --git a/package/bin/splunklib/modularinput/utils.py b/dist/escu/bin/splunklib/modularinput/utils.py similarity index 100% rename from package/bin/splunklib/modularinput/utils.py rename to dist/escu/bin/splunklib/modularinput/utils.py diff --git a/package/bin/splunklib/modularinput/validation_definition.py b/dist/escu/bin/splunklib/modularinput/validation_definition.py similarity index 100% rename from package/bin/splunklib/modularinput/validation_definition.py rename to dist/escu/bin/splunklib/modularinput/validation_definition.py diff --git a/package/bin/splunklib/ordereddict.py b/dist/escu/bin/splunklib/ordereddict.py similarity index 100% rename from package/bin/splunklib/ordereddict.py rename to dist/escu/bin/splunklib/ordereddict.py diff --git a/package/bin/splunklib/results.py b/dist/escu/bin/splunklib/results.py similarity index 100% rename from package/bin/splunklib/results.py rename to dist/escu/bin/splunklib/results.py diff --git a/package/bin/splunklib/searchcommands/__init__.py b/dist/escu/bin/splunklib/searchcommands/__init__.py similarity index 100% rename from package/bin/splunklib/searchcommands/__init__.py rename to dist/escu/bin/splunklib/searchcommands/__init__.py diff --git a/package/bin/splunklib/searchcommands/decorators.py b/dist/escu/bin/splunklib/searchcommands/decorators.py similarity index 100% rename from package/bin/splunklib/searchcommands/decorators.py rename to dist/escu/bin/splunklib/searchcommands/decorators.py diff --git a/package/bin/splunklib/searchcommands/environment.py b/dist/escu/bin/splunklib/searchcommands/environment.py similarity index 100% rename from package/bin/splunklib/searchcommands/environment.py rename to dist/escu/bin/splunklib/searchcommands/environment.py diff --git a/package/bin/splunklib/searchcommands/eventing_command.py b/dist/escu/bin/splunklib/searchcommands/eventing_command.py similarity index 100% rename from package/bin/splunklib/searchcommands/eventing_command.py rename to dist/escu/bin/splunklib/searchcommands/eventing_command.py diff --git a/package/bin/splunklib/searchcommands/external_search_command.py b/dist/escu/bin/splunklib/searchcommands/external_search_command.py similarity index 100% rename from package/bin/splunklib/searchcommands/external_search_command.py rename to dist/escu/bin/splunklib/searchcommands/external_search_command.py diff --git a/package/bin/splunklib/searchcommands/generating_command.py b/dist/escu/bin/splunklib/searchcommands/generating_command.py similarity index 100% rename from package/bin/splunklib/searchcommands/generating_command.py rename to dist/escu/bin/splunklib/searchcommands/generating_command.py diff --git a/package/bin/splunklib/searchcommands/internals.py b/dist/escu/bin/splunklib/searchcommands/internals.py similarity index 100% rename from package/bin/splunklib/searchcommands/internals.py rename to dist/escu/bin/splunklib/searchcommands/internals.py diff --git a/package/bin/splunklib/searchcommands/reporting_command.py b/dist/escu/bin/splunklib/searchcommands/reporting_command.py similarity index 100% rename from package/bin/splunklib/searchcommands/reporting_command.py rename to dist/escu/bin/splunklib/searchcommands/reporting_command.py diff --git a/package/bin/splunklib/searchcommands/search_command.py b/dist/escu/bin/splunklib/searchcommands/search_command.py similarity index 100% rename from package/bin/splunklib/searchcommands/search_command.py rename to dist/escu/bin/splunklib/searchcommands/search_command.py diff --git a/package/bin/splunklib/searchcommands/streaming_command.py b/dist/escu/bin/splunklib/searchcommands/streaming_command.py similarity index 100% rename from package/bin/splunklib/searchcommands/streaming_command.py rename to dist/escu/bin/splunklib/searchcommands/streaming_command.py diff --git a/package/bin/splunklib/searchcommands/validators.py b/dist/escu/bin/splunklib/searchcommands/validators.py similarity index 100% rename from package/bin/splunklib/searchcommands/validators.py rename to dist/escu/bin/splunklib/searchcommands/validators.py diff --git a/package/default/analytic_stories.conf b/dist/escu/default/analytic_stories.conf similarity index 100% rename from package/default/analytic_stories.conf rename to dist/escu/default/analytic_stories.conf diff --git a/package/default/analyticstories.conf b/dist/escu/default/analyticstories.conf similarity index 100% rename from package/default/analyticstories.conf rename to dist/escu/default/analyticstories.conf diff --git a/package/default/app.conf b/dist/escu/default/app.conf similarity index 100% rename from package/default/app.conf rename to dist/escu/default/app.conf diff --git a/package/default/collections.conf b/dist/escu/default/collections.conf similarity index 100% rename from package/default/collections.conf rename to dist/escu/default/collections.conf diff --git a/package/default/commands.conf b/dist/escu/default/commands.conf similarity index 100% rename from package/default/commands.conf rename to dist/escu/default/commands.conf diff --git a/package/default/content-version.conf b/dist/escu/default/content-version.conf similarity index 100% rename from package/default/content-version.conf rename to dist/escu/default/content-version.conf diff --git a/package/default/data/ui/nav/default.xml b/dist/escu/default/data/ui/nav/default.xml similarity index 74% rename from package/default/data/ui/nav/default.xml rename to dist/escu/default/data/ui/nav/default.xml index 6b3a7332fc..0f68796130 100644 --- a/package/default/data/ui/nav/default.xml +++ b/dist/escu/default/data/ui/nav/default.xml @@ -1,7 +1,5 @@