Branch was auto-updated.

This commit is contained in:
github-actions[bot]
2021-04-02 16:46:07 +00:00
committed by GitHub
356 changed files with 119579 additions and 35042 deletions
+46 -22
View File
@@ -124,47 +124,60 @@ jobs:
command: |
cd security-content
source venv/bin/activate
python contentctl.py --path . --verbose generate --output package
python contentctl.py --path . --verbose generate --product ESCU --output dist/escu
python contentctl.py --path . --verbose generate --product MUSTANG --output dist/mustang
# make a copy of use_case_lib in order to have ES work :-(
cp package/default/use_case_library.conf package/default/analyticstories.conf
cp dist/escu/default/use_case_library.conf dist/escu/default/analyticstories.conf
cp dist/mustang/default/use_case_library.conf dist/mustang/default/analyticstories.conf
- run:
name: copy lookups .csv files
command: |
cd security-content
# clean up current lookups
rm -rf package/lookups
mkdir package/lookups
rm -rf dist/escu/lookups
rm -rf dist/mustang/lookups
mkdir dist/escu/lookups
mkdir dist/mustang/lookups
# copy over lookups
cd lookups
cp -rv *.csv ../package/lookups
cp -rv *.csv ../dist/escu/lookups
cp -rv *.csv ../dist/mustang/lookups
- run:
name: update version and build number
command: |
cd security-content
# check if tag is set, get build number from the tag if set
if [ -z "${CIRCLE_TAG}" ]; then
CONTENT_VERSION=$(grep -oP "(\d+.\d+.\d+$)" package/default/content-version.conf)
CONTENT_VERSION=$(grep -oP "(\d+.\d+.\d+$)" dist/escu/default/content-version.conf)
echo "detected content version: $CONTENT_VERSION"
else
CONTENT_VERSION=$(echo $CIRCLE_TAG | grep -oP "\d+.\d+.\d+")
echo "content version: $CONTENT_VERSION, set by tag: $CIRCLE_TAG"
fi
# update build number and version
sed -i "s/build = .*$/build = $CIRCLE_BUILD_NUM/g" package/default/app.conf
sed -i "s/^version = .*$/version = $CONTENT_VERSION/g" package/default/app.conf
sed -i "s/\"version\": .*$/\"version\": \"$CONTENT_VERSION\"/g" package/app.manifest
sed -i "s/version = .*$/version = $CONTENT_VERSION/g" package/default/content-version.conf
tar -czf content-pack-build.tar.gz package/*
# update build number and version for ESCU
sed -i "s/build = .*$/build = $CIRCLE_BUILD_NUM/g" dist/escu/default/app.conf
sed -i "s/^version = .*$/version = $CONTENT_VERSION/g" dist/escu/default/app.conf
sed -i "s/\"version\": .*$/\"version\": \"$CONTENT_VERSION\"/g" dist/escu/app.manifest
sed -i "s/version = .*$/version = $CONTENT_VERSION/g" dist/escu/default/content-version.conf
tar -czf content-pack-build-escu.tar.gz dist/escu/*
# update build number and version for MUSTANG
sed -i "s/build = .*$/build = $CIRCLE_BUILD_NUM/g" dist/mustang/default/app.conf
sed -i "s/^version = .*$/version = $CONTENT_VERSION/g" dist/mustang/default/app.conf
sed -i "s/\"version\": .*$/\"version\": \"$CONTENT_VERSION\"/g" dist/mustang/app.manifest
sed -i "s/version = .*$/version = $CONTENT_VERSION/g" dist/mustang/default/content-version.conf
tar -czf content-pack-build-mustang.tar.gz dist/mustang/*
- persist_to_workspace:
root: security-content/
paths:
- content-pack-build.tar.gz
- content-pack-build-escu.tar.gz
- content-pack-build-mustang.tar.gz
build-package:
executor: content-executor
steps:
- attach_workspace:
at: ~/dist
at: ~/build
- run:
name: grab splunk packaging toolkit
command: |
@@ -185,18 +198,23 @@ jobs:
name: create a .spl for this build using slim
command: |
source ~/slim-latest/venv/bin/activate
cd ~/dist
tar -zxf content-pack-build.tar.gz
mv package DA-ESS-ContentUpdate
cd ~/build
tar -zxf content-pack-build-escu.tar.gz
tar -zxf content-pack-build-mustang.tar.gz
mv dist/escu DA-ESS-ContentUpdate
mv dist/mustang DA-ESS-ContentUpdateMustang
slim package -o upload DA-ESS-ContentUpdate
cp upload/*.tar.gz DA-ESS-ContentUpdate-latest.tar.gz
slim package -o upload DA-ESS-ContentUpdateMustang
cp upload/DA-ESS-ContentUpdate-*.tar.gz DA-ESS-ContentUpdate-latest.tar.gz
cp upload/DA-ESS-ContentUpdateMustang-*tar.gz DA-ESS-ContentUpdateMustang-latest.tar.gz
- store_artifacts:
path: ~/dist/upload
path: ~/build/upload
destination: package/
- persist_to_workspace:
root: ~/dist
root: ~/build
paths:
- DA-ESS-ContentUpdate-latest.tar.gz
- DA-ESS-ContentUpdateMustang-latest.tar.gz
run-appinspect:
executor: content-executor
@@ -218,10 +236,15 @@ jobs:
git clone --branch ${CIRCLE_BRANCH} https://${GITHUB_TOKEN}@github.com/splunk/security-content.git
fi
- run:
name: submit package to appinspect API
name: submit ESCU package to appinspect API
command: |
cd security-content/bin
./appinspect.sh ~/ $APPINSPECT_USERNAME $APPINSPECT_PASSWORD
./appinspect.sh ~/ DA-ESS-ContentUpdate-latest.tar.gz $APPINSPECT_USERNAME $APPINSPECT_PASSWORD
- run:
name: submit MUSTANG package to appinspect API
command: |
cd security-content/bin
./appinspect.sh ~/ DA-ESS-ContentUpdateMustang-latest.tar.gz $APPINSPECT_USERNAME $APPINSPECT_PASSWORD
- store_artifacts:
path: ~/report
destination: report/
@@ -229,6 +252,7 @@ jobs:
root: ~/
paths:
- DA-ESS-ContentUpdate-latest.tar.gz
- DA-ESS-ContentUpdateMustang-latest.tar.gz
update-sources-github:
executor: content-executor
steps:
-1
View File
@@ -20,7 +20,6 @@ __pycache__/
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
+14 -8
View File
@@ -1,12 +1,12 @@
#!/bin/bash
# simple script to run an appinspect API check
EXPECTED_ARGS=3
EXPECTED_ARGS=4
E_BADARGS=65
if [ $# -lt 3 ]
if [ $# -lt 4 ]
then
echo "Usage: `basename $0` <app_path> <username> <password>"
echo "Example `basename $0` ~/ doomguy R1p&T3ar"
echo "Usage: `basename $0` <app_path> <package_name> <username> <password>"
echo "Example `basename $0` ~/ DA-ESS-ContentUpdate-latest.tar.gz doomguy R1p&T3ar"
exit $E_BADARGS
fi
@@ -17,8 +17,13 @@ echo "Too many arguments"
fi
APP_PATH=$1
USERNAME=$2
PASSWORD=$3
PACKAGE_NAME=$2
USERNAME=$3
PASSWORD=$4
PACKAGE_PATH="/home/circleci/"$PACKAGE_NAME
cd $APP_PATH
mkdir report
# get a JWT token
@@ -26,7 +31,8 @@ AUTH_TOKEN=$(echo -n "$USERNAME:$PASSWORD" | base64)
APPINSPECT_TOKEN=$(curl -s --location --request GET 'https://api.splunk.com/2.0/rest/login/splunk' --header "Authorization: Basic $AUTH_TOKEN" | jq -r '.data | .token')
sleep 1
# submit a inspection job EXPECTS app on same directory
REQUEST_ID=$(curl -s --location --request POST 'https://appinspect.splunk.com/v1/app/validate' --header "Authorization: bearer $APPINSPECT_TOKEN" --form 'app_package=@"/home/circleci/DA-ESS-ContentUpdate-latest.tar.gz"' | jq -r '.request_id')
#REQUEST_ID=$(curl -s --location --request POST 'https://appinspect.splunk.com/v1/app/validate' --header "Authorization: bearer $APPINSPECT_TOKEN" --form 'app_package=@"/home/circleci/DA-ESS-ContentUpdate-latest.tar.gz"' | jq -r '.request_id')
REQUEST_ID=$(curl -s --location --request POST 'https://appinspect.splunk.com/v1/app/validate' --header "Authorization: bearer $APPINSPECT_TOKEN" --form 'app_package=@'$PACKAGE_PATH | jq -r '.request_id')
echo "app inspect request: $REQUEST_ID"
sleep 5
STATUS=$(curl -s --location --request GET https://appinspect.splunk.com/v1/app/validate/status/$REQUEST_ID --header "Authorization: bearer $APPINSPECT_TOKEN" | jq -r '.status')
@@ -40,7 +46,7 @@ do
# REPORT FINISHED CHECK RESULTS
then
echo "appinspect completed inspection"
curl -s --location --request GET https://appinspect.splunk.com/v1/app/report/$REQUEST_ID --header "Authorization: bearer $APPINSPECT_TOKEN" --header 'Content-Type: text/html' -o report/appinspect_report.html
curl -s --location --request GET https://appinspect.splunk.com/v1/app/report/$REQUEST_ID --header "Authorization: bearer $APPINSPECT_TOKEN" --header 'Content-Type: text/html' -o report/appinspect_report_$PACKAGE_NAME.html
FAILS=$(curl -s --location --request GET https://appinspect.splunk.com/v1/app/report/$REQUEST_ID --header "Authorization: bearer $APPINSPECT_TOKEN" --header 'Content-Type: application/json' | jq -r '.summary | .failure')
ERRORS=$(curl -s --location --request GET https://appinspect.splunk.com/v1/app/report/$REQUEST_ID --header "Authorization: bearer $APPINSPECT_TOKEN" --header 'Content-Type: application/json' | jq -r '.summary | .error')
if [ $FAILS -gt 0 -o $ERRORS -gt 0 ]
+13 -2
View File
@@ -602,7 +602,7 @@ def generate_mitre_lookup(OUTPUT_PATH):
def main(REPO_PATH, OUTPUT_PATH, VERBOSE):
def main(REPO_PATH, OUTPUT_PATH, PRODUCT, VERBOSE):
TEMPLATE_PATH = path.join(REPO_PATH, 'bin/jinja2_templates')
@@ -619,6 +619,12 @@ def main(REPO_PATH, OUTPUT_PATH, VERBOSE):
detections = load_objects("detections/*/*.yml", VERBOSE, REPO_PATH)
detections.extend(load_objects("detections/*/*/*.yml", VERBOSE, REPO_PATH))
if PRODUCT == "MUSTANG":
detections = [object for object in detections if 'Splunk Security Analytics for AWS' in object['tags']['product']]
stories = [object for object in stories if 'Splunk Security Analytics for AWS' in object['tags']['product']]
baselines = [object for object in baselines if 'Splunk Security Analytics for AWS' in object['tags']['product']]
response_tasks = [object for object in response_tasks if 'Splunk Security Analytics for AWS' in object['tags']['product']]
try:
if VERBOSE:
print("generating Mitre lookups")
@@ -641,6 +647,9 @@ def main(REPO_PATH, OUTPUT_PATH, VERBOSE):
# only use ESCU stories to the configuration
stories = sorted(filter(lambda s: s['type'].lower() == 'batch', stories), key=lambda s: s['name'])
story_path = generate_analytic_story_conf(stories, detections, response_tasks, baselines, TEMPLATE_PATH, OUTPUT_PATH)
use_case_lib_path = generate_use_case_library_conf(stories, detections, response_tasks, baselines, TEMPLATE_PATH, OUTPUT_PATH)
@@ -669,11 +678,13 @@ if __name__ == "__main__":
parser.add_argument("-p", "--path", required=True, help="path to security_content repo")
parser.add_argument("-o", "--output", required=True, help="path to the output directory")
parser.add_argument("-v", "--verbose", required=False, default=False, action='store_true', help="prints verbose output")
parser.add_argument("--product", required=True, default="ESCU", help="package type")
# parse them
args = parser.parse_args()
REPO_PATH = args.path
OUTPUT_PATH = args.output
VERBOSE = args.verbose
PRODUCT = args.product
main(REPO_PATH, OUTPUT_PATH, VERBOSE)
main(REPO_PATH, OUTPUT_PATH, PRODUCT, VERBOSE)
+3 -1
View File
@@ -74,7 +74,7 @@ def generate(args):
sys.exit(1)
print("contentctl is generating a new splunk_app under ".format(output))
generator.main(security_content_path, args.output, args.verbose)
generator.main(security_content_path, args.output, args.product, args.verbose)
def main(args):
@@ -111,6 +111,8 @@ def main(args):
help="Format of our deployment package, defaults to `splunk_app`.\n The deployment `splunk_app` runs on product Splunk Enterprise Security and Splunk Enterprise.")
generate_parser.add_argument("-o", "--output", required=False, type=str, default="package",
help="Path where to store the deployment package, defaults to `package`")
generate_parser.add_argument("--product", required=False, type=str, default="ESCU",
help="Type of package to create, choose between `ESCU`, or `MUSTANG`. Defaults to `ESCU`")
generate_parser.set_defaults(func=generate)
# # parse them
View File
View File
@@ -1,7 +1,5 @@
<nav search_view="search" color="#65A637">
<view name="escu_summary" default="true"/>
<view name="analytic_story_details"/>
<view name="explore_analytic_stories"/>
<view name="feedback"/>
<view name="search"/>
<view name="escu_usage"/>
+242
View File
@@ -0,0 +1,242 @@
<form theme="dark">
<label>Content Library</label>
<!-- Example uses stats transforming command -->
<!-- This limits evnts passed to post-process search -->
<title>Splunk Security Content</title>
<row>
<html>
<h2 style="color:red">Explore the Analytic Stories included with Splunk Security via <a href="https://www.splunk.com/en_us/resources/videos/splunk-enterprise-security-use-case-library.html">ES Use Case Library</a> or <a href="https://splunkbase.splunk.com/app/3435/">Splunk Security Essentials</a>.</h2>
</html>
</row>
<search id="baseSS">
<query>| rest /services/saved/searches splunk_server=local count=0 | search title="ESCU - *"</query>
</search>
<search id="baseAS">
<query>| rest /services/configs/conf-analytic_stories splunk_server=local count=0</query>
</search>
<init>
<set token="form.as_category">*</set>
<set token="form.detection">*</set>
<set token="form.as_story">*</set>
<set token="form.as_attack_id">*</set>
</init>
<!-- Rows for Analytic Story Stats -->
<!-- Rows for Analytic Story Table -->
<!-- Rows for Search Stats -->
<fieldset submitButton="false"></fieldset>
<row depends="$explore-use-case-es-show$">
<panel>
<html>
<div id="explore-use-case-es-info"/>
</html>
</panel>
</row>
<row id="analytic_stories_header_stats">
<panel>
<single>
<title>Total Analytic Stories</title>
<search base="baseAS">
<query>stats count</query>
</search>
<!-- post-process search -->
<option name="colorBy">value</option>
<option name="colorMode">block</option>
<option name="drilldown">none</option>
<option name="numberPrecision">0</option>
<option name="showSparkline">1</option>
<option name="showTrendIndicator">1</option>
<option name="trendColorInterpretation">standard</option>
<option name="trendDisplayMode">absolute</option>
<option name="unitPosition">after</option>
<option name="useColors">1</option>
<option name="useThousandSeparators">1</option>
<option name="rangeColors">["0x555","0x65a637"]</option>
<option name="rangeValues">[0]</option>
</single>
</panel>
<panel>
<single>
<title>Total Detections</title>
<search base="baseSS">
<query>stats count by action.correlationsearch.label| eventstats sum(count) as total_detection_count| fields total_detection_count</query>
</search>
<!-- post-process search -->
<option name="colorBy">value</option>
<option name="colorMode">block</option>
<option name="drilldown">none</option>
<option name="numberPrecision">0</option>
<option name="showSparkline">1</option>
<option name="showTrendIndicator">1</option>
<option name="trendColorInterpretation">standard</option>
<option name="trendDisplayMode">absolute</option>
<option name="unitPosition">after</option>
<option name="useColors">1</option>
<option name="useThousandSeparators">1</option>
<option name="rangeColors">["0x555","0x65a637"]</option>
<option name="rangeValues">[0]</option>
</single>
</panel>
<panel>
<single>
<title>ESCU App Version</title>
<search id="version">
<query>| rest /services/configs/conf-content-version splunk_server=local count=0 | table version</query>
</search>
<option name="colorBy">value</option>
<option name="colorMode">block</option>
<option name="drilldown">none</option>
<option name="numberPrecision">0</option>
<option name="rangeColors">["0x555","0x65a637"]</option>
<option name="rangeValues">[0]</option>
<option name="refresh.display">progressbar</option>
<option name="showSparkline">1</option>
<option name="showTrendIndicator">1</option>
<option name="trendColorInterpretation">standard</option>
<option name="trendDisplayMode">absolute</option>
<option name="unitPosition">after</option>
<option name="useColors">1</option>
<option name="useThousandSeparators">1</option>
</single>
</panel>
</row>
<row id="analytic_stories_viz">
<panel>
<title>Story Categories</title>
<chart>
<search>
<query>| rest /services/configs/conf-analytic_stories splunk_server=local count=0 | stats count by category</query>
</search>
<drilldown>
<set token="form.as_category">$click.value$</set>
<set token="as_category" prefix="&quot;" suffix="&quot;">$click.value$</set>
</drilldown>
<option name="charting.chart">bar</option>
<option name="charting.drilldown">all</option>
<option name="charting.legend.placement">none</option>
<option name="charting.axisLabelsX.integerUnits">true</option>
<option name="charting.axisTitleX.visibility">collapsed</option>
<option name="charting.axisTitleY.visibility">collapsed</option>
</chart>
</panel>
<panel>
<title>Analytic Stories by MITRE Technique ID</title>
<chart>
<search>
<query>
| rest /services/configs/conf-analytic_stories splunk_server=local count=0
| spath input=mappings path=mitre_attack{} output="MITRE Technique ID"
| stats dc(title) as "Analytic Stories" by "MITRE Technique ID"
</query>
</search>
<drilldown>
<set token="form.as_attack_id">$click.value$</set>
<set token="as_attack_id">$click.value$</set>
</drilldown>
<option name="charting.legend.placement">none</option>
</chart>
</panel>
</row>
<row id="analytic_stories_details_table">
<panel>
<input type="dropdown" token="as_story">
<label>Analytic Story</label>
<choice value="*">All</choice>
<search base="baseAS">
<latest>now</latest>
<query>| dedup title | rename title as story | fields story</query>
</search>
<fieldForLabel>story</fieldForLabel>
<fieldForValue>story</fieldForValue>
<default>*</default>
<prefix>"</prefix>
<suffix>"</suffix>
</input>
<input type="dropdown" token="detection">
<label>Detections</label>
<choice value="*">All</choice>
<search base="baseSS">
<latest>now</latest>
<query>rename action.correlationsearch.label as Detection | dedup Detection | fields Detection</query>
</search>
<fieldForLabel>Detection</fieldForLabel>
<fieldForValue>Detection</fieldForValue>
<prefix>"</prefix>
<suffix>"</suffix>
<default>*</default>
</input>
<input type="dropdown" token="as_category">
<label>Category</label>
<choice value="*">All</choice>
<search base="baseAS">
<latest>now</latest>
<query>| dedup category | fields category</query>
</search>
<fieldForLabel>category</fieldForLabel>
<fieldForValue>category</fieldForValue>
<default>*</default>
<prefix>"</prefix>
<suffix>"</suffix>
</input>
<input type="dropdown" token="as_attack_id">
<label>MITRE Technique ID</label>
<choice value="*">All</choice>
<search base="baseAS">
<latest>now</latest>
<query>| spath input=mappings path=mitre_attack{} output="MITRE Technique ID" | mvexpand "MITRE Technique ID"| dedup "MITRE Technique ID" | fields "MITRE Technique ID"</query>
</search>
<fieldForLabel>MITRE Technique ID</fieldForLabel>
<fieldForValue>MITRE Technique ID</fieldForValue>
<prefix>"</prefix>
<suffix>"</suffix>
<default>*</default>
</input>
<input type="dropdown" token="as_data_models">
<label>Data Models</label>
<choice value="*">All</choice>
<search base="baseAS">
<latest>now</latest>
<query>| spath input=data_models path={} output=dm | mvexpand dm | dedup dm | fields dm</query>
</search>
<fieldForLabel>dm</fieldForLabel>
<fieldForValue>dm</fieldForValue>
<default>*</default>
<prefix>"</prefix>
<suffix>"</suffix>
</input>
<html>
<input id="analytic_filter_clear" class="btn btn-primary" type="button" value="Clear All"/>
</html>
<table>
<title>Analytic Story Details</title>
<search base="baseAS">
<query>
spath input=data_models path={} output="Data Models"
| spath input=mappings path=kill_chain_phases{} output="Kill Chain Phases"
| spath input=detection_searches path={} output="Detections"
| spath input=mappings path=mitre_attack{} output="MITRE Technique ID"
| rename title as "Analytic Story" description as "Description" category as "Category" modification_date as "Last Updated"
| fillnull value="-"
| search "Analytic Story"=$as_story$
| search "Data Models"=$as_data_models$
| search "Category"=$as_category$
| search "MITRE Technique ID"=$as_attack_id$
| search "Detections"=$detection$
| table "Analytic Story", Description, Category, "MITRE Technique ID", "Data Models", Detections, "Last Updated"
</query>
</search>
<drilldown>
<link target="_blank">
<![CDATA[
/app/SplunkEnterpriseSecuritySuite/ess_analytic_story_details?analytic_story=$row.Analytic Story$
]]>
</link>
</drilldown>
<option name="drilldown">row</option>
<option name="wrap">true</option>
<option name="rowNumbers">true</option>
<option name="dataOverlayMode">none</option>
<option name="count">5</option>
</table>
</panel>
</row>
</form>

Some files were not shown because too many files have changed in this diff Show More