diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5ed12d9e6d..44319cbc2f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v2.1.0 # Use the ref you want to point at hooks: - - id: trailing-whitespace - id: check-executables-have-shebangs exclude: 'package/bin/da_ess_contentupdate/|package/bin/splunklib/|venv/' - id: check-json diff --git a/README.md b/README.md index dd4312b146..282e3f5448 100644 --- a/README.md +++ b/README.md @@ -11,32 +11,66 @@ Contains a collection of security stories with their corresponding detection, in # Usage Can be consumed using: +#### [Splunk App](https://github.com/splunk/security-content/releases) +grab the latest release of DA-ESS-ContentUpdate and install it in a Splunk Server. + #### [API](https://github.com/splunk/security-content-api) `curl https://g7jbilqdth.execute-api.us-west-2.amazonaws.com/api/` #### [CLI](https://github.com/splunk/security-content-api/blob/master/content-update.py) `python content-update.py -o $SPLUNK_HOME/etc/apps/DA-ESS-ContentUpdate --splunk_user admin --splunk_password xxxx` -# Structure +# Writing Content +Make sure you followed step 1 to 3 under [developing](https://github.com/splunk/security-content#developing) before starting. + +1. select which content [piece](https://github.com/splunk/security-content#content-parts) you want to write. +2. copy an example and edit to your needs, most sure you at minium write a [story](stories/), [detection](detections/) and [investigation](investigations/) +3. make a pull request .. if CI failed refer to [troubleshooting](https://github.com/splunk/security-content#troubleshooting) + + +# Security Content Layout +![](static/structure.png) + +#### Content Parts * [stories/](stories/) - contains all analytics stories/use cases for ESCU * [detections/](detections/) - splunk, uba and phantom detections that power stories * [investigations/](investigations/) - splunk, and phantom investigation content that are used in stories * [responses/](responses/) - automated splunk and phantom responses that are used in stories * [baselines/](baselines/) - phantom and Splunk baseline needed to support detections in stories + +#### Supporting parts * [package/](package/) - splunk content app source files, includes lookups, binaries, and defaul config files * [bin/](bin/) - where all binaries to produce, and test content lives -* [spec/](spec/) - location of all spec files that describe ESCU content + +# Docs + * [docs/](docs/) - documentation for all of the spec files +* [spec/](spec/) - location of all spec files that describe ESCU content # Developing For getting pre-commit checks, install the hooks see below for steps: -1. Install circleci [CLI Tool](https://circleci.com/docs/2.0/local-cli/#installation) -2. create virtualenv and install requirements: `virtualenv venv && source venv/bin/activate && pip install -r requirements.txt` -3. install pre-commit `pre-commit install` + +1. create virtualenv and install requirements: `virtualenv venv && source venv/bin/activate && pip install -r requirements.txt` +2. install pre-commit `pre-commit install` +3. Install circleci [CLI Tool](https://circleci.com/docs/2.0/local-cli/#installation) To test a local change to CI or build make sure you are running docker and then + `circleci local execute -e GITHUB_TOKEN=$GITHUB_TOKEN --branch ` To generate docs from schema automatically 1. install https://github.com/adobe/jsonschema2md 2. `jsonschema2md -d spec/v2/detections.json.spec -o docs` + +# Troubleshooting + +#### Our Automated Tests +1. [CI](https://github.com/splunk/security-content/blob/44946063173f7bc9921f0da0aa62139c084d1c51/.circleci/config.yml#L27) validates that the content was written to spec using [`validate.py`](https://github.com/splunk/security-content/blob/runstory/bin/generate.py), to run validation manually execute: `python bin/generate.py --path . --output package --storiesv1 --use_case_lib -v` +2. [CI](https://github.com/splunk/security-content/blob/44946063173f7bc9921f0da0aa62139c084d1c51/.circleci/config.yml#L60) generates splunk configuration files using [`generate.py`](https://github.com/splunk/security-content/blob/develop/bin/generate.py). If you want to export splunk conf files manually from the content run: `python bin/generate.py --path . --output package --storiesv1 --use_case_lib -v` +3. [CI](https://github.com/splunk/security-content/blob/44946063173f7bc9921f0da0aa62139c084d1c51/.circleci/config.yml#L107) builds a DA-ESS-ContentUpdate splunk package using the [Splunk Packaging Toolkit](http://dev.splunk.com/view/packaging-toolkit/SP-CAAAE9V) +4. [CI](https://github.com/splunk/security-content/blob/44946063173f7bc9921f0da0aa62139c084d1c51/.circleci/config.yml#L145) tests the newly produce package using [Splunk Appinspect](http://dev.splunk.com/view/appinspect/SP-CAAAE9U) + + + +# Todo's +* build cli for interacting and developing \ No newline at end of file diff --git a/bin/add_common_words.py b/bin/add_common_words.py deleted file mode 100644 index dac4e6031a..0000000000 --- a/bin/add_common_words.py +++ /dev/null @@ -1,44 +0,0 @@ -import enchant - -import nltk -nltk.download('stopwords') -nltk.download('punkt') -nltk.download('wordnet') - - -common_manifest_words = [ - "App", "Splunk", "AWS", "CVE", "IAM", "ESCU", "Netbackup", "CLI", "DNS", "CloudTrail", - "DNSTwist", "IP", "DHCP", "AMIs", "EventCode", "PowerShell", "dest", "instanceId", - "malware", "exfiltration", "exfiltrate", "Spectre", "Mitre", "ul", "li", "JBoss", "admin", - "subsearch", "cleartext", "WMI", "URL", "URLs", "ATT", "CK", "SMB", "ACL", "ACLs", "CIDR", - "VPC", "TXT", "USB", "SQL", "src", "VPN", "src", "br", "phishing", "weaponized", - "Ransomware", "ransomware", "VirusTotal", "RDP", "kerberized", "IMAP", "SSL", - "instanceType", "subsearch", "SPL", "rhaegal", "drogon", "BadRabbit", "WannaCry", - "CARBANAK", "wevtutil", "UAC", "ModifiedPersistence", "whoami", "tstats", "IPs", - "sourcetype", "JexBoss", "iptables", "wget", "NTFS", "noninteractive", "DLL", "DLLs", - "faux", "eval", "userName", "outlier", "DDoS", "logon", "noninteractive", "PID", "whois", - "misconfigured", "ARN", "MITM", "UDP", "MX", "Noriaki", "Iwasaki", "API", "APIs", - "awsRegion", "wmic", "vssadmin", "IDAPro", "AppData", "lifecycle", "prepending", - "prepended", "exfiltrating", "SDKs", "FQDN", "FQDNs", "DDNS", "Multipart", "CDN", - "cryptomining", "Cryptomining", "cryptojacking", "Cryptojacking", "typosquatting", - "DHL", "Samsam", "usernames", "antivirus", "NtLmSsP", "NTLM", "dnstwist", "url", - "netsh", "Netsh", "hostname", "hostnames", "csv", "amiID", "whitelist", "whitelisting", - "CVEs", "Cyber", "backticks", "backend", "backdoor", "backdoored", "wormable", - "RunDLL", "Schtasks", "logons", "spearphishing", "driveby", "cyber", "firstTime", - "lastTime", "ARNs", "GeoIP", "MaxMind", "DHS", "localgroup","PsExec", "psexec", - "accepteula", "cmdline", "apiCalls", "latestCount", "ol","internet","numberOfBlockedConnections", "arn", "AssumedRole", - "dataPointThreshold", "deviationThreshold", "eventNames","CloudWatch","VPCs", "ICMP" - "AssumeRole", "isnotnull", "requestingAccountId", - "http", "Sysmon", "CIM", "resourceId", "NetworkACLEvents", "securityGroupAPIs", - "CIDRs", "https", "username", "pst", "ost", "AccessKeyId","AssumeRole","isnotnull","AccessKeyId","redhat","bucketName","serviceName","javascript","VBScript", - "Mimikatz","SeDebugPrivilege","sekurlsa","lsass","offline","ExecutionPolicy","NirSoft","scomma","stext","Nirsoft","Clickspeed","clickstream","IIS","fraudster","splunk","DDOS","clickstreams", - "wscript", "cscript", "WinEventLog","filesystem","Recurly","whitelisted", "POC","Playbook", "SamSam", "Bruteforce", "POSTs", "java", "RCE" -] - - -d = enchant.Dict("en_US") - -for word in common_manifest_words: - if not d.is_added(word): - d.add(word) - diff --git a/bin/manifest_to_escu.py b/bin/manifest_to_escu.py deleted file mode 100644 index 592d5d91ae..0000000000 --- a/bin/manifest_to_escu.py +++ /dev/null @@ -1,383 +0,0 @@ -# Take the manifest files and build files for Enterprise Security Content Updates with markdown syntax - - -import datetime -import glob -import json -import os -import argparse - -ALL_UUIDS = [] - - -def markdown(x): - markdown = str(x) - markdown = markdown.replace("", "`") - markdown = markdown.replace("", "`") - markdown = markdown.replace("", "**") - markdown = markdown.replace("", "**") - # list tag replacements - markdown = markdown.replace("
  1. ", "\\\n\\\n1. ") - markdown = markdown.replace("
  2. ", "\\\n\\\n1. ") - markdown = markdown.replace("
", "") - markdown = markdown.replace("", "") - markdown = markdown.replace("", "") - markdown = markdown.replace("