diff --git a/.circleci/config.yml b/.circleci/config.yml index 35015aca46..35415b37e0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -100,6 +100,7 @@ jobs: # update build number and version sed -i "s/build = .*$/build = $CIRCLE_BUILD_NUM/g" package/default/app.conf sed -i "s/^version = .*$/version = $CONTENT_VERSION/g" package/default/app.conf + sed -i "s/\"version\": .*$/\"version\": \"$CONTENT_VERSION\"/g" package/app.manifest sed -i "s/version = .*$/version = $CONTENT_VERSION/g" package/default/content-version.conf tar -czf content-pack-build.tar.gz package/* - persist_to_workspace: @@ -192,6 +193,24 @@ jobs: else git clone --branch ${CIRCLE_BRANCH} https://${GITHUB_TOKEN}@github.com/splunk/security-content.git fi + - run: *apt-install + - run: + name: install python dependencies + command: | + cd security-content + rm -rf venv + virtualenv --python=/usr/bin/python2.7 --clear venv + source venv/bin/activate + pip install -q -r requirements.txt + - run: + name: run doc-gen + command: | + cd security-content + source venv/bin/activate + python bin/doc-gen.py --path . --output docs -v + - run: + name: update github with new docs and package bits + command: | cd security-content rm -rf package mv ~/latest package @@ -201,7 +220,8 @@ jobs: git config user.name "research bot" git config --global push.default simple git add package/* - git commit --allow-empty -m "updating package files [ci skip]" + git add docs/* + git commit --allow-empty -m "updating docs and package bits [ci skip]" # Push quietly to prevent showing the token in log git push https://${GITHUB_TOKEN}@github.com/splunk/security-content.git ${CIRCLE_BRANCH} CONTENT_VERSION=$(echo $CIRCLE_TAG | grep -oP "\d.\d.\d+") @@ -261,7 +281,7 @@ workflows: - build-sources - build-package - run-appinspect - # only update sources in develop + # update package and docs in github if is a tag filters: tags: only: /^v.*/ diff --git a/bin/doc-gen.py b/bin/doc-gen.py index 2d34dce392..8c9abc4124 100644 --- a/bin/doc-gen.py +++ b/bin/doc-gen.py @@ -36,6 +36,217 @@ def markdown(x): return markdown +def process_data_metadata(obj, complete_obj, name): + + # collect tagging + metadata = obj['data_metadata'] + if 'data_models' in metadata: + complete_obj[name]['data_models'] = metadata['data_models'] + if 'providing_technologies' in metadata: + complete_obj[name]['providing_technologies'] = metadata['providing_technologies'] + if 'data_source' in metadata: + complete_obj[name]['data_source'] = metadata['data_source'] + + if 'mappings' in obj: + complete_obj[name]['mappings'] = obj['mappings'] + if 'fields_required' in obj: + complete_obj[name]['entities'] = obj['fields_required'] + if 'entities' in obj: + complete_obj[name]['entities'] = obj['entities'] + + return complete_obj + + +def process_metadata(detections, story_name): + # grab mappings + mappings = dict() + + # grab provising technologies + providing_technologies = [] + + # grab datamodels + data_models = [] + + # process the above for detections + for detection_name, detection in sorted(detections.iteritems()): + for s in detection['stories']: + + # check if the detection is part of this story + if s == story_name: + # grab providing technologies + if 'providing_technologies' in detection: + for pt in detection['providing_technologies']: + providing_technologies.append(pt) + + # grab data models + if 'data_models' in detection: + for dm in detection['data_models']: + data_models.append(dm) + + for key in detection['mappings'].keys(): + mappings[key] = list(detection['mappings'][key]) + + return mappings, providing_technologies, data_models + + +def generate_detections(REPO_PATH, stories): + # first we process detections + + detections = [] + detections_manifest_files = path.join(path.expanduser(REPO_PATH), "detections/*.json") + for detections_manifest_file in glob.glob(detections_manifest_files): + # read in each story + try: + detection = json.loads( + open(detections_manifest_file, 'r').read()) + except IOError: + sys.exit("ERROR: reading {0}".format(detections_manifest_file)) + detections.append(detection) + + complete_detections = dict() + for detection in detections: + # lets process v1 detections + if detection['spec_version'] == 1: + if verbose: + print "processing v1 detection: {0}".format(detection['search_name']) + name = detection['search_name'] + type = 'splunk' + description = detection['search_description'] + id = detection['search_id'] + + # grab search information + correlation_rule = detection['correlation_rule'] + search = detection['search'] + schedule = detection['scheduling'] + earliest_time = schedule['earliest_time'] + latest_time = schedule['latest_time'] + cron = schedule['cron_schedule'] + + # grabbing entities + entities = [] + + investigations = [] + baselines = [] + responses = [] + for story_name, story in sorted(stories.iteritems()): + for d in story['detections']: + if d['name'] == name: + if 'investigations' in story: + investigations = story['investigations'] + if 'baselines' in story: + baselines = story['baselines'] + + # lets process v2 detections + if detection['spec_version'] == 2: + if verbose: + print "processing v2 detection: {0}".format(detection['name']) + name = detection['name'] + id = detection['id'] + entities = detection['entities'] + description = detection['description'] + + # splunk + if 'splunk' in detection['detect']: + type = 'splunk' + correlation_rule = detection['detect']['splunk']['correlation_rule'] + search = correlation_rule['search'] + earliest_time = correlation_rule['schedule']['earliest_time'] + latest_time = correlation_rule['schedule']['latest_time'] + cron = correlation_rule['schedule']['cron_schedule'] + + # uba + if 'uba' in detection['detect']: + uba = detection['detect']['uba'] + type = 'uba' + search = uba['search'] = 'CONSTRUCT DETECTION SEARCH HERE' + # earliest_time = uba['earliest_time'] + # latest_time = uba['latest_time'] + # cron = uba['cron_schedule'] + + # phantom + if 'phantom' in detection['detect']: + phantom = detection['detect']['phantom'] + type = 'phantom' + search = phantom['search'] = 'CONSTRUCT DETECTION SEARCH HERE' + # earliest_time = phantom['earliest_time'] + # latest_time = phantom['latest_time'] + # cron = phantom['cron_schedule'] + + baselines = [] + investigations = [] + responses = [] + if 'baselines' in detection: + for b in detection['baselines']: + baselines.append({"type": b['product_type'], "name": b['name']}) + if 'investigations' in detection: + for i in detection['investigations']: + investigations.append({"type": i['product_type'], "name": i['name']}) + if 'responses' in detection: + for r in detection['responses']: + responses.append({"type": r['product_type'], "name": r['name']}) + + complete_detections[name] = {} + complete_detections[name]['detection_name'] = name + complete_detections[name]['id'] = id + complete_detections[name]['search'] = search + complete_detections[name]['latest_time'] = latest_time + complete_detections[name]['earliest_time'] = earliest_time + complete_detections[name]['cron'] = cron + complete_detections[name]['investigations'] = investigations + complete_detections[name]['baselines'] = baselines + complete_detections[name]['responses'] = responses + complete_detections[name]['entities'] = entities + complete_detections[name]['description'] = description + complete_detections[name]['correlation_rule'] = correlation_rule + complete_detections[name]['type'] = type + complete_detections[name]['maintainers'] = detection['maintainers'] + if 'references' not in detection: + detection['references'] = [] + complete_detections[name]['references'] = detection['references'] + if 'channel' not in detection: + detection['channel'] = "" + complete_detections[name]['channel'] = detection['channel'] + if 'confidence' not in detection: + detection['confidence'] = "" + complete_detections[name]['confidence'] = detection['confidence'] + if 'eli5' not in detection: + detection['eli5'] = "" + complete_detections[name]['eli5'] = detection['eli5'] + if 'how_to_implement' not in detection: + detection['how_to_implement'] = "" + complete_detections[name]['how_to_implement'] = detection['how_to_implement'] + if 'asset_type' not in detection: + detection['asset_type'] = "" + complete_detections[name]['asset_type'] = detection['asset_type'] + if 'known_false_positives' not in detection: + detection['known_false_positives'] = "" + complete_detections[name]['known_false_positives'] = detection['known_false_positives'] + complete_detections[name]['security_domain'] = detection['security_domain'] + complete_detections[name]['version'] = detection['version'] + complete_detections[name]['spec_version'] = detection['spec_version'] + complete_detections[name]['creation_date'] = detection['creation_date'] + # set modification date to creation of there is not one + if 'modification_date' in detection: + complete_detections[name]['modification_date'] = detection['modification_date'] + else: + complete_detections[name]['modification_date'] = detection['creation_date'] + + # process its metadata + complete_detections = process_data_metadata(detection, complete_detections, name) + + # stories associated with the detection + complete_detections[name]['stories'] = [] + for story_name, story in sorted(stories.iteritems()): + for d in story['detections']: + if d['name'] == name: + complete_detections[name]['stories'].append(story['story_name']) + + # sort uniq the results + complete_detections[name]['stories'] = sorted(set(complete_detections[name]['stories'])) + + return complete_detections + + def generate_stories(REPO_PATH, verbose): story_files = [] story_manifest_files = path.join(path.expanduser(REPO_PATH), "stories/*.json") @@ -116,7 +327,7 @@ def generate_stories(REPO_PATH, verbose): return complete_stories -def write_splunk_docs(stories, OUTPUT_DIR): +def write_splunk_docs(stories, detections, OUTPUT_DIR): paths = [] # Create conf files from analytics stories files @@ -142,41 +353,68 @@ def write_splunk_docs(stories, OUTPUT_DIR): # if the category matches if story['category'] == c: output_file.write("\n==={0}===\n".format(story_name)) - # header information - output_file.write(""" -
Content-Type length, suspicious java classes and web servers executing suspicious processes--consistent with attempts to exploit Apache Struts vulnerabilities.
-'''Narrative''' Content-Type HTTP header to execute commands embedded in the header.Content-Type header consistent with attempts to exploit the vulnerability. This should be a relatively pertinent indicator, as the Content-Type header is generally consistent and does not have a large degree of variation.svchost.exe is found running from a location other than C:\Windows\System32, it is likely something malicious designed to hide in plain sight when simply reviewing process names. Similarly, if the process itself seems legitimate, but the parent process is running from the temporary browser cache, there may be activity initiated via a compromised website the user visited.C:\Windows\System32, it is likely something malicious designed to hide in plain sight when simply reviewing process names. DNSTwist, together with the support searches in this Analytic Story, to generate permutations of specified brands and external domains. Splunk can monitor email, DNS requests, and web traffic for these permutations and provide you with early warnings and situational awareness--powerful elements of an effective defense.ANY queries. This Analytic Story can help you detect attackers who may be abusing your company's DNS infrastructure to launch amplification attacks, causing Denial of Service to other victims.
-'''Narrative''' ANY queries are so much larger than the queries themselves--and can be made with a UDP packet, which does not require a handshake--attackers can spoof the source address of the packet and cause much more data to be sent to the victim than if they sent the traffic themselves. The ANY requests are will be larger than normal DNS server requests, due to the fact that the server provides significant details, such as MX records and associated IP addresses. A large volume of this traffic can result in a DOS on the victim's machine. This misconfiguration leads to two possible victims, the first being the DNS servers participating in an attack and the other being the hosts that are the targets of the DOS attack.netsh.exe, which can disable local firewall settings or set up a remote connection to a host from an infected system.
-'''Narrative''' netsh.exe,a command-line scripting utility that allows you to--either locally or remotely--display or modify the network configuration of a computer that is currently running. Netsh.exe can be used to discover and disable local firewall settings. It can also be used to set up a remote connection to a host from an infected system.netsh.exe.
+====Providing Technologies====
+* Carbon Black Response
+* CrowdStrike Falcon
+* Sysmon
+* Tanium
+* Ziften
- ''' Framework Mappings ''' .doc or .docx, they will assume that it is a Microsoft Word document and expect that double-clicking will open it using winword.exe. The user will typically also presume that the .docx file is safe. reg.exe processes, processes launching netsh, and many others.
-'''Narrative''' psexec.exe), unauthorized use of remote desktop services, file/admin$ shares, WMI, PowerShell, pass-the-hash, or the abuse of scheduled tasks. Organizations must be extra vigilant in detecting lateral movement techniques and look for suspicious activity in and around high-value strategic network assets, such as Active Directory, which are often considered the primary target or "crown jewels" to a persistent threat actor.svchost.exe is found running from a location other than C:\Windows\System32, it is likely something malicious designed to hide in plain sight when cursorily reviewing process names. Similarly, if the process itself seems legitimate, but the parent process is running from the temporary browser cache, that could be indicative of activity initiated via a compromised website a user visited.[HKEY_CURRENT_USER\SOFTWARE\Microsoft\Windows\CurrentVersion\Run]'help'='c:\\windows\\system32\\rundll32.exe c:\\windows\\system32\\zipfldr.dll,RouteTheCall c:\\programdata\\winapp.exe'. Though this technique is not exclusive to MUDCARP, it has been spotted in the group's arsenal of advanced techniques seen in the wild.reg.exe processes, files hidden with attrib.exe and disabling user-account control, among many others
-'''Narrative'''