admin/splunk-security_content
1
0
Fork 0
mirror of https://github.com/splunk/security_content synced 2026-06-08 17:32:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'exchange2shell' of https://github.com/splunk/security_content into exchange2shell

Browse Source
This commit is contained in:
Michael Haag
2022-10-03 20:39:22 -06:00
parent edd5db1f63 9b56a5f2d1
commit 2e57a73d89
11 changed files with 130 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/build-and-validate.yml
+9 -24
View File
@@ -144,15 +144,6 @@ jobs:
# update build number and version for ssa
tar -czf build/content-pack-build-ssa.tar.gz dist/ssa/*
- name: Download and Install Splunk Packaging Toolkit
run : |
source .venv/bin/activate
curl -Ls https://download.splunk.com/misc/packaging-toolkit/splunk-packaging-toolkit-1.0.1.tar.gz -o splunk-packaging-toolkit-latest.tar.gz
mkdir slim-latest
tar -zxf splunk-packaging-toolkit-latest.tar.gz -C slim-latest --strip-components=1
cd slim-latest
python -m pip install .
cd ..
- name: Build ESCU
run: |
@@ -162,25 +153,19 @@ jobs:
tar -zxf content-pack-build-ssa.tar.gz
mv dist/escu DA-ESS-ContentUpdate
mv dist/ssa SSA_Content
slim package -o upload DA-ESS-ContentUpdate
cp upload/DA-ESS-ContentUpdate-*.tar.gz DA-ESS-ContentUpdate-latest.tar.gz
#Build ESCU Content
#Do not use slim for speed, simplicity, and compatability
tar -zcf DA-ESS-ContentUpdate-latest.tar.gz DA-ESS-ContentUpdate
sha256sum DA-ESS-ContentUpdate-latest.tar.gz > checksum.txt
#Do this copy so that we conform as much as possible, and have to make
#as few changes as possible, once we start generating this as a real,
#properly packaged app
tar -zcf upload/SSA_Content-NO_SLIM.tar.gz SSA_Content
cp upload/SSA_Content-*.tar.gz SSA_Content-latest.tar.gz
#Build the SSA Content
#Do not use slim for speed, simplicity, and compatability
tar -zcf SSA_Content-latest.tar.gz SSA_Content
sha256sum SSA_Content-latest.tar.gz >> checksum.txt
- name: store_artifacts
uses: actions/upload-artifact@v2
with:
name: package
path: |
build/upload
- name: store_artifacts_two
- name: store_artifacts
uses: actions/upload-artifact@v2
with:
name: content-latest
bin/contentctl_project/contentctl_core/application/factory/ba_factory.py
+48 -39
View File
@@ -3,6 +3,8 @@ import sys
from pydantic import ValidationError
from dataclasses import dataclass
from typing import Tuple
import pathlib
from bin.contentctl_project.contentctl_core.domain.entities.enums.enums import SecurityContentType
from bin.contentctl_project.contentctl_core.application.builder.basic_builder import BasicBuilder
@@ -27,7 +29,7 @@ class BAFactoryOutputDto:
class BAFactory():
input_dto: BAFactoryInputDto
output_dto: BAFactoryOutputDto
ids: dict[str,list[str]] = {}
ids: dict[str,list[pathlib.Path]] = {}
def __init__(self, output_dto: BAFactoryOutputDto) -> None:
self.output_dto = output_dto
@@ -35,22 +37,32 @@ class BAFactory():
def execute(self, input_dto: BAFactoryInputDto) -> None:
self.input_dto = input_dto
print("Creating Security Content - SSA. This may take some time...")
self.createSecurityContent(SecurityContentType.unit_tests)
self.createSecurityContent(SecurityContentType.detections)
validation_errors = self.createSecurityContent(SecurityContentType.unit_tests)
validation_errors.extend(self.createSecurityContent(SecurityContentType.detections))
validation_errors.extend(Utils.check_ids_for_duplicates(self.ids))
if len(validation_errors) != 0:
print(f"There were [{len(validation_errors)}] error(s) found while parsing security_content")
for ve in validation_errors:
file_path = ve[0]
error = ve[1]
print(f'\nValidation Error for file [{file_path}]:\n{str(error)}')
raise(Exception("Error(s) validating Security Content"))
def createSecurityContent(self, type: SecurityContentType) -> list:
def createSecurityContent(self, type: SecurityContentType) -> list[Tuple[pathlib.Path, ValidationError]]:
objects = []
if type == SecurityContentType.unit_tests:
files = Utils.get_all_yml_files_from_directory(os.path.join(self.input_dto.input_path, 'tests'))
else:
files = Utils.get_all_yml_files_from_directory(os.path.join(self.input_dto.input_path, str(type.name)))
validation_error_found = False
validation_errors:list[Tuple[pathlib.Path, ValidationError]] = []
files_with_ssa = [f for f in files if 'ssa___' in f]
files_with_ssa = [f for f in files if f.name.startswith('ssa___')]
already_ran = False
progress_percent = 0
@@ -61,43 +73,40 @@ class BAFactory():
# that printouts end at 100%, not some other number
progress_percent = ((index+1)/len(files_with_ssa)) * 100
if 'ssa__' in file:
progress_percent = ((index+1)/len(files_with_ssa)) * 100
try:
type_string = "UNKNOWN TYPE"
if type == SecurityContentType.detections:
type_string = "Detections"
self.input_dto.director.constructDetection(self.input_dto.detection_builder, file, [], [], [], self.output_dto.tests, {}, [], [])
detection = self.input_dto.detection_builder.getObject()
Utils.add_id(self.ids, detection, file)
if not detection.deprecated and not detection.experimental:
self.output_dto.detections.append(detection)
elif type == SecurityContentType.unit_tests:
type_string = "Unit Tests"
self.input_dto.director.constructTest(self.input_dto.basic_builder, file)
test = self.input_dto.basic_builder.getObject()
Utils.add_id(self.ids, test, file)
self.output_dto.tests.append(test)
else:
raise(Exception(f"Unsupported content type: [{type}]"))
progress_percent = ((index+1)/len(files_with_ssa)) * 100
try:
type_string = "UNKNOWN TYPE"
if type == SecurityContentType.detections:
type_string = "Detections"
self.input_dto.director.constructDetection(self.input_dto.detection_builder, file, [], [], [], self.output_dto.tests, {}, [], [])
detection = self.input_dto.detection_builder.getObject()
Utils.add_id(self.ids, detection, file)
if not detection.deprecated and not detection.experimental:
self.output_dto.detections.append(detection)
elif type == SecurityContentType.unit_tests:
type_string = "Unit Tests"
self.input_dto.director.constructTest(self.input_dto.basic_builder, str(file))
test = self.input_dto.basic_builder.getObject()
Utils.add_id(self.ids, test, file)
self.output_dto.tests.append(test)
else:
raise(Exception(f"Unsupported content type: [{type}]"))
if (sys.stdout.isatty() and sys.stdin.isatty() and sys.stderr.isatty()) or not already_ran:
already_ran = True
print(f"\r{f'{type_string} Progress'.rjust(23)}: [{progress_percent:3.0f}%]...", end="", flush=True)
if (sys.stdout.isatty() and sys.stdin.isatty() and sys.stderr.isatty()) or not already_ran:
already_ran = True
print(f"\r{f'{type_string} Progress'.rjust(23)}: [{progress_percent:3.0f}%]...", end="", flush=True)
except ValidationError as e:
print('\nValidation Error for file ' + file)
print(e)
validation_error_found = True
except ValidationError as e:
validation_errors.append((pathlib.Path(file), e))
except Exception as e:
print(f"Unknown exception caught while Creating BA Security Content: {str(e)}")
sys.exit(1)
#Check for any duplicate IDs. The structure is uses
# to track them, self.ids, is populated previously in this
# function every time content is adde.
# This will also print out the duplicates if they exist.
validation_error_found |= Utils.check_ids_for_duplicates(self.ids)
print(f"\r{f'{type_string} Progress'.rjust(23)}: [{progress_percent:3.0f}%]...", end="", flush=True)
print("Done!")
if validation_error_found:
sys.exit(1)
return validation_errors
bin/contentctl_project/contentctl_core/application/factory/factory.py
+43 -35
View File
@@ -3,7 +3,8 @@ import sys
from pydantic import ValidationError
from dataclasses import dataclass
import pathlib
from typing import Tuple
from bin.contentctl_project.contentctl_core.domain.entities.enums.enums import SecurityContentProduct
from bin.contentctl_project.contentctl_core.domain.entities.enums.enums import SecurityContentType
from bin.contentctl_project.contentctl_core.application.builder.basic_builder import BasicBuilder
@@ -48,7 +49,7 @@ class FactoryOutputDto:
class Factory():
input_dto: FactoryInputDto
output_dto: FactoryOutputDto
ids: dict[str,list[str]] = {}
ids: dict[str,list[pathlib.Path]] = {}
def __init__(self, output_dto: FactoryOutputDto) -> None:
self.output_dto = output_dto
@@ -59,20 +60,30 @@ class Factory():
def execute(self, input_dto: FactoryInputDto) -> None:
self.input_dto = input_dto
print("Creating Security Content - ESCU. This may take some time...")
#Accumulate any validation errors that may occur while creating security_contnet
validation_errors = []
# order matters to load and enrich security content types
self.createSecurityContent(SecurityContentType.unit_tests)
self.createSecurityContent(SecurityContentType.lookups)
self.createSecurityContent(SecurityContentType.macros)
self.createSecurityContent(SecurityContentType.deployments)
self.createSecurityContent(SecurityContentType.baselines)
self.createSecurityContent(SecurityContentType.investigations)
self.createSecurityContent(SecurityContentType.playbooks)
self.createSecurityContent(SecurityContentType.detections)
self.createSecurityContent(SecurityContentType.stories)
validation_errors.extend(self.createSecurityContent(SecurityContentType.unit_tests))
validation_errors.extend(self.createSecurityContent(SecurityContentType.lookups))
validation_errors.extend(self.createSecurityContent(SecurityContentType.macros))
validation_errors.extend(self.createSecurityContent(SecurityContentType.deployments))
validation_errors.extend(self.createSecurityContent(SecurityContentType.baselines))
validation_errors.extend(self.createSecurityContent(SecurityContentType.investigations))
validation_errors.extend(self.createSecurityContent(SecurityContentType.playbooks))
validation_errors.extend(self.createSecurityContent(SecurityContentType.detections))
validation_errors.extend(self.createSecurityContent(SecurityContentType.stories))
validation_errors.extend(Utils.check_ids_for_duplicates(self.ids))
LinkValidator.print_link_validation_errors()
if len(validation_errors) != 0:
print(f"There were [{len(validation_errors)}] error(s) found while parsing security_content")
for ve in validation_errors:
file_path = ve[0]
error = ve[1]
print(f'\nValidation Error for file [{file_path}]:\n{str(error)}')
raise(Exception("Error(s) validating Security Content"))
def createSecurityContent(self, type: SecurityContentType) -> list:
def createSecurityContent(self, type: SecurityContentType) -> list[Tuple[pathlib.Path, ValidationError]]:
objects = []
if type == SecurityContentType.deployments:
files = Utils.get_all_yml_files_from_directory(os.path.join(self.input_dto.input_path, str(type.name), 'ESCU'))
@@ -82,17 +93,17 @@ class Factory():
files = Utils.get_all_yml_files_from_directory(os.path.join(self.input_dto.input_path, str(type.name)))
# Instead of failing on the first error, just keep track of
# whether or not an error was found. This way, we can
# report all of the errors on a single run so that the
# user can see all the errors they need to fix.
validation_error_found = False
# of all the exceptions that we generate. These exceptions
# will be returned from the function and should be printed
# by the caller.
validation_errors:list[Tuple[pathlib.Path, ValidationError]] = []
already_ran = False
progress_percent = 0
type_string = "UNKNOWN TYPE"
#Non threaded, production version of the construction code
files_without_ssa = [f for f in files if 'ssa___' not in f]
files_without_ssa = [f for f in files if not f.name.startswith('ssa___')]
for index,file in enumerate(files_without_ssa):
#Index + 1 because we are zero indexed, not 1 indexed. This ensures
@@ -102,35 +113,35 @@ class Factory():
type_string = "UNKNOWN TYPE"
if type == SecurityContentType.lookups:
type_string = "Lookups"
self.input_dto.director.constructLookup(self.input_dto.basic_builder, file)
self.input_dto.director.constructLookup(self.input_dto.basic_builder, str(file))
lookup = self.input_dto.basic_builder.getObject()
Utils.add_id(self.ids, lookup, file)
self.output_dto.lookups.append(lookup)
elif type == SecurityContentType.macros:
type_string = "Macros"
self.input_dto.director.constructMacro(self.input_dto.basic_builder, file)
self.input_dto.director.constructMacro(self.input_dto.basic_builder, str(file))
macro = self.input_dto.basic_builder.getObject()
Utils.add_id(self.ids, macro, file)
self.output_dto.macros.append(macro)
elif type == SecurityContentType.deployments:
type_string = "Deployments"
self.input_dto.director.constructDeployment(self.input_dto.basic_builder, file)
self.input_dto.director.constructDeployment(self.input_dto.basic_builder, str(file))
deployment = self.input_dto.basic_builder.getObject()
Utils.add_id(self.ids, deployment, file)
self.output_dto.deployments.append(deployment)
elif type == SecurityContentType.playbooks:
type_string = "Playbooks"
self.input_dto.director.constructPlaybook(self.input_dto.playbook_builder, file)
self.input_dto.director.constructPlaybook(self.input_dto.playbook_builder, str(file))
playbook = self.input_dto.playbook_builder.getObject()
Utils.add_id(self.ids, playbook, file)
self.output_dto.playbooks.append(playbook)
elif type == SecurityContentType.baselines:
type_string = "Baselines"
self.input_dto.director.constructBaseline(self.input_dto.baseline_builder, file, self.output_dto.deployments)
self.input_dto.director.constructBaseline(self.input_dto.baseline_builder, str(file), self.output_dto.deployments)
baseline = self.input_dto.baseline_builder.getObject()
Utils.add_id(self.ids, baseline, file)
self.output_dto.baselines.append(baseline)
@@ -144,7 +155,7 @@ class Factory():
elif type == SecurityContentType.stories:
type_string = "Stories"
self.input_dto.director.constructStory(self.input_dto.story_builder, file,
self.input_dto.director.constructStory(self.input_dto.story_builder, str(file),
self.output_dto.detections, self.output_dto.baselines, self.output_dto.investigations)
story = self.input_dto.story_builder.getObject()
Utils.add_id(self.ids, story, file)
@@ -162,7 +173,7 @@ class Factory():
elif type == SecurityContentType.unit_tests:
type_string = "Unit Tests"
self.input_dto.director.constructTest(self.input_dto.basic_builder, file)
self.input_dto.director.constructTest(self.input_dto.basic_builder, str(file))
test = self.input_dto.basic_builder.getObject()
Utils.add_id(self.ids, test, file)
self.output_dto.tests.append(test)
@@ -175,19 +186,16 @@ class Factory():
print(f"\r{f'{type_string} Progress'.rjust(23)}: [{progress_percent:3.0f}%]...", end="", flush=True)
except ValidationError as e:
print('\nValidation Error for file ' + file)
print(e)
validation_error_found = True
validation_errors.append((pathlib.Path(file), e))
except Exception as e:
print(f"Unknown exception caught while Creating Security Content: {str(e)}")
sys.exit(1)
#Check for any duplicate IDs. The structure is uses
# to track them, self.ids, is populated previously in this
# function every time content is adde.
# This will also print out the duplicates if they exist.
validation_error_found |= Utils.check_ids_for_duplicates(self.ids)
print(f"\r{f'{type_string} Progress'.rjust(23)}: [{progress_percent:3.0f}%]...", end="", flush=True)
print("Done!")
if validation_error_found:
sys.exit(1)
return validation_errors
bin/contentctl_project/contentctl_core/application/factory/object_factory.py
+1 -1
View File
@@ -25,5 +25,5 @@ class ObjectFactory():
files = Utils.get_all_yml_files_from_directory(input_dto.input_path)
for file in files:
input_dto.director.constructObjects(input_dto.builder, file)
input_dto.director.constructObjects(input_dto.builder, str(file))
self.objects.append(input_dto.builder.getObject())
bin/contentctl_project/contentctl_core/application/factory/utils/utils.py
+17 -10
View File
@@ -1,21 +1,25 @@
import os
import pathlib
from typing import Tuple
from pydantic import ValidationError
from bin.contentctl_project.contentctl_core.domain.entities.security_content_object import SecurityContentObject
class Utils:
@staticmethod
def get_all_yml_files_from_directory(path: str) -> list:
listOfFiles = list()
def get_all_yml_files_from_directory(path: str) -> list[pathlib.Path]:
listOfFiles:list[pathlib.Path] = []
for (dirpath, dirnames, filenames) in os.walk(path):
for file in filenames:
if file.endswith(".yml"):
listOfFiles.append(os.path.join(dirpath, file))
listOfFiles.append(pathlib.Path(os.path.join(dirpath, file)))
return sorted(listOfFiles)
@staticmethod
def add_id(id_dict:dict[str, list[str]], obj:SecurityContentObject, path:str) -> None:
def add_id(id_dict:dict[str, list[pathlib.Path]], obj:SecurityContentObject, path:pathlib.Path) -> None:
if hasattr(obj, "id"):
obj_id = obj.id
if obj_id in id_dict:
@@ -25,11 +29,14 @@ class Utils:
# Otherwise, no ID so nothing to add....
@staticmethod
def check_ids_for_duplicates(id_dict:dict[str, list[str]])->bool:
validation_error = False
def check_ids_for_duplicates(id_dict:dict[str, list[pathlib.Path]])->list[Tuple[pathlib.Path, ValidationError]]:
validation_errors:list[Tuple[pathlib.Path, ValidationError]] = []
for key, values in id_dict.items():
if len(values) > 1:
validation_error = True
id_conflicts_string = '\n\t* '.join(values)
print(f"\nError validating id [{key}] - duplicate ID is used for the following content: \n\t* {id_conflicts_string}")
return validation_error
error_file_path = pathlib.Path("MULTIPLE")
all_files = '\n\t'.join(str(pathlib.Path(p)) for p in values)
exception = ValueError(f"Error validating id [{key}] - duplicate ID was used in the following files: \n\t{all_files}")
validation_errors.append((error_file_path, exception))
return validation_errors
bin/contentctl_project/contentctl_core/domain/entities/detection_tags.py
+3 -2
View File
@@ -115,8 +115,9 @@ class DetectionTags(BaseModel):
@validator('risk_score')
def tags_calculate_risk_score(cls, v, values):
calculated_risk_score = (int(values['impact']))*(int(values['confidence']))/100
calculated_risk_score = round(values['impact'] * values['confidence'] / 100)
if calculated_risk_score != int(v):
raise ValueError('risk_score is calculated wrong: ' + values["name"])
raise ValueError(f"Risk Score must be calculated as round(confidence * impact / 100)"
f"\n Expected risk_score={calculated_risk_score}, found risk_score={int(v)}: {values['name']}")
return v
bin/docker_detection_tester/detection_testing_execution.py
+4 -2
View File
@@ -212,7 +212,8 @@ def generate_escu_app(persist_security_content: bool = False) -> str:
# There remove the latest file if it exists
commands = ["cd slim_packaging",
"cp -R ../dist/escu DA-ESS-ContentUpdate",
"slim package -o upload DA-ESS-ContentUpdate",
"mkdir upload",
"tar -czf upload/DA-ESS-ContentUpdate*.tar.gz DA-ESS-ContentUpdate",
"cp upload/DA-ESS-ContentUpdate*.tar.gz %s" % (output_file_path_from_slim_latest)]
else:
@@ -221,7 +222,8 @@ def generate_escu_app(persist_security_content: bool = False) -> str:
"mkdir slim_packaging",
"cd slim_packaging",
"cp -R ../dist/escu DA-ESS-ContentUpdate",
"slim package -o upload DA-ESS-ContentUpdate",
"mkdir upload",
"tar -czf upload/DA-ESS-ContentUpdate*.tar.gz DA-ESS-ContentUpdate",
"cp upload/DA-ESS-ContentUpdate*.tar.gz %s" % (output_file_path_from_slim_latest)]
ret = subprocess.run("; ".join(commands),
detections/experimental/application/splunk_account_discovery_drilldown_dashboard_disclosure.yml
+1 -1
View File
@@ -5,7 +5,7 @@ date: '2022-08-02'
author: Marissa Bower, Rod Soto, Splunk
type: TTP
datamodel: []
search: '| rest splunk_server=local /servicesNS/-/-/data/ui/views | search eai:data="*$env:*" eai:data="*url*" eai:data="*options*" | rename author AS Author eai:acl.sharing AS Permissions eai:appName AS App eai:data AS "Dashboard XML" | fields Author Permissions App "Dashboard XML" | `splunk_drilldown_dashboard_disclosure_filter`'
search: '| rest splunk_server=local /servicesNS/-/-/data/ui/views | search eai:data="*$env:*" eai:data="*url*" eai:data="*options*" | rename author AS Author eai:acl.sharing AS Permissions eai:appName AS App eai:data AS "Dashboard XML" | fields Author Permissions App "Dashboard XML" | `splunk_account_discovery_drilldown_dashboard_disclosure_filter`'
description: Splunk drilldown vulnerability disclosure in Dashboard application that can potentially allow exposure of tokens from privilege users. An attacker can create dashboard and share it to privileged user (admin) and detokenize variables using external urls within dashboards drilldown function.
how_to_implement: This search uses REST function to query for dashboards with environment variables present in URL options.
known_false_positives: This search may reveal non malicious URLs with environment variables used in organizations.
detections/experimental/endpoint/exchange_powershell_abuse_via_ssrf.yml
+3 -3
View File
@@ -1,7 +1,7 @@
name: Exchange PowerShell Abuse via SSRF
id: 29228ab4-0762-11ec-94aa-acde48001122
version: 1
date: '2021-08-27'
version: 2
date: '2022-10-02'
author: Michael Haag, Splunk
type: TTP
datamodel: []
@@ -20,7 +20,7 @@ description: 'This analytic identifies suspicious behavior related to ProxyShell
Review the source attempting to perform this activity against your environment.
In addition, review PowerShell logs and access recently granted to Exchange roles.'
search: '| `exchange` c_uri="*//autodiscover.json*" cs_uri_query="*PowerShell*" cs_method="POST"
search: '`exchange` c_uri="*//autodiscover.json*" cs_uri_query="*PowerShell*" cs_method="POST"
| stats count min(_time) as firstTime max(_time) as lastTime by dest, cs_uri_query,
cs_method, c_uri | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)`
| `exchange_powershell_abuse_via_ssrf_filter`'
detections/experimental/network/remote_desktop_network_traffic.yml
+1 -1
View File
@@ -13,7 +13,7 @@ description: This search looks for network traffic on TCP/3389, the default port
on your network.
search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time)
as lastTime from datamodel=Network_Traffic where All_Traffic.dest_port=3389 AND
All_Traffic.dest_category!=common_rdp_destination AND All_Traffic.src_category!=common_rdp_source
All_Traffic.dest_category!=common_rdp_destination AND All_Traffic.src_category!=common_rdp_source AND all_Traffic.action="allowed"
by All_Traffic.src All_Traffic.dest All_Traffic.dest_port | `drop_dm_object_name("All_Traffic")`
| `security_content_ctime(firstTime)`| `security_content_ctime(lastTime)` | `remote_desktop_network_traffic_filter` '
how_to_implement: To successfully implement this search you need to identify systems
requirements.txt
-1
View File
@@ -13,7 +13,6 @@ questionary==1.10.0
requests==2.28.1
six==1.16.0
splunk-appinspect==2.25.0
splunk-packaging-toolkit==1.0.1
splunk-sdk==1.7.2
wrapt-timeout-decorator==1.3.12.2
xmltodict==0.13.0