From 3029ba2dad748895187560a2551c4cd8c89070e6 Mon Sep 17 00:00:00 2001 From: Daniel Pauler Date: Sat, 20 Mar 2021 17:07:35 +0100 Subject: [PATCH 01/41] Added vscode config folder --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 0ac3d08ff8..46020d2814 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,9 @@ detections/*/.yml.example stories/*.yml.example tests/*/*.yml.example +# IDE +.vscode/ + # usual mac files .DS_Store #vim files From a48e0fcdbb4a8e90b1c34f4482943835f1aab726 Mon Sep 17 00:00:00 2001 From: Daniel Pauler Date: Sat, 20 Mar 2021 17:09:48 +0100 Subject: [PATCH 02/41] Fixed error handling --- bin/validate.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/bin/validate.py b/bin/validate.py index aebb7fe608..9c497e386f 100644 --- a/bin/validate.py +++ b/bin/validate.py @@ -70,7 +70,7 @@ def validate_objects(REPO_PATH, objects, verbose): errors = [] for lookup in objects['lookups']: - lookup_errors = validate_lookups_content(REPO_PATH, "lookups/%s", lookup) + errors = errors + validate_lookups_content(REPO_PATH, "lookups/%s", lookup) objects_array = objects['stories'] + objects['detections'] + objects['baselines'] + objects['response_tasks'] + objects['responses'] for object in objects_array: @@ -89,8 +89,6 @@ def validate_objects(REPO_PATH, objects, verbose): for object in objects['tests']: errors = errors + validate_tests(REPO_PATH, object) - errors = lookup_errors + errors - return errors From 70e25e225a7d7f5cf0ec21c8f0fb5037ac552c6a Mon Sep 17 00:00:00 2001 From: Daniel Pauler Date: Sat, 20 Mar 2021 17:18:43 +0100 Subject: [PATCH 03/41] Updated jsonschema validation --- bin/validate.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bin/validate.py b/bin/validate.py index 9c497e386f..5d9a5d5636 100644 --- a/bin/validate.py +++ b/bin/validate.py @@ -85,7 +85,6 @@ def validate_objects(REPO_PATH, objects, verbose): for object in objects['baselines']: errors = errors + validate_baseline_search(object, objects['macros']) - for object in objects['tests']: errors = errors + validate_tests(REPO_PATH, object) From 7a9f57dbfc0e8b3f25ce5d572bdd5c9ac0ec4019 Mon Sep 17 00:00:00 2001 From: Daniel Pauler Date: Sat, 20 Mar 2021 17:22:42 +0100 Subject: [PATCH 04/41] Updated jsonschema validation --- bin/validate.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/validate.py b/bin/validate.py index 5d9a5d5636..d0da8ab741 100644 --- a/bin/validate.py +++ b/bin/validate.py @@ -47,10 +47,10 @@ def validate_schema(REPO_PATH, type, objects, verbose): print("Error reading {0}".format(manifest_file)) error = True continue - try: - jsonschema.validate(instance=object, schema=schema) - except jsonschema.exceptions.ValidationError as json_ve: - errors.append("ERROR: {0} at:\n\t{1}".format(json.dumps(json_ve.message), manifest_file)) + + validator = jsonschema.Draft7Validator(schema, format_checker=jsonschema.FormatChecker()) + for schema_error in validator.iter_errors(object): + errors.append("ERROR: {0} at:\n\t{1}".format(json.dumps(schema_error.message), manifest_file)) error = True if type in objects: From da4bc607509d0a24f43d072e90653aba056f2786 Mon Sep 17 00:00:00 2001 From: mhaag-spl <5632822+MHaggis@users.noreply.github.com> Date: Tue, 30 Mar 2021 14:49:44 -0600 Subject: [PATCH 05/41] T1197 BITS Jobs --- detections/endpoint/bits_job_persistence.yml | 46 +++++++++++++++++ .../endpoint/bitsadmin_download_file.yml | 49 +++++++++++++++++++ .../powershell_start_bitstransfer.yml | 46 +++++++++++++++++ stories/bits_jobs.yml | 22 +++++++++ tests/endpoint/bits_job_persistence.test.yml | 12 +++++ .../endpoint/bitsadmin_download_file.test.yml | 12 +++++ .../powershell_start_bitstransfer.test.yml | 12 +++++ 7 files changed, 199 insertions(+) create mode 100644 detections/endpoint/bits_job_persistence.yml create mode 100644 detections/endpoint/bitsadmin_download_file.yml create mode 100644 detections/endpoint/powershell_start_bitstransfer.yml create mode 100644 stories/bits_jobs.yml create mode 100644 tests/endpoint/bits_job_persistence.test.yml create mode 100644 tests/endpoint/bitsadmin_download_file.test.yml create mode 100644 tests/endpoint/powershell_start_bitstransfer.test.yml diff --git a/detections/endpoint/bits_job_persistence.yml b/detections/endpoint/bits_job_persistence.yml new file mode 100644 index 0000000000..7935123803 --- /dev/null +++ b/detections/endpoint/bits_job_persistence.yml @@ -0,0 +1,46 @@ +name: BITS Job Persistence +id: e97a5ffe-90bf-11eb-928a-acde48001122 +version: 1 +date: '2021-03-29' +author: Michael Haag, Splunk +type: batch +datamodel: +- Endpoint +description: The following query identifies Microsoft Background Intelligent Transfer Service utility `bitsadmin.exe` scheduling a BITS job to persist on an endpoint. The query identifies the parameters used to create, resume or add a file to a BITS job +search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) + as lastTime from datamodel=Endpoint.Processes where Processes.process_name=bitsadmin.exe Processes.process IN + (*create*, *addfile*, *setnotifyflags*, *setnotifycmdline*, *setminretrydelay*, *setcustomheaders*, *resume* ) by Processes.dest Processes.user Processes.parent_process Processes.process_name + Processes.process Processes.process_id Processes.parent_process_id + | `drop_dm_object_name(Processes)` + | `security_content_ctime(firstTime)` + | `security_content_ctime(lastTime)` + | `bits_job_persistence_filter`' +how_to_implement: To successfully implement this search you need to be ingesting information + on process that include the name of the process responsible for the changes from + your endpoints into the `Endpoint` datamodel in the `Processes` node. +known_false_positives: Limited false positives will be present. Typically, applications will use `BitsAdmin.exe`. Any filtering should be done based on command-line arguments (legitimate applications) or parent process. +references: +- https://attack.mitre.org/techniques/T1197/ +- https://docs.microsoft.com/en-us/windows-server/administration/windows-commands/bitsadmin +- https://github.com/redcanaryco/atomic-red-team/blob/master/atomics/T1197/T1197.md#atomic-test-3---persist-download--execute +- https://lolbas-project.github.io/lolbas/Binaries/Bitsadmin/ +tags: + analytic_story: + - BITS Jobs + dataset: [] + kill_chain_phases: + - Exploitation + mitre_attack_id: + - T1197 + product: + - Splunk Enterprise + - Splunk Enterprise Security + - Splunk Cloud + required_fields: + - _time + - Processes.process + - Processes.parent_process + - Processes.process_name + - Processes.user + - Processes.dest + security_domain: endpoint \ No newline at end of file diff --git a/detections/endpoint/bitsadmin_download_file.yml b/detections/endpoint/bitsadmin_download_file.yml new file mode 100644 index 0000000000..f23b896e8e --- /dev/null +++ b/detections/endpoint/bitsadmin_download_file.yml @@ -0,0 +1,49 @@ +name: BITSAdmin Download File +id: 80630ff4-8e4c-11eb-aab5-acde48001122 +version: 1 +date: '2021-03-26' +author: Michael Haag, Splunk +type: batch +datamodel: +- Endpoint +description: The following query identifies Microsoft Background Intelligent Transfer Service utility `bitsadmin.exe` using the `transfer` parameter to download a remote object. In addition, look for `download` or `upload` on the command-line, the switches are not required to perform a transfer. Capture any files downloaded. Review the reputation of the IP or domain used. Typically once executed, a follow on command will be used to execute the dropped file. + Note that the network connection or file modification events related will not spawn or create from `bitsadmin.exe`, but the artifacts will appear in a parallel process of `svchost.exe` with a command-line similar to `svchost.exe -k netsvcs -s BITS`. It's important to review all parallel and child processes to capture any behaviors and artifacts. In some suspicious and malicious instances, BITS jobs will be created. You can use `bitsadmin /list /verbose` to list out the jobs during investigation. +search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) + as lastTime from datamodel=Endpoint.Processes where Processes.process_name=bitsadmin.exe + Processes.process=*transfer* by Processes.dest Processes.user Processes.parent_process Processes.process_name + Processes.process Processes.process_id Processes.parent_process_id + | `drop_dm_object_name(Processes)` + | `security_content_ctime(firstTime)` + | `security_content_ctime(lastTime)` + | `bitsadmin_download_file_filter`' +how_to_implement: To successfully implement this search you need to be ingesting information + on process that include the name of the process responsible for the changes from + your endpoints into the `Endpoint` datamodel in the `Processes` node. +known_false_positives: Limited false positives, however it may be required to filter based on parent process name or network connection. +references: +- https://github.com/redcanaryco/atomic-red-team/blob/8eb52117b748d378325f7719554a896e37bccec7/atomics/T1105/T1105.md#atomic-test-9---windows---bitsadmin-bits-download +- https://github.com/redcanaryco/atomic-red-team/blob/bc705cb7aaa5f26f2d96585fac8e4c7052df0ff9/atomics/T1197/T1197.md +- https://docs.microsoft.com/en-us/windows/win32/bits/bitsadmin-tool +- https://thedfirreport.com/2021/03/29/sodinokibi-aka-revil-ransomware/ +tags: + analytic_story: + - Ingress Tool Transfer + - BITS Jobs + dataset: [] + kill_chain_phases: + - Exploitation + mitre_attack_id: + - T1197 + - T1105 + product: + - Splunk Enterprise + - Splunk Enterprise Security + - Splunk Cloud + required_fields: + - _time + - Processes.process + - Processes.parent_process + - Processes.process_name + - Processes.user + - Processes.dest + security_domain: endpoint \ No newline at end of file diff --git a/detections/endpoint/powershell_start_bitstransfer.yml b/detections/endpoint/powershell_start_bitstransfer.yml new file mode 100644 index 0000000000..e374024756 --- /dev/null +++ b/detections/endpoint/powershell_start_bitstransfer.yml @@ -0,0 +1,46 @@ +name: PowerShell Start-BitsTransfer +id: 39e2605a-90d8-11eb-899e-acde48001122 +version: 1 +date: '2021-03-29' +author: Michael Haag, Splunk +type: batch +datamodel: +- Endpoint +description: Start-BitsTransfer is the PowerShell "version" of BitsAdmin.exe. Similar functionality is present. This technique variation is not as commonly used by adversaries, but has been abused in the past. + Lesser known uses include the ability to set the `-TransferType` to `Upload` for exfiltration of files. In an instance where `Upload` is used, it is highly possible files will be archived. + During triage, review parallel processes and process lineage. Capture any files on disk and review. For the remote domain or IP, what is the reputation? +search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) + as lastTime from datamodel=Endpoint.Processes where Processes.process_name=powershell.exe + Processes.process=*start-bitstransfer* by Processes.dest Processes.user Processes.parent_process Processes.process_name + Processes.process Processes.process_id Processes.parent_process_id + | `drop_dm_object_name(Processes)` + | `security_content_ctime(firstTime)` + | `security_content_ctime(lastTime)` + | `powershell_start_bitstransfer_filter`' +how_to_implement: To successfully implement this search you need to be ingesting information + on process that include the name of the process responsible for the changes from + your endpoints into the `Endpoint` datamodel in the `Processes` node. +known_false_positives: Limited false positives. It is possible administrators will utilize Start-BitsTransfer for administrative tasks, otherwise filter based parent process or command-line arguments. +references: +- https://isc.sans.edu/diary/Investigating+Microsoft+BITS+Activity/23281 +- https://docs.microsoft.com/en-us/windows/win32/bits/using-windows-powershell-to-create-bits-transfer-jobs +tags: + analytic_story: + - Bits Jobs + dataset: [] + kill_chain_phases: + - Exploitation + mitre_attack_id: + - T1197 + product: + - Splunk Enterprise + - Splunk Enterprise Security + - Splunk Cloud + required_fields: + - _time + - Processes.process + - Processes.parent_process + - Processes.process_name + - Processes.user + - Processes.dest + security_domain: endpoint \ No newline at end of file diff --git a/stories/bits_jobs.yml b/stories/bits_jobs.yml new file mode 100644 index 0000000000..655efdd27e --- /dev/null +++ b/stories/bits_jobs.yml @@ -0,0 +1,22 @@ +name: BITS Jobs +id: dbc7edce-8e4c-11eb-9f31-acde48001122 +version: 1 +date: '2021-03-26' +author: Michael Haag, Splunk +type: batch +description: Adversaries may abuse BITS jobs to persistently execute or clean up after malicious payloads. +narrative: Windows Background Intelligent Transfer Service (BITS) is a low-bandwidth, asynchronous file transfer mechanism exposed through Component Object Model (COM). BITS is commonly used by updaters, messengers, and other applications preferred to operate in the background (using available idle bandwidth) without interrupting other networked applications. File transfer tasks are implemented as BITS jobs, which contain a queue of one or more file operations. The interface to create and manage BITS jobs is accessible through PowerShell and the BITSAdmin tool. + Adversaries may abuse BITS to download, execute, and even clean up after running malicious code. BITS tasks are self-contained in the BITS job database, without new files or registry modifications, and often permitted by host firewalls. BITS enabled execution may also enable persistence by creating long-standing jobs (the default maximum lifetime is 90 days and extendable) or invoking an arbitrary program when a job completes or errors (including after system reboots). +references: +- https://attack.mitre.org/techniques/T1197/ +- https://docs.microsoft.com/en-us/windows/win32/bits/bitsadmin-tool +tags: + analytic_story: + - BITS Jobs + category: + - Adversary Tactics + product: + - Splunk Enterprise + - Splunk Enterprise Security + - Splunk Cloud + usecase: Advanced Threat Detection \ No newline at end of file diff --git a/tests/endpoint/bits_job_persistence.test.yml b/tests/endpoint/bits_job_persistence.test.yml new file mode 100644 index 0000000000..3e87301fc5 --- /dev/null +++ b/tests/endpoint/bits_job_persistence.test.yml @@ -0,0 +1,12 @@ +name: BITS Job Persistence Unit Test +tests: +- name: BITS Job Persistence + file: endpoint/bits_job_persistence.yml + pass_condition: '| stats count | where count > 0' + earliest_time: '-24h' + latest_time: 'now' + attack_data: + - file_name: windows-sysmon.log + data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1197/atomic_red_team/windows-sysmon.log + source: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational + sourcetype: xmlwineventlog diff --git a/tests/endpoint/bitsadmin_download_file.test.yml b/tests/endpoint/bitsadmin_download_file.test.yml new file mode 100644 index 0000000000..25663f7e79 --- /dev/null +++ b/tests/endpoint/bitsadmin_download_file.test.yml @@ -0,0 +1,12 @@ +name: BITSAdmin Download File Unit Test +tests: +- name: BITSAdmin Download File + file: endpoint/bitsadmin_download_file.yml + pass_condition: '| stats count | where count > 0' + earliest_time: '-24h' + latest_time: 'now' + attack_data: + - file_name: windows-sysmon.log + data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1197/atomic_red_team/windows-sysmon.log + source: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational + sourcetype: xmlwineventlog diff --git a/tests/endpoint/powershell_start_bitstransfer.test.yml b/tests/endpoint/powershell_start_bitstransfer.test.yml new file mode 100644 index 0000000000..ddfaf3ecc8 --- /dev/null +++ b/tests/endpoint/powershell_start_bitstransfer.test.yml @@ -0,0 +1,12 @@ +name: PowerShell Start-BitsTransfer Unit Test +tests: +- name: PowerShell Start-BitsTransfer + file: endpoint/powershell_start_bitstransfer.yml + pass_condition: '| stats count | where count > 0' + earliest_time: '-24h' + latest_time: 'now' + attack_data: + - file_name: windows-sysmon.log + data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1197/atomic_red_team/windows-sysmon.log + source: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational + sourcetype: xmlwineventlog From 86c0940069fb946380562d3788583690f87b821c Mon Sep 17 00:00:00 2001 From: root Date: Tue, 30 Mar 2021 21:48:03 +0000 Subject: [PATCH 06/41] Added detection testing service results inBITSAdmin Download File --- .../endpoint/bitsadmin_download_file.yml | 31 +++++++++++++------ 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/detections/endpoint/bitsadmin_download_file.yml b/detections/endpoint/bitsadmin_download_file.yml index f23b896e8e..a7fa7ba5c3 100644 --- a/detections/endpoint/bitsadmin_download_file.yml +++ b/detections/endpoint/bitsadmin_download_file.yml @@ -6,20 +6,29 @@ author: Michael Haag, Splunk type: batch datamodel: - Endpoint -description: The following query identifies Microsoft Background Intelligent Transfer Service utility `bitsadmin.exe` using the `transfer` parameter to download a remote object. In addition, look for `download` or `upload` on the command-line, the switches are not required to perform a transfer. Capture any files downloaded. Review the reputation of the IP or domain used. Typically once executed, a follow on command will be used to execute the dropped file. - Note that the network connection or file modification events related will not spawn or create from `bitsadmin.exe`, but the artifacts will appear in a parallel process of `svchost.exe` with a command-line similar to `svchost.exe -k netsvcs -s BITS`. It's important to review all parallel and child processes to capture any behaviors and artifacts. In some suspicious and malicious instances, BITS jobs will be created. You can use `bitsadmin /list /verbose` to list out the jobs during investigation. +description: The following query identifies Microsoft Background Intelligent Transfer + Service utility `bitsadmin.exe` using the `transfer` parameter to download a remote + object. In addition, look for `download` or `upload` on the command-line, the switches + are not required to perform a transfer. Capture any files downloaded. Review the + reputation of the IP or domain used. Typically once executed, a follow on command + will be used to execute the dropped file. Note that the network connection or file + modification events related will not spawn or create from `bitsadmin.exe`, but the + artifacts will appear in a parallel process of `svchost.exe` with a command-line + similar to `svchost.exe -k netsvcs -s BITS`. It's important to review all parallel + and child processes to capture any behaviors and artifacts. In some suspicious and + malicious instances, BITS jobs will be created. You can use `bitsadmin /list /verbose` + to list out the jobs during investigation. search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Processes where Processes.process_name=bitsadmin.exe - Processes.process=*transfer* by Processes.dest Processes.user Processes.parent_process Processes.process_name - Processes.process Processes.process_id Processes.parent_process_id - | `drop_dm_object_name(Processes)` - | `security_content_ctime(firstTime)` - | `security_content_ctime(lastTime)` + Processes.process=*transfer* by Processes.dest Processes.user Processes.parent_process + Processes.process_name Processes.process Processes.process_id Processes.parent_process_id + | `drop_dm_object_name(Processes)` | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `bitsadmin_download_file_filter`' how_to_implement: To successfully implement this search you need to be ingesting information on process that include the name of the process responsible for the changes from your endpoints into the `Endpoint` datamodel in the `Processes` node. -known_false_positives: Limited false positives, however it may be required to filter based on parent process name or network connection. +known_false_positives: Limited false positives, however it may be required to filter + based on parent process name or network connection. references: - https://github.com/redcanaryco/atomic-red-team/blob/8eb52117b748d378325f7719554a896e37bccec7/atomics/T1105/T1105.md#atomic-test-9---windows---bitsadmin-bits-download - https://github.com/redcanaryco/atomic-red-team/blob/bc705cb7aaa5f26f2d96585fac8e4c7052df0ff9/atomics/T1197/T1197.md @@ -29,7 +38,8 @@ tags: analytic_story: - Ingress Tool Transfer - BITS Jobs - dataset: [] + dataset: + - https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1197/atomic_red_team/windows-sysmon.log kill_chain_phases: - Exploitation mitre_attack_id: @@ -46,4 +56,5 @@ tags: - Processes.process_name - Processes.user - Processes.dest - security_domain: endpoint \ No newline at end of file + security_domain: endpoint + automated_detection_testing: passed From 412ece7dbc28df47a89144ae13e06bc01a2d3e48 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 30 Mar 2021 21:48:05 +0000 Subject: [PATCH 07/41] Added detection testing service results inBITS Job Persistence --- detections/endpoint/bits_job_persistence.yml | 28 ++++++++++++-------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/detections/endpoint/bits_job_persistence.yml b/detections/endpoint/bits_job_persistence.yml index 7935123803..55f10e87f9 100644 --- a/detections/endpoint/bits_job_persistence.yml +++ b/detections/endpoint/bits_job_persistence.yml @@ -6,19 +6,23 @@ author: Michael Haag, Splunk type: batch datamodel: - Endpoint -description: The following query identifies Microsoft Background Intelligent Transfer Service utility `bitsadmin.exe` scheduling a BITS job to persist on an endpoint. The query identifies the parameters used to create, resume or add a file to a BITS job +description: The following query identifies Microsoft Background Intelligent Transfer + Service utility `bitsadmin.exe` scheduling a BITS job to persist on an endpoint. + The query identifies the parameters used to create, resume or add a file to a BITS + job search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) - as lastTime from datamodel=Endpoint.Processes where Processes.process_name=bitsadmin.exe Processes.process IN - (*create*, *addfile*, *setnotifyflags*, *setnotifycmdline*, *setminretrydelay*, *setcustomheaders*, *resume* ) by Processes.dest Processes.user Processes.parent_process Processes.process_name - Processes.process Processes.process_id Processes.parent_process_id - | `drop_dm_object_name(Processes)` - | `security_content_ctime(firstTime)` - | `security_content_ctime(lastTime)` - | `bits_job_persistence_filter`' + as lastTime from datamodel=Endpoint.Processes where Processes.process_name=bitsadmin.exe + Processes.process IN (*create*, *addfile*, *setnotifyflags*, *setnotifycmdline*, + *setminretrydelay*, *setcustomheaders*, *resume* ) by Processes.dest Processes.user + Processes.parent_process Processes.process_name Processes.process Processes.process_id + Processes.parent_process_id | `drop_dm_object_name(Processes)` | `security_content_ctime(firstTime)` + | `security_content_ctime(lastTime)` | `bits_job_persistence_filter`' how_to_implement: To successfully implement this search you need to be ingesting information on process that include the name of the process responsible for the changes from your endpoints into the `Endpoint` datamodel in the `Processes` node. -known_false_positives: Limited false positives will be present. Typically, applications will use `BitsAdmin.exe`. Any filtering should be done based on command-line arguments (legitimate applications) or parent process. +known_false_positives: Limited false positives will be present. Typically, applications + will use `BitsAdmin.exe`. Any filtering should be done based on command-line arguments + (legitimate applications) or parent process. references: - https://attack.mitre.org/techniques/T1197/ - https://docs.microsoft.com/en-us/windows-server/administration/windows-commands/bitsadmin @@ -27,7 +31,8 @@ references: tags: analytic_story: - BITS Jobs - dataset: [] + dataset: + - https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1197/atomic_red_team/windows-sysmon.log kill_chain_phases: - Exploitation mitre_attack_id: @@ -43,4 +48,5 @@ tags: - Processes.process_name - Processes.user - Processes.dest - security_domain: endpoint \ No newline at end of file + security_domain: endpoint + automated_detection_testing: passed From 9598116f927cc8880b8be00390adf44ce67d05c8 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 30 Mar 2021 21:49:24 +0000 Subject: [PATCH 08/41] Added detection testing service results inPowerShell Start-BitsTransfer --- .../powershell_start_bitstransfer.yml | 28 +++++++++++-------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/detections/endpoint/powershell_start_bitstransfer.yml b/detections/endpoint/powershell_start_bitstransfer.yml index e374024756..4e48cafafc 100644 --- a/detections/endpoint/powershell_start_bitstransfer.yml +++ b/detections/endpoint/powershell_start_bitstransfer.yml @@ -6,28 +6,33 @@ author: Michael Haag, Splunk type: batch datamodel: - Endpoint -description: Start-BitsTransfer is the PowerShell "version" of BitsAdmin.exe. Similar functionality is present. This technique variation is not as commonly used by adversaries, but has been abused in the past. - Lesser known uses include the ability to set the `-TransferType` to `Upload` for exfiltration of files. In an instance where `Upload` is used, it is highly possible files will be archived. - During triage, review parallel processes and process lineage. Capture any files on disk and review. For the remote domain or IP, what is the reputation? +description: Start-BitsTransfer is the PowerShell "version" of BitsAdmin.exe. Similar + functionality is present. This technique variation is not as commonly used by adversaries, + but has been abused in the past. Lesser known uses include the ability to set the + `-TransferType` to `Upload` for exfiltration of files. In an instance where `Upload` + is used, it is highly possible files will be archived. During triage, review parallel + processes and process lineage. Capture any files on disk and review. For the remote + domain or IP, what is the reputation? search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Processes where Processes.process_name=powershell.exe - Processes.process=*start-bitstransfer* by Processes.dest Processes.user Processes.parent_process Processes.process_name - Processes.process Processes.process_id Processes.parent_process_id - | `drop_dm_object_name(Processes)` - | `security_content_ctime(firstTime)` - | `security_content_ctime(lastTime)` + Processes.process=*start-bitstransfer* by Processes.dest Processes.user Processes.parent_process + Processes.process_name Processes.process Processes.process_id Processes.parent_process_id + | `drop_dm_object_name(Processes)` | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `powershell_start_bitstransfer_filter`' how_to_implement: To successfully implement this search you need to be ingesting information on process that include the name of the process responsible for the changes from your endpoints into the `Endpoint` datamodel in the `Processes` node. -known_false_positives: Limited false positives. It is possible administrators will utilize Start-BitsTransfer for administrative tasks, otherwise filter based parent process or command-line arguments. +known_false_positives: Limited false positives. It is possible administrators will + utilize Start-BitsTransfer for administrative tasks, otherwise filter based parent + process or command-line arguments. references: - https://isc.sans.edu/diary/Investigating+Microsoft+BITS+Activity/23281 - https://docs.microsoft.com/en-us/windows/win32/bits/using-windows-powershell-to-create-bits-transfer-jobs tags: analytic_story: - Bits Jobs - dataset: [] + dataset: + - https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1197/atomic_red_team/windows-sysmon.log kill_chain_phases: - Exploitation mitre_attack_id: @@ -43,4 +48,5 @@ tags: - Processes.process_name - Processes.user - Processes.dest - security_domain: endpoint \ No newline at end of file + security_domain: endpoint + automated_detection_testing: passed From c214e8dc533a654b82ad56942d422d3693f59faf Mon Sep 17 00:00:00 2001 From: tcontreras Date: Wed, 31 Mar 2021 14:42:47 +0200 Subject: [PATCH 09/41] def_evasion_detections --- detections/endpoint/disable_registry_tool.yml | 44 +++++++++++++++++ .../endpoint/disable_show_hidden_files.yml | 47 +++++++++++++++++++ .../disable_windows_behavior_monitoring.yml | 45 ++++++++++++++++++ ...disable_windows_smartscreen_protection.yml | 44 +++++++++++++++++ .../endpoint/disabling_cmd_application.yml | 44 +++++++++++++++++ .../endpoint/disabling_controlpanel.yml | 43 +++++++++++++++++ .../disabling_firewall_with_netsh.yml | 46 ++++++++++++++++++ ...isabling_folderoptions_windows_feature.yml | 44 +++++++++++++++++ .../endpoint/disabling_norun_windows_app.yml | 45 ++++++++++++++++++ .../disabling_systemrestore_in_registry.yml | 44 +++++++++++++++++ .../endpoint/disabling_task_manager.yml | 44 +++++++++++++++++ tests/endpoint/disable_registry_tool.test.yml | 22 +++++++++ .../disable_show_hidden_files.test.yml | 22 +++++++++ ...sable_windows_behavior_monitoring.test.yml | 22 +++++++++ ...le_windows_smartscreen_protection.test.yml | 22 +++++++++ .../disabling_cmd_application.test.yml | 22 +++++++++ .../endpoint/disabling_controlpanel.test.yml | 22 +++++++++ .../disabling_firewall_with_netsh.test.yml | 22 +++++++++ ...ing_folderoptions_windows_feature.test.yml | 22 +++++++++ .../disabling_norun_windows_app.test.yml | 22 +++++++++ ...sabling_systemrestore_in_registry.test.yml | 22 +++++++++ .../endpoint/disabling_task_manager.test.yml | 22 +++++++++ 22 files changed, 732 insertions(+) create mode 100644 detections/endpoint/disable_registry_tool.yml create mode 100644 detections/endpoint/disable_show_hidden_files.yml create mode 100644 detections/endpoint/disable_windows_behavior_monitoring.yml create mode 100644 detections/endpoint/disable_windows_smartscreen_protection.yml create mode 100644 detections/endpoint/disabling_cmd_application.yml create mode 100644 detections/endpoint/disabling_controlpanel.yml create mode 100644 detections/endpoint/disabling_firewall_with_netsh.yml create mode 100644 detections/endpoint/disabling_folderoptions_windows_feature.yml create mode 100644 detections/endpoint/disabling_norun_windows_app.yml create mode 100644 detections/endpoint/disabling_systemrestore_in_registry.yml create mode 100644 detections/endpoint/disabling_task_manager.yml create mode 100644 tests/endpoint/disable_registry_tool.test.yml create mode 100644 tests/endpoint/disable_show_hidden_files.test.yml create mode 100644 tests/endpoint/disable_windows_behavior_monitoring.test.yml create mode 100644 tests/endpoint/disable_windows_smartscreen_protection.test.yml create mode 100644 tests/endpoint/disabling_cmd_application.test.yml create mode 100644 tests/endpoint/disabling_controlpanel.test.yml create mode 100644 tests/endpoint/disabling_firewall_with_netsh.test.yml create mode 100644 tests/endpoint/disabling_folderoptions_windows_feature.test.yml create mode 100644 tests/endpoint/disabling_norun_windows_app.test.yml create mode 100644 tests/endpoint/disabling_systemrestore_in_registry.test.yml create mode 100644 tests/endpoint/disabling_task_manager.test.yml diff --git a/detections/endpoint/disable_registry_tool.yml b/detections/endpoint/disable_registry_tool.yml new file mode 100644 index 0000000000..f9df9e40e9 --- /dev/null +++ b/detections/endpoint/disable_registry_tool.yml @@ -0,0 +1,44 @@ +name: Disable Registry Tool +id: cd2cf33c-9201-11eb-a10a-acde48001122 +version: 1 +date: '2021-03-31' +author: Teoderick Contreras, Splunk +type: batch +datamodel: +- Endpoint +description: This search is to identifies modification of registry to disable the regedit or registry tools of windows operating system. + Since registry tool is a swiss knife in analyzing registry, malware such as RAT or trojan Spy disable this application to prevent the removal + of their registry entry such as persistence, file less components and defense evasion. +search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Registry + where Registry.registry_path= "*\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Policies\\System\\DisableRegistryTools" + Registry.registry_value_name = "DWORD (0x00000001)" by Registry.registry_path Registry.registry_key_name Registry.registry_value_name Registry.dest + | `drop_dm_object_name(Registry)` + | `security_content_ctime(firstTime)` + |`security_content_ctime(lastTime)` + | `disable_registry_tool_filter`' +how_to_implement: To successfully implement this search you need to be ingesting information + on process that include the name of the process responsible for the changes from + your endpoints into the `Endpoint` datamodel in the `Registry` node. Also make sure that this registry was included in your config + files ex. sysmon config to be monitored. +known_false_positives: admin may disable this application for non technical user. +references: +- https://any.run/report/ea4ea08407d4ee72e009103a3b77e5a09412b722fdef67315ea63f22011152af/a866d7b1-c236-4f26-a391-5ae32213dfc4#registry +tags: + analytic_story: + - Windows Defense Evasion Tactics + kill_chain_phases: + - Exploitation + mitre_attack_id: + - T1562.001 + product: + - Splunk Enterprise + - Splunk Enterprise Security + - Splunk Cloud + required_fields: + - _time + - Registry.registry_key_name + - Registry.registry_path + - Registry.user + - Registry.dest + - Registry.registry_value_name + security_domain: endpoint \ No newline at end of file diff --git a/detections/endpoint/disable_show_hidden_files.yml b/detections/endpoint/disable_show_hidden_files.yml new file mode 100644 index 0000000000..f49cbc69db --- /dev/null +++ b/detections/endpoint/disable_show_hidden_files.yml @@ -0,0 +1,47 @@ +name: Disable Show Hidden Files +id: 6f3ccfa2-91fe-11eb-8f9b-acde48001122 +version: 1 +date: '2021-03-31' +author: Teoderick Contreras, Splunk +type: batch +datamodel: +- Endpoint +description: The following search is to idetifies a modification in registry to prevent the user seeing all the files with hidden attributes. + This event or techniques are known on some worm and trojan spy malware that will drop hidden files on the infected machine. +search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Registry + where (Registry.registry_path= "*\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Explorer\\Advanced\\Hidden" + OR Registry.registry_path= "*\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Explorer\\Advanced\\HideFileExt" + Registry.registry_value_name = "DWORD (0x00000001)") + OR (Registry.registry_path= "*\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Explorer\\Advanced\\ShowSuperHidden" Registry.registry_value_name = "DWORD (0x00000000)") + by Registry.registry_path Registry.registry_key_name Registry.registry_value_name Registry.dest + | `drop_dm_object_name(Registry)` + | `security_content_ctime(firstTime)` + |`security_content_ctime(lastTime)` + | `disable_show_hidden_files_filter`' +how_to_implement: To successfully implement this search you need to be ingesting information + on process that include the name of the process responsible for the changes from + your endpoints into the `Endpoint` datamodel in the `Registry` node. Also make sure that this registry was included in your config + files ex. sysmon config to be monitored. +known_false_positives: unknown +references: +- https://www.sophos.com/en-us/threat-center/threat-analyses/viruses-and-spyware/W32~Tiotua-P/detailed-analysis.aspx +tags: + analytic_story: + - Windows Defense Evasion Tactics + kill_chain_phases: + - Exploitation + mitre_attack_id: + - T1564.001 + - T1562.001 + product: + - Splunk Enterprise + - Splunk Enterprise Security + - Splunk Cloud + required_fields: + - _time + - Registry.registry_key_name + - Registry.registry_path + - Registry.user + - Registry.dest + - Registry.registry_value_nam + security_domain: endpoint \ No newline at end of file diff --git a/detections/endpoint/disable_windows_behavior_monitoring.yml b/detections/endpoint/disable_windows_behavior_monitoring.yml new file mode 100644 index 0000000000..d643a1b4a5 --- /dev/null +++ b/detections/endpoint/disable_windows_behavior_monitoring.yml @@ -0,0 +1,45 @@ +name: Disable Windows Behavior Monitoring +id: 79439cae-9200-11eb-a4d3-acde48001122 +version: 1 +date: '2021-03-31' +author: Teoderick Contreras, Splunk +type: batch +datamodel: +- Endpoint +description: This search is to identifies a modification in registry to disable the windows denfender real time behavior monitoring. + This event or technique is commonly seen in RAT, bot, or Trojan to disable AV to evade detections. +search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Registry + where Registry.registry_path= "*\\SOFTWARE\\Policies\\Microsoft\\Windows Defender\\Real-Time Protection\\DisableBehaviorMonitoring" + OR Registry.registry_path= "*\\SOFTWARE\\Policies\\Microsoft\\Windows Defender\\Real-Time Protection\\DisableOnAccessProtection" + OR Registry.registry_path= "*\\SOFTWARE\\Policies\\Microsoft\\Windows Defender\\Real-Time Protection\\DisableScanOnRealtimeEnable" + Registry.registry_value_name = "DWORD (0x00000001)" by Registry.registry_path Registry.registry_key_name Registry.registry_value_name Registry.dest + | `drop_dm_object_name(Registry)` + | `security_content_ctime(firstTime)` + |`security_content_ctime(lastTime)` + | `disable_windows_behavior_monitoring_filter`' +how_to_implement: To successfully implement this search you need to be ingesting information + on process that include the name of the process responsible for the changes from + your endpoints into the `Endpoint` datamodel in the `Registry` node. Also make sure that this registry was included in your config + files ex. sysmon config to be monitored. +known_false_positives: admin or user may choose to disable this windows features. +references: +- https://tccontre.blogspot.com/2020/01/remcos-rat-evading-windows-defender-av.html +tags: + analytic_story: + - Windows Defense Evasion Tactics + kill_chain_phases: + - Exploitation + mitre_attack_id: + - T1562.001 + product: + - Splunk Enterprise + - Splunk Enterprise Security + - Splunk Cloud + required_fields: + - _time + - Registry.registry_key_name + - Registry.registry_path + - Registry.user + - Registry.dest + - Registry.registry_value_name + security_domain: endpoint \ No newline at end of file diff --git a/detections/endpoint/disable_windows_smartscreen_protection.yml b/detections/endpoint/disable_windows_smartscreen_protection.yml new file mode 100644 index 0000000000..3e85bdad5a --- /dev/null +++ b/detections/endpoint/disable_windows_smartscreen_protection.yml @@ -0,0 +1,44 @@ +name: Disable Windows SmartScreen Protection +id: 664f0fd0-91ff-11eb-a56f-acde48001122 +version: 1 +date: '2021-03-31' +author: Teoderick Contreras, Splunk +type: batch +datamodel: +- Endpoint +description: The following search identifies a modification of registry to disable the smartscreen protection of windows machine. + This is windows feature provide an early warning system against website that might engage in phishing attack or malware distribution. + This modification are seen in RAT malware to cover their tracks upon downloading other of its component or other payload. +search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Registry + where Registry.registry_path= "*HKLM\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Explorer\\SmartScreenEnabled" Registry.registry_value_name = "Off" + by Registry.registry_path Registry.registry_key_name Registry.registry_value_name Registry.dest + | `drop_dm_object_name(Registry)` + | `security_content_ctime(firstTime)` + |`security_content_ctime(lastTime)` + | `disable_windows_smartscreen_protection_filter`' +how_to_implement: To successfully implement this search you need to be ingesting information + on process that include the name of the process responsible for the changes from + your endpoints into the `Endpoint` datamodel in the `Registry` node. Also make sure that this registry was included in your config + files ex. sysmon config to be monitored. +known_false_positives: admin or user may choose to disable this windows features. +references: +- https://tccontre.blogspot.com/2020/01/remcos-rat-evading-windows-defender-av.html +tags: + analytic_story: + - Windows Defense Evasion Tactics + kill_chain_phases: + - Exploitation + mitre_attack_id: + - T1562.001 + product: + - Splunk Enterprise + - Splunk Enterprise Security + - Splunk Cloud + required_fields: + - _time + - Registry.registry_key_name + - Registry.registry_path + - Registry.user + - Registry.dest + - Registry.registry_value_nam + security_domain: endpoint \ No newline at end of file diff --git a/detections/endpoint/disabling_cmd_application.yml b/detections/endpoint/disabling_cmd_application.yml new file mode 100644 index 0000000000..65955f97ae --- /dev/null +++ b/detections/endpoint/disabling_cmd_application.yml @@ -0,0 +1,44 @@ +name: Disabling CMD Application +id: ff86077c-9212-11eb-a1e6-acde48001122 +version: 1 +date: '2021-03-31' +author: Teoderick Contreras, Splunk +type: batch +datamodel: +- Endpoint +description: this search is to identify modification in registry to disable cmd prompt application. + This technique is commonly seen in RAT, Trojan or WORM to prevent triaging or deleting there samples through cmd application + which is one of the tool of analyst to traverse on directory and files. +search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Registry + where Registry.registry_path= "*\\SOFTWARE\\Policies\\Microsoft\\Windows\\System\\DisableCMD" Registry.registry_value_name = "DWORD (0x00000001)" + by Registry.registry_path Registry.registry_key_name Registry.registry_value_name Registry.dest + | `drop_dm_object_name(Registry)` + | `security_content_ctime(firstTime)` + |`security_content_ctime(lastTime)` + | `disabling_cmd_application_filter`' +how_to_implement: To successfully implement this search you need to be ingesting information + on process that include the name of the process responsible for the changes from + your endpoints into the `Endpoint` datamodel in the `Registry` node. Also make sure that this registry was included in your config + files ex. sysmon config to be monitored. +known_false_positives: admin may disable this application for non technical user. +references: +- https://any.run/report/ea4ea08407d4ee72e009103a3b77e5a09412b722fdef67315ea63f22011152af/a866d7b1-c236-4f26-a391-5ae32213dfc4#registry +tags: + analytic_story: + - Windows Defense Evasion Tactics + kill_chain_phases: + - Exploitation + mitre_attack_id: + - T1562.001 + product: + - Splunk Enterprise + - Splunk Enterprise Security + - Splunk Cloud + required_fields: + - _time + - Registry.registry_key_name + - Registry.registry_path + - Registry.user + - Registry.dest + - Registry.registry_value_name + security_domain: endpoint \ No newline at end of file diff --git a/detections/endpoint/disabling_controlpanel.yml b/detections/endpoint/disabling_controlpanel.yml new file mode 100644 index 0000000000..2c6709eea6 --- /dev/null +++ b/detections/endpoint/disabling_controlpanel.yml @@ -0,0 +1,43 @@ +name: Disabling ControlPanel +id: 6ae0148e-9215-11eb-a94a-acde48001122 +version: 1 +date: '2021-03-31' +author: Teoderick Contreras, Splunk +type: batch +datamodel: +- Endpoint +description: this search is to identify registry modification to disable control panel window. + This technique is commonly seen in malware to prevent their artifacts , persistence removed on the infected machine. +search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Registry + where Registry.registry_path= "*\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Policies\\Explorer\\NoControlPanel" + Registry.registry_value_name = "DWORD (0x00000001)" by Registry.registry_path Registry.registry_key_name Registry.registry_value_name Registry.dest + | `drop_dm_object_name(Registry)` + | `security_content_ctime(firstTime)` + | `security_content_ctime(lastTime)` + | `disabling_controlpanel_filter`' +how_to_implement: To successfully implement this search you need to be ingesting information + on process that include the name of the process responsible for the changes from + your endpoints into the `Endpoint` datamodel in the `Registry` node. Also make sure that this registry was included in your config + files ex. sysmon config to be monitored. +known_false_positives: admin may disable this application for non technical user. +references: +- https://any.run/report/ea4ea08407d4ee72e009103a3b77e5a09412b722fdef67315ea63f22011152af/a866d7b1-c236-4f26-a391-5ae32213dfc4#registry +tags: + analytic_story: + - Windows Defense Evasion Tactics + kill_chain_phases: + - Exploitation + mitre_attack_id: + - T1562.001 + product: + - Splunk Enterprise + - Splunk Enterprise Security + - Splunk Cloud + required_fields: + - _time + - Registry.registry_key_name + - Registry.registry_path + - Registry.user + - Registry.dest + - Registry.registry_value_name + security_domain: endpoint \ No newline at end of file diff --git a/detections/endpoint/disabling_firewall_with_netsh.yml b/detections/endpoint/disabling_firewall_with_netsh.yml new file mode 100644 index 0000000000..36ef1d5139 --- /dev/null +++ b/detections/endpoint/disabling_firewall_with_netsh.yml @@ -0,0 +1,46 @@ +name: Disabling Firewall with Netsh +id: 6860a62c-9203-11eb-9e05-acde48001122 +version: 1 +date: '2021-03-31' +author: Teoderick Contreras, Splunk +type: batch +datamodel: +- Endpoint +description: This search is to identifies suspicious firewall disabling using netsh application. + this technique is commonly seen in malware that tries to communicate or download its component or other payload to its C2 server. +search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Processes + where Processes.process_name=netsh.exe Processes.process= "*firewall*" (Processes.process= "*off*" OR Processes.process= "*disable*") + by Processes.dest Processes.user Processes.parent_process Processes.process_name Processes.process Processes.process_id Processes.parent_process_id + | `drop_dm_object_name(Processes)` + | `security_content_ctime(firstTime)` + | `security_content_ctime(lastTime)` + | `disabling_firewall_with_netsh_filter`' +how_to_implement: You must be ingesting data that records the filesystem activity + from your hosts to populate the Endpoint file-system data model node. If you are + using Sysmon, you will need a Splunk Universal Forwarder on each endpoint from which + you want to collect data. +known_false_positives: admin may disable firewall during testing or fixing network problem. +references: +- https://tccontre.blogspot.com/2020/01/remcos-rat-evading-windows-defender-av.htm +tags: + analytic_story: + - Windows Defense Evasion Tactics + kill_chain_phases: + - Exploitation + mitre_attack_id: + - T1562.001 + product: + - Splunk Enterprise + - Splunk Enterprise Security + - Splunk Cloud + required_fields: + - _time + - Processes.process_name + - Processes.process + - Processes.dest + - Processes.user + - Processes.parent_process + - Processes.process_name + - Processes.process_id + - Processes.parent_process_id + security_domain: endpoint \ No newline at end of file diff --git a/detections/endpoint/disabling_folderoptions_windows_feature.yml b/detections/endpoint/disabling_folderoptions_windows_feature.yml new file mode 100644 index 0000000000..17b3da5ab2 --- /dev/null +++ b/detections/endpoint/disabling_folderoptions_windows_feature.yml @@ -0,0 +1,44 @@ +name: Disabling FolderOptions Windows Feature +id: 83776de4-921a-11eb-868a-acde48001122 +version: 1 +date: '2021-03-31' +author: Teoderick Contreras, Splunk +type: batch +datamodel: +- Endpoint +description: This search is to identify registry modification to disable folder options feature of windows to show hidden files, file extension and etc. + This technique used by malware in combination if disabling show hidden files feature to hide their files and also to hide the file extension to lure the user + base on file icons or fake file extensions. +search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Registry + where Registry.registry_path= "*\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Policies\\Explorer\\NoFolderOptions" + Registry.registry_value_name = "DWORD (0x00000001)" by Registry.registry_path Registry.registry_key_name Registry.registry_value_name Registry.dest + | `drop_dm_object_name(Registry)` + | `security_content_ctime(firstTime)` + | `security_content_ctime(lastTime)` + | `disabling_folderoptions_windows_feature_filter`' +how_to_implement: To successfully implement this search you need to be ingesting information + on process that include the name of the process responsible for the changes from + your endpoints into the `Endpoint` datamodel in the `Registry` node. Also make sure that this registry was included in your config + files ex. sysmon config to be monitored. +known_false_positives: admin may disable this application for non technical user. +references: +- https://any.run/report/ea4ea08407d4ee72e009103a3b77e5a09412b722fdef67315ea63f22011152af/a866d7b1-c236-4f26-a391-5ae32213dfc4#registry +tags: + analytic_story: + - Windows Defense Evasion Tactics + kill_chain_phases: + - Exploitation + mitre_attack_id: + - T1562.001 + product: + - Splunk Enterprise + - Splunk Enterprise Security + - Splunk Cloud + required_fields: + - _time + - Registry.registry_key_name + - Registry.registry_path + - Registry.user + - Registry.dest + - Registry.registry_value_name + security_domain: endpoint \ No newline at end of file diff --git a/detections/endpoint/disabling_norun_windows_app.yml b/detections/endpoint/disabling_norun_windows_app.yml new file mode 100644 index 0000000000..50affd0447 --- /dev/null +++ b/detections/endpoint/disabling_norun_windows_app.yml @@ -0,0 +1,45 @@ +name: Disabling NoRun Windows App +id: de81bc46-9213-11eb-adc9-acde48001122 +version: 1 +date: '2021-03-31' +author: Teoderick Contreras, Splunk +type: batch +datamodel: +- Endpoint +description: This search is to identify modification of registry to disable run application in window start menu. + this application is known to be a helpful shortcut to windows OS user to run known application and also to execute some reg or batch script. + This technique is used malware to make cleaning of its infection more harder by preventing known application run easily through run shortcut. +search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Registry + where Registry.registry_path= "*\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Policies\\Explorer\\NoRun" + Registry.registry_value_name = "DWORD (0x00000001)" by Registry.registry_path Registry.registry_key_name Registry.registry_value_name Registry.dest + | `drop_dm_object_name(Registry)` + | `security_content_ctime(firstTime)` + | `security_content_ctime(lastTime)` + | `disabling_norun_windows_app_filter`' +how_to_implement: To successfully implement this search you need to be ingesting information + on process that include the name of the process responsible for the changes from + your endpoints into the `Endpoint` datamodel in the `Registry` node. Also make sure that this registry was included in your config + files ex. sysmon config to be monitored. +known_false_positives: admin may disable this application for non technical user. +references: +- https://any.run/report/ea4ea08407d4ee72e009103a3b77e5a09412b722fdef67315ea63f22011152af/a866d7b1-c236-4f26-a391-5ae32213dfc4#registry +- https://blog.malwarebytes.com/detections/pum-optional-norun/ +tags: + analytic_story: + - Windows Defense Evasion Tactics + kill_chain_phases: + - Exploitation + mitre_attack_id: + - T1562.001 + product: + - Splunk Enterprise + - Splunk Enterprise Security + - Splunk Cloud + required_fields: + - _time + - Registry.registry_key_name + - Registry.registry_path + - Registry.user + - Registry.dest + - Registry.registry_value_name + security_domain: endpoint \ No newline at end of file diff --git a/detections/endpoint/disabling_systemrestore_in_registry.yml b/detections/endpoint/disabling_systemrestore_in_registry.yml new file mode 100644 index 0000000000..eb6f008bf3 --- /dev/null +++ b/detections/endpoint/disabling_systemrestore_in_registry.yml @@ -0,0 +1,44 @@ +name: Disabling SystemRestore In Registry +id: f4f837e2-91fb-11eb-8bf6-acde48001122 +version: 1 +date: '2021-03-31' +author: Teoderick Contreras, Splunk +type: batch +datamodel: +- Endpoint +description: The following search identifies the modification of registry related in disabling the system restore of a machine. + This event or behavior are seen in some RAT malware to make the restore of the infected machine difficult and keep their infection on the box. +search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Registry + where Registry.registry_path= "*\\SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\SystemRestore\\DisableSR" + OR Registry.registry_path= "*\\SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\SystemRestore\\DisableConfig" + Registry.registry_value_name = "DWORD (0x00000001)" by Registry.registry_path Registry.registry_key_name Registry.registry_value_name Registry.dest + | `drop_dm_object_name(Registry)` + | `security_content_ctime(firstTime)` + |`security_content_ctime(lastTime)` + | `disabling_systemrestore_in_registry_filter`' +how_to_implement: To successfully implement this search you need to be ingesting information + on process that include the name of the process responsible for the changes from + your endpoints into the `Endpoint` datamodel in the `Registry` node. Also make sure that this registry was included in your config + files ex. sysmon config to be monitored. +known_false_positives: in some cases admin can disable systemrestore on a machine. +references: +- https://tccontre.blogspot.com/2020/01/remcos-rat-evading-windows-defender-av.html +tags: + analytic_story: + - Windows Defense Evasion Tactics + kill_chain_phases: + - Exploitation + mitre_attack_id: + - T1562.001 + product: + - Splunk Enterprise + - Splunk Enterprise Security + - Splunk Cloud + required_fields: + - _time + - Registry.registry_key_name + - Registry.registry_path + - Registry.user + - Registry.dest + - Registry.registry_value_name + security_domain: endpoint \ No newline at end of file diff --git a/detections/endpoint/disabling_task_manager.yml b/detections/endpoint/disabling_task_manager.yml new file mode 100644 index 0000000000..ccae7213a1 --- /dev/null +++ b/detections/endpoint/disabling_task_manager.yml @@ -0,0 +1,44 @@ +name: Disabling Task Manager +id: dac279bc-9202-11eb-b7fb-acde48001122 +version: 1 +date: '2021-03-31' +author: Teoderick Contreras, Splunk +type: batch +datamodel: +- Endpoint +description: This search is to identifies modification of registry to disable the task manager of windows operating system. + this event or technique are commonly seen in malware such as RAT, Trojan, TrojanSpy or worm to prevent the user to terminate their process. +search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Registry + where Registry.registry_path= "*\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Policies\\System\\DisableTaskMgr" + Registry.registry_value_name = "DWORD (0x00000001)" by Registry.registry_path Registry.registry_key_name Registry.registry_value_name Registry.dest + | `drop_dm_object_name(Registry)` + | `security_content_ctime(firstTime)` + | `security_content_ctime(lastTime)` + | `disabling_task_manager_filter`' +how_to_implement: To successfully implement this search you need to be ingesting information + on process that include the name of the process responsible for the changes from + your endpoints into the `Endpoint` datamodel in the `Registry` node. Also make sure that this registry was included in your config + files ex. sysmon config to be monitored. +known_false_positives: admin may disable this application for non technical user. +references: +- https://any.run/report/ea4ea08407d4ee72e009103a3b77e5a09412b722fdef67315ea63f22011152af/a866d7b1-c236-4f26-a391-5ae32213dfc4#registry +- https://blog.talosintelligence.com/2020/05/threat-roundup-0424-0501.html +tags: + analytic_story: + - Windows Defense Evasion Tactics + kill_chain_phases: + - Exploitation + mitre_attack_id: + - T1562.001 + product: + - Splunk Enterprise + - Splunk Enterprise Security + - Splunk Cloud + required_fields: + - _time + - Registry.registry_key_name + - Registry.registry_path + - Registry.user + - Registry.dest + - Registry.registry_value_name + security_domain: endpoint \ No newline at end of file diff --git a/tests/endpoint/disable_registry_tool.test.yml b/tests/endpoint/disable_registry_tool.test.yml new file mode 100644 index 0000000000..a433192706 --- /dev/null +++ b/tests/endpoint/disable_registry_tool.test.yml @@ -0,0 +1,22 @@ +name: Disable Registry Tool Unit Test +tests: +- name: Disable Registry Tool + file: detections/endpoint/disable_registry_tool.yml + pass_condition: '| stats count | where count > 0' + earliest_time: '-24h' + latest_time: 'now' + attack_data: + - file_name: windows-security.log + data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-security.log + source: WinEventLog:Security + sourcetype: WinEventLog + update_timestamp: True + - file_name: windows-system.log + data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-system.log + source: WinEventLog:System + sourcetype: WinEventLog + update_timestamp: True + - file_name: windows-sysmon.log + data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-sysmon.log + source: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational + sourcetype: xmlwineventlog \ No newline at end of file diff --git a/tests/endpoint/disable_show_hidden_files.test.yml b/tests/endpoint/disable_show_hidden_files.test.yml new file mode 100644 index 0000000000..a7c7c2510c --- /dev/null +++ b/tests/endpoint/disable_show_hidden_files.test.yml @@ -0,0 +1,22 @@ +name: Disable Show Hidden Files Unit Test +tests: +- name: Disable Show Hidden Files + file: detections/endpoint/disable_show_hidden_files.yml + pass_condition: '| stats count | where count > 0' + earliest_time: '-24h' + latest_time: 'now' + attack_data: + - file_name: windows-security.log + data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-security.log + source: WinEventLog:Security + sourcetype: WinEventLog + update_timestamp: True + - file_name: windows-system.log + data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-system.log + source: WinEventLog:System + sourcetype: WinEventLog + update_timestamp: True + - file_name: windows-sysmon.log + data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-sysmon.log + source: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational + sourcetype: xmlwineventlog \ No newline at end of file diff --git a/tests/endpoint/disable_windows_behavior_monitoring.test.yml b/tests/endpoint/disable_windows_behavior_monitoring.test.yml new file mode 100644 index 0000000000..f64902e257 --- /dev/null +++ b/tests/endpoint/disable_windows_behavior_monitoring.test.yml @@ -0,0 +1,22 @@ +name: Disable Windows Behavior Monitoring Unit Test +tests: +- name: Disable Windows Behavior Monitoring + file: detections/endpoint/disable_windows_behavior_monitoring.yml + pass_condition: '| stats count | where count > 0' + earliest_time: '-24h' + latest_time: 'now' + attack_data: + - file_name: windows-security.log + data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-security.log + source: WinEventLog:Security + sourcetype: WinEventLog + update_timestamp: True + - file_name: windows-system.log + data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-system.log + source: WinEventLog:System + sourcetype: WinEventLog + update_timestamp: True + - file_name: windows-sysmon.log + data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-sysmon.log + source: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational + sourcetype: xmlwineventlog \ No newline at end of file diff --git a/tests/endpoint/disable_windows_smartscreen_protection.test.yml b/tests/endpoint/disable_windows_smartscreen_protection.test.yml new file mode 100644 index 0000000000..32cdaae011 --- /dev/null +++ b/tests/endpoint/disable_windows_smartscreen_protection.test.yml @@ -0,0 +1,22 @@ +name: Disable Windows SmartScreen Protection Unit Test +tests: +- name: Disable Windows SmartScreen Protection + file: detections/endpoint/disable_windows_smartscreen_protection.yml + pass_condition: '| stats count | where count > 0' + earliest_time: '-24h' + latest_time: 'now' + attack_data: + - file_name: windows-security.log + data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-security.log + source: WinEventLog:Security + sourcetype: WinEventLog + update_timestamp: True + - file_name: windows-system.log + data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-system.log + source: WinEventLog:System + sourcetype: WinEventLog + update_timestamp: True + - file_name: windows-sysmon.log + data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-sysmon.log + source: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational + sourcetype: xmlwineventlog \ No newline at end of file diff --git a/tests/endpoint/disabling_cmd_application.test.yml b/tests/endpoint/disabling_cmd_application.test.yml new file mode 100644 index 0000000000..8ea1ce3c49 --- /dev/null +++ b/tests/endpoint/disabling_cmd_application.test.yml @@ -0,0 +1,22 @@ +name: Disabling CMD Application Unit Test +tests: +- name: Disabling CMD Application + file: detections/endpoint/disabling_cmd_application.yml + pass_condition: '| stats count | where count > 0' + earliest_time: '-24h' + latest_time: 'now' + attack_data: + - file_name: windows-security.log + data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-security.log + source: WinEventLog:Security + sourcetype: WinEventLog + update_timestamp: True + - file_name: windows-system.log + data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-system.log + source: WinEventLog:System + sourcetype: WinEventLog + update_timestamp: True + - file_name: windows-sysmon.log + data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-sysmon.log + source: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational + sourcetype: xmlwineventlog \ No newline at end of file diff --git a/tests/endpoint/disabling_controlpanel.test.yml b/tests/endpoint/disabling_controlpanel.test.yml new file mode 100644 index 0000000000..a86eeb0064 --- /dev/null +++ b/tests/endpoint/disabling_controlpanel.test.yml @@ -0,0 +1,22 @@ +name: Disabling ControlPanel Unit Test +tests: +- name: Disabling ControlPanel + file: detections/endpoint/disabling_controlpanel.yml + pass_condition: '| stats count | where count > 0' + earliest_time: '-24h' + latest_time: 'now' + attack_data: + - file_name: windows-security.log + data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-security.log + source: WinEventLog:Security + sourcetype: WinEventLog + update_timestamp: True + - file_name: windows-system.log + data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-system.log + source: WinEventLog:System + sourcetype: WinEventLog + update_timestamp: True + - file_name: windows-sysmon.log + data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-sysmon.log + source: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational + sourcetype: xmlwineventlog \ No newline at end of file diff --git a/tests/endpoint/disabling_firewall_with_netsh.test.yml b/tests/endpoint/disabling_firewall_with_netsh.test.yml new file mode 100644 index 0000000000..aec890f704 --- /dev/null +++ b/tests/endpoint/disabling_firewall_with_netsh.test.yml @@ -0,0 +1,22 @@ +name: Disabling Firewall with Netsh Unit Test +tests: +- name: Disabling Firewall with Netsh + file: detections/endpoint/disabling_firewall_with_netsh.yml + pass_condition: '| stats count | where count > 0' + earliest_time: '-24h' + latest_time: 'now' + attack_data: + - file_name: windows-security.log + data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-security.log + source: WinEventLog:Security + sourcetype: WinEventLog + update_timestamp: True + - file_name: windows-system.log + data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-system.log + source: WinEventLog:System + sourcetype: WinEventLog + update_timestamp: True + - file_name: windows-sysmon.log + data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-sysmon.log + source: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational + sourcetype: xmlwineventlog \ No newline at end of file diff --git a/tests/endpoint/disabling_folderoptions_windows_feature.test.yml b/tests/endpoint/disabling_folderoptions_windows_feature.test.yml new file mode 100644 index 0000000000..aba6e6225d --- /dev/null +++ b/tests/endpoint/disabling_folderoptions_windows_feature.test.yml @@ -0,0 +1,22 @@ +name: Disabling FolderOptions Windows Feature Unit Test +tests: +- name: Disabling FolderOptions Windows Feature + file: detections/endpoint/disabling_folderoptions_windows_feature.yml + pass_condition: '| stats count | where count > 0' + earliest_time: '-24h' + latest_time: 'now' + attack_data: + - file_name: windows-security.log + data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-security.log + source: WinEventLog:Security + sourcetype: WinEventLog + update_timestamp: True + - file_name: windows-system.log + data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-system.log + source: WinEventLog:System + sourcetype: WinEventLog + update_timestamp: True + - file_name: windows-sysmon.log + data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-sysmon.log + source: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational + sourcetype: xmlwineventlogE \ No newline at end of file diff --git a/tests/endpoint/disabling_norun_windows_app.test.yml b/tests/endpoint/disabling_norun_windows_app.test.yml new file mode 100644 index 0000000000..f3b5ac705d --- /dev/null +++ b/tests/endpoint/disabling_norun_windows_app.test.yml @@ -0,0 +1,22 @@ +name: Disabling NoRun Windows App Unit Test +tests: +- name: Disabling NoRun Windows App + file: detections/endpoint/disabling_norun_windows_app.yml + pass_condition: '| stats count | where count > 0' + earliest_time: '-24h' + latest_time: 'now' + attack_data: + - file_name: windows-security.log + data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-security.log + source: WinEventLog:Security + sourcetype: WinEventLog + update_timestamp: True + - file_name: windows-system.log + data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-system.log + source: WinEventLog:System + sourcetype: WinEventLog + update_timestamp: True + - file_name: windows-sysmon.log + data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-sysmon.log + source: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational + sourcetype: xmlwineventlog \ No newline at end of file diff --git a/tests/endpoint/disabling_systemrestore_in_registry.test.yml b/tests/endpoint/disabling_systemrestore_in_registry.test.yml new file mode 100644 index 0000000000..7d5d2d4a03 --- /dev/null +++ b/tests/endpoint/disabling_systemrestore_in_registry.test.yml @@ -0,0 +1,22 @@ +name: Disabling SystemRestore In Registry Unit Test +tests: +- name: Disabling SystemRestore In Registry + file: detections/endpoint/disabling_systemrestore_in_registry.yml + pass_condition: '| stats count | where count > 0' + earliest_time: '-24h' + latest_time: 'now' + attack_data: + - file_name: windows-security.log + data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-security.log + source: WinEventLog:Security + sourcetype: WinEventLog + update_timestamp: True + - file_name: windows-system.log + data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-system.log + source: WinEventLog:System + sourcetype: WinEventLog + update_timestamp: True + - file_name: windows-sysmon.log + data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-sysmon.log + source: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational + sourcetype: xmlwineventlog \ No newline at end of file diff --git a/tests/endpoint/disabling_task_manager.test.yml b/tests/endpoint/disabling_task_manager.test.yml new file mode 100644 index 0000000000..bd8ede11ad --- /dev/null +++ b/tests/endpoint/disabling_task_manager.test.yml @@ -0,0 +1,22 @@ +name: Disabling Task Manager Unit Test +tests: +- name: Disabling Task Manager + file: detections/endpoint/disabling_task_manager.yml + pass_condition: '| stats count | where count > 0' + earliest_time: '-24h' + latest_time: 'now' + attack_data: + - file_name: windows-security.log + data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-security.log + source: WinEventLog:Security + sourcetype: WinEventLog + update_timestamp: True + - file_name: windows-system.log + data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-system.log + source: WinEventLog:System + sourcetype: WinEventLog + update_timestamp: True + - file_name: windows-sysmon.log + data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-sysmon.log + source: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational + sourcetype: xmlwineventlog \ No newline at end of file From fa58ca006b4caf19961036dd603d7c978cd7d54d Mon Sep 17 00:00:00 2001 From: tcontreras Date: Wed, 31 Mar 2021 15:00:36 +0200 Subject: [PATCH 10/41] def_evasion_detections --- tests/endpoint/disable_registry_tool.test.yml | 2 +- tests/endpoint/disable_show_hidden_files.test.yml | 2 +- tests/endpoint/disable_windows_behavior_monitoring.test.yml | 2 +- tests/endpoint/disable_windows_smartscreen_protection.test.yml | 2 +- tests/endpoint/disabling_cmd_application.test.yml | 2 +- tests/endpoint/disabling_controlpanel.test.yml | 2 +- tests/endpoint/disabling_firewall_with_netsh.test.yml | 2 +- tests/endpoint/disabling_folderoptions_windows_feature.test.yml | 2 +- tests/endpoint/disabling_norun_windows_app.test.yml | 2 +- tests/endpoint/disabling_systemrestore_in_registry.test.yml | 2 +- tests/endpoint/disabling_task_manager.test.yml | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/endpoint/disable_registry_tool.test.yml b/tests/endpoint/disable_registry_tool.test.yml index a433192706..f2f6fd210c 100644 --- a/tests/endpoint/disable_registry_tool.test.yml +++ b/tests/endpoint/disable_registry_tool.test.yml @@ -1,7 +1,7 @@ name: Disable Registry Tool Unit Test tests: - name: Disable Registry Tool - file: detections/endpoint/disable_registry_tool.yml + file: endpoint/disable_registry_tool.yml pass_condition: '| stats count | where count > 0' earliest_time: '-24h' latest_time: 'now' diff --git a/tests/endpoint/disable_show_hidden_files.test.yml b/tests/endpoint/disable_show_hidden_files.test.yml index a7c7c2510c..98db777275 100644 --- a/tests/endpoint/disable_show_hidden_files.test.yml +++ b/tests/endpoint/disable_show_hidden_files.test.yml @@ -1,7 +1,7 @@ name: Disable Show Hidden Files Unit Test tests: - name: Disable Show Hidden Files - file: detections/endpoint/disable_show_hidden_files.yml + file: endpoint/disable_show_hidden_files.yml pass_condition: '| stats count | where count > 0' earliest_time: '-24h' latest_time: 'now' diff --git a/tests/endpoint/disable_windows_behavior_monitoring.test.yml b/tests/endpoint/disable_windows_behavior_monitoring.test.yml index f64902e257..01c556ef9c 100644 --- a/tests/endpoint/disable_windows_behavior_monitoring.test.yml +++ b/tests/endpoint/disable_windows_behavior_monitoring.test.yml @@ -1,7 +1,7 @@ name: Disable Windows Behavior Monitoring Unit Test tests: - name: Disable Windows Behavior Monitoring - file: detections/endpoint/disable_windows_behavior_monitoring.yml + file: endpoint/disable_windows_behavior_monitoring.yml pass_condition: '| stats count | where count > 0' earliest_time: '-24h' latest_time: 'now' diff --git a/tests/endpoint/disable_windows_smartscreen_protection.test.yml b/tests/endpoint/disable_windows_smartscreen_protection.test.yml index 32cdaae011..b1174e469c 100644 --- a/tests/endpoint/disable_windows_smartscreen_protection.test.yml +++ b/tests/endpoint/disable_windows_smartscreen_protection.test.yml @@ -1,7 +1,7 @@ name: Disable Windows SmartScreen Protection Unit Test tests: - name: Disable Windows SmartScreen Protection - file: detections/endpoint/disable_windows_smartscreen_protection.yml + file: endpoint/disable_windows_smartscreen_protection.yml pass_condition: '| stats count | where count > 0' earliest_time: '-24h' latest_time: 'now' diff --git a/tests/endpoint/disabling_cmd_application.test.yml b/tests/endpoint/disabling_cmd_application.test.yml index 8ea1ce3c49..42cb6a6770 100644 --- a/tests/endpoint/disabling_cmd_application.test.yml +++ b/tests/endpoint/disabling_cmd_application.test.yml @@ -1,7 +1,7 @@ name: Disabling CMD Application Unit Test tests: - name: Disabling CMD Application - file: detections/endpoint/disabling_cmd_application.yml + file: endpoint/disabling_cmd_application.yml pass_condition: '| stats count | where count > 0' earliest_time: '-24h' latest_time: 'now' diff --git a/tests/endpoint/disabling_controlpanel.test.yml b/tests/endpoint/disabling_controlpanel.test.yml index a86eeb0064..b0f8a184d4 100644 --- a/tests/endpoint/disabling_controlpanel.test.yml +++ b/tests/endpoint/disabling_controlpanel.test.yml @@ -1,7 +1,7 @@ name: Disabling ControlPanel Unit Test tests: - name: Disabling ControlPanel - file: detections/endpoint/disabling_controlpanel.yml + file: endpoint/disabling_controlpanel.yml pass_condition: '| stats count | where count > 0' earliest_time: '-24h' latest_time: 'now' diff --git a/tests/endpoint/disabling_firewall_with_netsh.test.yml b/tests/endpoint/disabling_firewall_with_netsh.test.yml index aec890f704..cfad109c1e 100644 --- a/tests/endpoint/disabling_firewall_with_netsh.test.yml +++ b/tests/endpoint/disabling_firewall_with_netsh.test.yml @@ -1,7 +1,7 @@ name: Disabling Firewall with Netsh Unit Test tests: - name: Disabling Firewall with Netsh - file: detections/endpoint/disabling_firewall_with_netsh.yml + file: endpoint/disabling_firewall_with_netsh.yml pass_condition: '| stats count | where count > 0' earliest_time: '-24h' latest_time: 'now' diff --git a/tests/endpoint/disabling_folderoptions_windows_feature.test.yml b/tests/endpoint/disabling_folderoptions_windows_feature.test.yml index aba6e6225d..213be9e4cf 100644 --- a/tests/endpoint/disabling_folderoptions_windows_feature.test.yml +++ b/tests/endpoint/disabling_folderoptions_windows_feature.test.yml @@ -1,7 +1,7 @@ name: Disabling FolderOptions Windows Feature Unit Test tests: - name: Disabling FolderOptions Windows Feature - file: detections/endpoint/disabling_folderoptions_windows_feature.yml + file: endpoint/disabling_folderoptions_windows_feature.yml pass_condition: '| stats count | where count > 0' earliest_time: '-24h' latest_time: 'now' diff --git a/tests/endpoint/disabling_norun_windows_app.test.yml b/tests/endpoint/disabling_norun_windows_app.test.yml index f3b5ac705d..93cb1186b8 100644 --- a/tests/endpoint/disabling_norun_windows_app.test.yml +++ b/tests/endpoint/disabling_norun_windows_app.test.yml @@ -1,7 +1,7 @@ name: Disabling NoRun Windows App Unit Test tests: - name: Disabling NoRun Windows App - file: detections/endpoint/disabling_norun_windows_app.yml + file: endpoint/disabling_norun_windows_app.yml pass_condition: '| stats count | where count > 0' earliest_time: '-24h' latest_time: 'now' diff --git a/tests/endpoint/disabling_systemrestore_in_registry.test.yml b/tests/endpoint/disabling_systemrestore_in_registry.test.yml index 7d5d2d4a03..61d8cfbf10 100644 --- a/tests/endpoint/disabling_systemrestore_in_registry.test.yml +++ b/tests/endpoint/disabling_systemrestore_in_registry.test.yml @@ -1,7 +1,7 @@ name: Disabling SystemRestore In Registry Unit Test tests: - name: Disabling SystemRestore In Registry - file: detections/endpoint/disabling_systemrestore_in_registry.yml + file: endpoint/disabling_systemrestore_in_registry.yml pass_condition: '| stats count | where count > 0' earliest_time: '-24h' latest_time: 'now' diff --git a/tests/endpoint/disabling_task_manager.test.yml b/tests/endpoint/disabling_task_manager.test.yml index bd8ede11ad..b169662e6b 100644 --- a/tests/endpoint/disabling_task_manager.test.yml +++ b/tests/endpoint/disabling_task_manager.test.yml @@ -1,7 +1,7 @@ name: Disabling Task Manager Unit Test tests: - name: Disabling Task Manager - file: detections/endpoint/disabling_task_manager.yml + file: endpoint/disabling_task_manager.yml pass_condition: '| stats count | where count > 0' earliest_time: '-24h' latest_time: 'now' From 10b51469c3f7c7582f5338aa01dd8fbb861f3c89 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 31 Mar 2021 15:47:04 +0000 Subject: [PATCH 11/41] Added detection testing service results inDisable Registry Tool --- detections/endpoint/disable_registry_tool.yml | 33 +++++++++++-------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/detections/endpoint/disable_registry_tool.yml b/detections/endpoint/disable_registry_tool.yml index f9df9e40e9..3f31a7ffa2 100644 --- a/detections/endpoint/disable_registry_tool.yml +++ b/detections/endpoint/disable_registry_tool.yml @@ -6,26 +6,26 @@ author: Teoderick Contreras, Splunk type: batch datamodel: - Endpoint -description: This search is to identifies modification of registry to disable the regedit or registry tools of windows operating system. - Since registry tool is a swiss knife in analyzing registry, malware such as RAT or trojan Spy disable this application to prevent the removal - of their registry entry such as persistence, file less components and defense evasion. -search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Registry - where Registry.registry_path= "*\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Policies\\System\\DisableRegistryTools" - Registry.registry_value_name = "DWORD (0x00000001)" by Registry.registry_path Registry.registry_key_name Registry.registry_value_name Registry.dest - | `drop_dm_object_name(Registry)` - | `security_content_ctime(firstTime)` - |`security_content_ctime(lastTime)` - | `disable_registry_tool_filter`' +description: This search is to identifies modification of registry to disable the + regedit or registry tools of windows operating system. Since registry tool is a + swiss knife in analyzing registry, malware such as RAT or trojan Spy disable this + application to prevent the removal of their registry entry such as persistence, + file less components and defense evasion. +search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) + as lastTime from datamodel=Endpoint.Registry where Registry.registry_path= "*\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Policies\\System\\DisableRegistryTools" + Registry.registry_value_name = "DWORD (0x00000001)" by Registry.registry_path Registry.registry_key_name + Registry.registry_value_name Registry.dest | `drop_dm_object_name(Registry)` | `security_content_ctime(firstTime)` + |`security_content_ctime(lastTime)` | `disable_registry_tool_filter`' how_to_implement: To successfully implement this search you need to be ingesting information on process that include the name of the process responsible for the changes from - your endpoints into the `Endpoint` datamodel in the `Registry` node. Also make sure that this registry was included in your config - files ex. sysmon config to be monitored. + your endpoints into the `Endpoint` datamodel in the `Registry` node. Also make sure + that this registry was included in your config files ex. sysmon config to be monitored. known_false_positives: admin may disable this application for non technical user. references: - https://any.run/report/ea4ea08407d4ee72e009103a3b77e5a09412b722fdef67315ea63f22011152af/a866d7b1-c236-4f26-a391-5ae32213dfc4#registry tags: analytic_story: - - Windows Defense Evasion Tactics + - Windows Defense Evasion Tactics kill_chain_phases: - Exploitation mitre_attack_id: @@ -41,4 +41,9 @@ tags: - Registry.user - Registry.dest - Registry.registry_value_name - security_domain: endpoint \ No newline at end of file + security_domain: endpoint + automated_detection_testing: passed + dataset: + - https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-security.log + - https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-system.log + - https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-sysmon.log From ae45f7a3ae507b6752aae820b54a6ba27fc214ff Mon Sep 17 00:00:00 2001 From: root Date: Wed, 31 Mar 2021 16:05:22 +0000 Subject: [PATCH 12/41] Added detection testing service results inDisable Show Hidden Files --- .../endpoint/disable_show_hidden_files.yml | 37 +++++++++++-------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/detections/endpoint/disable_show_hidden_files.yml b/detections/endpoint/disable_show_hidden_files.yml index f49cbc69db..036ea045f8 100644 --- a/detections/endpoint/disable_show_hidden_files.yml +++ b/detections/endpoint/disable_show_hidden_files.yml @@ -6,28 +6,28 @@ author: Teoderick Contreras, Splunk type: batch datamodel: - Endpoint -description: The following search is to idetifies a modification in registry to prevent the user seeing all the files with hidden attributes. - This event or techniques are known on some worm and trojan spy malware that will drop hidden files on the infected machine. -search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Registry - where (Registry.registry_path= "*\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Explorer\\Advanced\\Hidden" - OR Registry.registry_path= "*\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Explorer\\Advanced\\HideFileExt" - Registry.registry_value_name = "DWORD (0x00000001)") - OR (Registry.registry_path= "*\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Explorer\\Advanced\\ShowSuperHidden" Registry.registry_value_name = "DWORD (0x00000000)") - by Registry.registry_path Registry.registry_key_name Registry.registry_value_name Registry.dest - | `drop_dm_object_name(Registry)` - | `security_content_ctime(firstTime)` - |`security_content_ctime(lastTime)` - | `disable_show_hidden_files_filter`' +description: The following search is to idetifies a modification in registry to prevent + the user seeing all the files with hidden attributes. This event or techniques are + known on some worm and trojan spy malware that will drop hidden files on the infected + machine. +search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) + as lastTime from datamodel=Endpoint.Registry where (Registry.registry_path= "*\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Explorer\\Advanced\\Hidden" + OR Registry.registry_path= "*\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Explorer\\Advanced\\HideFileExt" + Registry.registry_value_name = "DWORD (0x00000001)") OR (Registry.registry_path= + "*\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Explorer\\Advanced\\ShowSuperHidden" + Registry.registry_value_name = "DWORD (0x00000000)") by Registry.registry_path Registry.registry_key_name + Registry.registry_value_name Registry.dest | `drop_dm_object_name(Registry)` | `security_content_ctime(firstTime)` + |`security_content_ctime(lastTime)` | `disable_show_hidden_files_filter`' how_to_implement: To successfully implement this search you need to be ingesting information on process that include the name of the process responsible for the changes from - your endpoints into the `Endpoint` datamodel in the `Registry` node. Also make sure that this registry was included in your config - files ex. sysmon config to be monitored. + your endpoints into the `Endpoint` datamodel in the `Registry` node. Also make sure + that this registry was included in your config files ex. sysmon config to be monitored. known_false_positives: unknown references: - https://www.sophos.com/en-us/threat-center/threat-analyses/viruses-and-spyware/W32~Tiotua-P/detailed-analysis.aspx tags: analytic_story: - - Windows Defense Evasion Tactics + - Windows Defense Evasion Tactics kill_chain_phases: - Exploitation mitre_attack_id: @@ -44,4 +44,9 @@ tags: - Registry.user - Registry.dest - Registry.registry_value_nam - security_domain: endpoint \ No newline at end of file + security_domain: endpoint + automated_detection_testing: passed + dataset: + - https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-security.log + - https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-system.log + - https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-sysmon.log From 71139506d750a40edc138dc2d2eaa00b7b06bd73 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 31 Mar 2021 17:23:30 +0000 Subject: [PATCH 13/41] Added detection testing service results inDisable Windows Behavior Monitoring --- .../disable_windows_behavior_monitoring.yml | 36 +++++++++++-------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/detections/endpoint/disable_windows_behavior_monitoring.yml b/detections/endpoint/disable_windows_behavior_monitoring.yml index d643a1b4a5..7c5ab7d25e 100644 --- a/detections/endpoint/disable_windows_behavior_monitoring.yml +++ b/detections/endpoint/disable_windows_behavior_monitoring.yml @@ -6,27 +6,28 @@ author: Teoderick Contreras, Splunk type: batch datamodel: - Endpoint -description: This search is to identifies a modification in registry to disable the windows denfender real time behavior monitoring. - This event or technique is commonly seen in RAT, bot, or Trojan to disable AV to evade detections. -search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Registry - where Registry.registry_path= "*\\SOFTWARE\\Policies\\Microsoft\\Windows Defender\\Real-Time Protection\\DisableBehaviorMonitoring" - OR Registry.registry_path= "*\\SOFTWARE\\Policies\\Microsoft\\Windows Defender\\Real-Time Protection\\DisableOnAccessProtection" - OR Registry.registry_path= "*\\SOFTWARE\\Policies\\Microsoft\\Windows Defender\\Real-Time Protection\\DisableScanOnRealtimeEnable" - Registry.registry_value_name = "DWORD (0x00000001)" by Registry.registry_path Registry.registry_key_name Registry.registry_value_name Registry.dest - | `drop_dm_object_name(Registry)` - | `security_content_ctime(firstTime)` - |`security_content_ctime(lastTime)` - | `disable_windows_behavior_monitoring_filter`' +description: This search is to identifies a modification in registry to disable the + windows denfender real time behavior monitoring. This event or technique is commonly + seen in RAT, bot, or Trojan to disable AV to evade detections. +search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) + as lastTime from datamodel=Endpoint.Registry where Registry.registry_path= "*\\SOFTWARE\\Policies\\Microsoft\\Windows + Defender\\Real-Time Protection\\DisableBehaviorMonitoring" OR Registry.registry_path= + "*\\SOFTWARE\\Policies\\Microsoft\\Windows Defender\\Real-Time Protection\\DisableOnAccessProtection" + OR Registry.registry_path= "*\\SOFTWARE\\Policies\\Microsoft\\Windows Defender\\Real-Time + Protection\\DisableScanOnRealtimeEnable" Registry.registry_value_name = "DWORD (0x00000001)" + by Registry.registry_path Registry.registry_key_name Registry.registry_value_name + Registry.dest | `drop_dm_object_name(Registry)` | `security_content_ctime(firstTime)` + |`security_content_ctime(lastTime)` | `disable_windows_behavior_monitoring_filter`' how_to_implement: To successfully implement this search you need to be ingesting information on process that include the name of the process responsible for the changes from - your endpoints into the `Endpoint` datamodel in the `Registry` node. Also make sure that this registry was included in your config - files ex. sysmon config to be monitored. + your endpoints into the `Endpoint` datamodel in the `Registry` node. Also make sure + that this registry was included in your config files ex. sysmon config to be monitored. known_false_positives: admin or user may choose to disable this windows features. references: - https://tccontre.blogspot.com/2020/01/remcos-rat-evading-windows-defender-av.html tags: analytic_story: - - Windows Defense Evasion Tactics + - Windows Defense Evasion Tactics kill_chain_phases: - Exploitation mitre_attack_id: @@ -42,4 +43,9 @@ tags: - Registry.user - Registry.dest - Registry.registry_value_name - security_domain: endpoint \ No newline at end of file + security_domain: endpoint + automated_detection_testing: passed + dataset: + - https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-security.log + - https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-system.log + - https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-sysmon.log From 2db2f77d69f71d17b9805ea8ce1d84d697fb1fea Mon Sep 17 00:00:00 2001 From: mhaag-spl <5632822+MHaggis@users.noreply.github.com> Date: Wed, 31 Mar 2021 11:32:45 -0600 Subject: [PATCH 14/41] Ingress Tool Transfer Story Adds --- detections/endpoint/any_powershell_downloadfile.yml | 1 + detections/endpoint/any_powershell_downloadstring.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/detections/endpoint/any_powershell_downloadfile.yml b/detections/endpoint/any_powershell_downloadfile.yml index 55cbd2377c..afe2f0cf0b 100644 --- a/detections/endpoint/any_powershell_downloadfile.yml +++ b/detections/endpoint/any_powershell_downloadfile.yml @@ -31,6 +31,7 @@ references: tags: analytic_story: - Malicious PowerShell + - Ingress Tool Transfer dataset: - https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1059.001/atomic_red_team/windows-sysmon.log kill_chain_phases: diff --git a/detections/endpoint/any_powershell_downloadstring.yml b/detections/endpoint/any_powershell_downloadstring.yml index 9b335941a7..018246a866 100644 --- a/detections/endpoint/any_powershell_downloadstring.yml +++ b/detections/endpoint/any_powershell_downloadstring.yml @@ -32,6 +32,7 @@ tags: analytic_story: - Malicious PowerShell - HAFNIUM Group + - Ingress Tool Transfer automated_detection_testing: passed dataset: - https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1059.001/atomic_red_team/windows-sysmon.log From 1ab256593f78656d87accc86953fbbb895e17f2d Mon Sep 17 00:00:00 2001 From: mhaag-spl <5632822+MHaggis@users.noreply.github.com> Date: Wed, 31 Mar 2021 13:24:32 -0600 Subject: [PATCH 15/41] Domain Trust Discovery --- .../endpoint/dsquery_domain_discovery.yml | 54 +++++++++++++++++++ .../nltest_domain_trust_discovery.yml | 1 + .../endpoint/windows_adfind_exe.yml | 1 + stories/domain_trust_discovery.yml | 20 +++++++ .../dsquery_domain_discovery.test.yml | 12 +++++ 5 files changed, 88 insertions(+) create mode 100644 detections/endpoint/dsquery_domain_discovery.yml create mode 100644 stories/domain_trust_discovery.yml create mode 100644 tests/endpoint/dsquery_domain_discovery.test.yml diff --git a/detections/endpoint/dsquery_domain_discovery.yml b/detections/endpoint/dsquery_domain_discovery.yml new file mode 100644 index 0000000000..d64d6de108 --- /dev/null +++ b/detections/endpoint/dsquery_domain_discovery.yml @@ -0,0 +1,54 @@ +name: DSQuery Domain Discovery +id: cc316032-924a-11eb-91a2-acde48001122 +version: 1 +date: '2021-03-31' +author: Michael Haag, Splunk +type: batch +datamodel: +- Endpoint +description: 'The following analytic identifies "dsquery.exe" execution with arguments looking for `TrustedDomain` query directly on the command-line. This is typically indicative of an Administrator or adversary perform domain trust discovery. Note that this query does not identify any other variations of "Dsquery.exe" usage.\ + + Within this detection, it is assumed `dsquery.exe` is not moved or renamed. \ + + The search will return the first time and last time these command-line arguments were used for these executions, as well as the target system, the user, process "dsquery.exe" and its parent process.\ + + DSQuery.exe is natively found in `C:\Windows\system32` and `C:\Windows\syswow64` and only on Server operation systems.\ + + The following DLL(s) are loaded when DSQuery.exe is launched `dsquery.dll`. If found loaded by another process, it's possible dsquery is running within that process context in memory.\ + + In addition to trust discovery, review parallel processes for additional behaviors performed. Identify the parent process and capture any files (batch files, for example) being used.' +search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) + as lastTime from datamodel=Endpoint.Processes where Processes.process_name=dsquery.exe + Processes.process=*trustedDomain* by Processes.dest Processes.user Processes.parent_process Processes.process_name + Processes.process Processes.process_id Processes.parent_process_id + | `drop_dm_object_name(Processes)` + | `security_content_ctime(firstTime)` + | `security_content_ctime(lastTime)` | `dsquery_domain_discovery_filter`' +how_to_implement: To successfully implement this search you need to be ingesting information + on process that include the name of the process responsible for the changes from + your endpoints into the `Endpoint` datamodel in the `Processes` node. +known_false_positives: Limited false positives. If there is a true false positive, filter based on command-line or parent process. +references: +- https://github.com/redcanaryco/atomic-red-team/blob/master/atomics/T1482/T1482.md +- http://www.harmj0y.net/blog/redteaming/a-guide-to-attacking-domain-trusts/ +- https://docs.microsoft.com/en-us/previous-versions/windows/it-pro/windows-server-2012-R2-and-2012/cc732952(v=ws.11) +- https://docs.microsoft.com/en-us/previous-versions/windows/it-pro/windows-server-2012-R2-and-2012/cc754232(v=ws.11) +tags: + analytic_story: + - Domain Trust Discovery + dataset: [] + kill_chain_phases: + - Exploitation + mitre_attack_id: + - T1482 + product: + - Splunk Enterprise + - Splunk Enterprise Security + - Splunk Cloud + required_fields: + - _time + - Processes.process_name + - Processes.process + - Processes.user + - Processes.dest + security_domain: endpoint \ No newline at end of file diff --git a/detections/endpoint/nltest_domain_trust_discovery.yml b/detections/endpoint/nltest_domain_trust_discovery.yml index e6e45673e1..e63be64c0a 100644 --- a/detections/endpoint/nltest_domain_trust_discovery.yml +++ b/detections/endpoint/nltest_domain_trust_discovery.yml @@ -33,6 +33,7 @@ references: tags: analytic_story: - Ryuk Ransomware + - Domain Trust Discovery asset_type: Endpoint automated_detection_testing: passed cis20: diff --git a/detections/experimental/endpoint/windows_adfind_exe.yml b/detections/experimental/endpoint/windows_adfind_exe.yml index c02fe85f37..cd184c74e6 100644 --- a/detections/experimental/endpoint/windows_adfind_exe.yml +++ b/detections/experimental/endpoint/windows_adfind_exe.yml @@ -30,6 +30,7 @@ references: tags: analytic_story: - NOBELIUM Group + - Domain Trust Discovery asset_type: Endpoint cis20: - CIS 8 diff --git a/stories/domain_trust_discovery.yml b/stories/domain_trust_discovery.yml new file mode 100644 index 0000000000..35565cf124 --- /dev/null +++ b/stories/domain_trust_discovery.yml @@ -0,0 +1,20 @@ +pyhonname: Domain Trust Discovery +id: e6f30f14-8daf-11eb-a017-acde48001122 +version: 1 +date: '2021-03-25' +author: Michael Haag, Splunk +type: batch +description: Adversaries may attempt to gather information on domain trust relationships that may be used to identify lateral movement opportunities in Windows multi-domain/forest environments. +narrative: Domain trusts provide a mechanism for a domain to allow access to resources based on the authentication procedures of another domain. Domain trusts allow the users of the trusted domain to access resources in the trusting domain. The information discovered may help the adversary conduct SID-History Injection, Pass the Ticket, and Kerberoasting. Domain trusts can be enumerated using the DSEnumerateDomainTrusts() Win32 API call, .NET methods, and LDAP. The Windows utility Nltest is known to be used by adversaries to enumerate domain trusts. +references: +- https://attack.mitre.org/techniques/T1482/ +tags: + analytic_story: + - Domain Trust Discovery + category: + - Adversary Tactics + product: + - Splunk Enterprise + - Splunk Enterprise Security + - Splunk Cloud + usecase: Advanced Threat Detection \ No newline at end of file diff --git a/tests/endpoint/dsquery_domain_discovery.test.yml b/tests/endpoint/dsquery_domain_discovery.test.yml new file mode 100644 index 0000000000..4d43b04ae4 --- /dev/null +++ b/tests/endpoint/dsquery_domain_discovery.test.yml @@ -0,0 +1,12 @@ +name: DSQuery Domain Discovery Unit Test +tests: +- name: DSQuery Domain Discovery + file: detections/endpoint/dsquery_domain_discovery.yml + pass_condition: '| stats count | where count > 0' + earliest_time: '-24h' + latest_time: 'now' + attack_data: + - file_name: windows-sysmon.log + data: https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1482/atomic_red_team/windows-sysmon.log + source: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational + sourcetype: xmlwineventlog \ No newline at end of file From f2c6e23be666cafd67aa36d5b1280ec28d339825 Mon Sep 17 00:00:00 2001 From: mhaag-spl <5632822+MHaggis@users.noreply.github.com> Date: Wed, 31 Mar 2021 13:41:56 -0600 Subject: [PATCH 16/41] Update dsquery_domain_discovery.yml --- detections/endpoint/dsquery_domain_discovery.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/detections/endpoint/dsquery_domain_discovery.yml b/detections/endpoint/dsquery_domain_discovery.yml index d64d6de108..1cd75a5616 100644 --- a/detections/endpoint/dsquery_domain_discovery.yml +++ b/detections/endpoint/dsquery_domain_discovery.yml @@ -8,7 +8,7 @@ datamodel: - Endpoint description: 'The following analytic identifies "dsquery.exe" execution with arguments looking for `TrustedDomain` query directly on the command-line. This is typically indicative of an Administrator or adversary perform domain trust discovery. Note that this query does not identify any other variations of "Dsquery.exe" usage.\ - Within this detection, it is assumed `dsquery.exe` is not moved or renamed. \ + Within this detection, it is assumed `dsquery.exe` is not moved or renamed.\ The search will return the first time and last time these command-line arguments were used for these executions, as well as the target system, the user, process "dsquery.exe" and its parent process.\ @@ -23,7 +23,8 @@ search: '| tstats `security_content_summariesonly` count min(_time) as firstTime Processes.process Processes.process_id Processes.parent_process_id | `drop_dm_object_name(Processes)` | `security_content_ctime(firstTime)` - | `security_content_ctime(lastTime)` | `dsquery_domain_discovery_filter`' + | `security_content_ctime(lastTime)` + | `dsquery_domain_discovery_filter`' how_to_implement: To successfully implement this search you need to be ingesting information on process that include the name of the process responsible for the changes from your endpoints into the `Endpoint` datamodel in the `Processes` node. From 501a979aefb18e6e3763b8ff1a0a5c7b9e857eea Mon Sep 17 00:00:00 2001 From: mhaag-spl <5632822+MHaggis@users.noreply.github.com> Date: Wed, 31 Mar 2021 13:43:51 -0600 Subject: [PATCH 17/41] Update dsquery_domain_discovery.yml --- detections/endpoint/dsquery_domain_discovery.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/detections/endpoint/dsquery_domain_discovery.yml b/detections/endpoint/dsquery_domain_discovery.yml index 1cd75a5616..c279f98b38 100644 --- a/detections/endpoint/dsquery_domain_discovery.yml +++ b/detections/endpoint/dsquery_domain_discovery.yml @@ -14,7 +14,7 @@ description: 'The following analytic identifies "dsquery.exe" execution with arg DSQuery.exe is natively found in `C:\Windows\system32` and `C:\Windows\syswow64` and only on Server operation systems.\ - The following DLL(s) are loaded when DSQuery.exe is launched `dsquery.dll`. If found loaded by another process, it's possible dsquery is running within that process context in memory.\ + The following DLL(s) are loaded when DSQuery.exe is launched `dsquery.dll`. If found loaded by another process, it is possible dsquery is running within that process context in memory.\ In addition to trust discovery, review parallel processes for additional behaviors performed. Identify the parent process and capture any files (batch files, for example) being used.' search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) From 5b2c3a2bd58cfd81c7b1b9f5b1e09bcbdda9590d Mon Sep 17 00:00:00 2001 From: mhaag-spl <5632822+MHaggis@users.noreply.github.com> Date: Wed, 31 Mar 2021 14:09:38 -0600 Subject: [PATCH 18/41] Possible DSQuery fix --- detections/endpoint/dsquery_domain_discovery.yml | 7 +------ tests/endpoint/dsquery_domain_discovery.test.yml | 2 +- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/detections/endpoint/dsquery_domain_discovery.yml b/detections/endpoint/dsquery_domain_discovery.yml index c279f98b38..620b51154f 100644 --- a/detections/endpoint/dsquery_domain_discovery.yml +++ b/detections/endpoint/dsquery_domain_discovery.yml @@ -7,15 +7,10 @@ type: batch datamodel: - Endpoint description: 'The following analytic identifies "dsquery.exe" execution with arguments looking for `TrustedDomain` query directly on the command-line. This is typically indicative of an Administrator or adversary perform domain trust discovery. Note that this query does not identify any other variations of "Dsquery.exe" usage.\ - Within this detection, it is assumed `dsquery.exe` is not moved or renamed.\ - The search will return the first time and last time these command-line arguments were used for these executions, as well as the target system, the user, process "dsquery.exe" and its parent process.\ - - DSQuery.exe is natively found in `C:\Windows\system32` and `C:\Windows\syswow64` and only on Server operation systems.\ - + DSQuery.exe is natively found in `C:\Windows\system32` and `C:\Windows\syswow64` and only on Server operating systems.\ The following DLL(s) are loaded when DSQuery.exe is launched `dsquery.dll`. If found loaded by another process, it is possible dsquery is running within that process context in memory.\ - In addition to trust discovery, review parallel processes for additional behaviors performed. Identify the parent process and capture any files (batch files, for example) being used.' search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Processes where Processes.process_name=dsquery.exe diff --git a/tests/endpoint/dsquery_domain_discovery.test.yml b/tests/endpoint/dsquery_domain_discovery.test.yml index 4d43b04ae4..b863b03676 100644 --- a/tests/endpoint/dsquery_domain_discovery.test.yml +++ b/tests/endpoint/dsquery_domain_discovery.test.yml @@ -1,7 +1,7 @@ name: DSQuery Domain Discovery Unit Test tests: - name: DSQuery Domain Discovery - file: detections/endpoint/dsquery_domain_discovery.yml + file: endpoint/dsquery_domain_discovery.yml pass_condition: '| stats count | where count > 0' earliest_time: '-24h' latest_time: 'now' From 77d618a37b66d7f70c24238b8715812e8ed591b9 Mon Sep 17 00:00:00 2001 From: mhaag-spl <5632822+MHaggis@users.noreply.github.com> Date: Wed, 31 Mar 2021 14:18:05 -0600 Subject: [PATCH 19/41] Found error - typo --- detections/endpoint/dsquery_domain_discovery.yml | 6 +++++- stories/domain_trust_discovery.yml | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/detections/endpoint/dsquery_domain_discovery.yml b/detections/endpoint/dsquery_domain_discovery.yml index 620b51154f..43e3c794be 100644 --- a/detections/endpoint/dsquery_domain_discovery.yml +++ b/detections/endpoint/dsquery_domain_discovery.yml @@ -8,9 +8,13 @@ datamodel: - Endpoint description: 'The following analytic identifies "dsquery.exe" execution with arguments looking for `TrustedDomain` query directly on the command-line. This is typically indicative of an Administrator or adversary perform domain trust discovery. Note that this query does not identify any other variations of "Dsquery.exe" usage.\ Within this detection, it is assumed `dsquery.exe` is not moved or renamed.\ + The search will return the first time and last time these command-line arguments were used for these executions, as well as the target system, the user, process "dsquery.exe" and its parent process.\ - DSQuery.exe is natively found in `C:\Windows\system32` and `C:\Windows\syswow64` and only on Server operating systems.\ + + DSQuery.exe is natively found in `C:\Windows\system32` and `C:\Windows\syswow64` and only on Server operating system.\ + The following DLL(s) are loaded when DSQuery.exe is launched `dsquery.dll`. If found loaded by another process, it is possible dsquery is running within that process context in memory.\ + In addition to trust discovery, review parallel processes for additional behaviors performed. Identify the parent process and capture any files (batch files, for example) being used.' search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Processes where Processes.process_name=dsquery.exe diff --git a/stories/domain_trust_discovery.yml b/stories/domain_trust_discovery.yml index 35565cf124..23ac481d24 100644 --- a/stories/domain_trust_discovery.yml +++ b/stories/domain_trust_discovery.yml @@ -1,4 +1,4 @@ -pyhonname: Domain Trust Discovery +name: Domain Trust Discovery id: e6f30f14-8daf-11eb-a017-acde48001122 version: 1 date: '2021-03-25' From 9140952d065fc393bdd0d4cebe71b3d6b2195c42 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 31 Mar 2021 20:51:45 +0000 Subject: [PATCH 20/41] Added detection testing service results inDSQuery Domain Discovery --- .../endpoint/dsquery_domain_discovery.yml | 39 ++++++++++++------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/detections/endpoint/dsquery_domain_discovery.yml b/detections/endpoint/dsquery_domain_discovery.yml index 43e3c794be..1b9185bb19 100644 --- a/detections/endpoint/dsquery_domain_discovery.yml +++ b/detections/endpoint/dsquery_domain_discovery.yml @@ -6,28 +6,37 @@ author: Michael Haag, Splunk type: batch datamodel: - Endpoint -description: 'The following analytic identifies "dsquery.exe" execution with arguments looking for `TrustedDomain` query directly on the command-line. This is typically indicative of an Administrator or adversary perform domain trust discovery. Note that this query does not identify any other variations of "Dsquery.exe" usage.\ +description: 'The following analytic identifies "dsquery.exe" execution with arguments + looking for `TrustedDomain` query directly on the command-line. This is typically + indicative of an Administrator or adversary perform domain trust discovery. Note + that this query does not identify any other variations of "Dsquery.exe" usage.\ Within this detection, it is assumed `dsquery.exe` is not moved or renamed.\ - The search will return the first time and last time these command-line arguments were used for these executions, as well as the target system, the user, process "dsquery.exe" and its parent process.\ + The search will return the first time and last time these command-line arguments + were used for these executions, as well as the target system, the user, process + "dsquery.exe" and its parent process.\ - DSQuery.exe is natively found in `C:\Windows\system32` and `C:\Windows\syswow64` and only on Server operating system.\ + DSQuery.exe is natively found in `C:\Windows\system32` and `C:\Windows\syswow64` + and only on Server operating system.\ - The following DLL(s) are loaded when DSQuery.exe is launched `dsquery.dll`. If found loaded by another process, it is possible dsquery is running within that process context in memory.\ - - In addition to trust discovery, review parallel processes for additional behaviors performed. Identify the parent process and capture any files (batch files, for example) being used.' + The following DLL(s) are loaded when DSQuery.exe is launched `dsquery.dll`. If found + loaded by another process, it is possible dsquery is running within that process + context in memory.\ + + In addition to trust discovery, review parallel processes for additional behaviors + performed. Identify the parent process and capture any files (batch files, for example) + being used.' search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Processes where Processes.process_name=dsquery.exe - Processes.process=*trustedDomain* by Processes.dest Processes.user Processes.parent_process Processes.process_name - Processes.process Processes.process_id Processes.parent_process_id - | `drop_dm_object_name(Processes)` - | `security_content_ctime(firstTime)` - | `security_content_ctime(lastTime)` + Processes.process=*trustedDomain* by Processes.dest Processes.user Processes.parent_process + Processes.process_name Processes.process Processes.process_id Processes.parent_process_id + | `drop_dm_object_name(Processes)` | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `dsquery_domain_discovery_filter`' how_to_implement: To successfully implement this search you need to be ingesting information on process that include the name of the process responsible for the changes from your endpoints into the `Endpoint` datamodel in the `Processes` node. -known_false_positives: Limited false positives. If there is a true false positive, filter based on command-line or parent process. +known_false_positives: Limited false positives. If there is a true false positive, + filter based on command-line or parent process. references: - https://github.com/redcanaryco/atomic-red-team/blob/master/atomics/T1482/T1482.md - http://www.harmj0y.net/blog/redteaming/a-guide-to-attacking-domain-trusts/ @@ -36,7 +45,8 @@ references: tags: analytic_story: - Domain Trust Discovery - dataset: [] + dataset: + - https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1482/atomic_red_team/windows-sysmon.log kill_chain_phases: - Exploitation mitre_attack_id: @@ -51,4 +61,5 @@ tags: - Processes.process - Processes.user - Processes.dest - security_domain: endpoint \ No newline at end of file + security_domain: endpoint + automated_detection_testing: passed From 6d17629359d4196e82a97cc63b816148afbdfb57 Mon Sep 17 00:00:00 2001 From: mhaag-spl <5632822+MHaggis@users.noreply.github.com> Date: Wed, 31 Mar 2021 15:01:44 -0600 Subject: [PATCH 21/41] Text updates --- detections/endpoint/bits_job_persistence.yml | 3 ++- detections/endpoint/dsquery_domain_discovery.yml | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/detections/endpoint/bits_job_persistence.yml b/detections/endpoint/bits_job_persistence.yml index 55f10e87f9..f0444e99c5 100644 --- a/detections/endpoint/bits_job_persistence.yml +++ b/detections/endpoint/bits_job_persistence.yml @@ -9,7 +9,8 @@ datamodel: description: The following query identifies Microsoft Background Intelligent Transfer Service utility `bitsadmin.exe` scheduling a BITS job to persist on an endpoint. The query identifies the parameters used to create, resume or add a file to a BITS - job + job. Typically seen combined in a oneliner or ran in sequence. If identified, review the BITS job created and capture any files written to disk. It is possible for BITS to be used to upload files and this may require further network data analysis to identify. You can use `bitsadmin /list /verbose` + to list out the jobs during investigation. search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Processes where Processes.process_name=bitsadmin.exe Processes.process IN (*create*, *addfile*, *setnotifyflags*, *setnotifycmdline*, diff --git a/detections/endpoint/dsquery_domain_discovery.yml b/detections/endpoint/dsquery_domain_discovery.yml index 1b9185bb19..8e3d815a65 100644 --- a/detections/endpoint/dsquery_domain_discovery.yml +++ b/detections/endpoint/dsquery_domain_discovery.yml @@ -10,6 +10,7 @@ description: 'The following analytic identifies "dsquery.exe" execution with arg looking for `TrustedDomain` query directly on the command-line. This is typically indicative of an Administrator or adversary perform domain trust discovery. Note that this query does not identify any other variations of "Dsquery.exe" usage.\ + Within this detection, it is assumed `dsquery.exe` is not moved or renamed.\ The search will return the first time and last time these command-line arguments From d35bf6fdaba25f0498d3d7eed5d1e8a8a01e4f32 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 1 Apr 2021 07:59:18 +0000 Subject: [PATCH 22/41] Added detection testing service results inDisable Windows SmartScreen Protection --- ...disable_windows_smartscreen_protection.yml | 33 +++++++++++-------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/detections/endpoint/disable_windows_smartscreen_protection.yml b/detections/endpoint/disable_windows_smartscreen_protection.yml index 3e85bdad5a..b127627da0 100644 --- a/detections/endpoint/disable_windows_smartscreen_protection.yml +++ b/detections/endpoint/disable_windows_smartscreen_protection.yml @@ -6,26 +6,26 @@ author: Teoderick Contreras, Splunk type: batch datamodel: - Endpoint -description: The following search identifies a modification of registry to disable the smartscreen protection of windows machine. - This is windows feature provide an early warning system against website that might engage in phishing attack or malware distribution. - This modification are seen in RAT malware to cover their tracks upon downloading other of its component or other payload. -search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Registry - where Registry.registry_path= "*HKLM\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Explorer\\SmartScreenEnabled" Registry.registry_value_name = "Off" - by Registry.registry_path Registry.registry_key_name Registry.registry_value_name Registry.dest - | `drop_dm_object_name(Registry)` - | `security_content_ctime(firstTime)` - |`security_content_ctime(lastTime)` - | `disable_windows_smartscreen_protection_filter`' +description: The following search identifies a modification of registry to disable + the smartscreen protection of windows machine. This is windows feature provide an + early warning system against website that might engage in phishing attack or malware + distribution. This modification are seen in RAT malware to cover their tracks upon + downloading other of its component or other payload. +search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) + as lastTime from datamodel=Endpoint.Registry where Registry.registry_path= "*HKLM\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Explorer\\SmartScreenEnabled" Registry.registry_value_name + = "Off" by Registry.registry_path Registry.registry_key_name Registry.registry_value_name + Registry.dest | `drop_dm_object_name(Registry)` | `security_content_ctime(firstTime)` + |`security_content_ctime(lastTime)` | `disable_windows_smartscreen_protection_filter`' how_to_implement: To successfully implement this search you need to be ingesting information on process that include the name of the process responsible for the changes from - your endpoints into the `Endpoint` datamodel in the `Registry` node. Also make sure that this registry was included in your config - files ex. sysmon config to be monitored. + your endpoints into the `Endpoint` datamodel in the `Registry` node. Also make sure + that this registry was included in your config files ex. sysmon config to be monitored. known_false_positives: admin or user may choose to disable this windows features. references: - https://tccontre.blogspot.com/2020/01/remcos-rat-evading-windows-defender-av.html tags: analytic_story: - - Windows Defense Evasion Tactics + - Windows Defense Evasion Tactics kill_chain_phases: - Exploitation mitre_attack_id: @@ -41,4 +41,9 @@ tags: - Registry.user - Registry.dest - Registry.registry_value_nam - security_domain: endpoint \ No newline at end of file + security_domain: endpoint + automated_detection_testing: passed + dataset: + - https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-security.log + - https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-system.log + - https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-sysmon.log From 7e29850f481ea8189b69c938cb2cb869c16e288f Mon Sep 17 00:00:00 2001 From: root Date: Thu, 1 Apr 2021 08:27:41 +0000 Subject: [PATCH 23/41] Added detection testing service results inDisabling CMD Application --- .../endpoint/disabling_cmd_application.yml | 32 +++++++++++-------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/detections/endpoint/disabling_cmd_application.yml b/detections/endpoint/disabling_cmd_application.yml index 65955f97ae..60d544c9bb 100644 --- a/detections/endpoint/disabling_cmd_application.yml +++ b/detections/endpoint/disabling_cmd_application.yml @@ -6,26 +6,25 @@ author: Teoderick Contreras, Splunk type: batch datamodel: - Endpoint -description: this search is to identify modification in registry to disable cmd prompt application. - This technique is commonly seen in RAT, Trojan or WORM to prevent triaging or deleting there samples through cmd application - which is one of the tool of analyst to traverse on directory and files. -search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Registry - where Registry.registry_path= "*\\SOFTWARE\\Policies\\Microsoft\\Windows\\System\\DisableCMD" Registry.registry_value_name = "DWORD (0x00000001)" - by Registry.registry_path Registry.registry_key_name Registry.registry_value_name Registry.dest - | `drop_dm_object_name(Registry)` - | `security_content_ctime(firstTime)` - |`security_content_ctime(lastTime)` - | `disabling_cmd_application_filter`' +description: this search is to identify modification in registry to disable cmd prompt + application. This technique is commonly seen in RAT, Trojan or WORM to prevent triaging + or deleting there samples through cmd application which is one of the tool of analyst + to traverse on directory and files. +search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) + as lastTime from datamodel=Endpoint.Registry where Registry.registry_path= "*\\SOFTWARE\\Policies\\Microsoft\\Windows\\System\\DisableCMD" + Registry.registry_value_name = "DWORD (0x00000001)" by Registry.registry_path Registry.registry_key_name + Registry.registry_value_name Registry.dest | `drop_dm_object_name(Registry)` | `security_content_ctime(firstTime)` + |`security_content_ctime(lastTime)` | `disabling_cmd_application_filter`' how_to_implement: To successfully implement this search you need to be ingesting information on process that include the name of the process responsible for the changes from - your endpoints into the `Endpoint` datamodel in the `Registry` node. Also make sure that this registry was included in your config - files ex. sysmon config to be monitored. + your endpoints into the `Endpoint` datamodel in the `Registry` node. Also make sure + that this registry was included in your config files ex. sysmon config to be monitored. known_false_positives: admin may disable this application for non technical user. references: - https://any.run/report/ea4ea08407d4ee72e009103a3b77e5a09412b722fdef67315ea63f22011152af/a866d7b1-c236-4f26-a391-5ae32213dfc4#registry tags: analytic_story: - - Windows Defense Evasion Tactics + - Windows Defense Evasion Tactics kill_chain_phases: - Exploitation mitre_attack_id: @@ -41,4 +40,9 @@ tags: - Registry.user - Registry.dest - Registry.registry_value_name - security_domain: endpoint \ No newline at end of file + security_domain: endpoint + automated_detection_testing: passed + dataset: + - https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-security.log + - https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-system.log + - https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-sysmon.log From e0cdb5bb1c6054883db739fcdf26701f0635f870 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 1 Apr 2021 08:46:08 +0000 Subject: [PATCH 24/41] Added detection testing service results inDisabling ControlPanel --- .../endpoint/disabling_controlpanel.yml | 30 +++++++++++-------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/detections/endpoint/disabling_controlpanel.yml b/detections/endpoint/disabling_controlpanel.yml index 2c6709eea6..f4c03a35b8 100644 --- a/detections/endpoint/disabling_controlpanel.yml +++ b/detections/endpoint/disabling_controlpanel.yml @@ -6,25 +6,24 @@ author: Teoderick Contreras, Splunk type: batch datamodel: - Endpoint -description: this search is to identify registry modification to disable control panel window. - This technique is commonly seen in malware to prevent their artifacts , persistence removed on the infected machine. -search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Registry - where Registry.registry_path= "*\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Policies\\Explorer\\NoControlPanel" - Registry.registry_value_name = "DWORD (0x00000001)" by Registry.registry_path Registry.registry_key_name Registry.registry_value_name Registry.dest - | `drop_dm_object_name(Registry)` - | `security_content_ctime(firstTime)` - | `security_content_ctime(lastTime)` - | `disabling_controlpanel_filter`' +description: this search is to identify registry modification to disable control panel + window. This technique is commonly seen in malware to prevent their artifacts , + persistence removed on the infected machine. +search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) + as lastTime from datamodel=Endpoint.Registry where Registry.registry_path= "*\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Policies\\Explorer\\NoControlPanel" + Registry.registry_value_name = "DWORD (0x00000001)" by Registry.registry_path Registry.registry_key_name + Registry.registry_value_name Registry.dest | `drop_dm_object_name(Registry)` | `security_content_ctime(firstTime)` + | `security_content_ctime(lastTime)` | `disabling_controlpanel_filter`' how_to_implement: To successfully implement this search you need to be ingesting information on process that include the name of the process responsible for the changes from - your endpoints into the `Endpoint` datamodel in the `Registry` node. Also make sure that this registry was included in your config - files ex. sysmon config to be monitored. + your endpoints into the `Endpoint` datamodel in the `Registry` node. Also make sure + that this registry was included in your config files ex. sysmon config to be monitored. known_false_positives: admin may disable this application for non technical user. references: - https://any.run/report/ea4ea08407d4ee72e009103a3b77e5a09412b722fdef67315ea63f22011152af/a866d7b1-c236-4f26-a391-5ae32213dfc4#registry tags: analytic_story: - - Windows Defense Evasion Tactics + - Windows Defense Evasion Tactics kill_chain_phases: - Exploitation mitre_attack_id: @@ -40,4 +39,9 @@ tags: - Registry.user - Registry.dest - Registry.registry_value_name - security_domain: endpoint \ No newline at end of file + security_domain: endpoint + automated_detection_testing: passed + dataset: + - https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-security.log + - https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-system.log + - https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-sysmon.log From d294ac965c42c38bb9c293666983515c946b8bd3 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 1 Apr 2021 09:11:18 +0000 Subject: [PATCH 25/41] Added detection testing service results inDisabling Firewall with Netsh --- .../disabling_firewall_with_netsh.yml | 40 +++++++++++-------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/detections/endpoint/disabling_firewall_with_netsh.yml b/detections/endpoint/disabling_firewall_with_netsh.yml index 36ef1d5139..03fff72233 100644 --- a/detections/endpoint/disabling_firewall_with_netsh.yml +++ b/detections/endpoint/disabling_firewall_with_netsh.yml @@ -6,20 +6,21 @@ author: Teoderick Contreras, Splunk type: batch datamodel: - Endpoint -description: This search is to identifies suspicious firewall disabling using netsh application. - this technique is commonly seen in malware that tries to communicate or download its component or other payload to its C2 server. -search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Processes - where Processes.process_name=netsh.exe Processes.process= "*firewall*" (Processes.process= "*off*" OR Processes.process= "*disable*") - by Processes.dest Processes.user Processes.parent_process Processes.process_name Processes.process Processes.process_id Processes.parent_process_id - | `drop_dm_object_name(Processes)` - | `security_content_ctime(firstTime)` - | `security_content_ctime(lastTime)` - | `disabling_firewall_with_netsh_filter`' +description: This search is to identifies suspicious firewall disabling using netsh + application. this technique is commonly seen in malware that tries to communicate + or download its component or other payload to its C2 server. +search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) + as lastTime from datamodel=Endpoint.Processes where Processes.process_name=netsh.exe + Processes.process= "*firewall*" (Processes.process= "*off*" OR Processes.process= + "*disable*") by Processes.dest Processes.user Processes.parent_process Processes.process_name + Processes.process Processes.process_id Processes.parent_process_id | `drop_dm_object_name(Processes)` + | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `disabling_firewall_with_netsh_filter`' how_to_implement: You must be ingesting data that records the filesystem activity from your hosts to populate the Endpoint file-system data model node. If you are using Sysmon, you will need a Splunk Universal Forwarder on each endpoint from which you want to collect data. -known_false_positives: admin may disable firewall during testing or fixing network problem. +known_false_positives: admin may disable firewall during testing or fixing network + problem. references: - https://tccontre.blogspot.com/2020/01/remcos-rat-evading-windows-defender-av.htm tags: @@ -37,10 +38,15 @@ tags: - _time - Processes.process_name - Processes.process - - Processes.dest - - Processes.user - - Processes.parent_process - - Processes.process_name - - Processes.process_id - - Processes.parent_process_id - security_domain: endpoint \ No newline at end of file + - Processes.dest + - Processes.user + - Processes.parent_process + - Processes.process_name + - Processes.process_id + - Processes.parent_process_id + security_domain: endpoint + automated_detection_testing: passed + dataset: + - https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-security.log + - https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-system.log + - https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-sysmon.log From 031a8a99451160a7f87312dd96193cdcade567fa Mon Sep 17 00:00:00 2001 From: root Date: Thu, 1 Apr 2021 10:06:43 +0000 Subject: [PATCH 26/41] Added detection testing service results inDisabling FolderOptions Windows Feature --- ...isabling_folderoptions_windows_feature.yml | 33 +++++++++++-------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/detections/endpoint/disabling_folderoptions_windows_feature.yml b/detections/endpoint/disabling_folderoptions_windows_feature.yml index 17b3da5ab2..59adce28fc 100644 --- a/detections/endpoint/disabling_folderoptions_windows_feature.yml +++ b/detections/endpoint/disabling_folderoptions_windows_feature.yml @@ -6,26 +6,26 @@ author: Teoderick Contreras, Splunk type: batch datamodel: - Endpoint -description: This search is to identify registry modification to disable folder options feature of windows to show hidden files, file extension and etc. - This technique used by malware in combination if disabling show hidden files feature to hide their files and also to hide the file extension to lure the user - base on file icons or fake file extensions. -search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Registry - where Registry.registry_path= "*\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Policies\\Explorer\\NoFolderOptions" - Registry.registry_value_name = "DWORD (0x00000001)" by Registry.registry_path Registry.registry_key_name Registry.registry_value_name Registry.dest - | `drop_dm_object_name(Registry)` - | `security_content_ctime(firstTime)` - | `security_content_ctime(lastTime)` - | `disabling_folderoptions_windows_feature_filter`' +description: This search is to identify registry modification to disable folder options + feature of windows to show hidden files, file extension and etc. This technique + used by malware in combination if disabling show hidden files feature to hide their + files and also to hide the file extension to lure the user base on file icons or + fake file extensions. +search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) + as lastTime from datamodel=Endpoint.Registry where Registry.registry_path= "*\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Policies\\Explorer\\NoFolderOptions" + Registry.registry_value_name = "DWORD (0x00000001)" by Registry.registry_path Registry.registry_key_name + Registry.registry_value_name Registry.dest | `drop_dm_object_name(Registry)` | `security_content_ctime(firstTime)` + | `security_content_ctime(lastTime)` | `disabling_folderoptions_windows_feature_filter`' how_to_implement: To successfully implement this search you need to be ingesting information on process that include the name of the process responsible for the changes from - your endpoints into the `Endpoint` datamodel in the `Registry` node. Also make sure that this registry was included in your config - files ex. sysmon config to be monitored. + your endpoints into the `Endpoint` datamodel in the `Registry` node. Also make sure + that this registry was included in your config files ex. sysmon config to be monitored. known_false_positives: admin may disable this application for non technical user. references: - https://any.run/report/ea4ea08407d4ee72e009103a3b77e5a09412b722fdef67315ea63f22011152af/a866d7b1-c236-4f26-a391-5ae32213dfc4#registry tags: analytic_story: - - Windows Defense Evasion Tactics + - Windows Defense Evasion Tactics kill_chain_phases: - Exploitation mitre_attack_id: @@ -41,4 +41,9 @@ tags: - Registry.user - Registry.dest - Registry.registry_value_name - security_domain: endpoint \ No newline at end of file + security_domain: endpoint + automated_detection_testing: passed + dataset: + - https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-security.log + - https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-system.log + - https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-sysmon.log From 5ef291663669a4140bc61f4594f8cf131c07c663 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 1 Apr 2021 11:25:29 +0000 Subject: [PATCH 27/41] Added detection testing service results inDisabling SystemRestore In Registry --- .../disabling_systemrestore_in_registry.yml | 34 +++++++++++-------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/detections/endpoint/disabling_systemrestore_in_registry.yml b/detections/endpoint/disabling_systemrestore_in_registry.yml index eb6f008bf3..f0ed94188f 100644 --- a/detections/endpoint/disabling_systemrestore_in_registry.yml +++ b/detections/endpoint/disabling_systemrestore_in_registry.yml @@ -6,26 +6,27 @@ author: Teoderick Contreras, Splunk type: batch datamodel: - Endpoint -description: The following search identifies the modification of registry related in disabling the system restore of a machine. - This event or behavior are seen in some RAT malware to make the restore of the infected machine difficult and keep their infection on the box. -search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Registry - where Registry.registry_path= "*\\SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\SystemRestore\\DisableSR" - OR Registry.registry_path= "*\\SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\SystemRestore\\DisableConfig" - Registry.registry_value_name = "DWORD (0x00000001)" by Registry.registry_path Registry.registry_key_name Registry.registry_value_name Registry.dest - | `drop_dm_object_name(Registry)` - | `security_content_ctime(firstTime)` - |`security_content_ctime(lastTime)` - | `disabling_systemrestore_in_registry_filter`' +description: The following search identifies the modification of registry related + in disabling the system restore of a machine. This event or behavior are seen in + some RAT malware to make the restore of the infected machine difficult and keep + their infection on the box. +search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) + as lastTime from datamodel=Endpoint.Registry where Registry.registry_path= "*\\SOFTWARE\\Microsoft\\Windows + NT\\CurrentVersion\\SystemRestore\\DisableSR" OR Registry.registry_path= "*\\SOFTWARE\\Microsoft\\Windows + NT\\CurrentVersion\\SystemRestore\\DisableConfig" Registry.registry_value_name = + "DWORD (0x00000001)" by Registry.registry_path Registry.registry_key_name Registry.registry_value_name + Registry.dest | `drop_dm_object_name(Registry)` | `security_content_ctime(firstTime)` + |`security_content_ctime(lastTime)` | `disabling_systemrestore_in_registry_filter`' how_to_implement: To successfully implement this search you need to be ingesting information on process that include the name of the process responsible for the changes from - your endpoints into the `Endpoint` datamodel in the `Registry` node. Also make sure that this registry was included in your config - files ex. sysmon config to be monitored. + your endpoints into the `Endpoint` datamodel in the `Registry` node. Also make sure + that this registry was included in your config files ex. sysmon config to be monitored. known_false_positives: in some cases admin can disable systemrestore on a machine. references: - https://tccontre.blogspot.com/2020/01/remcos-rat-evading-windows-defender-av.html tags: analytic_story: - - Windows Defense Evasion Tactics + - Windows Defense Evasion Tactics kill_chain_phases: - Exploitation mitre_attack_id: @@ -41,4 +42,9 @@ tags: - Registry.user - Registry.dest - Registry.registry_value_name - security_domain: endpoint \ No newline at end of file + security_domain: endpoint + automated_detection_testing: passed + dataset: + - https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-security.log + - https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-system.log + - https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-sysmon.log From b0173344e3c1a1938955706b938448d520428206 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 1 Apr 2021 11:58:27 +0000 Subject: [PATCH 28/41] Added detection testing service results inDisabling NoRun Windows App --- .../endpoint/disabling_norun_windows_app.yml | 33 +++++++++++-------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/detections/endpoint/disabling_norun_windows_app.yml b/detections/endpoint/disabling_norun_windows_app.yml index 50affd0447..e77ffbd10c 100644 --- a/detections/endpoint/disabling_norun_windows_app.yml +++ b/detections/endpoint/disabling_norun_windows_app.yml @@ -6,27 +6,27 @@ author: Teoderick Contreras, Splunk type: batch datamodel: - Endpoint -description: This search is to identify modification of registry to disable run application in window start menu. - this application is known to be a helpful shortcut to windows OS user to run known application and also to execute some reg or batch script. - This technique is used malware to make cleaning of its infection more harder by preventing known application run easily through run shortcut. -search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Registry - where Registry.registry_path= "*\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Policies\\Explorer\\NoRun" - Registry.registry_value_name = "DWORD (0x00000001)" by Registry.registry_path Registry.registry_key_name Registry.registry_value_name Registry.dest - | `drop_dm_object_name(Registry)` - | `security_content_ctime(firstTime)` - | `security_content_ctime(lastTime)` - | `disabling_norun_windows_app_filter`' +description: This search is to identify modification of registry to disable run application + in window start menu. this application is known to be a helpful shortcut to windows + OS user to run known application and also to execute some reg or batch script. This + technique is used malware to make cleaning of its infection more harder by preventing + known application run easily through run shortcut. +search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) + as lastTime from datamodel=Endpoint.Registry where Registry.registry_path= "*\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Policies\\Explorer\\NoRun" + Registry.registry_value_name = "DWORD (0x00000001)" by Registry.registry_path Registry.registry_key_name + Registry.registry_value_name Registry.dest | `drop_dm_object_name(Registry)` | `security_content_ctime(firstTime)` + | `security_content_ctime(lastTime)` | `disabling_norun_windows_app_filter`' how_to_implement: To successfully implement this search you need to be ingesting information on process that include the name of the process responsible for the changes from - your endpoints into the `Endpoint` datamodel in the `Registry` node. Also make sure that this registry was included in your config - files ex. sysmon config to be monitored. + your endpoints into the `Endpoint` datamodel in the `Registry` node. Also make sure + that this registry was included in your config files ex. sysmon config to be monitored. known_false_positives: admin may disable this application for non technical user. references: - https://any.run/report/ea4ea08407d4ee72e009103a3b77e5a09412b722fdef67315ea63f22011152af/a866d7b1-c236-4f26-a391-5ae32213dfc4#registry - https://blog.malwarebytes.com/detections/pum-optional-norun/ tags: analytic_story: - - Windows Defense Evasion Tactics + - Windows Defense Evasion Tactics kill_chain_phases: - Exploitation mitre_attack_id: @@ -42,4 +42,9 @@ tags: - Registry.user - Registry.dest - Registry.registry_value_name - security_domain: endpoint \ No newline at end of file + security_domain: endpoint + automated_detection_testing: passed + dataset: + - https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-security.log + - https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-system.log + - https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-sysmon.log From fa1c0e2c627cc8930fdfab6916c3812c205fde43 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 1 Apr 2021 12:20:50 +0000 Subject: [PATCH 29/41] Added detection testing service results inDisabling Task Manager --- .../endpoint/disabling_task_manager.yml | 31 +++++++++++-------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/detections/endpoint/disabling_task_manager.yml b/detections/endpoint/disabling_task_manager.yml index ccae7213a1..f1ce08549d 100644 --- a/detections/endpoint/disabling_task_manager.yml +++ b/detections/endpoint/disabling_task_manager.yml @@ -6,26 +6,26 @@ author: Teoderick Contreras, Splunk type: batch datamodel: - Endpoint -description: This search is to identifies modification of registry to disable the task manager of windows operating system. - this event or technique are commonly seen in malware such as RAT, Trojan, TrojanSpy or worm to prevent the user to terminate their process. -search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Registry - where Registry.registry_path= "*\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Policies\\System\\DisableTaskMgr" - Registry.registry_value_name = "DWORD (0x00000001)" by Registry.registry_path Registry.registry_key_name Registry.registry_value_name Registry.dest - | `drop_dm_object_name(Registry)` - | `security_content_ctime(firstTime)` - | `security_content_ctime(lastTime)` - | `disabling_task_manager_filter`' +description: This search is to identifies modification of registry to disable the + task manager of windows operating system. this event or technique are commonly seen + in malware such as RAT, Trojan, TrojanSpy or worm to prevent the user to terminate + their process. +search: '| tstats `security_content_summariesonly` count min(_time) as firstTime max(_time) + as lastTime from datamodel=Endpoint.Registry where Registry.registry_path= "*\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Policies\\System\\DisableTaskMgr" + Registry.registry_value_name = "DWORD (0x00000001)" by Registry.registry_path Registry.registry_key_name + Registry.registry_value_name Registry.dest | `drop_dm_object_name(Registry)` | `security_content_ctime(firstTime)` + | `security_content_ctime(lastTime)` | `disabling_task_manager_filter`' how_to_implement: To successfully implement this search you need to be ingesting information on process that include the name of the process responsible for the changes from - your endpoints into the `Endpoint` datamodel in the `Registry` node. Also make sure that this registry was included in your config - files ex. sysmon config to be monitored. + your endpoints into the `Endpoint` datamodel in the `Registry` node. Also make sure + that this registry was included in your config files ex. sysmon config to be monitored. known_false_positives: admin may disable this application for non technical user. references: - https://any.run/report/ea4ea08407d4ee72e009103a3b77e5a09412b722fdef67315ea63f22011152af/a866d7b1-c236-4f26-a391-5ae32213dfc4#registry - https://blog.talosintelligence.com/2020/05/threat-roundup-0424-0501.html tags: analytic_story: - - Windows Defense Evasion Tactics + - Windows Defense Evasion Tactics kill_chain_phases: - Exploitation mitre_attack_id: @@ -41,4 +41,9 @@ tags: - Registry.user - Registry.dest - Registry.registry_value_name - security_domain: endpoint \ No newline at end of file + security_domain: endpoint + automated_detection_testing: passed + dataset: + - https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-security.log + - https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-system.log + - https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1562.001/win_app_defender_disabling/windows-sysmon.log From 131c5901165752bc0c26d738d3fc8abe1c698e20 Mon Sep 17 00:00:00 2001 From: research bot Date: Fri, 2 Apr 2021 17:16:34 +0000 Subject: [PATCH 30/41] updating docs and package bits [ci skip] --- docs/detections.md | 2 +- docs/detections.wiki | 4 +- docs/stories.md | 24 +- docs/stories.wiki | 26 +- package/README.md | 7 + package/README/essoc_story_detail.txt | 15 + package/README/essoc_summary.txt | 24 + package/README/essoc_usage_dashboard.txt | 51 + package/app.manifest | 46 + package/bin/dnstwist.py | 400 + package/bin/escu_contextualize.py | 1 + package/bin/escu_investigate.py | 1 + package/bin/runstory.py | 1 + package/bin/splunklib/__init__.py | 19 + package/bin/splunklib/binding.py | 1373 + package/bin/splunklib/client.py | 3718 + package/bin/splunklib/data.py | 258 + .../bin/splunklib/modularinput/__init__.py | 12 + .../bin/splunklib/modularinput/argument.py | 102 + package/bin/splunklib/modularinput/event.py | 107 + .../splunklib/modularinput/event_writer.py | 84 + .../modularinput/input_definition.py | 59 + package/bin/splunklib/modularinput/scheme.py | 84 + package/bin/splunklib/modularinput/script.py | 176 + package/bin/splunklib/modularinput/utils.py | 72 + .../modularinput/validation_definition.py | 83 + package/bin/splunklib/ordereddict.py | 128 + package/bin/splunklib/results.py | 288 + .../bin/splunklib/searchcommands/__init__.py | 155 + .../splunklib/searchcommands/decorators.py | 447 + .../splunklib/searchcommands/environment.py | 122 + .../searchcommands/eventing_command.py | 142 + .../searchcommands/external_search_command.py | 227 + .../searchcommands/generating_command.py | 320 + .../bin/splunklib/searchcommands/internals.py | 786 + .../searchcommands/reporting_command.py | 280 + .../searchcommands/search_command.py | 1101 + .../searchcommands/streaming_command.py | 188 + .../splunklib/searchcommands/validators.py | 384 + package/default/analytic_stories.conf | 1741 + package/default/analyticstories.conf | 5681 + package/default/app.conf | 30 + package/default/collections.conf | 55 + package/default/commands.conf | 10 + package/default/content-version.conf | 2 + package/default/data/ui/nav/default.xml | 7 + ...l_backup_logs_for_host___response_task.xml | 9 + ...tes_activity_by_src_ip___response_task.xml | 9 + ...ity_hub_alerts_by_dest___response_task.xml | 9 + ...ivities_by_accesskeyid___response_task.xml | 9 + ...user_activities_by_arn___response_task.xml | 9 + ...rk_acl_details_from_id___response_task.xml | 9 + ...details_via_resourceid___response_task.xml | 9 + ...details_via_bucketname___response_task.xml | 9 + ...tes_activity_by_src_ip___response_task.xml | 9 + ...aws_activity_from_city___response_task.xml | 9 + ..._activity_from_country___response_task.xml | 9 + ...tivity_from_ip_address___response_task.xml | 9 + ...s_activity_from_region___response_task.xml | 9 + ...ckup_logs_for_endpoint___response_task.xml | 9 + ...cate_logs_for_a_domain___response_task.xml | 9 + ...ver_history_for_a_host___response_task.xml | 9 + ..._get_dns_traffic_ratio___response_task.xml | 9 + ..._details_by_instanceid___response_task.xml | 9 + ...get_ec2_launch_details___response_task.xml | 9 + ...h_panel_get_email_info___response_task.xml | 9 + ...s_from_specific_sender___response_task.xml | 9 + ...rence_of_a_mac_address___response_task.xml | 9 + ...story_of_email_sources___response_task.xml | 9 + ...fications_for_endpoint___response_task.xml | 9 + ...modifications_for_user___response_task.xml | 9 + ...el_get_notable_history___response_task.xml | 9 + ...en_cobra_threat_actors___response_task.xml | 9 + ...et_parent_process_info___response_task.xml | 9 + ..._process_file_activity___response_task.xml | 9 + ...panel_get_process_info___response_task.xml | 9 + ...tion_for_port_activity___response_task.xml | 9 + ...le_for_the_dns_traffic___response_task.xml | 9 + ..._wmi_activity_for_host___response_task.xml | 9 + ...rmation_via_session_id___response_task.xml | 9 + ...vities_via_region_name___response_task.xml | 9 + ...tivities_by_user_field___response_task.xml | 9 + ..._multiple_destinations___response_task.xml | 9 + ...rk_traffic_from_src_ip___response_task.xml | 9 + ...e_okta_activity_by_app___response_task.xml | 9 + ...activity_by_ip_address___response_task.xml | 9 + ...pass_the_hash_attempts___response_task.xml | 9 + ...ss_the_ticket_attempts___response_task.xml | 9 + ...e_previous_unseen_user___response_task.xml | 9 + ...esktop_authentications___response_task.xml | 9 + ...strings_in_http_header___response_task.xml | 9 + ...ser_activities_in_okta___response_task.xml | 9 + ...ate_web_posts_from_src___response_task.xml | 9 + .../default/data/ui/views/escu_summary.xml | 242 + package/default/data/ui/views/escu_usage.xml | 152 + package/default/data/ui/views/feedback.xml | 13 + package/default/distsearch.conf | 5 + package/default/es_investigations.conf | 1484 + package/default/macros.conf | 1685 + package/default/restmap.conf | 6 + package/default/savedsearches.conf | 14681 +++ package/default/searchbnf.conf | 26 + package/default/transforms.conf | 254 + package/default/usage_searches.conf | 73 + package/default/use_case_library.conf | 5681 + package/default/workflow_actions.conf | 360 + package/lookups/aws_service_accounts.csv | 1 + .../baseline_blocked_outbound_connections.csv | 1 + package/lookups/brand_monitoring.csv | 1 + package/lookups/csc_lookup.csv | 21 + package/lookups/discovered_dns_records.csv | 1 + package/lookups/domains.csv | 1 + .../lookups/dynamic_dns_providers_default.csv | 91976 ++++++++++++++++ .../lookups/dynamic_dns_providers_local.csv | 1 + package/lookups/escu_search_id.csv | 1 + .../is_suspicious_file_extension_lookup.csv | 52 + package/lookups/is_windows_system_file.csv | 563 + package/lookups/legit_domains.csv | 20 + package/lookups/mitre_enrichment.csv | 440 + .../lookups/network_acl_activity_baseline.csv | 1 + ...eviously_seen_S3_access_from_remote_ip.csv | 1 + ...viously_seen_api_calls_from_user_roles.csv | 1 + ...iously_seen_aws_cross_account_activity.csv | 1 + .../lookups/previously_seen_aws_regions.csv | 1 + .../previously_seen_cmd_line_arguments.csv | 1 + ...viously_seen_ec2_modifications_by_user.csv | 1 + .../lookups/prohibited_apps_launching_cmd.csv | 17 + package/lookups/prohibited_processes.csv | 20 + package/lookups/prohibited_softwares.csv | 20 + package/lookups/ransomware_extensions.csv | 290 + package/lookups/ransomware_notes.csv | 61 + .../rare_process_allow_list_default.csv | 7 + .../lookups/rare_process_allow_list_local.csv | 1 + package/lookups/s3_deletion_baseline.csv | 1 + .../security_group_activity_baseline.csv | 1 + package/lookups/security_services.csv | 5 + package/lookups/suspicious_files.csv | 4 + .../lookups/uncommon_processes_default.csv | 9 + package/lookups/uncommon_processes_local.csv | 1 + package/metadata/default.meta | 23 + package/static/appIcon.png | Bin 0 -> 3658 bytes package/static/appIconAlt.png | Bin 0 -> 2656 bytes package/static/appIconAlt_2x.png | Bin 0 -> 7442 bytes package/static/appIcon_2x.png | Bin 0 -> 3657 bytes 144 files changed, 137446 insertions(+), 28 deletions(-) create mode 100644 package/README.md create mode 100644 package/README/essoc_story_detail.txt create mode 100644 package/README/essoc_summary.txt create mode 100644 package/README/essoc_usage_dashboard.txt create mode 100644 package/app.manifest create mode 100755 package/bin/dnstwist.py create mode 100644 package/bin/escu_contextualize.py create mode 100644 package/bin/escu_investigate.py create mode 100644 package/bin/runstory.py create mode 100755 package/bin/splunklib/__init__.py create mode 100755 package/bin/splunklib/binding.py create mode 100755 package/bin/splunklib/client.py create mode 100755 package/bin/splunklib/data.py create mode 100755 package/bin/splunklib/modularinput/__init__.py create mode 100755 package/bin/splunklib/modularinput/argument.py create mode 100755 package/bin/splunklib/modularinput/event.py create mode 100755 package/bin/splunklib/modularinput/event_writer.py create mode 100755 package/bin/splunklib/modularinput/input_definition.py create mode 100755 package/bin/splunklib/modularinput/scheme.py create mode 100755 package/bin/splunklib/modularinput/script.py create mode 100755 package/bin/splunklib/modularinput/utils.py create mode 100755 package/bin/splunklib/modularinput/validation_definition.py create mode 100755 package/bin/splunklib/ordereddict.py create mode 100755 package/bin/splunklib/results.py create mode 100755 package/bin/splunklib/searchcommands/__init__.py create mode 100755 package/bin/splunklib/searchcommands/decorators.py create mode 100755 package/bin/splunklib/searchcommands/environment.py create mode 100755 package/bin/splunklib/searchcommands/eventing_command.py create mode 100755 package/bin/splunklib/searchcommands/external_search_command.py create mode 100755 package/bin/splunklib/searchcommands/generating_command.py create mode 100755 package/bin/splunklib/searchcommands/internals.py create mode 100755 package/bin/splunklib/searchcommands/reporting_command.py create mode 100755 package/bin/splunklib/searchcommands/search_command.py create mode 100755 package/bin/splunklib/searchcommands/streaming_command.py create mode 100755 package/bin/splunklib/searchcommands/validators.py create mode 100644 package/default/analytic_stories.conf create mode 100644 package/default/analyticstories.conf create mode 100644 package/default/app.conf create mode 100644 package/default/collections.conf create mode 100644 package/default/commands.conf create mode 100644 package/default/content-version.conf create mode 100644 package/default/data/ui/nav/default.xml create mode 100644 package/default/data/ui/panels/workbench_panel_all_backup_logs_for_host___response_task.xml create mode 100644 package/default/data/ui/panels/workbench_panel_amazon_eks_kubernetes_activity_by_src_ip___response_task.xml create mode 100644 package/default/data/ui/panels/workbench_panel_aws_investigate_security_hub_alerts_by_dest___response_task.xml create mode 100644 package/default/data/ui/panels/workbench_panel_aws_investigate_user_activities_by_accesskeyid___response_task.xml create mode 100644 package/default/data/ui/panels/workbench_panel_aws_investigate_user_activities_by_arn___response_task.xml create mode 100644 package/default/data/ui/panels/workbench_panel_aws_network_acl_details_from_id___response_task.xml create mode 100644 package/default/data/ui/panels/workbench_panel_aws_network_interface_details_via_resourceid___response_task.xml create mode 100644 package/default/data/ui/panels/workbench_panel_aws_s3_bucket_details_via_bucketname___response_task.xml create mode 100644 package/default/data/ui/panels/workbench_panel_gcp_kubernetes_activity_by_src_ip___response_task.xml create mode 100644 package/default/data/ui/panels/workbench_panel_get_all_aws_activity_from_city___response_task.xml create mode 100644 package/default/data/ui/panels/workbench_panel_get_all_aws_activity_from_country___response_task.xml create mode 100644 package/default/data/ui/panels/workbench_panel_get_all_aws_activity_from_ip_address___response_task.xml create mode 100644 package/default/data/ui/panels/workbench_panel_get_all_aws_activity_from_region___response_task.xml create mode 100644 package/default/data/ui/panels/workbench_panel_get_backup_logs_for_endpoint___response_task.xml create mode 100644 package/default/data/ui/panels/workbench_panel_get_certificate_logs_for_a_domain___response_task.xml create mode 100644 package/default/data/ui/panels/workbench_panel_get_dns_server_history_for_a_host___response_task.xml create mode 100644 package/default/data/ui/panels/workbench_panel_get_dns_traffic_ratio___response_task.xml create mode 100644 package/default/data/ui/panels/workbench_panel_get_ec2_instance_details_by_instanceid___response_task.xml create mode 100644 package/default/data/ui/panels/workbench_panel_get_ec2_launch_details___response_task.xml create mode 100644 package/default/data/ui/panels/workbench_panel_get_email_info___response_task.xml create mode 100644 package/default/data/ui/panels/workbench_panel_get_emails_from_specific_sender___response_task.xml create mode 100644 package/default/data/ui/panels/workbench_panel_get_first_occurrence_and_last_occurrence_of_a_mac_address___response_task.xml create mode 100644 package/default/data/ui/panels/workbench_panel_get_history_of_email_sources___response_task.xml create mode 100644 package/default/data/ui/panels/workbench_panel_get_logon_rights_modifications_for_endpoint___response_task.xml create mode 100644 package/default/data/ui/panels/workbench_panel_get_logon_rights_modifications_for_user___response_task.xml create mode 100644 package/default/data/ui/panels/workbench_panel_get_notable_history___response_task.xml create mode 100644 package/default/data/ui/panels/workbench_panel_get_outbound_emails_to_hidden_cobra_threat_actors___response_task.xml create mode 100644 package/default/data/ui/panels/workbench_panel_get_parent_process_info___response_task.xml create mode 100644 package/default/data/ui/panels/workbench_panel_get_process_file_activity___response_task.xml create mode 100644 package/default/data/ui/panels/workbench_panel_get_process_info___response_task.xml create mode 100644 package/default/data/ui/panels/workbench_panel_get_process_information_for_port_activity___response_task.xml create mode 100644 package/default/data/ui/panels/workbench_panel_get_process_responsible_for_the_dns_traffic___response_task.xml create mode 100644 package/default/data/ui/panels/workbench_panel_get_sysmon_wmi_activity_for_host___response_task.xml create mode 100644 package/default/data/ui/panels/workbench_panel_get_web_session_information_via_session_id___response_task.xml create mode 100644 package/default/data/ui/panels/workbench_panel_investigate_aws_activities_via_region_name___response_task.xml create mode 100644 package/default/data/ui/panels/workbench_panel_investigate_aws_user_activities_by_user_field___response_task.xml create mode 100644 package/default/data/ui/panels/workbench_panel_investigate_failed_logins_for_multiple_destinations___response_task.xml create mode 100644 package/default/data/ui/panels/workbench_panel_investigate_network_traffic_from_src_ip___response_task.xml create mode 100644 package/default/data/ui/panels/workbench_panel_investigate_okta_activity_by_app___response_task.xml create mode 100644 package/default/data/ui/panels/workbench_panel_investigate_okta_activity_by_ip_address___response_task.xml create mode 100644 package/default/data/ui/panels/workbench_panel_investigate_pass_the_hash_attempts___response_task.xml create mode 100644 package/default/data/ui/panels/workbench_panel_investigate_pass_the_ticket_attempts___response_task.xml create mode 100644 package/default/data/ui/panels/workbench_panel_investigate_previous_unseen_user___response_task.xml create mode 100644 package/default/data/ui/panels/workbench_panel_investigate_successful_remote_desktop_authentications___response_task.xml create mode 100644 package/default/data/ui/panels/workbench_panel_investigate_suspicious_strings_in_http_header___response_task.xml create mode 100644 package/default/data/ui/panels/workbench_panel_investigate_user_activities_in_okta___response_task.xml create mode 100644 package/default/data/ui/panels/workbench_panel_investigate_web_posts_from_src___response_task.xml create mode 100644 package/default/data/ui/views/escu_summary.xml create mode 100644 package/default/data/ui/views/escu_usage.xml create mode 100644 package/default/data/ui/views/feedback.xml create mode 100644 package/default/distsearch.conf create mode 100644 package/default/es_investigations.conf create mode 100644 package/default/macros.conf create mode 100644 package/default/restmap.conf create mode 100644 package/default/savedsearches.conf create mode 100644 package/default/searchbnf.conf create mode 100644 package/default/transforms.conf create mode 100644 package/default/usage_searches.conf create mode 100644 package/default/use_case_library.conf create mode 100644 package/default/workflow_actions.conf create mode 100644 package/lookups/aws_service_accounts.csv create mode 100644 package/lookups/baseline_blocked_outbound_connections.csv create mode 100644 package/lookups/brand_monitoring.csv create mode 100644 package/lookups/csc_lookup.csv create mode 100644 package/lookups/discovered_dns_records.csv create mode 100644 package/lookups/domains.csv create mode 100644 package/lookups/dynamic_dns_providers_default.csv create mode 100644 package/lookups/dynamic_dns_providers_local.csv create mode 100644 package/lookups/escu_search_id.csv create mode 100644 package/lookups/is_suspicious_file_extension_lookup.csv create mode 100644 package/lookups/is_windows_system_file.csv create mode 100644 package/lookups/legit_domains.csv create mode 100644 package/lookups/mitre_enrichment.csv create mode 100644 package/lookups/network_acl_activity_baseline.csv create mode 100644 package/lookups/previously_seen_S3_access_from_remote_ip.csv create mode 100644 package/lookups/previously_seen_api_calls_from_user_roles.csv create mode 100644 package/lookups/previously_seen_aws_cross_account_activity.csv create mode 100644 package/lookups/previously_seen_aws_regions.csv create mode 100644 package/lookups/previously_seen_cmd_line_arguments.csv create mode 100644 package/lookups/previously_seen_ec2_modifications_by_user.csv create mode 100644 package/lookups/prohibited_apps_launching_cmd.csv create mode 100644 package/lookups/prohibited_processes.csv create mode 100644 package/lookups/prohibited_softwares.csv create mode 100644 package/lookups/ransomware_extensions.csv create mode 100644 package/lookups/ransomware_notes.csv create mode 100644 package/lookups/rare_process_allow_list_default.csv create mode 100644 package/lookups/rare_process_allow_list_local.csv create mode 100644 package/lookups/s3_deletion_baseline.csv create mode 100644 package/lookups/security_group_activity_baseline.csv create mode 100644 package/lookups/security_services.csv create mode 100644 package/lookups/suspicious_files.csv create mode 100644 package/lookups/uncommon_processes_default.csv create mode 100644 package/lookups/uncommon_processes_local.csv create mode 100644 package/metadata/default.meta create mode 100644 package/static/appIcon.png create mode 100644 package/static/appIconAlt.png create mode 100644 package/static/appIconAlt_2x.png create mode 100644 package/static/appIcon_2x.png diff --git a/docs/detections.md b/docs/detections.md index e3669fbc23..e71e6fc596 100644 --- a/docs/detections.md +++ b/docs/detections.md @@ -8190,7 +8190,7 @@ The following analytics are designed to identifies some CLOP ransomware variant #### Search ``` -| tstats `security_content_summariesonly` values(Processes.process) as cmdline values(Processes.parent_process_name) as parent_process values(Processes.process_name) count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Processes where Processes.process = "*runrun*" OR Processes.process = "*temp.dat*" by Processes.parent_process_name Processes.process_name Processes.process Processes.dest Processes.user Processes.process_id Processes.process_guid +| tstats `security_content_summariesonly` values(Processes.process) as cmdline values(Processes.parent_process_name) as parent_process values(Processes.process_name) count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Processes where Processes.process_name != "*temp.dat*" Processes.process = "*runrun*" OR Processes.process = "*temp.dat*" by Processes.parent_process_name Processes.process_name Processes.process Processes.dest Processes.user Processes.process_id Processes.process_guid | `drop_dm_object_name(Processes)` | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` diff --git a/docs/detections.wiki b/docs/detections.wiki index f9a66a7e2f..9a59c287b8 100644 --- a/docs/detections.wiki +++ b/docs/detections.wiki @@ -13106,7 +13106,7 @@ The following analytics are designed to identifies some CLOP ransomware variant ====Search==== -| tstats `security_content_summariesonly` values(Processes.process) as cmdline values(Processes.parent_process_name) as parent_process values(Processes.process_name) count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Processes where Processes.process = "*runrun*" OR Processes.process = "*temp.dat*" by Processes.parent_process_name Processes.process_name Processes.process Processes.dest Processes.user Processes.process_id Processes.process_guid +| tstats `security_content_summariesonly` values(Processes.process) as cmdline values(Processes.parent_process_name) as parent_process values(Processes.process_name) count min(_time) as firstTime max(_time) as lastTime from datamodel=Endpoint.Processes where Processes.process_name != "*temp.dat*" Processes.process = "*runrun*" OR Processes.process = "*temp.dat*" by Processes.parent_process_name Processes.process_name Processes.process Processes.dest Processes.user Processes.process_id Processes.process_guid | `drop_dm_object_name(Processes)` | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` @@ -31671,7 +31671,7 @@ There might be false positives associted with this detection since items like ar ''#############'' ''# Automatically generated by doc_gen.py in https://github.com/splunk/security_content'' -''# On Date: 2021-03-29 18:42:22.264486 UTC'' +''# On Date: 2021-04-02 17:10:21.330169 UTC'' ''# Author: Splunk Security Research'' ''# Contact: research@splunk.com'' ''#############'' diff --git a/docs/stories.md b/docs/stories.md index f5f0bab171..6a9447404c 100644 --- a/docs/stories.md +++ b/docs/stories.md @@ -3096,7 +3096,7 @@ _version_: 1 ### AWS Network ACL Activity Monitor your AWS network infrastructure for bad configurations and malicious activity. Investigative searches help you probe deeper, when the facts warrant it. -- **Product**: Splunk Enterprise, Splunk Enterprise Security, Splunk Cloud +- **Product**: Splunk Security Analytics for AWS, Splunk Enterprise, Splunk Enterprise Security, Splunk Cloud - **Datamodel**: - **ATT&CK**: [T1562.007](https://attack.mitre.org/techniques/T1562.007/) - **Last Updated**: 2018-05-21 @@ -3143,7 +3143,7 @@ _version_: 2 ### AWS Security Hub Alerts This story is focused around detecting Security Hub alerts generated from AWS -- **Product**: Splunk Enterprise, Splunk Enterprise Security, Splunk Cloud +- **Product**: Splunk Security Analytics for AWS, Splunk Enterprise, Splunk Enterprise Security, Splunk Cloud - **Datamodel**: - **ATT&CK**: - **Last Updated**: 2020-08-04 @@ -3267,7 +3267,7 @@ _version_: 1 ### Cloud Cryptomining Monitor your cloud compute instances for activities related to cryptojacking/cryptomining. New instances that originate from previously unseen regions, users who launch abnormally high numbers of instances, or compute instances started by previously unseen users are just a few examples of potentially malicious behavior. -- **Product**: Splunk Enterprise, Splunk Enterprise Security, Splunk Cloud +- **Product**: Splunk Security Analytics for AWS, Splunk Enterprise, Splunk Enterprise Security, Splunk Cloud - **Datamodel**: Change - **ATT&CK**: [T1078.004](https://attack.mitre.org/techniques/T1078.004/), [T1535](https://attack.mitre.org/techniques/T1535/) - **Last Updated**: 2019-10-02 @@ -3313,7 +3313,7 @@ _version_: 1 ### Cloud Federated Credential Abuse This analytical story addresses events that indicate abuse of cloud federated credentials. These credentials are usually extracted from endpoint desktop or servers specially those servers that provide federation services such as Windows Active Directory Federation Services. Identity Federation relies on objects such as Oauth2 tokens, cookies or SAML assertions in order to provide seamless access between cloud and perimeter environments. If these objects are either hijacked or forged then attackers will be able to pivot into victim's cloud environements. -- **Product**: Splunk Enterprise, Splunk Enterprise Security, Splunk Cloud +- **Product**: Splunk Security Analytics for AWS, Splunk Enterprise, Splunk Enterprise Security, Splunk Cloud - **Datamodel**: Endpoint - **ATT&CK**: [T1003.001](https://attack.mitre.org/techniques/T1003.001/), [T1078](https://attack.mitre.org/techniques/T1078/), [T1136.003](https://attack.mitre.org/techniques/T1136.003/), [T1546.012](https://attack.mitre.org/techniques/T1546.012/), [T1556](https://attack.mitre.org/techniques/T1556/) - **Last Updated**: 2021-01-26 @@ -3615,7 +3615,7 @@ _version_: 1 ### Office 365 Detections This story is focused around detecting Office 365 Attacks. -- **Product**: Splunk Enterprise, Splunk Enterprise Security, Splunk Cloud +- **Product**: Splunk Security Analytics for AWS, Splunk Enterprise, Splunk Enterprise Security, Splunk Cloud - **Datamodel**: - **ATT&CK**: [T1110](https://attack.mitre.org/techniques/T1110/), [T1110.001](https://attack.mitre.org/techniques/T1110.001/), [T1114](https://attack.mitre.org/techniques/T1114/), [T1114.002](https://attack.mitre.org/techniques/T1114.002/), [T1114.003](https://attack.mitre.org/techniques/T1114.003/), [T1136.003](https://attack.mitre.org/techniques/T1136.003/), [T1556](https://attack.mitre.org/techniques/T1556/), [T1562.007](https://attack.mitre.org/techniques/T1562.007/) - **Last Updated**: 2020-12-16 @@ -3733,7 +3733,7 @@ _version_: 1 ### Suspicious AWS Login Activities Monitor your AWS authentication events using your CloudTrail logs. Searches within this Analytic Story will help you stay aware of and investigate suspicious logins. -- **Product**: Splunk Enterprise, Splunk Enterprise Security, Splunk Cloud +- **Product**: Splunk Security Analytics for AWS, Splunk Enterprise, Splunk Enterprise Security, Splunk Cloud - **Datamodel**: Authentication - **ATT&CK**: [T1078.004](https://attack.mitre.org/techniques/T1078.004/), [T1535](https://attack.mitre.org/techniques/T1535/) - **Last Updated**: 2019-05-01 @@ -3777,7 +3777,7 @@ _version_: 1 ### Suspicious AWS S3 Activities Use the searches in this Analytic Story to monitor your AWS S3 buckets for evidence of anomalous activity and suspicious behaviors, such as detecting open S3 buckets and buckets being accessed from a new IP. The contextual and investigative searches will give you more information, when required. -- **Product**: Splunk Enterprise, Splunk Enterprise Security, Splunk Cloud +- **Product**: Splunk Security Analytics for AWS, Splunk Enterprise, Splunk Enterprise Security, Splunk Cloud - **Datamodel**: - **ATT&CK**: [T1530](https://attack.mitre.org/techniques/T1530/) - **Last Updated**: 2018-07-24 @@ -3860,7 +3860,7 @@ _version_: 1 ### Suspicious Cloud Authentication Activities Monitor your cloud authentication events. Searches within this Analytic Story leverage the recent cloud updates to the Authentication data model to help you stay aware of and investigate suspicious login activity. -- **Product**: Splunk Enterprise, Splunk Enterprise Security, Splunk Cloud +- **Product**: Splunk Security Analytics for AWS, Splunk Enterprise, Splunk Enterprise Security, Splunk Cloud - **Datamodel**: Authentication - **ATT&CK**: [T1535](https://attack.mitre.org/techniques/T1535/) - **Last Updated**: 2020-06-04 @@ -3908,7 +3908,7 @@ _version_: 1 ### Suspicious Cloud Instance Activities Monitor your cloud infrastructure provisioning activities for behaviors originating from unfamiliar or unusual locations. These behaviors may indicate that malicious activities are occurring somewhere within your cloud environment. -- **Product**: Splunk Enterprise, Splunk Enterprise Security, Splunk Cloud +- **Product**: Splunk Security Analytics for AWS, Splunk Enterprise, Splunk Enterprise Security, Splunk Cloud - **Datamodel**: Change - **ATT&CK**: [T1078.004](https://attack.mitre.org/techniques/T1078.004/) - **Last Updated**: 2020-08-25 @@ -3949,7 +3949,7 @@ _version_: 1 ### Suspicious Cloud Provisioning Activities Monitor your cloud infrastructure provisioning activities for behaviors originating from unfamiliar or unusual locations. These behaviors may indicate that malicious activities are occurring somewhere within your cloud environment. -- **Product**: Splunk Enterprise, Splunk Enterprise Security, Splunk Cloud +- **Product**: Splunk Security Analytics for AWS, Splunk Enterprise, Splunk Enterprise Security, Splunk Cloud - **Datamodel**: Change - **ATT&CK**: [T1078](https://attack.mitre.org/techniques/T1078/) - **Last Updated**: 2018-08-20 @@ -3990,7 +3990,7 @@ _version_: 1 ### Suspicious Cloud User Activities Detect and investigate suspicious activities by users and roles in your cloud environments. -- **Product**: Splunk Enterprise, Splunk Enterprise Security, Splunk Cloud +- **Product**: Splunk Security Analytics for AWS, Splunk Security Analytics for AWS, Splunk Enterprise, Splunk Enterprise Security, Splunk Cloud - **Datamodel**: Change - **ATT&CK**: [T1078](https://attack.mitre.org/techniques/T1078/), [T1078.004](https://attack.mitre.org/techniques/T1078.004/) - **Last Updated**: 2020-09-04 @@ -4658,7 +4658,7 @@ _version_: 1 ### Ransomware Cloud Leverage searches that allow you to detect and investigate unusual activities that might relate to ransomware. These searches include cloud related objects that may be targeted by malicious actors via cloud providers own encryption features. -- **Product**: Splunk Enterprise, Splunk Enterprise Security, Splunk Cloud +- **Product**: Splunk Security Analytics for AWS, Splunk Enterprise, Splunk Enterprise Security, Splunk Cloud - **Datamodel**: - **ATT&CK**: [T1486](https://attack.mitre.org/techniques/T1486/) - **Last Updated**: 2020-10-27 diff --git a/docs/stories.wiki b/docs/stories.wiki index c5e92f7821..6df32b70ff 100644 --- a/docs/stories.wiki +++ b/docs/stories.wiki @@ -4081,7 +4081,7 @@ This analytic story contains detections that query your AWS Cloudtrail for activ ===Aws network acl activity=== Monitor your AWS network infrastructure for bad configurations and malicious activity. Investigative searches help you probe deeper, when the facts warrant it. -* '''Product''': Splunk Enterprise, Splunk Enterprise Security, Splunk Cloud +* '''Product''': Splunk Security Analytics for AWS, Splunk Enterprise, Splunk Enterprise Security, Splunk Cloud * '''Datamodel''': * '''ATT&CK''': [https://attack.mitre.org/techniques/T1562.007/ T1562.007] * '''Last Updated''': 2018-05-21 @@ -4136,7 +4136,7 @@ Monitor your AWS network infrastructure for bad configurations and malicious act ===Aws security hub alerts=== This story is focused around detecting Security Hub alerts generated from AWS -* '''Product''': Splunk Enterprise, Splunk Enterprise Security, Splunk Cloud +* '''Product''': Splunk Security Analytics for AWS, Splunk Enterprise, Splunk Enterprise Security, Splunk Cloud * '''Datamodel''': * '''ATT&CK''': * '''Last Updated''': 2020-08-04 @@ -4274,7 +4274,7 @@ Detect and investigate dormant user accounts for your AWS environment that have ===Cloud cryptomining=== Monitor your cloud compute instances for activities related to cryptojacking/cryptomining. New instances that originate from previously unseen regions, users who launch abnormally high numbers of instances, or compute instances started by previously unseen users are just a few examples of potentially malicious behavior. -* '''Product''': Splunk Enterprise, Splunk Enterprise Security, Splunk Cloud +* '''Product''': Splunk Security Analytics for AWS, Splunk Enterprise, Splunk Enterprise Security, Splunk Cloud * '''Datamodel''': Change * '''ATT&CK''': [https://attack.mitre.org/techniques/T1078.004/ T1078.004], [https://attack.mitre.org/techniques/T1535/ T1535] * '''Last Updated''': 2019-10-02 @@ -4331,7 +4331,7 @@ Monitor your cloud compute instances for activities related to cryptojacking/cry ===Cloud federated credential abuse=== This analytical story addresses events that indicate abuse of cloud federated credentials. These credentials are usually extracted from endpoint desktop or servers specially those servers that provide federation services such as Windows Active Directory Federation Services. Identity Federation relies on objects such as Oauth2 tokens, cookies or SAML assertions in order to provide seamless access between cloud and perimeter environments. If these objects are either hijacked or forged then attackers will be able to pivot into victim's cloud environements. -* '''Product''': Splunk Enterprise, Splunk Enterprise Security, Splunk Cloud +* '''Product''': Splunk Security Analytics for AWS, Splunk Enterprise, Splunk Enterprise Security, Splunk Cloud * '''Datamodel''': Endpoint * '''ATT&CK''': [https://attack.mitre.org/techniques/T1078/ T1078], [https://attack.mitre.org/techniques/T1003.001/ T1003.001], [https://attack.mitre.org/techniques/T1136.003/ T1136.003], [https://attack.mitre.org/techniques/T1556/ T1556], [https://attack.mitre.org/techniques/T1546.012/ T1546.012] * '''Last Updated''': 2021-01-26 @@ -4673,7 +4673,7 @@ This story addresses detection and response around Sensitive Role usage within a ===Office 365 detections=== This story is focused around detecting Office 365 Attacks. -* '''Product''': Splunk Enterprise, Splunk Enterprise Security, Splunk Cloud +* '''Product''': Splunk Security Analytics for AWS, Splunk Enterprise, Splunk Enterprise Security, Splunk Cloud * '''Datamodel''': * '''ATT&CK''': [https://attack.mitre.org/techniques/T1110.001/ T1110.001], [https://attack.mitre.org/techniques/T1136.003/ T1136.003], [https://attack.mitre.org/techniques/T1562.007/ T1562.007], [https://attack.mitre.org/techniques/T1556/ T1556], [https://attack.mitre.org/techniques/T1110/ T1110], [https://attack.mitre.org/techniques/T1114/ T1114], [https://attack.mitre.org/techniques/T1114.003/ T1114.003], [https://attack.mitre.org/techniques/T1114.002/ T1114.002] * '''Last Updated''': 2020-12-16 @@ -4831,7 +4831,7 @@ Use the searches in this Analytic Story to monitor your AWS EC2 instances for ev ===Suspicious aws login activities=== Monitor your AWS authentication events using your CloudTrail logs. Searches within this Analytic Story will help you stay aware of and investigate suspicious logins. -* '''Product''': Splunk Enterprise, Splunk Enterprise Security, Splunk Cloud +* '''Product''': Splunk Security Analytics for AWS, Splunk Enterprise, Splunk Enterprise Security, Splunk Cloud * '''Datamodel''': Authentication * '''ATT&CK''': [https://attack.mitre.org/techniques/T1535/ T1535], [https://attack.mitre.org/techniques/T1078.004/ T1078.004] * '''Last Updated''': 2019-05-01 @@ -4886,7 +4886,7 @@ Monitor your AWS authentication events using your CloudTrail logs. Searches with ===Suspicious aws s3 activities=== Use the searches in this Analytic Story to monitor your AWS S3 buckets for evidence of anomalous activity and suspicious behaviors, such as detecting open S3 buckets and buckets being accessed from a new IP. The contextual and investigative searches will give you more information, when required. -* '''Product''': Splunk Enterprise, Splunk Enterprise Security, Splunk Cloud +* '''Product''': Splunk Security Analytics for AWS, Splunk Enterprise, Splunk Enterprise Security, Splunk Cloud * '''Datamodel''': * '''ATT&CK''': [https://attack.mitre.org/techniques/T1530/ T1530] * '''Last Updated''': 2018-07-24 @@ -4975,7 +4975,7 @@ Leverage these searches to monitor your AWS network traffic for evidence of anom ===Suspicious cloud authentication activities=== Monitor your cloud authentication events. Searches within this Analytic Story leverage the recent cloud updates to the Authentication data model to help you stay aware of and investigate suspicious login activity. -* '''Product''': Splunk Enterprise, Splunk Enterprise Security, Splunk Cloud +* '''Product''': Splunk Security Analytics for AWS, Splunk Enterprise, Splunk Enterprise Security, Splunk Cloud * '''Datamodel''': Authentication * '''ATT&CK''': [https://attack.mitre.org/techniques/T1535/ T1535], [https://attack.mitre.org/techniques/T1078.004/ T1078.004] * '''Last Updated''': 2020-06-04 @@ -5034,7 +5034,7 @@ Monitor your cloud authentication events. Searches within this Analytic Story le ===Suspicious cloud instance activities=== Monitor your cloud infrastructure provisioning activities for behaviors originating from unfamiliar or unusual locations. These behaviors may indicate that malicious activities are occurring somewhere within your cloud environment. -* '''Product''': Splunk Enterprise, Splunk Enterprise Security, Splunk Cloud +* '''Product''': Splunk Security Analytics for AWS, Splunk Enterprise, Splunk Enterprise Security, Splunk Cloud * '''Datamodel''': Change * '''ATT&CK''': [https://attack.mitre.org/techniques/T1078.004/ T1078.004] * '''Last Updated''': 2020-08-25 @@ -5083,7 +5083,7 @@ Monitor your cloud infrastructure provisioning activities for behaviors originat ===Suspicious cloud provisioning activities=== Monitor your cloud infrastructure provisioning activities for behaviors originating from unfamiliar or unusual locations. These behaviors may indicate that malicious activities are occurring somewhere within your cloud environment. -* '''Product''': Splunk Enterprise, Splunk Enterprise Security, Splunk Cloud +* '''Product''': Splunk Security Analytics for AWS, Splunk Enterprise, Splunk Enterprise Security, Splunk Cloud * '''Datamodel''': Change * '''ATT&CK''': [https://attack.mitre.org/techniques/T1078/ T1078] * '''Last Updated''': 2018-08-20 @@ -5132,7 +5132,7 @@ Monitor your cloud infrastructure provisioning activities for behaviors originat ===Suspicious cloud user activities=== Detect and investigate suspicious activities by users and roles in your cloud environments. -* '''Product''': Splunk Enterprise, Splunk Enterprise Security, Splunk Cloud +* '''Product''': Splunk Security Analytics for AWS, Splunk Security Analytics for AWS, Splunk Enterprise, Splunk Enterprise Security, Splunk Cloud * '''Datamodel''': Change * '''ATT&CK''': [https://attack.mitre.org/techniques/T1078.004/ T1078.004], [https://attack.mitre.org/techniques/T1078/ T1078] * '''Last Updated''': 2020-09-04 @@ -6045,7 +6045,7 @@ Leverage searches that allow you to detect and investigate unusual activities th ===Ransomware cloud=== Leverage searches that allow you to detect and investigate unusual activities that might relate to ransomware. These searches include cloud related objects that may be targeted by malicious actors via cloud providers own encryption features. -* '''Product''': Splunk Enterprise, Splunk Enterprise Security, Splunk Cloud +* '''Product''': Splunk Security Analytics for AWS, Splunk Enterprise, Splunk Enterprise Security, Splunk Cloud * '''Datamodel''': * '''ATT&CK''': [https://attack.mitre.org/techniques/T1486/ T1486] * '''Last Updated''': 2020-10-27 @@ -6827,7 +6827,7 @@ Reduce the risk of CVE-2018-11409, an information disclosure vulnerability withi '' ############# # Automatically generated by doc_gen.py in https://github.com/splunk/security_content -# On Date: 2021-03-29 18:42:22.600999 UTC +# On Date: 2021-04-02 17:10:21.639044 UTC # Author: Splunk Security Research # Contact: research@splunk.com ############# diff --git a/package/README.md b/package/README.md new file mode 100644 index 0000000000..d0a87c3f9b --- /dev/null +++ b/package/README.md @@ -0,0 +1,7 @@ +# Splunk ES Content Update + +This subscription service delivers pre-packaged Security Content for use with Splunk Enterprise Security. Subscribers get regular updates to help security practitioners more quickly address ongoing and time-sensitive customer problems and threats. + +Requires Splunk Enterprise Security version 4.5 or greater. + +For more information please visit the [Splunk ES Content Update user documentation](https://docs.splunk.com/Documentation/ESSOC). \ No newline at end of file diff --git a/package/README/essoc_story_detail.txt b/package/README/essoc_story_detail.txt new file mode 100644 index 0000000000..5333752842 --- /dev/null +++ b/package/README/essoc_story_detail.txt @@ -0,0 +1,15 @@ +The Analytic Story Details dashboard renders all the details of the content related to a specific analytic story which +can be chose via the drop down + +Each analytic story has attributes associated with it and the following: +______________________________________________________________________ + + + Analytic Story: name of the analytic story + Description ; description of the analytic story + Search Name : The name of the searches belonging to the chosen analytic story + Search : The search query which looks for an attack pattern corresponding to the analytic story + Search Description: The description of the search query + Asset Type: The analytic story specifies what asset in the infrastructure may be compromised + Category: The category that the search belongs to (malware, vulnerabilities, best practices, abuse) + Kill Chain Phase: The kill chain phase of the attack that the search is after. \ No newline at end of file diff --git a/package/README/essoc_summary.txt b/package/README/essoc_summary.txt new file mode 100644 index 0000000000..d7dde31ec6 --- /dev/null +++ b/package/README/essoc_summary.txt @@ -0,0 +1,24 @@ +The ES_SOC Summary Dashboard provides you a summarized view of the analytic story contents of the ES-SOC app. +The dashboard has the following panels gives you following details + +1) Analytic story Summary + - Total Analytic Stories : The total number of Analytic stories in the ES-SOC application + - Total Searches: The total number of searches in ES-SOC + - Searches added last week: Number of searches added to ES-SOC in the last week. + + 2) Analytic story Category: This dashboard panel summarizes the categories of the searches that the ES-SOC app contains. The categories of the analytic stories are as follow + -Malware: These searches detect specific malware behavior for a particular phase of the attack kill chain. E.g. a malware’s delivery method via email or a malware’s installation behavior via registry key changes + -Vulnerability: These searches detect behavior or a signature of a vulnerable software in use. These searches are not designed to replace vulnerability management or scanning systems. The purpose of these searches is to discover a vulnerability through side effects or behaviors. + -Abuse: Some actions can be deemed malicious because they are unexpected, violate corporate policy or are significantly different than the actions of other users. E.g. A USB disk that is seen on multiple systems or a user that uploads excessive files to a cloud service or a database query that dumps an entire table + -Best Practices: Searches that correspond to specific guidelines from organizations like SANS or OWASP + + 3) Kill Chain phases: Every analytic story has one or more searches which look for a certain kind of attack pattern/behavior. These searches have an attribute which essentially tells you what Kill chain phase does the search correspond to. + The numbers on the dashboard represents the number of searches correponding to each kill chain phase + + 4) Analytic story table: This table gives the user a comprehensive view of some of the details of the analytic story. Some of the listed attributes are: + - Analytic Story : The name of the analytic story + - Description: The description of the analyttic story + - Search names: The name of the searches in each analytic story + - Datamodels: The name of the datamodel that the search is querying against. + - Technology Examples: This field represent some examples related to the technologies required to populate the datamodels(Nessues, Cisco Firewall,etc) + - Kill chain phase: The name of the kill chain phase that the search belongs to \ No newline at end of file diff --git a/package/README/essoc_usage_dashboard.txt b/package/README/essoc_usage_dashboard.txt new file mode 100644 index 0000000000..fcbc842881 --- /dev/null +++ b/package/README/essoc_usage_dashboard.txt @@ -0,0 +1,51 @@ +###################### +ESSOC Usage Dashboard# +###################### + +The ESSOC Usage dashboard is designed to provide high-level insight into the usage of the ES-SOC app. It is suitable for display when providing feedback to the Splunk team or for identifying how the ES-SOC app is being used. This dashboard has two time selectors that work independently - the top time selector determines the search time range for all the single-value. And the lower time selector, determines the time range for the usage table. + +IMPORTANT: The user loading this dashboard must have permission to search the _audit index + +################## +#Dashboard panels# +################## + +Searches Ran + +The total number of searches in ES-SOC that were executed. This number includes scheduled searches and ad hoc searches run from the search bar using the '| savedsearch ‘ syntax + +Unique Searches + +The unique/distinct searches executed on the deployment. This is equivalent to the distinct count of searches run in the ES-SOC app. + +Most Run + +The total number of searches in ES-SOC that were executed. This number includes scheduled searches and ad hoc searches run from the search bar using the '| savedsearch ‘ syntax. + +Ad hoc Searches + +The total number of searches run from the search bar using the '| savedsearch ‘ syntax. + +Scheduled + +The total number of ESSOC searches run that were scheduled. + +Most Active User + +The user who executed the highest number/count of searches. This calculation includes scheduled searches and ad hoc searches run from the search bar using the '| savedsearch ‘ syntax. + +Search Run Time (seconds) + +Total run time of all searches executed in seconds. This calculation includes scheduled searches and ad hoc searches run from the search bar using the '| savedsearch ‘ syntax. + +Average Run Time (seconds) + +Average run time of all searches executed in seconds. This calculation includes scheduled searches and ad hoc searches run from the search bar using the '| savedsearch ‘ syntax. + +Max Run Time (seconds) + +The run time of the longest running search. This calculation includes scheduled searches and ad hoc searches run from the search bar using the '| savedsearch ‘ syntax. + +Search summary + +This table provides details on each search that was executed in the ESSOC app. \ No newline at end of file diff --git a/package/app.manifest b/package/app.manifest new file mode 100644 index 0000000000..4bc6dbc051 --- /dev/null +++ b/package/app.manifest @@ -0,0 +1,46 @@ +{ + "schemaVersion": "1.0.0", + "info": { + "title": "ES Content Updates", + "id": { + "group": null, + "name": "DA-ESS-ContentUpdate", + "version": "3.19.0" + }, + "author": [ + { + "name": "Splunk Security Research Team", + "email": "research@splunk.com", + "company": "Splunk" + } + ], + "releaseDate": null, + "description": "Explore the Analytic Stories included with ES Content Updates.", + "classification": { + "intendedAudience": null, + "categories": [], + "developmentStatus": null + }, + "commonInformationModels": null, + "license": { + "name": null, + "text": null, + "uri": null + }, + "privacyPolicy": { + "name": null, + "text": null, + "uri": null + }, + "releaseNotes": { + "name": null, + "text": "./README.md", + "uri": null + } + }, + "dependencies": null, + "tasks": null, + "inputGroups": null, + "incompatibleApps": null, + "platformRequirements": null +} \ No newline at end of file diff --git a/package/bin/dnstwist.py b/package/bin/dnstwist.py new file mode 100755 index 0000000000..ccbf79824e --- /dev/null +++ b/package/bin/dnstwist.py @@ -0,0 +1,400 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# +# +# Licensed under the Apache License, Version 2.0 (the "License"): you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# +# Code modified from DNSTwist (https://github.com/elceef/dnstwist) +# Thanks elceef! +# +# Changes made: +# Just kept the DomainFuzz class and passing the domain to the fuzzer. Then added +# the Splunk specific code around it +# + +from __future__ import absolute_import, division, print_function, unicode_literals + +import sys +import re +import csv +import time +import os + +from splunklib.searchcommands import dispatch, GeneratingCommand, \ + Configuration, Option, Boolean +from splunk.clilib.bundle_paths import make_splunkhome_path + + +class DomainFuzz(object): + + def __init__(self, domain): + self.domain, self.tld = self.__domain_tld(domain) + self.domains = [] + self.qwerty = { + '1': '2q', '2': '3wq1', '3': '4ew2', '4': '5re3', + '5': '6tr4', '6': '7yt5', '7': '8uy6', '8': '9iu7', + '9': '0oi8', '0': 'po9', 'q': '12wa', 'w': '3esaq2', + 'e': '4rdsw3', 'r': '5tfde4', 't': '6ygfr5', 'y': '7uhgt6', + 'u': '8ijhy7', 'i': '9okju8', 'o': '0plki9', 'p': 'lo0', + 'a': 'qwsz', 's': 'edxzaw', 'd': 'rfcxse', 'f': 'tgvcdr', + 'g': 'yhbvft', 'h': 'ujnbgy', 'j': 'ikmnhu', 'k': 'olmji', + 'l': 'kop', 'z': 'asx', 'x': 'zsdc', 'c': 'xdfv', + 'v': 'cfgb', 'b': 'vghn', 'n': 'bhjm', 'm': 'njk' + } + self.qwertz = { + '1': '2q', '2': '3wq1', '3': '4ew2', '4': '5re3', + '5': '6tr4', '6': '7zt5', '7': '8uz6', '8': '9iu7', + '9': '0oi8', '0': 'po9', 'q': '12wa', 'w': '3esaq2', + 'e': '4rdsw3', 'r': '5tfde4', 't': '6zgfr5', + 'z': '7uhgt6', 'u': '8ijhz7', 'i': '9okju8', + 'o': '0plki9', 'p': 'lo0', 'a': 'qwsy', 's': 'edxyaw', + 'd': 'rfcxse', 'f': 'tgvcdr', 'g': 'zhbvft', + 'h': 'ujnbgz', 'j': 'ikmnhu', 'k': 'olmji', 'l': 'kop', + 'y': 'asx', 'x': 'ysdc', 'c': 'xdfv', 'v': 'cfgb', + 'b': 'vghn', 'n': 'bhjm', 'm': 'njk' + } + self.azerty = { + '1': '2a', '2': '3za1', '3': '4ez2', '4': '5re3', + '5': '6tr4', '6': '7yt5', '7': '8uy6', '8': '9iu7', + '9': '0oi8', '0': 'po9', 'a': '2zq1', 'z': '3esqa2', + 'e': '4rdsz3', 'r': '5tfde4', 't': '6ygfr5', + 'y': '7uhgt6', 'u': '8ijhy7', 'i': '9okju8', + 'o': '0plki9', 'p': 'lo0m', 'q': 'zswa', 's': 'edxwqz', + 'd': 'rfcxse', 'f': 'tgvcdr', 'g': 'yhbvft', + 'h': 'ujnbgy', 'j': 'iknhu', 'k': 'olji', 'l': 'kopm', + 'm': 'lp', 'w': 'sxq', 'x': 'zsdc', 'c': 'xdfv', + 'v': 'cfgb', 'b': 'vghn', 'n': 'bhj' + } + self.keyboards = [self.qwerty, self.qwertz, self.azerty] + + def __domain_tld(self, domain): + domain = domain.rsplit('.', 2) + + if len(domain) == 2: + return domain[0], domain[1] + + return domain[0] + '.' + domain[1], domain[2] + + def __validate_domain(self, domain): + if len(domain) == len(domain.encode('idna')) and domain != domain.encode('idna'): + return False + allowed = re.compile(b'(?=^.{4,253}$)(^((?!-)[a-zA-Z0-9-]{1,63}(?= 48 and o <= 57) or (o >= 97 and o <= 122) or o == 45: + result.append(self.domain[:i] + b + self.domain[i+1:]) + + return result + + def __homoglyph(self): + glyphs = { + 'a': [u'à', u'á', u'â', u'ã', u'ä', u'å', u'ɑ', u'а', u'ạ', u'ǎ', u'ă', u'ȧ', u'ӓ'], + 'b': ['d', 'lb', 'ib', u'ʙ', u'Ь', u'b̔', u'ɓ', u'Б'], + 'c': [u'ϲ', u'с', u'ƈ', u'ċ', u'ć', u'ç'], + 'd': ['b', 'cl', 'dl', 'di', u'ԁ', u'ժ', u'ɗ', u'đ'], + 'e': [u'é', u'ê', u'ë', u'ē', u'ĕ', u'ě', u'ė', u'е', u'ẹ', u'ę', u'є', u'ϵ', u'ҽ'], + 'f': [u'Ϝ', u'ƒ', u'Ғ'], + 'g': ['q', u'ɢ', u'ɡ', u'Ԍ', u'Ԍ', u'ġ', u'ğ', u'ց', u'ǵ', u'ģ'], + 'h': ['lh', 'ih', u'һ', u'հ', u'Ꮒ', u'н'], + 'i': ['1', 'l', u'Ꭵ', u'í', u'ï', u'ı', u'ɩ', u'ι', u'ꙇ', u'ǐ', u'ĭ'], + 'j': [u'ј', u'ʝ', u'ϳ', u'ɉ'], + 'k': ['lk', 'ik', 'lc', u'κ', u'ⲕ', u'κ'], + 'l': ['1', 'i', u'ɫ', u'ł'], + 'm': ['n', 'nn', 'rn', 'rr', u'ṃ', u'ᴍ', u'м', u'ɱ'], + 'n': ['m', 'r', u'ń'], + 'o': ['0', u'Ο', u'ο', u'О', u'о', u'Օ', u'ȯ', u'ọ', u'ỏ', u'ơ', u'ó', u'ö', u'ӧ'], + 'p': [u'ρ', u'р', u'ƿ', u'Ϸ', u'Þ'], + 'q': ['g', u'զ', u'ԛ', u'գ', u'ʠ'], + 'r': [u'ʀ', u'Г', u'ᴦ', u'ɼ', u'ɽ'], + 's': [u'Ⴝ', u'Ꮪ', u'ʂ', u'ś', u'ѕ'], + 't': [u'τ', u'т', u'ţ'], + 'u': [u'μ', u'υ', u'Ս', u'ս', u'ц', u'ᴜ', u'ǔ', u'ŭ'], + 'v': [u'ѵ', u'ν', u'v̇'], + 'w': ['vv', u'ѡ', u'ա', u'ԝ'], + 'x': [u'х', u'ҳ', u'ẋ'], + 'y': [u'ʏ', u'γ', u'у', u'Ү', u'ý'], + 'z': [u'ʐ', u'ż', u'ź', u'ʐ', u'ᴢ'] + } + + result = [] + + for ws in range(0, len(self.domain)): + for i in range(0, (len(self.domain)-ws)+1): + win = self.domain[i:i+ws] + + j = 0 + while j < ws: + c = win[j] + if c in glyphs: + win_copy = win + for g in glyphs[c]: + win = win.replace(c, g) + result.append(self.domain[:i] + win + self.domain[i+ws:]) + win = win_copy + j += 1 + + return list(set(result)) + + def __hyphenation(self): + result = [] + + for i in range(1, len(self.domain)): + result.append(self.domain[:i] + '-' + self.domain[i:]) + + return result + + def __insertion(self): + result = [] + + for i in range(1, len(self.domain)-1): + for keys in self.keyboards: + if self.domain[i] in keys: + for c in keys[self.domain[i]]: + result.append(self.domain[:i] + c + self.domain[i] + self.domain[i+1:]) + result.append(self.domain[:i] + self.domain[i] + c + self.domain[i+1:]) + + return list(set(result)) + + def __omission(self): + result = [] + + for i in range(0, len(self.domain)): + result.append(self.domain[:i] + self.domain[i+1:]) + + n = re.sub(r'(.)\1+', r'\1', self.domain) + + if n not in result and n != self.domain: + result.append(n) + + return list(set(result)) + + def __repetition(self): + result = [] + + for i in range(0, len(self.domain)): + if self.domain[i].isalpha(): + result.append(self.domain[:i] + self.domain[i] + self.domain[i] + self.domain[i+1:]) + + return list(set(result)) + + def __replacement(self): + result = [] + + for i in range(0, len(self.domain)): + for keys in self.keyboards: + if self.domain[i] in keys: + for c in keys[self.domain[i]]: + result.append(self.domain[:i] + c + self.domain[i+1:]) + + return list(set(result)) + + def __subdomain(self): + result = [] + + for i in range(1, len(self.domain)): + if self.domain[i] not in ['-', '.'] and self.domain[i-1] not in ['-', '.']: + result.append(self.domain[:i] + '.' + self.domain[i:]) + + return result + + def __transposition(self): + result = [] + + for i in range(0, len(self.domain)-1): + if self.domain[i+1] != self.domain[i]: + result.append(self.domain[:i] + self.domain[i+1] + self.domain[i] + self.domain[i+2:]) + + return result + + def __vowel_swap(self): + vowels = 'aeiou' + result = [] + + for i in range(0, len(self.domain)): + for vowel in vowels: + if self.domain[i] in vowels: + result.append(self.domain[:i] + vowel + self.domain[i+1:]) + + return list(set(result)) + + def __addition(self): + result = [] + + for i in range(97, 123): + result.append(self.domain + chr(i)) + + return result + + def generate(self): + self.domains.append({'fuzzer': 'Original*', 'domain-name': self.domain + '.' + self.tld}) + + for domain in self.__addition(): + self.domains.append({'fuzzer': 'Addition', 'domain-name': domain + '.' + self.tld}) + for domain in self.__bitsquatting(): + self.domains.append({'fuzzer': 'Bitsquatting', 'domain-name': domain + '.' + self.tld}) + for domain in self.__homoglyph(): + self.domains.append({'fuzzer': 'Homoglyph', 'domain-name': domain + '.' + self.tld}) + for domain in self.__hyphenation(): + self.domains.append({'fuzzer': 'Hyphenation', 'domain-name': domain + '.' + self.tld}) + for domain in self.__insertion(): + self.domains.append({'fuzzer': 'Insertion', 'domain-name': domain + '.' + self.tld}) + for domain in self.__omission(): + self.domains.append({'fuzzer': 'Omission', 'domain-name': domain + '.' + self.tld}) + for domain in self.__repetition(): + self.domains.append({'fuzzer': 'Repetition', 'domain-name': domain + '.' + self.tld}) + for domain in self.__replacement(): + self.domains.append({'fuzzer': 'Replacement', 'domain-name': domain + '.' + self.tld}) + for domain in self.__subdomain(): + self.domains.append({'fuzzer': 'Subdomain', 'domain-name': domain + '.' + self.tld}) + for domain in self.__transposition(): + self.domains.append({'fuzzer': 'Transposition', 'domain-name': domain + '.' + self.tld}) + for domain in self.__vowel_swap(): + self.domains.append({'fuzzer': 'Vowel-swap', 'domain-name': domain + '.' + self.tld}) + + if not self.domain.startswith('www.'): + self.domains.append({'fuzzer': 'Various', 'domain-name': 'ww' + self.domain + '.' + self.tld}) + self.domains.append({'fuzzer': 'Various', 'domain-name': 'www' + self.domain + '.' + self.tld}) + self.domains.append({'fuzzer': 'Various', 'domain-name': 'www-' + self.domain + '.' + self.tld}) + if '.' in self.tld: + self.domains.append({'fuzzer': 'Various', 'domain-name': self.domain + '.' + self.tld.split('.')[-1]}) + self.domains.append({'fuzzer': 'Various', 'domain-name': self.domain + self.tld}) + if '.' not in self.tld: + self.domains.append({'fuzzer': 'Various', 'domain-name': self.domain + self.tld + '.' + self.tld}) + if self.tld != 'com' and '.' not in self.tld: + self.domains.append({'fuzzer': 'Various', 'domain-name': self.domain + '-' + self.tld + '.com'}) + + self.__filter_domains() + + +@Configuration(distributed=True) +class DnsTwistCommand(GeneratingCommand): + + domainlist_file_name = Option(doc=''' + **Syntax:** **domainlist=**** + **Description:** CSV file from which repeated random samples will be drawn + ''', name='domainlist', require=False) + + populate_from_cim = Option(doc=''' + **Syntax: populate_cim= + **Description:** When `true`, populates Splunk_SA_CIM lookups cim_corporate_email_domains.csv + and cim_corporate_web_domains.csv with dnstwisted domains. Defaults to `false`. + ''', name='populate_from_cim', default=False, validate=Boolean()) + + domain = Option(doc=''' + **Syntax:** **domain=**** + **Description:** Domain to DNS generated twisted entries for. + ''', name='domain', require=False, default='') + + def generate(self): + event_count = 0 + csv_file_names = [] + + if self.populate_from_cim: + csv_file_names.append(make_splunkhome_path([ + 'etc', + 'apps', + 'Splunk_SA_CIM', + 'lookups', + 'cim_corporate_email_domains.csv'])) + csv_file_names.append(make_splunkhome_path([ + 'etc', + 'apps', + 'Splunk_SA_CIM', + 'lookups', + 'cim_corporate_web_domains.csv'])) + + # Make sure we just get the base file name from file. In case there was some directory traversal going on. + if self.domainlist_file_name: + sanitized_file_name = os.path.basename(self.domainlist_file_name) + lookup_path = make_splunkhome_path(['etc', 'apps', 'DA-ESS-ContentUpdate', 'lookups', sanitized_file_name]) + + # Make sure there really isn't any directory traversal going on. + valid_path = True + if "../" in lookup_path: + valid_path = False + + # Make sure the path that is created by adding the file name to the path is the same as the + # absolute path + if lookup_path != os.path.abspath(lookup_path): + valid_path = False + + if valid_path: + csv_file_names.append(lookup_path) + + domains_to_twist = [] + + for csv_file_name in csv_file_names: + if os.path.exists(csv_file_name): + # this is nasty but works .. please forgive me + if sys.version_info >= (3, 0): + csv_file = open(csv_file_name, "r", newline='') + else: + csv_file = open(csv_file_name, "r") + for input_domain in csv.DictReader(csv_file): + if input_domain['domain'] not in domains_to_twist: + domains_to_twist.append(input_domain['domain']) + + # if a single domain is passed lets just calculate that + if self.domain != '': + domains_to_twist = [] + domains_to_twist.append(self.domain) + + for domain_to_twist in domains_to_twist: + domain_to_twist = domain_to_twist.lstrip('*') + dfuzz = DomainFuzz(domain_to_twist) + dfuzz.generate() + domains = dfuzz.domains + for domain in domains: + # We don't want to keep the original domain + if domain['domain-name'] in domain_to_twist: + continue + event_count += 1 + yield { + '_time': time.time(), + 'event_no': event_count, + '_raw': domain['domain-name'], + 'domain': '*'+domain['domain-name']+'*', + 'original_domain': domain_to_twist + } + + def __init__(self): + super(DnsTwistCommand, self).__init__() + + +dispatch(DnsTwistCommand, sys.argv, sys.stdin, sys.stdout, __name__) diff --git a/package/bin/escu_contextualize.py b/package/bin/escu_contextualize.py new file mode 100644 index 0000000000..48ae9fcf5e --- /dev/null +++ b/package/bin/escu_contextualize.py @@ -0,0 +1 @@ +# dropped AR action support due to python 3 dependency, we leverage playbooks in stories as an alternative. diff --git a/package/bin/escu_investigate.py b/package/bin/escu_investigate.py new file mode 100644 index 0000000000..48ae9fcf5e --- /dev/null +++ b/package/bin/escu_investigate.py @@ -0,0 +1 @@ +# dropped AR action support due to python 3 dependency, we leverage playbooks in stories as an alternative. diff --git a/package/bin/runstory.py b/package/bin/runstory.py new file mode 100644 index 0000000000..2e3141eac8 --- /dev/null +++ b/package/bin/runstory.py @@ -0,0 +1 @@ +# runstory was deprecated, its functionality was moved to: https://github.com/splunk/analytic_story_execution diff --git a/package/bin/splunklib/__init__.py b/package/bin/splunklib/__init__.py new file mode 100755 index 0000000000..dd448e0d3a --- /dev/null +++ b/package/bin/splunklib/__init__.py @@ -0,0 +1,19 @@ +# Copyright 2011-2015 Splunk, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"): you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Python library for Splunk.""" + +__version_info__ = (1, 6, 2) +__version__ = ".".join(map(str, __version_info__)) + diff --git a/package/bin/splunklib/binding.py b/package/bin/splunklib/binding.py new file mode 100755 index 0000000000..e184c3ade0 --- /dev/null +++ b/package/bin/splunklib/binding.py @@ -0,0 +1,1373 @@ +# Copyright 2011-2015 Splunk, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"): you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""The **splunklib.binding** module provides a low-level binding interface to the +`Splunk REST API `_. + +This module handles the wire details of calling the REST API, such as +authentication tokens, prefix paths, URL encoding, and so on. Actual path +segments, ``GET`` and ``POST`` arguments, and the parsing of responses is left +to the user. + +If you want a friendlier interface to the Splunk REST API, use the +:mod:`splunklib.client` module. +""" + +import httplib +import logging +import socket +import ssl +import urllib +import io +import sys +import Cookie + +from base64 import b64encode +from datetime import datetime +from functools import wraps +from StringIO import StringIO + +from contextlib import contextmanager + +from xml.etree.ElementTree import XML +try: + from xml.etree.ElementTree import ParseError +except ImportError, e: + from xml.parsers.expat import ExpatError as ParseError + +from .data import record + +__all__ = [ + "AuthenticationError", + "connect", + "Context", + "handler", + "HTTPError" +] + +# If you change these, update the docstring +# on _authority as well. +DEFAULT_HOST = "localhost" +DEFAULT_PORT = "8089" +DEFAULT_SCHEME = "https" + +def _log_duration(f): + @wraps(f) + def new_f(*args, **kwargs): + start_time = datetime.now() + val = f(*args, **kwargs) + end_time = datetime.now() + logging.debug("Operation took %s", end_time-start_time) + return val + return new_f + + +def _parse_cookies(cookie_str, dictionary): + """Tries to parse any key-value pairs of cookies in a string, + then updates the the dictionary with any key-value pairs found. + + **Example**:: + dictionary = {} + _parse_cookies('my=value', dictionary) + # Now the following is True + dictionary['my'] == 'value' + + :param cookie_str: A string containing "key=value" pairs from an HTTP "Set-Cookie" header. + :type cookie_str: ``str`` + :param dictionary: A dictionary to update with any found key-value pairs. + :type dictionary: ``dict`` + """ + parsed_cookie = Cookie.SimpleCookie(cookie_str) + for cookie in parsed_cookie.values(): + dictionary[cookie.key] = cookie.coded_value + + +def _make_cookie_header(cookies): + """ + Takes a list of 2-tuples of key-value pairs of + cookies, and returns a valid HTTP ``Cookie`` + header. + + **Example**:: + + header = _make_cookie_header([("key", "value"), ("key_2", "value_2")]) + # Now the following is True + header == "key=value; key_2=value_2" + + :param cookies: A list of 2-tuples of cookie key-value pairs. + :type cookies: ``list`` of 2-tuples + :return: ``str` An HTTP header cookie string. + :rtype: ``str`` + """ + return "; ".join("%s=%s" % (key, value) for key, value in cookies) + +# Singleton values to eschew None +class _NoAuthenticationToken(object): + """The value stored in a :class:`Context` or :class:`splunklib.client.Service` + class that is not logged in. + + If a ``Context`` or ``Service`` object is created without an authentication + token, and there has not yet been a call to the ``login`` method, the token + field of the ``Context`` or ``Service`` object is set to + ``_NoAuthenticationToken``. + + Likewise, after a ``Context`` or ``Service`` object has been logged out, the + token is set to this value again. + """ + pass + + +class UrlEncoded(str): + """This class marks URL-encoded strings. + It should be considered an SDK-private implementation detail. + + Manually tracking whether strings are URL encoded can be difficult. Avoid + calling ``urllib.quote`` to replace special characters with escapes. When + you receive a URL-encoded string, *do* use ``urllib.unquote`` to replace + escapes with single characters. Then, wrap any string you want to use as a + URL in ``UrlEncoded``. Note that because the ``UrlEncoded`` class is + idempotent, making multiple calls to it is OK. + + ``UrlEncoded`` objects are identical to ``str`` objects (including being + equal if their contents are equal) except when passed to ``UrlEncoded`` + again. + + ``UrlEncoded`` removes the ``str`` type support for interpolating values + with ``%`` (doing that raises a ``TypeError``). There is no reliable way to + encode values this way, so instead, interpolate into a string, quoting by + hand, and call ``UrlEncode`` with ``skip_encode=True``. + + **Example**:: + + import urllib + UrlEncoded('%s://%s' % (scheme, urllib.quote(host)), skip_encode=True) + + If you append ``str`` strings and ``UrlEncoded`` strings, the result is also + URL encoded. + + **Example**:: + + UrlEncoded('ab c') + 'de f' == UrlEncoded('ab cde f') + 'ab c' + UrlEncoded('de f') == UrlEncoded('ab cde f') + """ + def __new__(self, val='', skip_encode=False, encode_slash=False): + if isinstance(val, UrlEncoded): + # Don't urllib.quote something already URL encoded. + return val + elif skip_encode: + return str.__new__(self, val) + elif encode_slash: + return str.__new__(self, urllib.quote_plus(val)) + else: + # When subclassing str, just call str's __new__ method + # with your class and the value you want to have in the + # new string. + return str.__new__(self, urllib.quote(val)) + + def __add__(self, other): + """self + other + + If *other* is not a ``UrlEncoded``, URL encode it before + adding it. + """ + if isinstance(other, UrlEncoded): + return UrlEncoded(str.__add__(self, other), skip_encode=True) + else: + return UrlEncoded(str.__add__(self, urllib.quote(other)), skip_encode=True) + + def __radd__(self, other): + """other + self + + If *other* is not a ``UrlEncoded``, URL _encode it before + adding it. + """ + if isinstance(other, UrlEncoded): + return UrlEncoded(str.__radd__(self, other), skip_encode=True) + else: + return UrlEncoded(str.__add__(urllib.quote(other), self), skip_encode=True) + + def __mod__(self, fields): + """Interpolation into ``UrlEncoded``s is disabled. + + If you try to write ``UrlEncoded("%s") % "abc", will get a + ``TypeError``. + """ + raise TypeError("Cannot interpolate into a UrlEncoded object.") + def __repr__(self): + return "UrlEncoded(%s)" % repr(urllib.unquote(str(self))) + +@contextmanager +def _handle_auth_error(msg): + """Handle reraising HTTP authentication errors as something clearer. + + If an ``HTTPError`` is raised with status 401 (access denied) in + the body of this context manager, reraise it as an + ``AuthenticationError`` instead, with *msg* as its message. + + This function adds no round trips to the server. + + :param msg: The message to be raised in ``AuthenticationError``. + :type msg: ``str`` + + **Example**:: + + with _handle_auth_error("Your login failed."): + ... # make an HTTP request + """ + try: + yield + except HTTPError as he: + if he.status == 401: + raise AuthenticationError(msg, he) + else: + raise + +def _authentication(request_fun): + """Decorator to handle autologin and authentication errors. + + *request_fun* is a function taking no arguments that needs to + be run with this ``Context`` logged into Splunk. + + ``_authentication``'s behavior depends on whether the + ``autologin`` field of ``Context`` is set to ``True`` or + ``False``. If it's ``False``, then ``_authentication`` + aborts if the ``Context`` is not logged in, and raises an + ``AuthenticationError`` if an ``HTTPError`` of status 401 is + raised in *request_fun*. If it's ``True``, then + ``_authentication`` will try at all sensible places to + log in before issuing the request. + + If ``autologin`` is ``False``, ``_authentication`` makes + one roundtrip to the server if the ``Context`` is logged in, + or zero if it is not. If ``autologin`` is ``True``, it's less + deterministic, and may make at most three roundtrips (though + that would be a truly pathological case). + + :param request_fun: A function of no arguments encapsulating + the request to make to the server. + + **Example**:: + + import splunklib.binding as binding + c = binding.connect(..., autologin=True) + c.logout() + def f(): + c.get("/services") + return 42 + print _authentication(f) + """ + @wraps(request_fun) + def wrapper(self, *args, **kwargs): + if self.token is _NoAuthenticationToken and \ + not self.has_cookies(): + # Not yet logged in. + if self.autologin and self.username and self.password: + # This will throw an uncaught + # AuthenticationError if it fails. + self.login() + else: + # Try the request anyway without authentication. + # Most requests will fail. Some will succeed, such as + # 'GET server/info'. + with _handle_auth_error("Request aborted: not logged in."): + return request_fun(self, *args, **kwargs) + try: + # Issue the request + return request_fun(self, *args, **kwargs) + except HTTPError as he: + if he.status == 401 and self.autologin: + # Authentication failed. Try logging in, and then + # rerunning the request. If either step fails, throw + # an AuthenticationError and give up. + with _handle_auth_error("Autologin failed."): + self.login() + with _handle_auth_error( + "Autologin succeeded, but there was an auth error on " + "next request. Something is very wrong."): + return request_fun(self, *args, **kwargs) + elif he.status == 401 and not self.autologin: + raise AuthenticationError( + "Request failed: Session is not logged in.", he) + else: + raise + + return wrapper + + +def _authority(scheme=DEFAULT_SCHEME, host=DEFAULT_HOST, port=DEFAULT_PORT): + """Construct a URL authority from the given *scheme*, *host*, and *port*. + + Named in accordance with RFC2396_, which defines URLs as:: + + ://? + + .. _RFC2396: http://www.ietf.org/rfc/rfc2396.txt + + So ``https://localhost:8000/a/b/b?boris=hilda`` would be parsed as:: + + scheme := https + authority := localhost:8000 + path := /a/b/c + query := boris=hilda + + :param scheme: URL scheme (the default is "https") + :type scheme: "http" or "https" + :param host: The host name (the default is "localhost") + :type host: string + :param port: The port number (the default is 8089) + :type port: integer + :return: The URL authority. + :rtype: UrlEncoded (subclass of ``str``) + + **Example**:: + + _authority() == "https://localhost:8089" + + _authority(host="splunk.utopia.net") == "https://splunk.utopia.net:8089" + + _authority(host="2001:0db8:85a3:0000:0000:8a2e:0370:7334") == \ + "https://[2001:0db8:85a3:0000:0000:8a2e:0370:7334]:8089" + + _authority(scheme="http", host="splunk.utopia.net", port="471") == \ + "http://splunk.utopia.net:471" + + """ + if ':' in host: + # IPv6 addresses must be enclosed in [ ] in order to be well + # formed. + host = '[' + host + ']' + return UrlEncoded("%s://%s:%s" % (scheme, host, port), skip_encode=True) + +# kwargs: sharing, owner, app +def namespace(sharing=None, owner=None, app=None, **kwargs): + """This function constructs a Splunk namespace. + + Every Splunk resource belongs to a namespace. The namespace is specified by + the pair of values ``owner`` and ``app`` and is governed by a ``sharing`` mode. + The possible values for ``sharing`` are: "user", "app", "global" and "system", + which map to the following combinations of ``owner`` and ``app`` values: + + "user" => {owner}, {app} + + "app" => nobody, {app} + + "global" => nobody, {app} + + "system" => nobody, system + + "nobody" is a special user name that basically means no user, and "system" + is the name reserved for system resources. + + "-" is a wildcard that can be used for both ``owner`` and ``app`` values and + refers to all users and all apps, respectively. + + In general, when you specify a namespace you can specify any combination of + these three values and the library will reconcile the triple, overriding the + provided values as appropriate. + + Finally, if no namespacing is specified the library will make use of the + ``/services`` branch of the REST API, which provides a namespaced view of + Splunk resources equivelent to using ``owner={currentUser}`` and + ``app={defaultApp}``. + + The ``namespace`` function returns a representation of the namespace from + reconciling the values you provide. It ignores any keyword arguments other + than ``owner``, ``app``, and ``sharing``, so you can provide ``dicts`` of + configuration information without first having to extract individual keys. + + :param sharing: The sharing mode (the default is "user"). + :type sharing: "system", "global", "app", or "user" + :param owner: The owner context (the default is "None"). + :type owner: ``string`` + :param app: The app context (the default is "None"). + :type app: ``string`` + :returns: A :class:`splunklib.data.Record` containing the reconciled + namespace. + + **Example**:: + + import splunklib.binding as binding + n = binding.namespace(sharing="user", owner="boris", app="search") + n = binding.namespace(sharing="global", app="search") + """ + if sharing in ["system"]: + return record({'sharing': sharing, 'owner': "nobody", 'app': "system" }) + if sharing in ["global", "app"]: + return record({'sharing': sharing, 'owner': "nobody", 'app': app}) + if sharing in ["user", None]: + return record({'sharing': sharing, 'owner': owner, 'app': app}) + raise ValueError("Invalid value for argument: 'sharing'") + + +class Context(object): + """This class represents a context that encapsulates a splunkd connection. + + The ``Context`` class encapsulates the details of HTTP requests, + authentication, a default namespace, and URL prefixes to simplify access to + the REST API. + + After creating a ``Context`` object, you must call its :meth:`login` + method before you can issue requests to splunkd. Or, use the :func:`connect` + function to create an already-authenticated ``Context`` object. You can + provide a session token explicitly (the same token can be shared by multiple + ``Context`` objects) to provide authentication. + + :param host: The host name (the default is "localhost"). + :type host: ``string`` + :param port: The port number (the default is 8089). + :type port: ``integer`` + :param scheme: The scheme for accessing the service (the default is "https"). + :type scheme: "https" or "http" + :param sharing: The sharing mode for the namespace (the default is "user"). + :type sharing: "global", "system", "app", or "user" + :param owner: The owner context of the namespace (optional, the default is "None"). + :type owner: ``string`` + :param app: The app context of the namespace (optional, the default is "None"). + :type app: ``string`` + :param token: A session token. When provided, you don't need to call :meth:`login`. + :type token: ``string`` + :param cookie: A session cookie. When provided, you don't need to call :meth:`login`. + This parameter is only supported for Splunk 6.2+. + :type cookie: ``string`` + :param username: The Splunk account username, which is used to + authenticate the Splunk instance. + :type username: ``string`` + :param password: The password for the Splunk account. + :type password: ``string`` + :param handler: The HTTP request handler (optional). + :returns: A ``Context`` instance. + + **Example**:: + + import splunklib.binding as binding + c = binding.Context(username="boris", password="natasha", ...) + c.login() + # Or equivalently + c = binding.connect(username="boris", password="natasha") + # Or if you already have a session token + c = binding.Context(token="atg232342aa34324a") + # Or if you already have a valid cookie + c = binding.Context(cookie="splunkd_8089=...") + """ + def __init__(self, handler=None, **kwargs): + self.http = HttpLib(handler) + self.token = kwargs.get("token", _NoAuthenticationToken) + if self.token is None: # In case someone explicitly passes token=None + self.token = _NoAuthenticationToken + self.scheme = kwargs.get("scheme", DEFAULT_SCHEME) + self.host = kwargs.get("host", DEFAULT_HOST) + self.port = int(kwargs.get("port", DEFAULT_PORT)) + self.authority = _authority(self.scheme, self.host, self.port) + self.namespace = namespace(**kwargs) + self.username = kwargs.get("username", "") + self.password = kwargs.get("password", "") + self.basic = kwargs.get("basic", False) + self.autologin = kwargs.get("autologin", False) + + # Store any cookies in the self.http._cookies dict + if kwargs.has_key("cookie") and kwargs['cookie'] not in [None, _NoAuthenticationToken]: + _parse_cookies(kwargs["cookie"], self.http._cookies) + + def get_cookies(self): + """Gets the dictionary of cookies from the ``HttpLib`` member of this instance. + + :return: Dictionary of cookies stored on the ``self.http``. + :rtype: ``dict`` + """ + return self.http._cookies + + def has_cookies(self): + """Returns true if the ``HttpLib`` member of this instance has at least + one cookie stored. + + :return: ``True`` if there is at least one cookie, else ``False`` + :rtype: ``bool`` + """ + return len(self.get_cookies()) > 0 + + # Shared per-context request headers + @property + def _auth_headers(self): + """Headers required to authenticate a request. + + Assumes your ``Context`` already has a authentication token or + cookie, either provided explicitly or obtained by logging + into the Splunk instance. + + :returns: A list of 2-tuples containing key and value + """ + if self.has_cookies(): + return [("Cookie", _make_cookie_header(self.get_cookies().items()))] + elif self.basic and (self.username and self.password): + token = 'Basic %s' % b64encode("%s:%s" % (self.username, self.password)) + return [("Authorization", token)] + elif self.token is _NoAuthenticationToken: + return [] + else: + # Ensure the token is properly formatted + if self.token.startswith('Splunk '): + token = self.token + else: + token = 'Splunk %s' % self.token + return [("Authorization", token)] + + def connect(self): + """Returns an open connection (socket) to the Splunk instance. + + This method is used for writing bulk events to an index or similar tasks + where the overhead of opening a connection multiple times would be + prohibitive. + + :returns: A socket. + + **Example**:: + + import splunklib.binding as binding + c = binding.connect(...) + socket = c.connect() + socket.write("POST %s HTTP/1.1\\r\\n" % "some/path/to/post/to") + socket.write("Host: %s:%s\\r\\n" % (c.host, c.port)) + socket.write("Accept-Encoding: identity\\r\\n") + socket.write("Authorization: %s\\r\\n" % c.token) + socket.write("X-Splunk-Input-Mode: Streaming\\r\\n") + socket.write("\\r\\n") + """ + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + if self.scheme == "https": + sock = ssl.wrap_socket(sock) + sock.connect((socket.gethostbyname(self.host), self.port)) + return sock + + @_authentication + @_log_duration + def delete(self, path_segment, owner=None, app=None, sharing=None, **query): + """Performs a DELETE operation at the REST path segment with the given + namespace and query. + + This method is named to match the HTTP method. ``delete`` makes at least + one round trip to the server, one additional round trip for each 303 + status returned, and at most two additional round trips if + the ``autologin`` field of :func:`connect` is set to ``True``. + + If *owner*, *app*, and *sharing* are omitted, this method uses the + default :class:`Context` namespace. All other keyword arguments are + included in the URL as query parameters. + + :raises AuthenticationError: Raised when the ``Context`` object is not + logged in. + :raises HTTPError: Raised when an error occurred in a GET operation from + *path_segment*. + :param path_segment: A REST path segment. + :type path_segment: ``string`` + :param owner: The owner context of the namespace (optional). + :type owner: ``string`` + :param app: The app context of the namespace (optional). + :type app: ``string`` + :param sharing: The sharing mode of the namespace (optional). + :type sharing: ``string`` + :param query: All other keyword arguments, which are used as query + parameters. + :type query: ``string`` + :return: The response from the server. + :rtype: ``dict`` with keys ``body``, ``headers``, ``reason``, + and ``status`` + + **Example**:: + + c = binding.connect(...) + c.delete('saved/searches/boris') == \\ + {'body': ...a response reader object..., + 'headers': [('content-length', '1786'), + ('expires', 'Fri, 30 Oct 1998 00:00:00 GMT'), + ('server', 'Splunkd'), + ('connection', 'close'), + ('cache-control', 'no-store, max-age=0, must-revalidate, no-cache'), + ('date', 'Fri, 11 May 2012 16:53:06 GMT'), + ('content-type', 'text/xml; charset=utf-8')], + 'reason': 'OK', + 'status': 200} + c.delete('nonexistant/path') # raises HTTPError + c.logout() + c.delete('apps/local') # raises AuthenticationError + """ + path = self.authority + self._abspath(path_segment, owner=owner, + app=app, sharing=sharing) + logging.debug("DELETE request to %s (body: %s)", path, repr(query)) + response = self.http.delete(path, self._auth_headers, **query) + return response + + @_authentication + @_log_duration + def get(self, path_segment, owner=None, app=None, sharing=None, **query): + """Performs a GET operation from the REST path segment with the given + namespace and query. + + This method is named to match the HTTP method. ``get`` makes at least + one round trip to the server, one additional round trip for each 303 + status returned, and at most two additional round trips if + the ``autologin`` field of :func:`connect` is set to ``True``. + + If *owner*, *app*, and *sharing* are omitted, this method uses the + default :class:`Context` namespace. All other keyword arguments are + included in the URL as query parameters. + + :raises AuthenticationError: Raised when the ``Context`` object is not + logged in. + :raises HTTPError: Raised when an error occurred in a GET operation from + *path_segment*. + :param path_segment: A REST path segment. + :type path_segment: ``string`` + :param owner: The owner context of the namespace (optional). + :type owner: ``string`` + :param app: The app context of the namespace (optional). + :type app: ``string`` + :param sharing: The sharing mode of the namespace (optional). + :type sharing: ``string`` + :param query: All other keyword arguments, which are used as query + parameters. + :type query: ``string`` + :return: The response from the server. + :rtype: ``dict`` with keys ``body``, ``headers``, ``reason``, + and ``status`` + + **Example**:: + + c = binding.connect(...) + c.get('apps/local') == \\ + {'body': ...a response reader object..., + 'headers': [('content-length', '26208'), + ('expires', 'Fri, 30 Oct 1998 00:00:00 GMT'), + ('server', 'Splunkd'), + ('connection', 'close'), + ('cache-control', 'no-store, max-age=0, must-revalidate, no-cache'), + ('date', 'Fri, 11 May 2012 16:30:35 GMT'), + ('content-type', 'text/xml; charset=utf-8')], + 'reason': 'OK', + 'status': 200} + c.get('nonexistant/path') # raises HTTPError + c.logout() + c.get('apps/local') # raises AuthenticationError + """ + path = self.authority + self._abspath(path_segment, owner=owner, + app=app, sharing=sharing) + logging.debug("GET request to %s (body: %s)", path, repr(query)) + response = self.http.get(path, self._auth_headers, **query) + return response + + @_authentication + @_log_duration + def post(self, path_segment, owner=None, app=None, sharing=None, headers=None, **query): + """Performs a POST operation from the REST path segment with the given + namespace and query. + + This method is named to match the HTTP method. ``post`` makes at least + one round trip to the server, one additional round trip for each 303 + status returned, and at most two additional round trips if + the ``autologin`` field of :func:`connect` is set to ``True``. + + If *owner*, *app*, and *sharing* are omitted, this method uses the + default :class:`Context` namespace. All other keyword arguments are + included in the URL as query parameters. + + Some of Splunk's endpoints, such as ``receivers/simple`` and + ``receivers/stream``, require unstructured data in the POST body + and all metadata passed as GET-style arguments. If you provide + a ``body`` argument to ``post``, it will be used as the POST + body, and all other keyword arguments will be passed as + GET-style arguments in the URL. + + :raises AuthenticationError: Raised when the ``Context`` object is not + logged in. + :raises HTTPError: Raised when an error occurred in a GET operation from + *path_segment*. + :param path_segment: A REST path segment. + :type path_segment: ``string`` + :param owner: The owner context of the namespace (optional). + :type owner: ``string`` + :param app: The app context of the namespace (optional). + :type app: ``string`` + :param sharing: The sharing mode of the namespace (optional). + :type sharing: ``string`` + :param headers: List of extra HTTP headers to send (optional). + :type headers: ``list`` of 2-tuples. + :param query: All other keyword arguments, which are used as query + parameters. + :type query: ``string`` + :return: The response from the server. + :rtype: ``dict`` with keys ``body``, ``headers``, ``reason``, + and ``status`` + + **Example**:: + + c = binding.connect(...) + c.post('saved/searches', name='boris', + search='search * earliest=-1m | head 1') == \\ + {'body': ...a response reader object..., + 'headers': [('content-length', '10455'), + ('expires', 'Fri, 30 Oct 1998 00:00:00 GMT'), + ('server', 'Splunkd'), + ('connection', 'close'), + ('cache-control', 'no-store, max-age=0, must-revalidate, no-cache'), + ('date', 'Fri, 11 May 2012 16:46:06 GMT'), + ('content-type', 'text/xml; charset=utf-8')], + 'reason': 'Created', + 'status': 201} + c.post('nonexistant/path') # raises HTTPError + c.logout() + # raises AuthenticationError: + c.post('saved/searches', name='boris', + search='search * earliest=-1m | head 1') + """ + if headers is None: + headers = [] + + path = self.authority + self._abspath(path_segment, owner=owner, app=app, sharing=sharing) + logging.debug("POST request to %s (body: %s)", path, repr(query)) + all_headers = headers + self._auth_headers + response = self.http.post(path, all_headers, **query) + return response + + @_authentication + @_log_duration + def request(self, path_segment, method="GET", headers=None, body="", + owner=None, app=None, sharing=None): + """Issues an arbitrary HTTP request to the REST path segment. + + This method is named to match ``httplib.request``. This function + makes a single round trip to the server. + + If *owner*, *app*, and *sharing* are omitted, this method uses the + default :class:`Context` namespace. All other keyword arguments are + included in the URL as query parameters. + + :raises AuthenticationError: Raised when the ``Context`` object is not + logged in. + :raises HTTPError: Raised when an error occurred in a GET operation from + *path_segment*. + :param path_segment: A REST path segment. + :type path_segment: ``string`` + :param method: The HTTP method to use (optional). + :type method: ``string`` + :param headers: List of extra HTTP headers to send (optional). + :type headers: ``list`` of 2-tuples. + :param body: Content of the HTTP request (optional). + :type body: ``string`` + :param owner: The owner context of the namespace (optional). + :type owner: ``string`` + :param app: The app context of the namespace (optional). + :type app: ``string`` + :param sharing: The sharing mode of the namespace (optional). + :type sharing: ``string`` + :param query: All other keyword arguments, which are used as query + parameters. + :type query: ``string`` + :return: The response from the server. + :rtype: ``dict`` with keys ``body``, ``headers``, ``reason``, + and ``status`` + + **Example**:: + + c = binding.connect(...) + c.request('saved/searches', method='GET') == \\ + {'body': ...a response reader object..., + 'headers': [('content-length', '46722'), + ('expires', 'Fri, 30 Oct 1998 00:00:00 GMT'), + ('server', 'Splunkd'), + ('connection', 'close'), + ('cache-control', 'no-store, max-age=0, must-revalidate, no-cache'), + ('date', 'Fri, 11 May 2012 17:24:19 GMT'), + ('content-type', 'text/xml; charset=utf-8')], + 'reason': 'OK', + 'status': 200} + c.request('nonexistant/path', method='GET') # raises HTTPError + c.logout() + c.get('apps/local') # raises AuthenticationError + """ + if headers is None: + headers = [] + + path = self.authority \ + + self._abspath(path_segment, owner=owner, + app=app, sharing=sharing) + all_headers = headers + self._auth_headers + logging.debug("%s request to %s (headers: %s, body: %s)", + method, path, str(all_headers), repr(body)) + response = self.http.request(path, + {'method': method, + 'headers': all_headers, + 'body': body}) + return response + + def login(self): + """Logs into the Splunk instance referred to by the :class:`Context` + object. + + Unless a ``Context`` is created with an explicit authentication token + (probably obtained by logging in from a different ``Context`` object) + you must call :meth:`login` before you can issue requests. + The authentication token obtained from the server is stored in the + ``token`` field of the ``Context`` object. + + :raises AuthenticationError: Raised when login fails. + :returns: The ``Context`` object, so you can chain calls. + + **Example**:: + + import splunklib.binding as binding + c = binding.Context(...).login() + # Then issue requests... + """ + + if self.has_cookies() and \ + (not self.username and not self.password): + # If we were passed session cookie(s), but no username or + # password, then login is a nop, since we're automatically + # logged in. + return + + if self.token is not _NoAuthenticationToken and \ + (not self.username and not self.password): + # If we were passed a session token, but no username or + # password, then login is a nop, since we're automatically + # logged in. + return + + if self.basic and (self.username and self.password): + # Basic auth mode requested, so this method is a nop as long + # as credentials were passed in. + return + + # Only try to get a token and updated cookie if username & password are specified + try: + response = self.http.post( + self.authority + self._abspath("/services/auth/login"), + username=self.username, + password=self.password, + cookie="1") # In Splunk 6.2+, passing "cookie=1" will return the "set-cookie" header + + body = response.body.read() + session = XML(body).findtext("./sessionKey") + self.token = "Splunk %s" % session + return self + except HTTPError as he: + if he.status == 401: + raise AuthenticationError("Login failed.", he) + else: + raise + + def logout(self): + """Forgets the current session token, and cookies.""" + self.token = _NoAuthenticationToken + self.http._cookies = {} + return self + + def _abspath(self, path_segment, + owner=None, app=None, sharing=None): + """Qualifies *path_segment* into an absolute path for a URL. + + If *path_segment* is already absolute, returns it unchanged. + If *path_segment* is relative, then qualifies it with either + the provided namespace arguments or the ``Context``'s default + namespace. Any forbidden characters in *path_segment* are URL + encoded. This function has no network activity. + + Named to be consistent with RFC2396_. + + .. _RFC2396: http://www.ietf.org/rfc/rfc2396.txt + + :param path_segment: A relative or absolute URL path segment. + :type path_segment: ``string`` + :param owner, app, sharing: Components of a namespace (defaults + to the ``Context``'s namespace if all + three are omitted) + :type owner, app, sharing: ``string`` + :return: A ``UrlEncoded`` (a subclass of ``str``). + :rtype: ``string`` + + **Example**:: + + import splunklib.binding as binding + c = binding.connect(owner='boris', app='search', sharing='user') + c._abspath('/a/b/c') == '/a/b/c' + c._abspath('/a/b c/d') == '/a/b%20c/d' + c._abspath('apps/local/search') == \ + '/servicesNS/boris/search/apps/local/search' + c._abspath('apps/local/search', sharing='system') == \ + '/servicesNS/nobody/system/apps/local/search' + url = c.authority + c._abspath('apps/local/sharing') + """ + skip_encode = isinstance(path_segment, UrlEncoded) + # If path_segment is absolute, escape all forbidden characters + # in it and return it. + if path_segment.startswith('/'): + return UrlEncoded(path_segment, skip_encode=skip_encode) + + # path_segment is relative, so we need a namespace to build an + # absolute path. + if owner or app or sharing: + ns = namespace(owner=owner, app=app, sharing=sharing) + else: + ns = self.namespace + + # If no app or owner are specified, then use the /services + # endpoint. Otherwise, use /servicesNS with the specified + # namespace. If only one of app and owner is specified, use + # '-' for the other. + if ns.app is None and ns.owner is None: + return UrlEncoded("/services/%s" % path_segment, skip_encode=skip_encode) + + oname = "nobody" if ns.owner is None else ns.owner + aname = "system" if ns.app is None else ns.app + path = UrlEncoded("/servicesNS/%s/%s/%s" % (oname, aname, path_segment), + skip_encode=skip_encode) + return path + + +def connect(**kwargs): + """This function returns an authenticated :class:`Context` object. + + This function is a shorthand for calling :meth:`Context.login`. + + This function makes one round trip to the server. + + :param host: The host name (the default is "localhost"). + :type host: ``string`` + :param port: The port number (the default is 8089). + :type port: ``integer`` + :param scheme: The scheme for accessing the service (the default is "https"). + :type scheme: "https" or "http" + :param owner: The owner context of the namespace (the default is "None"). + :type owner: ``string`` + :param app: The app context of the namespace (the default is "None"). + :type app: ``string`` + :param sharing: The sharing mode for the namespace (the default is "user"). + :type sharing: "global", "system", "app", or "user" + :param token: The current session token (optional). Session tokens can be + shared across multiple service instances. + :type token: ``string`` + :param cookie: A session cookie. When provided, you don't need to call :meth:`login`. + This parameter is only supported for Splunk 6.2+. + :type cookie: ``string`` + :param username: The Splunk account username, which is used to + authenticate the Splunk instance. + :type username: ``string`` + :param password: The password for the Splunk account. + :type password: ``string`` + :param autologin: When ``True``, automatically tries to log in again if the + session terminates. + :type autologin: ``Boolean`` + :return: An initialized :class:`Context` instance. + + **Example**:: + + import splunklib.binding as binding + c = binding.connect(...) + response = c.get("apps/local") + """ + c = Context(**kwargs) + c.login() + return c + +# Note: the error response schema supports multiple messages but we only +# return the first, although we do return the body so that an exception +# handler that wants to read multiple messages can do so. +class HTTPError(Exception): + """This exception is raised for HTTP responses that return an error.""" + def __init__(self, response, _message=None): + status = response.status + reason = response.reason + body = response.body.read() + try: + detail = XML(body).findtext("./messages/msg") + except ParseError as err: + detail = body + message = "HTTP %d %s%s" % ( + status, reason, "" if detail is None else " -- %s" % detail) + Exception.__init__(self, _message or message) + self.status = status + self.reason = reason + self.headers = response.headers + self.body = body + self._response = response + +class AuthenticationError(HTTPError): + """Raised when a login request to Splunk fails. + + If your username was unknown or you provided an incorrect password + in a call to :meth:`Context.login` or :meth:`splunklib.client.Service.login`, + this exception is raised. + """ + def __init__(self, message, cause): + # Put the body back in the response so that HTTPError's constructor can + # read it again. + cause._response.body = StringIO(cause.body) + + HTTPError.__init__(self, cause._response, message) + +# +# The HTTP interface used by the Splunk binding layer abstracts the underlying +# HTTP library using request & response 'messages' which are implemented as +# dictionaries with the following structure: +# +# # HTTP request message (only method required) +# request { +# method : str, +# headers? : [(str, str)*], +# body? : str, +# } +# +# # HTTP response message (all keys present) +# response { +# status : int, +# reason : str, +# headers : [(str, str)*], +# body : file, +# } +# + +# Encode the given kwargs as a query string. This wrapper will also _encode +# a list value as a sequence of assignemnts to the corresponding arg name, +# for example an argument such as 'foo=[1,2,3]' will be encoded as +# 'foo=1&foo=2&foo=3'. +def _encode(**kwargs): + items = [] + for key, value in kwargs.iteritems(): + if isinstance(value, list): + items.extend([(key, item) for item in value]) + else: + items.append((key, value)) + return urllib.urlencode(items) + +# Crack the given url into (scheme, host, port, path) +def _spliturl(url): + scheme, opaque = urllib.splittype(url) + netloc, path = urllib.splithost(opaque) + host, port = urllib.splitport(netloc) + # Strip brackets if its an IPv6 address + if host.startswith('[') and host.endswith(']'): host = host[1:-1] + if port is None: port = DEFAULT_PORT + return scheme, host, port, path + +# Given an HTTP request handler, this wrapper objects provides a related +# family of convenience methods built using that handler. +class HttpLib(object): + """A set of convenient methods for making HTTP calls. + + ``HttpLib`` provides a general :meth:`request` method, and :meth:`delete`, + :meth:`post`, and :meth:`get` methods for the three HTTP methods that Splunk + uses. + + By default, ``HttpLib`` uses Python's built-in ``httplib`` library, + but you can replace it by passing your own handling function to the + constructor for ``HttpLib``. + + The handling function should have the type: + + ``handler(`url`, `request_dict`) -> response_dict`` + + where `url` is the URL to make the request to (including any query and + fragment sections) as a dictionary with the following keys: + + - method: The method for the request, typically ``GET``, ``POST``, or ``DELETE``. + + - headers: A list of pairs specifying the HTTP headers (for example: ``[('key': value), ...]``). + + - body: A string containing the body to send with the request (this string + should default to ''). + + and ``response_dict`` is a dictionary with the following keys: + + - status: An integer containing the HTTP status code (such as 200 or 404). + + - reason: The reason phrase, if any, returned by the server. + + - headers: A list of pairs containing the response headers (for example, ``[('key': value), ...]``). + + - body: A stream-like object supporting ``read(size=None)`` and ``close()`` + methods to get the body of the response. + + The response dictionary is returned directly by ``HttpLib``'s methods with + no further processing. By default, ``HttpLib`` calls the :func:`handler` function + to get a handler function. + """ + def __init__(self, custom_handler=None): + self.handler = handler() if custom_handler is None else custom_handler + self._cookies = {} + + def delete(self, url, headers=None, **kwargs): + """Sends a DELETE request to a URL. + + :param url: The URL. + :type url: ``string`` + :param headers: A list of pairs specifying the headers for the HTTP + response (for example, ``[('Content-Type': 'text/cthulhu'), ('Token': 'boris')]``). + :type headers: ``list`` + :param kwargs: Additional keyword arguments (optional). These arguments + are interpreted as the query part of the URL. The order of keyword + arguments is not preserved in the request, but the keywords and + their arguments will be URL encoded. + :type kwargs: ``dict`` + :returns: A dictionary describing the response (see :class:`HttpLib` for + its structure). + :rtype: ``dict`` + """ + if headers is None: headers = [] + if kwargs: + # url is already a UrlEncoded. We have to manually declare + # the query to be encoded or it will get automatically URL + # encoded by being appended to url. + url = url + UrlEncoded('?' + _encode(**kwargs), skip_encode=True) + message = { + 'method': "DELETE", + 'headers': headers, + } + return self.request(url, message) + + def get(self, url, headers=None, **kwargs): + """Sends a GET request to a URL. + + :param url: The URL. + :type url: ``string`` + :param headers: A list of pairs specifying the headers for the HTTP + response (for example, ``[('Content-Type': 'text/cthulhu'), ('Token': 'boris')]``). + :type headers: ``list`` + :param kwargs: Additional keyword arguments (optional). These arguments + are interpreted as the query part of the URL. The order of keyword + arguments is not preserved in the request, but the keywords and + their arguments will be URL encoded. + :type kwargs: ``dict`` + :returns: A dictionary describing the response (see :class:`HttpLib` for + its structure). + :rtype: ``dict`` + """ + if headers is None: headers = [] + if kwargs: + # url is already a UrlEncoded. We have to manually declare + # the query to be encoded or it will get automatically URL + # encoded by being appended to url. + url = url + UrlEncoded('?' + _encode(**kwargs), skip_encode=True) + return self.request(url, { 'method': "GET", 'headers': headers }) + + def post(self, url, headers=None, **kwargs): + """Sends a POST request to a URL. + + :param url: The URL. + :type url: ``string`` + :param headers: A list of pairs specifying the headers for the HTTP + response (for example, ``[('Content-Type': 'text/cthulhu'), ('Token': 'boris')]``). + :type headers: ``list`` + :param kwargs: Additional keyword arguments (optional). If the argument + is ``body``, the value is used as the body for the request, and the + keywords and their arguments will be URL encoded. If there is no + ``body`` keyword argument, all the keyword arguments are encoded + into the body of the request in the format ``x-www-form-urlencoded``. + :type kwargs: ``dict`` + :returns: A dictionary describing the response (see :class:`HttpLib` for + its structure). + :rtype: ``dict`` + """ + if headers is None: headers = [] + + # We handle GET-style arguments and an unstructured body. This is here + # to support the receivers/stream endpoint. + if 'body' in kwargs: + # We only use application/x-www-form-urlencoded if there is no other + # Content-Type header present. This can happen in cases where we + # send requests as application/json, e.g. for KV Store. + if len(filter(lambda x: x[0].lower() == "content-type", headers)) == 0: + headers.append(("Content-Type", "application/x-www-form-urlencoded")) + + body = kwargs.pop('body') + if len(kwargs) > 0: + url = url + UrlEncoded('?' + _encode(**kwargs), skip_encode=True) + else: + body = _encode(**kwargs) + message = { + 'method': "POST", + 'headers': headers, + 'body': body + } + return self.request(url, message) + + def request(self, url, message, **kwargs): + """Issues an HTTP request to a URL. + + :param url: The URL. + :type url: ``string`` + :param message: A dictionary with the format as described in + :class:`HttpLib`. + :type message: ``dict`` + :param kwargs: Additional keyword arguments (optional). These arguments + are passed unchanged to the handler. + :type kwargs: ``dict`` + :returns: A dictionary describing the response (see :class:`HttpLib` for + its structure). + :rtype: ``dict`` + """ + response = self.handler(url, message, **kwargs) + response = record(response) + if 400 <= response.status: + raise HTTPError(response) + + # Update the cookie with any HTTP request + # Initially, assume list of 2-tuples + key_value_tuples = response.headers + # If response.headers is a dict, get the key-value pairs as 2-tuples + # this is the case when using urllib2 + if isinstance(response.headers, dict): + key_value_tuples = response.headers.items() + for key, value in key_value_tuples: + if key.lower() == "set-cookie": + _parse_cookies(value, self._cookies) + + return response + + +# Converts an httplib response into a file-like object. +class ResponseReader(io.RawIOBase): + """This class provides a file-like interface for :class:`httplib` responses. + + The ``ResponseReader`` class is intended to be a layer to unify the different + types of HTTP libraries used with this SDK. This class also provides a + preview of the stream and a few useful predicates. + """ + # For testing, you can use a StringIO as the argument to + # ``ResponseReader`` instead of an ``httplib.HTTPResponse``. It + # will work equally well. + def __init__(self, response, connection=None): + self._response = response + self._connection = connection + self._buffer = '' + + def __str__(self): + return self.read() + + @property + def empty(self): + """Indicates whether there is any more data in the response.""" + return self.peek(1) == "" + + def peek(self, size): + """Nondestructively retrieves a given number of characters. + + The next :meth:`read` operation behaves as though this method was never + called. + + :param size: The number of characters to retrieve. + :type size: ``integer`` + """ + c = self.read(size) + self._buffer = self._buffer + c + return c + + def close(self): + """Closes this response.""" + if _connection: + _connection.close() + self._response.close() + + def read(self, size = None): + """Reads a given number of characters from the response. + + :param size: The number of characters to read, or "None" to read the + entire response. + :type size: ``integer`` or "None" + + """ + r = self._buffer + self._buffer = '' + if size is not None: + size -= len(r) + r = r + self._response.read(size) + return r + + def readable(self): + """ Indicates that the response reader is readable.""" + return True + + def readinto(self, byte_array): + """ Read data into a byte array, upto the size of the byte array. + + :param byte_array: A byte array/memory view to pour bytes into. + :type byte_array: ``bytearray`` or ``memoryview`` + + """ + max_size = len(byte_array) + data = self.read(max_size) + bytes_read = len(data) + byte_array[:bytes_read] = data + return bytes_read + + +def handler(key_file=None, cert_file=None, timeout=None): + """This class returns an instance of the default HTTP request handler using + the values you provide. + + :param `key_file`: A path to a PEM (Privacy Enhanced Mail) formatted file containing your private key (optional). + :type key_file: ``string`` + :param `cert_file`: A path to a PEM (Privacy Enhanced Mail) formatted file containing a certificate chain file (optional). + :type cert_file: ``string`` + :param `timeout`: The request time-out period, in seconds (optional). + :type timeout: ``integer`` or "None" + """ + + def connect(scheme, host, port): + kwargs = {} + if timeout is not None: kwargs['timeout'] = timeout + if scheme == "http": + return httplib.HTTPConnection(host, port, **kwargs) + if scheme == "https": + if key_file is not None: kwargs['key_file'] = key_file + if cert_file is not None: kwargs['cert_file'] = cert_file + + # If running Python 2.7.9+, disable SSL certificate validation + if sys.version_info >= (2,7,9) and key_file is None and cert_file is None: + kwargs['context'] = ssl._create_unverified_context() + return httplib.HTTPSConnection(host, port, **kwargs) + raise ValueError("unsupported scheme: %s" % scheme) + + def request(url, message, **kwargs): + scheme, host, port, path = _spliturl(url) + body = message.get("body", "") + head = { + "Content-Length": str(len(body)), + "Host": host, + "User-Agent": "splunk-sdk-python/1.6.2", + "Accept": "*/*", + "Connection": "Close", + } # defaults + for key, value in message["headers"]: + head[key] = value + method = message.get("method", "GET") + + connection = connect(scheme, host, port) + is_keepalive = False + try: + connection.request(method, path, body, head) + if timeout is not None: + connection.sock.settimeout(timeout) + response = connection.getresponse() + is_keepalive = "keep-alive" in response.getheader("connection", default="close").lower() + finally: + if not is_keepalive: + connection.close() + + return { + "status": response.status, + "reason": response.reason, + "headers": response.getheaders(), + "body": ResponseReader(response, connection if is_keepalive else None), + } + + return request diff --git a/package/bin/splunklib/client.py b/package/bin/splunklib/client.py new file mode 100755 index 0000000000..d618e8a406 --- /dev/null +++ b/package/bin/splunklib/client.py @@ -0,0 +1,3718 @@ +# Copyright 2011-2015 Splunk, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"): you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# The purpose of this module is to provide a friendlier domain interface to +# various Splunk endpoints. The approach here is to leverage the binding +# layer to capture endpoint context and provide objects and methods that +# offer simplified access their corresponding endpoints. The design avoids +# caching resource state. From the perspective of this module, the 'policy' +# for caching resource state belongs in the application or a higher level +# framework, and its the purpose of this module to provide simplified +# access to that resource state. +# +# A side note, the objects below that provide helper methods for updating eg: +# Entity state, are written so that they may be used in a fluent style. +# + +"""The **splunklib.client** module provides a Pythonic interface to the +`Splunk REST API `_, +allowing you programmatically access Splunk's resources. + +**splunklib.client** wraps a Pythonic layer around the wire-level +binding of the **splunklib.binding** module. The core of the library is the +:class:`Service` class, which encapsulates a connection to the server, and +provides access to the various aspects of Splunk's functionality, which are +exposed via the REST API. Typically you connect to a running Splunk instance +with the :func:`connect` function:: + + import splunklib.client as client + service = client.connect(host='localhost', port=8089, + username='admin', password='...') + assert isinstance(service, client.Service) + +:class:`Service` objects have fields for the various Splunk resources (such as apps, +jobs, saved searches, inputs, and indexes). All of these fields are +:class:`Collection` objects:: + + appcollection = service.apps + my_app = appcollection.create('my_app') + my_app = appcollection['my_app'] + appcollection.delete('my_app') + +The individual elements of the collection, in this case *applications*, +are subclasses of :class:`Entity`. An ``Entity`` object has fields for its +attributes, and methods that are specific to each kind of entity. For example:: + + print my_app['author'] # Or: print my_app.author + my_app.package() # Creates a compressed package of this application +""" + +import datetime +import json +import urllib +import logging +from time import sleep +from datetime import datetime, timedelta +import socket +import contextlib + +from .binding import Context, HTTPError, AuthenticationError, namespace, UrlEncoded, _encode, _make_cookie_header, _NoAuthenticationToken +from .data import record +from . import data + +__all__ = [ + "connect", + "NotSupportedError", + "OperationError", + "IncomparableException", + "Service", + "namespace" +] + +PATH_APPS = "apps/local/" +PATH_CAPABILITIES = "authorization/capabilities/" +PATH_CONF = "configs/conf-%s/" +PATH_PROPERTIES = "properties/" +PATH_DEPLOYMENT_CLIENTS = "deployment/client/" +PATH_DEPLOYMENT_TENANTS = "deployment/tenants/" +PATH_DEPLOYMENT_SERVERS = "deployment/server/" +PATH_DEPLOYMENT_SERVERCLASSES = "deployment/serverclass/" +PATH_EVENT_TYPES = "saved/eventtypes/" +PATH_FIRED_ALERTS = "alerts/fired_alerts/" +PATH_INDEXES = "data/indexes/" +PATH_INPUTS = "data/inputs/" +PATH_JOBS = "search/jobs/" +PATH_LOGGER = "/services/server/logger/" +PATH_MESSAGES = "messages/" +PATH_MODULAR_INPUTS = "data/modular-inputs" +PATH_ROLES = "authorization/roles/" +PATH_SAVED_SEARCHES = "saved/searches/" +PATH_STANZA = "configs/conf-%s/%s" # (file, stanza) +PATH_USERS = "authentication/users/" +PATH_RECEIVERS_STREAM = "/services/receivers/stream" +PATH_RECEIVERS_SIMPLE = "/services/receivers/simple" +PATH_STORAGE_PASSWORDS = "storage/passwords" + +XNAMEF_ATOM = "{http://www.w3.org/2005/Atom}%s" +XNAME_ENTRY = XNAMEF_ATOM % "entry" +XNAME_CONTENT = XNAMEF_ATOM % "content" + +MATCH_ENTRY_CONTENT = "%s/%s/*" % (XNAME_ENTRY, XNAME_CONTENT) + + +class IllegalOperationException(Exception): + """Thrown when an operation is not possible on the Splunk instance that a + :class:`Service` object is connected to.""" + pass + + +class IncomparableException(Exception): + """Thrown when trying to compare objects (using ``==``, ``<``, ``>``, and + so on) of a type that doesn't support it.""" + pass + + +class AmbiguousReferenceException(ValueError): + """Thrown when the name used to fetch an entity matches more than one entity.""" + pass + + +class InvalidNameException(Exception): + """Thrown when the specified name contains characters that are not allowed + in Splunk entity names.""" + pass + + +class NoSuchCapability(Exception): + """Thrown when the capability that has been referred to doesn't exist.""" + pass + + +class OperationError(Exception): + """Raised for a failed operation, such as a time out.""" + pass + + +class NotSupportedError(Exception): + """Raised for operations that are not supported on a given object.""" + pass + + +def _trailing(template, *targets): + """Substring of *template* following all *targets*. + + **Example**:: + + template = "this is a test of the bunnies." + _trailing(template, "is", "est", "the") == " bunnies" + + Each target is matched successively in the string, and the string + remaining after the last target is returned. If one of the targets + fails to match, a ValueError is raised. + + :param template: Template to extract a trailing string from. + :type template: ``string`` + :param targets: Strings to successively match in *template*. + :type targets: list of ``string``s + :return: Trailing string after all targets are matched. + :rtype: ``string`` + :raises ValueError: Raised when one of the targets does not match. + """ + s = template + for t in targets: + n = s.find(t) + if n == -1: + raise ValueError("Target " + t + " not found in template.") + s = s[n + len(t):] + return s + + +# Filter the given state content record according to the given arg list. +def _filter_content(content, *args): + if len(args) > 0: + return record((k, content[k]) for k in args) + return record((k, v) for k, v in content.iteritems() + if k not in ['eai:acl', 'eai:attributes', 'type']) + +# Construct a resource path from the given base path + resource name +def _path(base, name): + if not base.endswith('/'): base = base + '/' + return base + name + + +# Load an atom record from the body of the given response +def _load_atom(response, match=None): + return data.load(response.body.read(), match) + + +# Load an array of atom entries from the body of the given response +def _load_atom_entries(response): + r = _load_atom(response) + if 'feed' in r: + # Need this to handle a random case in the REST API + if r.feed.get('totalResults') in [0, '0']: + return [] + entries = r.feed.get('entry', None) + if entries is None: return None + return entries if isinstance(entries, list) else [entries] + # Unlike most other endpoints, the jobs endpoint does not return + # its state wrapped in another element, but at the top level. + # For example, in XML, it returns ... instead of + # .... + else: + entries = r.get('entry', None) + if entries is None: return None + return entries if isinstance(entries, list) else [entries] + + +# Load the sid from the body of the given response +def _load_sid(response): + return _load_atom(response).response.sid + + +# Parse the given atom entry record into a generic entity state record +def _parse_atom_entry(entry): + title = entry.get('title', None) + + elink = entry.get('link', []) + elink = elink if isinstance(elink, list) else [elink] + links = record((link.rel, link.href) for link in elink) + + # Retrieve entity content values + content = entry.get('content', {}) + + # Host entry metadata + metadata = _parse_atom_metadata(content) + + # Filter some of the noise out of the content record + content = record((k, v) for k, v in content.iteritems() + if k not in ['eai:acl', 'eai:attributes']) + + if 'type' in content: + if isinstance(content['type'], list): + content['type'] = [t for t in content['type'] if t != 'text/xml'] + # Unset type if it was only 'text/xml' + if len(content['type']) == 0: + content.pop('type', None) + # Flatten 1 element list + if len(content['type']) == 1: + content['type'] = content['type'][0] + else: + content.pop('type', None) + + return record({ + 'title': title, + 'links': links, + 'access': metadata.access, + 'fields': metadata.fields, + 'content': content, + 'updated': entry.get("updated") + }) + + +# Parse the metadata fields out of the given atom entry content record +def _parse_atom_metadata(content): + # Hoist access metadata + access = content.get('eai:acl', None) + + # Hoist content metadata (and cleanup some naming) + attributes = content.get('eai:attributes', {}) + fields = record({ + 'required': attributes.get('requiredFields', []), + 'optional': attributes.get('optionalFields', []), + 'wildcard': attributes.get('wildcardFields', [])}) + + return record({'access': access, 'fields': fields}) + +# kwargs: scheme, host, port, app, owner, username, password +def connect(**kwargs): + """This function connects and logs in to a Splunk instance. + + This function is a shorthand for :meth:`Service.login`. + The ``connect`` function makes one round trip to the server (for logging in). + + :param host: The host name (the default is "localhost"). + :type host: ``string`` + :param port: The port number (the default is 8089). + :type port: ``integer`` + :param scheme: The scheme for accessing the service (the default is "https"). + :type scheme: "https" or "http" + :param `owner`: The owner context of the namespace (optional). + :type owner: ``string`` + :param `app`: The app context of the namespace (optional). + :type app: ``string`` + :param sharing: The sharing mode for the namespace (the default is "user"). + :type sharing: "global", "system", "app", or "user" + :param `token`: The current session token (optional). Session tokens can be + shared across multiple service instances. + :type token: ``string`` + :param cookie: A session cookie. When provided, you don't need to call :meth:`login`. + This parameter is only supported for Splunk 6.2+. + :type cookie: ``string`` + :param autologin: When ``True``, automatically tries to log in again if the + session terminates. + :type autologin: ``boolean`` + :param `username`: The Splunk account username, which is used to + authenticate the Splunk instance. + :type username: ``string`` + :param `password`: The password for the Splunk account. + :type password: ``string`` + :return: An initialized :class:`Service` connection. + + **Example**:: + + import splunklib.client as client + s = client.connect(...) + a = s.apps["my_app"] + ... + """ + s = Service(**kwargs) + s.login() + return s + + +# In preparation for adding Storm support, we added an +# intermediary class between Service and Context. Storm's +# API is not going to be the same as enterprise Splunk's +# API, so we will derive both Service (for enterprise Splunk) +# and StormService for (Splunk Storm) from _BaseService, and +# put any shared behavior on it. +class _BaseService(Context): + pass + + +class Service(_BaseService): + """A Pythonic binding to Splunk instances. + + A :class:`Service` represents a binding to a Splunk instance on an + HTTP or HTTPS port. It handles the details of authentication, wire + formats, and wraps the REST API endpoints into something more + Pythonic. All of the low-level operations on the instance from + :class:`splunklib.binding.Context` are also available in case you need + to do something beyond what is provided by this class. + + After creating a ``Service`` object, you must call its :meth:`login` + method before you can issue requests to Splunk. + Alternately, use the :func:`connect` function to create an already + authenticated :class:`Service` object, or provide a session token + when creating the :class:`Service` object explicitly (the same + token may be shared by multiple :class:`Service` objects). + + :param host: The host name (the default is "localhost"). + :type host: ``string`` + :param port: The port number (the default is 8089). + :type port: ``integer`` + :param scheme: The scheme for accessing the service (the default is "https"). + :type scheme: "https" or "http" + :param `owner`: The owner context of the namespace (optional; use "-" for wildcard). + :type owner: ``string`` + :param `app`: The app context of the namespace (optional; use "-" for wildcard). + :type app: ``string`` + :param `token`: The current session token (optional). Session tokens can be + shared across multiple service instances. + :type token: ``string`` + :param cookie: A session cookie. When provided, you don't need to call :meth:`login`. + This parameter is only supported for Splunk 6.2+. + :type cookie: ``string`` + :param `username`: The Splunk account username, which is used to + authenticate the Splunk instance. + :type username: ``string`` + :param `password`: The password, which is used to authenticate the Splunk + instance. + :type password: ``string`` + :return: A :class:`Service` instance. + + **Example**:: + + import splunklib.client as client + s = client.Service(username="boris", password="natasha", ...) + s.login() + # Or equivalently + s = client.connect(username="boris", password="natasha") + # Or if you already have a session token + s = client.Service(token="atg232342aa34324a") + # Or if you already have a valid cookie + s = client.Service(cookie="splunkd_8089=...") + """ + def __init__(self, **kwargs): + super(Service, self).__init__(**kwargs) + self._splunk_version = None + + @property + def apps(self): + """Returns the collection of applications that are installed on this instance of Splunk. + + :return: A :class:`Collection` of :class:`Application` entities. + """ + return Collection(self, PATH_APPS, item=Application) + + @property + def confs(self): + """Returns the collection of configuration files for this Splunk instance. + + :return: A :class:`Configurations` collection of + :class:`ConfigurationFile` entities. + """ + return Configurations(self) + + @property + def capabilities(self): + """Returns the list of system capabilities. + + :return: A ``list`` of capabilities. + """ + response = self.get(PATH_CAPABILITIES) + return _load_atom(response, MATCH_ENTRY_CONTENT).capabilities + + @property + def event_types(self): + """Returns the collection of event types defined in this Splunk instance. + + :return: An :class:`Entity` containing the event types. + """ + return Collection(self, PATH_EVENT_TYPES) + + @property + def fired_alerts(self): + """Returns the collection of alerts that have been fired on the Splunk + instance, grouped by saved search. + + :return: A :class:`Collection` of :class:`AlertGroup` entities. + """ + return Collection(self, PATH_FIRED_ALERTS, item=AlertGroup) + + @property + def indexes(self): + """Returns the collection of indexes for this Splunk instance. + + :return: An :class:`Indexes` collection of :class:`Index` entities. + """ + return Indexes(self, PATH_INDEXES, item=Index) + + @property + def info(self): + """Returns the information about this instance of Splunk. + + :return: The system information, as key-value pairs. + :rtype: ``dict`` + """ + response = self.get("/services/server/info") + return _filter_content(_load_atom(response, MATCH_ENTRY_CONTENT)) + + @property + def inputs(self): + """Returns the collection of inputs configured on this Splunk instance. + + :return: An :class:`Inputs` collection of :class:`Input` entities. + """ + return Inputs(self) + + def job(self, sid): + """Retrieves a search job by sid. + + :return: A :class:`Job` object. + """ + return Job(self, sid).refresh() + + @property + def jobs(self): + """Returns the collection of current search jobs. + + :return: A :class:`Jobs` collection of :class:`Job` entities. + """ + return Jobs(self) + + @property + def loggers(self): + """Returns the collection of logging level categories and their status. + + :return: A :class:`Loggers` collection of logging levels. + """ + return Loggers(self) + + @property + def messages(self): + """Returns the collection of service messages. + + :return: A :class:`Collection` of :class:`Message` entities. + """ + return Collection(self, PATH_MESSAGES, item=Message) + + @property + def modular_input_kinds(self): + """Returns the collection of the modular input kinds on this Splunk instance. + + :return: A :class:`ReadOnlyCollection` of :class:`ModularInputKind` entities. + """ + if self.splunk_version >= (5,): + return ReadOnlyCollection(self, PATH_MODULAR_INPUTS, item=ModularInputKind) + else: + raise IllegalOperationException("Modular inputs are not supported before Splunk version 5.") + + @property + def storage_passwords(self): + """Returns the collection of the storage passwords on this Splunk instance. + + :return: A :class:`ReadOnlyCollection` of :class:`StoragePasswords` entities. + """ + return StoragePasswords(self) + + # kwargs: enable_lookups, reload_macros, parse_only, output_mode + def parse(self, query, **kwargs): + """Parses a search query and returns a semantic map of the search. + + :param query: The search query to parse. + :type query: ``string`` + :param kwargs: Arguments to pass to the ``search/parser`` endpoint + (optional). Valid arguments are: + + * "enable_lookups" (``boolean``): If ``True``, performs reverse lookups + to expand the search expression. + + * "output_mode" (``string``): The output format (XML or JSON). + + * "parse_only" (``boolean``): If ``True``, disables the expansion of + search due to evaluation of subsearches, time term expansion, + lookups, tags, eventtypes, and sourcetype alias. + + * "reload_macros" (``boolean``): If ``True``, reloads macro + definitions from macros.conf. + + :type kwargs: ``dict`` + :return: A semantic map of the parsed search query. + """ + return self.get("search/parser", q=query, **kwargs) + + def restart(self, timeout=None): + """Restarts this Splunk instance. + + The service is unavailable until it has successfully restarted. + + If a *timeout* value is specified, ``restart`` blocks until the service + resumes or the timeout period has been exceeded. Otherwise, ``restart`` returns + immediately. + + :param timeout: A timeout period, in seconds. + :type timeout: ``integer`` + """ + msg = { "value": "Restart requested by " + self.username + "via the Splunk SDK for Python"} + # This message will be deleted once the server actually restarts. + self.messages.create(name="restart_required", **msg) + result = self.post("/services/server/control/restart") + if timeout is None: + return result + start = datetime.now() + diff = timedelta(seconds=timeout) + while datetime.now() - start < diff: + try: + self.login() + if not self.restart_required: + return result + except Exception, e: + sleep(1) + raise Exception, "Operation time out." + + @property + def restart_required(self): + """Indicates whether splunkd is in a state that requires a restart. + + :return: A ``boolean`` that indicates whether a restart is required. + + """ + response = self.get("messages").body.read() + messages = data.load(response)['feed'] + if 'entry' not in messages: + result = False + else: + if isinstance(messages['entry'], dict): + titles = [messages['entry']['title']] + else: + titles = [x['title'] for x in messages['entry']] + result = 'restart_required' in titles + return result + + @property + def roles(self): + """Returns the collection of user roles. + + :return: A :class:`Roles` collection of :class:`Role` entities. + """ + return Roles(self) + + def search(self, query, **kwargs): + """Runs a search using a search query and any optional arguments you + provide, and returns a `Job` object representing the search. + + :param query: A search query. + :type query: ``string`` + :param kwargs: Arguments for the search (optional): + + * "output_mode" (``string``): Specifies the output format of the + results. + + * "earliest_time" (``string``): Specifies the earliest time in the + time range to + search. The time string can be a UTC time (with fractional + seconds), a relative time specifier (to now), or a formatted + time string. + + * "latest_time" (``string``): Specifies the latest time in the time + range to + search. The time string can be a UTC time (with fractional + seconds), a relative time specifier (to now), or a formatted + time string. + + * "rf" (``string``): Specifies one or more fields to add to the + search. + + :type kwargs: ``dict`` + :rtype: class:`Job` + :returns: An object representing the created job. + """ + return self.jobs.create(query, **kwargs) + + @property + def saved_searches(self): + """Returns the collection of saved searches. + + :return: A :class:`SavedSearches` collection of :class:`SavedSearch` + entities. + """ + return SavedSearches(self) + + @property + def settings(self): + """Returns the configuration settings for this instance of Splunk. + + :return: A :class:`Settings` object containing configuration settings. + """ + return Settings(self) + + @property + def splunk_version(self): + """Returns the version of the splunkd instance this object is attached + to. + + The version is returned as a tuple of the version components as + integers (for example, `(4,3,3)` or `(5,)`). + + :return: A ``tuple`` of ``integers``. + """ + if self._splunk_version is None: + self._splunk_version = tuple([int(p) for p in self.info['version'].split('.')]) + return self._splunk_version + + @property + def kvstore(self): + """Returns the collection of KV Store collections. + + :return: A :class:`KVStoreCollections` collection of :class:`KVStoreCollection` entities. + """ + return KVStoreCollections(self) + + @property + def users(self): + """Returns the collection of users. + + :return: A :class:`Users` collection of :class:`User` entities. + """ + return Users(self) + + +class Endpoint(object): + """This class represents individual Splunk resources in the Splunk REST API. + + An ``Endpoint`` object represents a URI, such as ``/services/saved/searches``. + This class provides the common functionality of :class:`Collection` and + :class:`Entity` (essentially HTTP GET and POST methods). + """ + def __init__(self, service, path): + self.service = service + self.path = path if path.endswith('/') else path + '/' + + def get(self, path_segment="", owner=None, app=None, sharing=None, **query): + """Performs a GET operation on the path segment relative to this endpoint. + + This method is named to match the HTTP method. This method makes at least + one roundtrip to the server, one additional round trip for + each 303 status returned, plus at most two additional round + trips if + the ``autologin`` field of :func:`connect` is set to ``True``. + + If *owner*, *app*, and *sharing* are omitted, this method takes a + default namespace from the :class:`Service` object for this :class:`Endpoint`. + All other keyword arguments are included in the URL as query parameters. + + :raises AuthenticationError: Raised when the ``Service`` is not logged in. + :raises HTTPError: Raised when an error in the request occurs. + :param path_segment: A path segment relative to this endpoint. + :type path_segment: ``string`` + :param owner: The owner context of the namespace (optional). + :type owner: ``string`` + :param app: The app context of the namespace (optional). + :type app: ``string`` + :param sharing: The sharing mode for the namespace (optional). + :type sharing: "global", "system", "app", or "user" + :param query: All other keyword arguments, which are used as query + parameters. + :type query: ``string`` + :return: The response from the server. + :rtype: ``dict`` with keys ``body``, ``headers``, ``reason``, + and ``status`` + + **Example**:: + + import splunklib.client + s = client.service(...) + apps = s.apps + apps.get() == \\ + {'body': ...a response reader object..., + 'headers': [('content-length', '26208'), + ('expires', 'Fri, 30 Oct 1998 00:00:00 GMT'), + ('server', 'Splunkd'), + ('connection', 'close'), + ('cache-control', 'no-store, max-age=0, must-revalidate, no-cache'), + ('date', 'Fri, 11 May 2012 16:30:35 GMT'), + ('content-type', 'text/xml; charset=utf-8')], + 'reason': 'OK', + 'status': 200} + apps.get('nonexistant/path') # raises HTTPError + s.logout() + apps.get() # raises AuthenticationError + """ + # self.path to the Endpoint is relative in the SDK, so passing + # owner, app, sharing, etc. along will produce the correct + # namespace in the final request. + if path_segment.startswith('/'): + path = path_segment + else: + path = self.service._abspath(self.path + path_segment, owner=owner, + app=app, sharing=sharing) + # ^-- This was "%s%s" % (self.path, path_segment). + # That doesn't work, because self.path may be UrlEncoded. + return self.service.get(path, + owner=owner, app=app, sharing=sharing, + **query) + + def post(self, path_segment="", owner=None, app=None, sharing=None, **query): + """Performs a POST operation on the path segment relative to this endpoint. + + This method is named to match the HTTP method. This method makes at least + one roundtrip to the server, one additional round trip for + each 303 status returned, plus at most two additional round trips if + the ``autologin`` field of :func:`connect` is set to ``True``. + + If *owner*, *app*, and *sharing* are omitted, this method takes a + default namespace from the :class:`Service` object for this :class:`Endpoint`. + All other keyword arguments are included in the URL as query parameters. + + :raises AuthenticationError: Raised when the ``Service`` is not logged in. + :raises HTTPError: Raised when an error in the request occurs. + :param path_segment: A path segment relative to this endpoint. + :type path_segment: ``string`` + :param owner: The owner context of the namespace (optional). + :type owner: ``string`` + :param app: The app context of the namespace (optional). + :type app: ``string`` + :param sharing: The sharing mode of the namespace (optional). + :type sharing: ``string`` + :param query: All other keyword arguments, which are used as query + parameters. + :type query: ``string`` + :return: The response from the server. + :rtype: ``dict`` with keys ``body``, ``headers``, ``reason``, + and ``status`` + + **Example**:: + + import splunklib.client + s = client.service(...) + apps = s.apps + apps.post(name='boris') == \\ + {'body': ...a response reader object..., + 'headers': [('content-length', '2908'), + ('expires', 'Fri, 30 Oct 1998 00:00:00 GMT'), + ('server', 'Splunkd'), + ('connection', 'close'), + ('cache-control', 'no-store, max-age=0, must-revalidate, no-cache'), + ('date', 'Fri, 11 May 2012 18:34:50 GMT'), + ('content-type', 'text/xml; charset=utf-8')], + 'reason': 'Created', + 'status': 201} + apps.get('nonexistant/path') # raises HTTPError + s.logout() + apps.get() # raises AuthenticationError + """ + if path_segment.startswith('/'): + path = path_segment + else: + path = self.service._abspath(self.path + path_segment, owner=owner, app=app, sharing=sharing) + return self.service.post(path, owner=owner, app=app, sharing=sharing, **query) + + +# kwargs: path, app, owner, sharing, state +class Entity(Endpoint): + """This class is a base class for Splunk entities in the REST API, such as + saved searches, jobs, indexes, and inputs. + + ``Entity`` provides the majority of functionality required by entities. + Subclasses only implement the special cases for individual entities. + For example for deployment serverclasses, the subclass makes whitelists and + blacklists into Python lists. + + An ``Entity`` is addressed like a dictionary, with a few extensions, + so the following all work:: + + ent['email.action'] + ent['disabled'] + ent['whitelist'] + + Many endpoints have values that share a prefix, such as + ``email.to``, ``email.action``, and ``email.subject``. You can extract + the whole fields, or use the key ``email`` to get a dictionary of + all the subelements. That is, ``ent['email']`` returns a + dictionary with the keys ``to``, ``action``, ``subject``, and so on. If + there are multiple levels of dots, each level is made into a + subdictionary, so ``email.body.salutation`` can be accessed at + ``ent['email']['body']['salutation']`` or + ``ent['email.body.salutation']``. + + You can also access the fields as though they were the fields of a Python + object, as in:: + + ent.email.action + ent.disabled + ent.whitelist + + However, because some of the field names are not valid Python identifiers, + the dictionary-like syntax is preferrable. + + The state of an :class:`Entity` object is cached, so accessing a field + does not contact the server. If you think the values on the + server have changed, call the :meth:`Entity.refresh` method. + """ + # Not every endpoint in the API is an Entity or a Collection. For + # example, a saved search at saved/searches/{name} has an additional + # method saved/searches/{name}/scheduled_times, but this isn't an + # entity in its own right. In these cases, subclasses should + # implement a method that uses the get and post methods inherited + # from Endpoint, calls the _load_atom function (it's elsewhere in + # client.py, but not a method of any object) to read the + # information, and returns the extracted data in a Pythonesque form. + # + # The primary use of subclasses of Entity is to handle specially + # named fields in the Entity. If you only need to provide a default + # value for an optional field, subclass Entity and define a + # dictionary ``defaults``. For instance,:: + # + # class Hypothetical(Entity): + # defaults = {'anOptionalField': 'foo', + # 'anotherField': 'bar'} + # + # If you have to do more than provide a default, such as rename or + # actually process values, then define a new method with the + # ``@property`` decorator. + # + # class Hypothetical(Entity): + # @property + # def foobar(self): + # return self.content['foo'] + "-" + self.content["bar"] + + # Subclasses can override defaults the default values for + # optional fields. See above. + defaults = {} + + def __init__(self, service, path, **kwargs): + Endpoint.__init__(self, service, path) + self._state = None + if not kwargs.get('skip_refresh', False): + self.refresh(kwargs.get('state', None)) # "Prefresh" + return + + def __contains__(self, item): + try: + self[item] + return True + except KeyError, AttributeError: + return False + + def __eq__(self, other): + """Raises IncomparableException. + + Since Entity objects are snapshots of times on the server, no + simple definition of equality will suffice beyond instance + equality, and instance equality leads to strange situations + such as:: + + import splunklib.client as client + c = client.connect(...) + saved_searches = c.saved_searches + x = saved_searches['asearch'] + + but then ``x != saved_searches['asearch']``. + + whether or not there was a change on the server. Rather than + try to do something fancy, we simple declare that equality is + undefined for Entities. + + Makes no roundtrips to the server. + """ + raise IncomparableException( + "Equality is undefined for objects of class %s" % \ + self.__class__.__name__) + + def __getattr__(self, key): + # Called when an attribute was not found by the normal method. In this + # case we try to find it in self.content and then self.defaults. + if key in self.state.content: + return self.state.content[key] + elif key in self.defaults: + return self.defaults[key] + else: + raise AttributeError(key) + + def __getitem__(self, key): + # getattr attempts to find a field on the object in the normal way, + # then calls __getattr__ if it cannot. + return getattr(self, key) + + # Load the Atom entry record from the given response - this is a method + # because the "entry" record varies slightly by entity and this allows + # for a subclass to override and handle any special cases. + def _load_atom_entry(self, response): + elem = _load_atom(response, XNAME_ENTRY) + if isinstance(elem, list): + raise AmbiguousReferenceException("Fetch from server returned multiple entries for name %s." % self.name) + else: + return elem.entry + + # Load the entity state record from the given response + def _load_state(self, response): + entry = self._load_atom_entry(response) + return _parse_atom_entry(entry) + + def _run_action(self, path_segment, **kwargs): + """Run a method and return the content Record from the returned XML. + + A method is a relative path from an Entity that is not itself + an Entity. _run_action assumes that the returned XML is an + Atom field containing one Entry, and the contents of Entry is + what should be the return value. This is right in enough cases + to make this method useful. + """ + response = self.get(path_segment, **kwargs) + data = self._load_atom_entry(response) + rec = _parse_atom_entry(data) + return rec.content + + def _proper_namespace(self, owner=None, app=None, sharing=None): + """Produce a namespace sans wildcards for use in entity requests. + + This method tries to fill in the fields of the namespace which are `None` + or wildcard (`'-'`) from the entity's namespace. If that fails, it uses + the service's namespace. + + :param owner: + :param app: + :param sharing: + :return: + """ + if owner is None and app is None and sharing is None: # No namespace provided + if self._state is not None and 'access' in self._state: + return (self._state.access.owner, + self._state.access.app, + self._state.access.sharing) + else: + return (self.service.namespace['owner'], + self.service.namespace['app'], + self.service.namespace['sharing']) + else: + return (owner,app,sharing) + + def delete(self): + owner, app, sharing = self._proper_namespace() + return self.service.delete(self.path, owner=owner, app=app, sharing=sharing) + + def get(self, path_segment="", owner=None, app=None, sharing=None, **query): + owner, app, sharing = self._proper_namespace(owner, app, sharing) + return super(Entity, self).get(path_segment, owner=owner, app=app, sharing=sharing, **query) + + def post(self, path_segment="", owner=None, app=None, sharing=None, **query): + owner, app, sharing = self._proper_namespace(owner, app, sharing) + return super(Entity, self).post(path_segment, owner=owner, app=app, sharing=sharing, **query) + + def refresh(self, state=None): + """Refreshes the state of this entity. + + If *state* is provided, load it as the new state for this + entity. Otherwise, make a roundtrip to the server (by calling + the :meth:`read` method of ``self``) to fetch an updated state, + plus at most two additional round trips if + the ``autologin`` field of :func:`connect` is set to ``True``. + + :param state: Entity-specific arguments (optional). + :type state: ``dict`` + :raises EntityDeletedException: Raised if the entity no longer exists on + the server. + + **Example**:: + + import splunklib.client as client + s = client.connect(...) + search = s.apps['search'] + search.refresh() + """ + if state is not None: + self._state = state + else: + self._state = self.read(self.get()) + return self + + @property + def access(self): + """Returns the access metadata for this entity. + + :return: A :class:`splunklib.data.Record` object with three keys: + ``owner``, ``app``, and ``sharing``. + """ + return self.state.access + + @property + def content(self): + """Returns the contents of the entity. + + :return: A ``dict`` containing values. + """ + return self.state.content + + def disable(self): + """Disables the entity at this endpoint.""" + self.post("disable") + if self.service.restart_required: + self.service.restart(120) + return self + + def enable(self): + """Enables the entity at this endpoint.""" + self.post("enable") + return self + + @property + def fields(self): + """Returns the content metadata for this entity. + + :return: A :class:`splunklib.data.Record` object with three keys: + ``required``, ``optional``, and ``wildcard``. + """ + return self.state.fields + + @property + def links(self): + """Returns a dictionary of related resources. + + :return: A ``dict`` with keys and corresponding URLs. + """ + return self.state.links + + @property + def name(self): + """Returns the entity name. + + :return: The entity name. + :rtype: ``string`` + """ + return self.state.title + + def read(self, response): + """ Reads the current state of the entity from the server. """ + results = self._load_state(response) + # In lower layers of the SDK, we end up trying to URL encode + # text to be dispatched via HTTP. However, these links are already + # URL encoded when they arrive, and we need to mark them as such. + unquoted_links = dict([(k, UrlEncoded(v, skip_encode=True)) + for k,v in results['links'].iteritems()]) + results['links'] = unquoted_links + return results + + def reload(self): + """Reloads the entity.""" + self.post("_reload") + return self + + @property + def state(self): + """Returns the entity's state record. + + :return: A ``dict`` containing fields and metadata for the entity. + """ + if self._state is None: self.refresh() + return self._state + + def update(self, **kwargs): + """Updates the server with any changes you've made to the current entity + along with any additional arguments you specify. + + **Note**: You cannot update the ``name`` field of an entity. + + Many of the fields in the REST API are not valid Python + identifiers, which means you cannot pass them as keyword + arguments. That is, Python will fail to parse the following:: + + # This fails + x.update(check-new=False, email.to='boris@utopia.net') + + However, you can always explicitly use a dictionary to pass + such keys:: + + # This works + x.update(**{'check-new': False, 'email.to': 'boris@utopia.net'}) + + :param kwargs: Additional entity-specific arguments (optional). + :type kwargs: ``dict`` + + :return: The entity this method is called on. + :rtype: class:`Entity` + """ + # The peculiarity in question: the REST API creates a new + # Entity if we pass name in the dictionary, instead of the + # expected behavior of updating this Entity. Therefore we + # check for 'name' in kwargs and throw an error if it is + # there. + if 'name' in kwargs: + raise IllegalOperationException('Cannot update the name of an Entity via the REST API.') + self.post(**kwargs) + return self + + +class ReadOnlyCollection(Endpoint): + """This class represents a read-only collection of entities in the Splunk + instance. + """ + def __init__(self, service, path, item=Entity): + Endpoint.__init__(self, service, path) + self.item = item # Item accessor + self.null_count = -1 + + def __contains__(self, name): + """Is there at least one entry called *name* in this collection? + + Makes a single roundtrip to the server, plus at most two more + if + the ``autologin`` field of :func:`connect` is set to ``True``. + """ + try: + self[name] + return True + except KeyError: + return False + except AmbiguousReferenceException: + return True + + def __getitem__(self, key): + """Fetch an item named *key* from this collection. + + A name is not a unique identifier in a collection. The unique + identifier is a name plus a namespace. For example, there can + be a saved search named ``'mysearch'`` with sharing ``'app'`` + in application ``'search'``, and another with sharing + ``'user'`` with owner ``'boris'`` and application + ``'search'``. If the ``Collection`` is attached to a + ``Service`` that has ``'-'`` (wildcard) as user and app in its + namespace, then both of these may be visible under the same + name. + + Where there is no conflict, ``__getitem__`` will fetch the + entity given just the name. If there is a conflict and you + pass just a name, it will raise a ``ValueError``. In that + case, add the namespace as a second argument. + + This function makes a single roundtrip to the server, plus at + most two additional round trips if + the ``autologin`` field of :func:`connect` is set to ``True``. + + :param key: The name to fetch, or a tuple (name, namespace). + :return: An :class:`Entity` object. + :raises KeyError: Raised if *key* does not exist. + :raises ValueError: Raised if no namespace is specified and *key* + does not refer to a unique name. + + *Example*:: + + s = client.connect(...) + saved_searches = s.saved_searches + x1 = saved_searches.create( + 'mysearch', 'search * | head 1', + owner='admin', app='search', sharing='app') + x2 = saved_searches.create( + 'mysearch', 'search * | head 1', + owner='admin', app='search', sharing='user') + # Raises ValueError: + saved_searches['mysearch'] + # Fetches x1 + saved_searches[ + 'mysearch', + client.namespace(sharing='app', app='search')] + # Fetches x2 + saved_searches[ + 'mysearch', + client.namespace(sharing='user', owner='boris', app='search')] + """ + try: + if isinstance(key, tuple) and len(key) == 2: + # x[a,b] is translated to x.__getitem__( (a,b) ), so we + # have to extract values out. + key, ns = key + key = UrlEncoded(key, encode_slash=True) + response = self.get(key, owner=ns.owner, app=ns.app) + else: + key = UrlEncoded(key, encode_slash=True) + response = self.get(key) + entries = self._load_list(response) + if len(entries) > 1: + raise AmbiguousReferenceException("Found multiple entities named '%s'; please specify a namespace." % key) + elif len(entries) == 0: + raise KeyError(key) + else: + return entries[0] + except HTTPError as he: + if he.status == 404: # No entity matching key and namespace. + raise KeyError(key) + else: + raise + + def __iter__(self, **kwargs): + """Iterate over the entities in the collection. + + :param kwargs: Additional arguments. + :type kwargs: ``dict`` + :rtype: iterator over entities. + + Implemented to give Collection a listish interface. This + function always makes a roundtrip to the server, plus at most + two additional round trips if + the ``autologin`` field of :func:`connect` is set to ``True``. + + **Example**:: + + import splunklib.client as client + c = client.connect(...) + saved_searches = c.saved_searches + for entity in saved_searches: + print "Saved search named %s" % entity.name + """ + + for item in self.iter(**kwargs): + yield item + + def __len__(self): + """Enable ``len(...)`` for ``Collection`` objects. + + Implemented for consistency with a listish interface. No + further failure modes beyond those possible for any method on + an Endpoint. + + This function always makes a round trip to the server, plus at + most two additional round trips if + the ``autologin`` field of :func:`connect` is set to ``True``. + + **Example**:: + + import splunklib.client as client + c = client.connect(...) + saved_searches = c.saved_searches + n = len(saved_searches) + """ + return len(self.list()) + + def _entity_path(self, state): + """Calculate the path to an entity to be returned. + + *state* should be the dictionary returned by + :func:`_parse_atom_entry`. :func:`_entity_path` extracts the + link to this entity from *state*, and strips all the namespace + prefixes from it to leave only the relative path of the entity + itself, sans namespace. + + :rtype: ``string`` + :return: an absolute path + """ + # This has been factored out so that it can be easily + # overloaded by Configurations, which has to switch its + # entities' endpoints from its own properties/ to configs/. + raw_path = urllib.unquote(state.links.alternate) + if 'servicesNS/' in raw_path: + return _trailing(raw_path, 'servicesNS/', '/', '/') + elif 'services/' in raw_path: + return _trailing(raw_path, 'services/') + else: + return raw_path + + def _load_list(self, response): + """Converts *response* to a list of entities. + + *response* is assumed to be a :class:`Record` containing an + HTTP response, of the form:: + + {'status': 200, + 'headers': [('content-length', '232642'), + ('expires', 'Fri, 30 Oct 1998 00:00:00 GMT'), + ('server', 'Splunkd'), + ('connection', 'close'), + ('cache-control', 'no-store, max-age=0, must-revalidate, no-cache'), + ('date', 'Tue, 29 May 2012 15:27:08 GMT'), + ('content-type', 'text/xml; charset=utf-8')], + 'reason': 'OK', + 'body': ...a stream implementing .read()...} + + The ``'body'`` key refers to a stream containing an Atom feed, + that is, an XML document with a toplevel element ````, + and within that element one or more ```` elements. + """ + # Some subclasses of Collection have to override this because + # splunkd returns something that doesn't match + # . + entries = _load_atom_entries(response) + if entries is None: return [] + entities = [] + for entry in entries: + state = _parse_atom_entry(entry) + entity = self.item( + self.service, + self._entity_path(state), + state=state) + entities.append(entity) + + return entities + + def itemmeta(self): + """Returns metadata for members of the collection. + + Makes a single roundtrip to the server, plus two more at most if + the ``autologin`` field of :func:`connect` is set to ``True``. + + :return: A :class:`splunklib.data.Record` object containing the metadata. + + **Example**:: + + import splunklib.client as client + import pprint + s = client.connect(...) + pprint.pprint(s.apps.itemmeta()) + {'access': {'app': 'search', + 'can_change_perms': '1', + 'can_list': '1', + 'can_share_app': '1', + 'can_share_global': '1', + 'can_share_user': '1', + 'can_write': '1', + 'modifiable': '1', + 'owner': 'admin', + 'perms': {'read': ['*'], 'write': ['admin']}, + 'removable': '0', + 'sharing': 'user'}, + 'fields': {'optional': ['author', + 'configured', + 'description', + 'label', + 'manageable', + 'template', + 'visible'], + 'required': ['name'], 'wildcard': []}} + """ + response = self.get("_new") + content = _load_atom(response, MATCH_ENTRY_CONTENT) + return _parse_atom_metadata(content) + + def iter(self, offset=0, count=None, pagesize=None, **kwargs): + """Iterates over the collection. + + This method is equivalent to the :meth:`list` method, but + it returns an iterator and can load a certain number of entities at a + time from the server. + + :param offset: The index of the first entity to return (optional). + :type offset: ``integer`` + :param count: The maximum number of entities to return (optional). + :type count: ``integer`` + :param pagesize: The number of entities to load (optional). + :type pagesize: ``integer`` + :param kwargs: Additional arguments (optional): + + - "search" (``string``): The search query to filter responses. + + - "sort_dir" (``string``): The direction to sort returned items: + "asc" or "desc". + + - "sort_key" (``string``): The field to use for sorting (optional). + + - "sort_mode" (``string``): The collating sequence for sorting + returned items: "auto", "alpha", "alpha_case", or "num". + + :type kwargs: ``dict`` + + **Example**:: + + import splunklib.client as client + s = client.connect(...) + for saved_search in s.saved_searches.iter(pagesize=10): + # Loads 10 saved searches at a time from the + # server. + ... + """ + assert pagesize is None or pagesize > 0 + if count is None: + count = self.null_count + fetched = 0 + while count == self.null_count or fetched < count: + response = self.get(count=pagesize or count, offset=offset, **kwargs) + items = self._load_list(response) + N = len(items) + fetched += N + for item in items: + yield item + if pagesize is None or N < pagesize: + break + offset += N + logging.debug("pagesize=%d, fetched=%d, offset=%d, N=%d, kwargs=%s", pagesize, fetched, offset, N, kwargs) + + # kwargs: count, offset, search, sort_dir, sort_key, sort_mode + def list(self, count=None, **kwargs): + """Retrieves a list of entities in this collection. + + The entire collection is loaded at once and is returned as a list. This + function makes a single roundtrip to the server, plus at most two more if + the ``autologin`` field of :func:`connect` is set to ``True``. + There is no caching--every call makes at least one round trip. + + :param count: The maximum number of entities to return (optional). + :type count: ``integer`` + :param kwargs: Additional arguments (optional): + + - "offset" (``integer``): The offset of the first item to return. + + - "search" (``string``): The search query to filter responses. + + - "sort_dir" (``string``): The direction to sort returned items: + "asc" or "desc". + + - "sort_key" (``string``): The field to use for sorting (optional). + + - "sort_mode" (``string``): The collating sequence for sorting + returned items: "auto", "alpha", "alpha_case", or "num". + + :type kwargs: ``dict`` + :return: A ``list`` of entities. + """ + # response = self.get(count=count, **kwargs) + # return self._load_list(response) + return list(self.iter(count=count, **kwargs)) + + + + +class Collection(ReadOnlyCollection): + """A collection of entities. + + Splunk provides a number of different collections of distinct + entity types: applications, saved searches, fired alerts, and a + number of others. Each particular type is available separately + from the Splunk instance, and the entities of that type are + returned in a :class:`Collection`. + + The interface for :class:`Collection` does not quite match either + ``list`` or ``dict`` in Python, because there are enough semantic + mismatches with either to make its behavior surprising. A unique + element in a :class:`Collection` is defined by a string giving its + name plus namespace (although the namespace is optional if the name is + unique). + + **Example**:: + + import splunklib.client as client + service = client.connect(...) + mycollection = service.saved_searches + mysearch = mycollection['my_search', client.namespace(owner='boris', app='natasha', sharing='user')] + # Or if there is only one search visible named 'my_search' + mysearch = mycollection['my_search'] + + Similarly, ``name`` in ``mycollection`` works as you might expect (though + you cannot currently pass a namespace to the ``in`` operator), as does + ``len(mycollection)``. + + However, as an aggregate, :class:`Collection` behaves more like a + list. If you iterate over a :class:`Collection`, you get an + iterator over the entities, not the names and namespaces. + + **Example**:: + + for entity in mycollection: + assert isinstance(entity, client.Entity) + + Use the :meth:`create` and :meth:`delete` methods to create and delete + entities in this collection. To view the access control list and other + metadata of the collection, use the :meth:`ReadOnlyCollection.itemmeta` method. + + :class:`Collection` does no caching. Each call makes at least one + round trip to the server to fetch data. + """ + + def create(self, name, **params): + """Creates a new entity in this collection. + + This function makes either one or two roundtrips to the + server, depending on the type of entities in this + collection, plus at most two more if + the ``autologin`` field of :func:`connect` is set to ``True``. + + :param name: The name of the entity to create. + :type name: ``string`` + :param namespace: A namespace, as created by the :func:`splunklib.binding.namespace` + function (optional). You can also set ``owner``, ``app``, and + ``sharing`` in ``params``. + :type namespace: A :class:`splunklib.data.Record` object with keys ``owner``, ``app``, + and ``sharing``. + :param params: Additional entity-specific arguments (optional). + :type params: ``dict`` + :return: The new entity. + :rtype: A subclass of :class:`Entity`, chosen by :meth:`Collection.self.item`. + + **Example**:: + + import splunklib.client as client + s = client.connect(...) + applications = s.apps + new_app = applications.create("my_fake_app") + """ + if not isinstance(name, basestring): + raise InvalidNameException("%s is not a valid name for an entity." % name) + if 'namespace' in params: + namespace = params.pop('namespace') + params['owner'] = namespace.owner + params['app'] = namespace.app + params['sharing'] = namespace.sharing + response = self.post(name=name, **params) + atom = _load_atom(response, XNAME_ENTRY) + if atom is None: + # This endpoint doesn't return the content of the new + # item. We have to go fetch it ourselves. + return self[name] + else: + entry = atom.entry + state = _parse_atom_entry(entry) + entity = self.item( + self.service, + self._entity_path(state), + state=state) + return entity + + def delete(self, name, **params): + """Deletes a specified entity from the collection. + + :param name: The name of the entity to delete. + :type name: ``string`` + :return: The collection. + :rtype: ``self`` + + This method is implemented for consistency with the REST API's DELETE + method. + + If there is no *name* entity on the server, a ``KeyError`` is + thrown. This function always makes a roundtrip to the server. + + **Example**:: + + import splunklib.client as client + c = client.connect(...) + saved_searches = c.saved_searches + saved_searches.create('my_saved_search', + 'search * | head 1') + assert 'my_saved_search' in saved_searches + saved_searches.delete('my_saved_search') + assert 'my_saved_search' not in saved_searches + """ + name = UrlEncoded(name, encode_slash=True) + if 'namespace' in params: + namespace = params.pop('namespace') + params['owner'] = namespace.owner + params['app'] = namespace.app + params['sharing'] = namespace.sharing + try: + self.service.delete(_path(self.path, name), **params) + except HTTPError as he: + # An HTTPError with status code 404 means that the entity + # has already been deleted, and we reraise it as a + # KeyError. + if he.status == 404: + raise KeyError("No such entity %s" % name) + else: + raise + return self + + def get(self, name="", owner=None, app=None, sharing=None, **query): + """Performs a GET request to the server on the collection. + + If *owner*, *app*, and *sharing* are omitted, this method takes a + default namespace from the :class:`Service` object for this :class:`Endpoint`. + All other keyword arguments are included in the URL as query parameters. + + :raises AuthenticationError: Raised when the ``Service`` is not logged in. + :raises HTTPError: Raised when an error in the request occurs. + :param path_segment: A path segment relative to this endpoint. + :type path_segment: ``string`` + :param owner: The owner context of the namespace (optional). + :type owner: ``string`` + :param app: The app context of the namespace (optional). + :type app: ``string`` + :param sharing: The sharing mode for the namespace (optional). + :type sharing: "global", "system", "app", or "user" + :param query: All other keyword arguments, which are used as query + parameters. + :type query: ``string`` + :return: The response from the server. + :rtype: ``dict`` with keys ``body``, ``headers``, ``reason``, + and ``status`` + + Example: + + import splunklib.client + s = client.service(...) + saved_searches = s.saved_searches + saved_searches.get("my/saved/search") == \\ + {'body': ...a response reader object..., + 'headers': [('content-length', '26208'), + ('expires', 'Fri, 30 Oct 1998 00:00:00 GMT'), + ('server', 'Splunkd'), + ('connection', 'close'), + ('cache-control', 'no-store, max-age=0, must-revalidate, no-cache'), + ('date', 'Fri, 11 May 2012 16:30:35 GMT'), + ('content-type', 'text/xml; charset=utf-8')], + 'reason': 'OK', + 'status': 200} + saved_searches.get('nonexistant/search') # raises HTTPError + s.logout() + saved_searches.get() # raises AuthenticationError + + """ + name = UrlEncoded(name, encode_slash=True) + return super(Collection, self).get(name, owner, app, sharing, **query) + + + + +class ConfigurationFile(Collection): + """This class contains all of the stanzas from one configuration file. + """ + # __init__'s arguments must match those of an Entity, not a + # Collection, since it is being created as the elements of a + # Configurations, which is a Collection subclass. + def __init__(self, service, path, **kwargs): + Collection.__init__(self, service, path, item=Stanza) + self.name = kwargs['state']['title'] + + +class Configurations(Collection): + """This class provides access to the configuration files from this Splunk + instance. Retrieve this collection using :meth:`Service.confs`. + + Splunk's configuration is divided into files, and each file into + stanzas. This collection is unusual in that the values in it are + themselves collections of :class:`ConfigurationFile` objects. + """ + def __init__(self, service): + Collection.__init__(self, service, PATH_PROPERTIES, item=ConfigurationFile) + if self.service.namespace.owner == '-' or self.service.namespace.app == '-': + raise ValueError("Configurations cannot have wildcards in namespace.") + + def __getitem__(self, key): + # The superclass implementation is designed for collections that contain + # entities. This collection (Configurations) contains collections + # (ConfigurationFile). + # + # The configurations endpoint returns multiple entities when we ask for a single file. + # This screws up the default implementation of __getitem__ from Collection, which thinks + # that multiple entities means a name collision, so we have to override it here. + try: + response = self.get(key) + return ConfigurationFile(self.service, PATH_CONF % key, state={'title': key}) + except HTTPError as he: + if he.status == 404: # No entity matching key + raise KeyError(key) + else: + raise + + def __contains__(self, key): + # configs/conf-{name} never returns a 404. We have to post to properties/{name} + # in order to find out if a configuration exists. + try: + response = self.get(key) + return True + except HTTPError as he: + if he.status == 404: # No entity matching key + return False + else: + raise + + def create(self, name): + """ Creates a configuration file named *name*. + + If there is already a configuration file with that name, + the existing file is returned. + + :param name: The name of the configuration file. + :type name: ``string`` + + :return: The :class:`ConfigurationFile` object. + """ + # This has to be overridden to handle the plumbing of creating + # a ConfigurationFile (which is a Collection) instead of some + # Entity. + if not isinstance(name, basestring): + raise ValueError("Invalid name: %s" % repr(name)) + response = self.post(__conf=name) + if response.status == 303: + return self[name] + elif response.status == 201: + return ConfigurationFile(self.service, PATH_CONF % name, item=Stanza, state={'title': name}) + else: + raise ValueError("Unexpected status code %s returned from creating a stanza" % response.status) + + def delete(self, key): + """Raises `IllegalOperationException`.""" + raise IllegalOperationException("Cannot delete configuration files from the REST API.") + + def _entity_path(self, state): + # Overridden to make all the ConfigurationFile objects + # returned refer to the configs/ path instead of the + # properties/ path used by Configrations. + return PATH_CONF % state['title'] + + +class Stanza(Entity): + """This class contains a single configuration stanza.""" + + def submit(self, stanza): + """Adds keys to the current configuration stanza as a + dictionary of key-value pairs. + + :param stanza: A dictionary of key-value pairs for the stanza. + :type stanza: ``dict`` + :return: The :class:`Stanza` object. + """ + body = _encode(**stanza) + self.service.post(self.path, body=body) + return self + + def __len__(self): + # The stanza endpoint returns all the keys at the same level in the XML as the eai information + # and 'disabled', so to get an accurate length, we have to filter those out and have just + # the stanza keys. + return len([x for x in self._state.content.keys() + if not x.startswith('eai') and x != 'disabled']) + + +class StoragePassword(Entity): + """This class contains a storage password. + """ + def __init__(self, service, path, **kwargs): + state = kwargs.get('state', None) + kwargs['skip_refresh'] = kwargs.get('skip_refresh', state is not None) + super(StoragePassword, self).__init__(service, path, **kwargs) + self._state = state + + @property + def clear_password(self): + return self.content.get('clear_password') + + @property + def encrypted_password(self): + return self.content.get('encr_password') + + @property + def realm(self): + return self.content.get('realm') + + @property + def username(self): + return self.content.get('username') + + +class StoragePasswords(Collection): + """This class provides access to the storage passwords from this Splunk + instance. Retrieve this collection using :meth:`Service.storage_passwords`. + """ + def __init__(self, service): + if service.namespace.owner == '-' or service.namespace.app == '-': + raise ValueError("StoragePasswords cannot have wildcards in namespace.") + super(StoragePasswords, self).__init__(service, PATH_STORAGE_PASSWORDS, item=StoragePassword) + + def create(self, password, username, realm=None): + """ Creates a storage password. + + A `StoragePassword` can be identified by , or by : if the + optional realm parameter is also provided. + + :param password: The password for the credentials - this is the only part of the credentials that will be stored securely. + :type name: ``string`` + :param username: The username for the credentials. + :type name: ``string`` + :param realm: The credential realm. (optional) + :type name: ``string`` + + :return: The :class:`StoragePassword` object created. + """ + if not isinstance(username, basestring): + raise ValueError("Invalid name: %s" % repr(username)) + + if realm is None: + response = self.post(password=password, name=username) + else: + response = self.post(password=password, realm=realm, name=username) + + if response.status != 201: + raise ValueError("Unexpected status code %s returned from creating a stanza" % response.status) + + entries = _load_atom_entries(response) + state = _parse_atom_entry(entries[0]) + storage_password = StoragePassword(self.service, self._entity_path(state), state=state, skip_refresh=True) + + return storage_password + + def delete(self, username, realm=None): + """Delete a storage password by username and/or realm. + + The identifier can be passed in through the username parameter as + or :, but the preferred way is by + passing in the username and realm parameters. + + :param username: The username for the credentials, or : if the realm parameter is omitted. + :type name: ``string`` + :param realm: The credential realm. (optional) + :type name: ``string`` + :return: The `StoragePassword` collection. + :rtype: ``self`` + """ + if realm is None: + # This case makes the username optional, so + # the full name can be passed in as realm. + # Assume it's already encoded. + name = username + else: + # Encode each component separately + name = UrlEncoded(realm, encode_slash=True) + ":" + UrlEncoded(username, encode_slash=True) + + # Append the : expected at the end of the name + if name[-1] is not ":": + name = name + ":" + return Collection.delete(self, name) + + +class AlertGroup(Entity): + """This class represents a group of fired alerts for a saved search. Access + it using the :meth:`alerts` property.""" + def __init__(self, service, path, **kwargs): + Entity.__init__(self, service, path, **kwargs) + + def __len__(self): + return self.count + + @property + def alerts(self): + """Returns a collection of triggered alerts. + + :return: A :class:`Collection` of triggered alerts. + """ + return Collection(self.service, self.path) + + @property + def count(self): + """Returns the count of triggered alerts. + + :return: The triggered alert count. + :rtype: ``integer`` + """ + return int(self.content.get('triggered_alert_count', 0)) + + +class Indexes(Collection): + """This class contains the collection of indexes in this Splunk instance. + Retrieve this collection using :meth:`Service.indexes`. + """ + def get_default(self): + """ Returns the name of the default index. + + :return: The name of the default index. + + """ + index = self['_audit'] + return index['defaultDatabase'] + + def delete(self, name): + """ Deletes a given index. + + **Note**: This method is only supported in Splunk 5.0 and later. + + :param name: The name of the index to delete. + :type name: ``string`` + """ + if self.service.splunk_version >= (5,): + Collection.delete(self, name) + else: + raise IllegalOperationException("Deleting indexes via the REST API is " + "not supported before Splunk version 5.") + + +class Index(Entity): + """This class represents an index and provides different operations, such as + cleaning the index, writing to the index, and so forth.""" + def __init__(self, service, path, **kwargs): + Entity.__init__(self, service, path, **kwargs) + + def attach(self, host=None, source=None, sourcetype=None): + """Opens a stream (a writable socket) for writing events to the index. + + :param host: The host value for events written to the stream. + :type host: ``string`` + :param source: The source value for events written to the stream. + :type source: ``string`` + :param sourcetype: The sourcetype value for events written to the + stream. + :type sourcetype: ``string`` + + :return: A writable socket. + """ + args = { 'index': self.name } + if host is not None: args['host'] = host + if source is not None: args['source'] = source + if sourcetype is not None: args['sourcetype'] = sourcetype + path = UrlEncoded(PATH_RECEIVERS_STREAM + "?" + urllib.urlencode(args), skip_encode=True) + + cookie_or_auth_header = "Authorization: Splunk %s\r\n" % \ + (self.service.token if self.service.token is _NoAuthenticationToken + else self.service.token.replace("Splunk ", "")) + + # If we have cookie(s), use them instead of "Authorization: ..." + if self.service.has_cookies(): + cookie_or_auth_header = "Cookie: %s\r\n" % _make_cookie_header(self.service.get_cookies().items()) + + # Since we need to stream to the index connection, we have to keep + # the connection open and use the Splunk extension headers to note + # the input mode + sock = self.service.connect() + headers = ["POST %s HTTP/1.1\r\n" % self.service._abspath(path), + "Host: %s:%s\r\n" % (self.service.host, int(self.service.port)), + "Accept-Encoding: identity\r\n", + cookie_or_auth_header, + "X-Splunk-Input-Mode: Streaming\r\n", + "\r\n"] + + for h in headers: + sock.write(h) + return sock + + @contextlib.contextmanager + def attached_socket(self, *args, **kwargs): + """Opens a raw socket in a ``with`` block to write data to Splunk. + + The arguments are identical to those for :meth:`attach`. The socket is + automatically closed at the end of the ``with`` block, even if an + exception is raised in the block. + + :param host: The host value for events written to the stream. + :type host: ``string`` + :param source: The source value for events written to the stream. + :type source: ``string`` + :param sourcetype: The sourcetype value for events written to the + stream. + :type sourcetype: ``string`` + + :returns: Nothing. + + **Example**:: + + import splunklib.client as client + s = client.connect(...) + index = s.indexes['some_index'] + with index.attached_socket(sourcetype='test') as sock: + sock.send('Test event\\r\\n') + + """ + try: + sock = self.attach(*args, **kwargs) + yield sock + finally: + sock.shutdown(socket.SHUT_RDWR) + sock.close() + + def clean(self, timeout=60): + """Deletes the contents of the index. + + This method blocks until the index is empty, because it needs to restore + values at the end of the operation. + + :param timeout: The time-out period for the operation, in seconds (the + default is 60). + :type timeout: ``integer`` + + :return: The :class:`Index`. + """ + self.refresh() + + tds = self['maxTotalDataSizeMB'] + ftp = self['frozenTimePeriodInSecs'] + was_disabled_initially = self.disabled + try: + if (not was_disabled_initially and \ + self.service.splunk_version < (5,)): + # Need to disable the index first on Splunk 4.x, + # but it doesn't work to disable it on 5.0. + self.disable() + self.update(maxTotalDataSizeMB=1, frozenTimePeriodInSecs=1) + self.roll_hot_buckets() + + # Wait until event count goes to 0. + start = datetime.now() + diff = timedelta(seconds=timeout) + while self.content.totalEventCount != '0' and datetime.now() < start+diff: + sleep(1) + self.refresh() + + if self.content.totalEventCount != '0': + raise OperationError, "Cleaning index %s took longer than %s seconds; timing out." %\ + (self.name, timeout) + finally: + # Restore original values + self.update(maxTotalDataSizeMB=tds, frozenTimePeriodInSecs=ftp) + if (not was_disabled_initially and \ + self.service.splunk_version < (5,)): + # Re-enable the index if it was originally enabled and we messed with it. + self.enable() + + return self + + def roll_hot_buckets(self): + """Performs rolling hot buckets for this index. + + :return: The :class:`Index`. + """ + self.post("roll-hot-buckets") + return self + + def submit(self, event, host=None, source=None, sourcetype=None): + """Submits a single event to the index using ``HTTP POST``. + + :param event: The event to submit. + :type event: ``string`` + :param `host`: The host value of the event. + :type host: ``string`` + :param `source`: The source value of the event. + :type source: ``string`` + :param `sourcetype`: The sourcetype value of the event. + :type sourcetype: ``string`` + + :return: The :class:`Index`. + """ + args = { 'index': self.name } + if host is not None: args['host'] = host + if source is not None: args['source'] = source + if sourcetype is not None: args['sourcetype'] = sourcetype + + # The reason we use service.request directly rather than POST + # is that we are not sending a POST request encoded using + # x-www-form-urlencoded (as we do not have a key=value body), + # because we aren't really sending a "form". + self.service.post(PATH_RECEIVERS_SIMPLE, body=event, **args) + return self + + # kwargs: host, host_regex, host_segment, rename-source, sourcetype + def upload(self, filename, **kwargs): + """Uploads a file for immediate indexing. + + **Note**: The file must be locally accessible from the server. + + :param filename: The name of the file to upload. The file can be a + plain, compressed, or archived file. + :type filename: ``string`` + :param kwargs: Additional arguments (optional). For more about the + available parameters, see `Index parameters `_ on Splunk Developer Portal. + :type kwargs: ``dict`` + + :return: The :class:`Index`. + """ + kwargs['index'] = self.name + path = 'data/inputs/oneshot' + self.service.post(path, name=filename, **kwargs) + return self + + +class Input(Entity): + """This class represents a Splunk input. This class is the base for all + typed input classes and is also used when the client does not recognize an + input kind. + """ + def __init__(self, service, path, kind=None, **kwargs): + # kind can be omitted (in which case it is inferred from the path) + # Otherwise, valid values are the paths from data/inputs ("udp", + # "monitor", "tcp/raw"), or two special cases: "tcp" (which is "tcp/raw") + # and "splunktcp" (which is "tcp/cooked"). + Entity.__init__(self, service, path, **kwargs) + if kind is None: + path_segments = path.split('/') + i = path_segments.index('inputs') + 1 + if path_segments[i] == 'tcp': + self.kind = path_segments[i] + '/' + path_segments[i+1] + else: + self.kind = path_segments[i] + else: + self.kind = kind + + # Handle old input kind names. + if self.kind == 'tcp': + self.kind = 'tcp/raw' + if self.kind == 'splunktcp': + self.kind = 'tcp/cooked' + + def update(self, **kwargs): + """Updates the server with any changes you've made to the current input + along with any additional arguments you specify. + + :param kwargs: Additional arguments (optional). For more about the + available parameters, see `Input parameters `_ on Splunk Developer Portal. + :type kwargs: ``dict`` + + :return: The input this method was called on. + :rtype: class:`Input` + """ + # UDP and TCP inputs require special handling due to their restrictToHost + # field. For all other inputs kinds, we can dispatch to the superclass method. + if self.kind not in ['tcp', 'splunktcp', 'tcp/raw', 'tcp/cooked', 'udp']: + return super(Input, self).update(**kwargs) + else: + # The behavior of restrictToHost is inconsistent across input kinds and versions of Splunk. + # In Splunk 4.x, the name of the entity is only the port, independent of the value of + # restrictToHost. In Splunk 5.0 this changed so the name will be of the form :. + # In 5.0 and 5.0.1, if you don't supply the restrictToHost value on every update, it will + # remove the host restriction from the input. As of 5.0.2 you simply can't change restrictToHost + # on an existing input. + + # The logic to handle all these cases: + # - Throw an exception if the user tries to set restrictToHost on an existing input + # for *any* version of Splunk. + # - Set the existing restrictToHost value on the update args internally so we don't + # cause it to change in Splunk 5.0 and 5.0.1. + to_update = kwargs.copy() + + if 'restrictToHost' in kwargs: + raise IllegalOperationException("Cannot set restrictToHost on an existing input with the SDK.") + elif 'restrictToHost' in self._state.content and self.kind != 'udp': + to_update['restrictToHost'] = self._state.content['restrictToHost'] + + # Do the actual update operation. + return super(Input, self).update(**to_update) + + +# Inputs is a "kinded" collection, which is a heterogenous collection where +# each item is tagged with a kind, that provides a single merged view of all +# input kinds. +class Inputs(Collection): + """This class represents a collection of inputs. The collection is + heterogeneous and each member of the collection contains a *kind* property + that indicates the specific type of input. + Retrieve this collection using :meth:`Service.inputs`.""" + + def __init__(self, service, kindmap=None): + Collection.__init__(self, service, PATH_INPUTS, item=Input) + + def __getitem__(self, key): + # The key needed to retrieve the input needs it's parenthesis to be URL encoded + # based on the REST API for input + # + if isinstance(key, tuple) and len(key) == 2: + # Fetch a single kind + key, kind = key + key = UrlEncoded(key, encode_slash=True) + try: + response = self.get(self.kindpath(kind) + "/" + key) + entries = self._load_list(response) + if len(entries) > 1: + raise AmbiguousReferenceException("Found multiple inputs of kind %s named %s." % (kind, key)) + elif len(entries) == 0: + raise KeyError((key, kind)) + else: + return entries[0] + except HTTPError as he: + if he.status == 404: # No entity matching kind and key + raise KeyError((key, kind)) + else: + raise + else: + # Iterate over all the kinds looking for matches. + kind = None + candidate = None + key = UrlEncoded(key, encode_slash=True) + for kind in self.kinds: + try: + response = self.get(kind + "/" + key) + entries = self._load_list(response) + if len(entries) > 1: + raise AmbiguousReferenceException("Found multiple inputs of kind %s named %s." % (kind, key)) + elif len(entries) == 0: + pass + else: + if candidate is not None: # Already found at least one candidate + raise AmbiguousReferenceException("Found multiple inputs named %s, please specify a kind" % key) + candidate = entries[0] + except HTTPError as he: + if he.status == 404: + pass # Just carry on to the next kind. + else: + raise + if candidate is None: + raise KeyError(key) # Never found a match. + else: + return candidate + + def __contains__(self, key): + if isinstance(key, tuple) and len(key) == 2: + # If we specify a kind, this will shortcut properly + try: + self.__getitem__(key) + return True + except KeyError: + return False + else: + # Without a kind, we want to minimize the number of round trips to the server, so we + # reimplement some of the behavior of __getitem__ in order to be able to stop searching + # on the first hit. + for kind in self.kinds: + try: + response = self.get(self.kindpath(kind) + "/" + key) + entries = self._load_list(response) + if len(entries) > 0: + return True + else: + pass + except HTTPError as he: + if he.status == 404: + pass # Just carry on to the next kind. + else: + raise + return False + + def create(self, name, kind, **kwargs): + """Creates an input of a specific kind in this collection, with any + arguments you specify. + + :param `name`: The input name. + :type name: ``string`` + :param `kind`: The kind of input: + + - "ad": Active Directory + + - "monitor": Files and directories + + - "registry": Windows Registry + + - "script": Scripts + + - "splunktcp": TCP, processed + + - "tcp": TCP, unprocessed + + - "udp": UDP + + - "win-event-log-collections": Windows event log + + - "win-perfmon": Performance monitoring + + - "win-wmi-collections": WMI + + :type kind: ``string`` + :param `kwargs`: Additional arguments (optional). For more about the + available parameters, see `Input parameters `_ on Splunk Developer Portal. + + :type kwargs: ``dict`` + + :return: The new :class:`Input`. + """ + kindpath = self.kindpath(kind) + self.post(kindpath, name=name, **kwargs) + + # If we created an input with restrictToHost set, then + # its path will be :, not just , + # and we have to adjust accordingly. + + # Url encodes the name of the entity. + name = UrlEncoded(name, encode_slash=True) + path = _path( + self.path + kindpath, + '%s:%s' % (kwargs['restrictToHost'], name) \ + if kwargs.has_key('restrictToHost') else name + ) + return Input(self.service, path, kind) + + def delete(self, name, kind=None): + """Removes an input from the collection. + + :param `kind`: The kind of input: + + - "ad": Active Directory + + - "monitor": Files and directories + + - "registry": Windows Registry + + - "script": Scripts + + - "splunktcp": TCP, processed + + - "tcp": TCP, unprocessed + + - "udp": UDP + + - "win-event-log-collections": Windows event log + + - "win-perfmon": Performance monitoring + + - "win-wmi-collections": WMI + + :type kind: ``string`` + :param name: The name of the input to remove. + :type name: ``string`` + + :return: The :class:`Inputs` collection. + """ + if kind is None: + self.service.delete(self[name].path) + else: + self.service.delete(self[name, kind].path) + return self + + def itemmeta(self, kind): + """Returns metadata for the members of a given kind. + + :param `kind`: The kind of input: + + - "ad": Active Directory + + - "monitor": Files and directories + + - "registry": Windows Registry + + - "script": Scripts + + - "splunktcp": TCP, processed + + - "tcp": TCP, unprocessed + + - "udp": UDP + + - "win-event-log-collections": Windows event log + + - "win-perfmon": Performance monitoring + + - "win-wmi-collections": WMI + + :type kind: ``string`` + + :return: The metadata. + :rtype: class:``splunklib.data.Record`` + """ + response = self.get("%s/_new" % self._kindmap[kind]) + content = _load_atom(response, MATCH_ENTRY_CONTENT) + return _parse_atom_metadata(content) + + def _get_kind_list(self, subpath=None): + if subpath is None: + subpath = [] + + kinds = [] + response = self.get('/'.join(subpath)) + content = _load_atom_entries(response) + for entry in content: + this_subpath = subpath + [entry.title] + # The "all" endpoint doesn't work yet. + # The "tcp/ssl" endpoint is not a real input collection. + if entry.title == 'all' or this_subpath == ['tcp','ssl']: + continue + elif 'create' in [x.rel for x in entry.link]: + path = '/'.join(subpath + [entry.title]) + kinds.append(path) + else: + subkinds = self._get_kind_list(subpath + [entry.title]) + kinds.extend(subkinds) + return kinds + + @property + def kinds(self): + """Returns the input kinds on this Splunk instance. + + :return: The list of input kinds. + :rtype: ``list`` + """ + return self._get_kind_list() + + def kindpath(self, kind): + """Returns a path to the resources for a given input kind. + + :param `kind`: The kind of input: + + - "ad": Active Directory + + - "monitor": Files and directories + + - "registry": Windows Registry + + - "script": Scripts + + - "splunktcp": TCP, processed + + - "tcp": TCP, unprocessed + + - "udp": UDP + + - "win-event-log-collections": Windows event log + + - "win-perfmon": Performance monitoring + + - "win-wmi-collections": WMI + + :type kind: ``string`` + + :return: The relative endpoint path. + :rtype: ``string`` + """ + if kind == 'tcp': + return UrlEncoded('tcp/raw', skip_encode=True) + elif kind == 'splunktcp': + return UrlEncoded('tcp/cooked', skip_encode=True) + else: + return UrlEncoded(kind, skip_encode=True) + + def list(self, *kinds, **kwargs): + """Returns a list of inputs that are in the :class:`Inputs` collection. + You can also filter by one or more input kinds. + + This function iterates over all possible inputs, regardless of any arguments you + specify. Because the :class:`Inputs` collection is the union of all the inputs of each + kind, this method implements parameters such as "count", "search", and so + on at the Python level once all the data has been fetched. The exception + is when you specify a single input kind, and then this method makes a single request + with the usual semantics for parameters. + + :param kinds: The input kinds to return (optional). + + - "ad": Active Directory + + - "monitor": Files and directories + + - "registry": Windows Registry + + - "script": Scripts + + - "splunktcp": TCP, processed + + - "tcp": TCP, unprocessed + + - "udp": UDP + + - "win-event-log-collections": Windows event log + + - "win-perfmon": Performance monitoring + + - "win-wmi-collections": WMI + + :type kinds: ``string`` + :param kwargs: Additional arguments (optional): + + - "count" (``integer``): The maximum number of items to return. + + - "offset" (``integer``): The offset of the first item to return. + + - "search" (``string``): The search query to filter responses. + + - "sort_dir" (``string``): The direction to sort returned items: + "asc" or "desc". + + - "sort_key" (``string``): The field to use for sorting (optional). + + - "sort_mode" (``string``): The collating sequence for sorting + returned items: "auto", "alpha", "alpha_case", or "num". + + :type kwargs: ``dict`` + + :return: A list of input kinds. + :rtype: ``list`` + """ + if len(kinds) == 0: + kinds = self.kinds + if len(kinds) == 1: + kind = kinds[0] + logging.debug("Inputs.list taking short circuit branch for single kind.") + path = self.kindpath(kind) + logging.debug("Path for inputs: %s", path) + try: + path = UrlEncoded(path, skip_encode=True) + response = self.get(path, **kwargs) + except HTTPError, he: + if he.status == 404: # No inputs of this kind + return [] + entities = [] + entries = _load_atom_entries(response) + if entries is None: + return [] # No inputs in a collection comes back with no feed or entry in the XML + for entry in entries: + state = _parse_atom_entry(entry) + # Unquote the URL, since all URL encoded in the SDK + # should be of type UrlEncoded, and all str should not + # be URL encoded. + path = urllib.unquote(state.links.alternate) + entity = Input(self.service, path, kind, state=state) + entities.append(entity) + return entities + + search = kwargs.get('search', '*') + + entities = [] + for kind in kinds: + response = None + try: + kind = UrlEncoded(kind, skip_encode=True) + response = self.get(self.kindpath(kind), search=search) + except HTTPError as e: + if e.status == 404: + continue # No inputs of this kind + else: + raise + + entries = _load_atom_entries(response) + if entries is None: continue # No inputs to process + for entry in entries: + state = _parse_atom_entry(entry) + # Unquote the URL, since all URL encoded in the SDK + # should be of type UrlEncoded, and all str should not + # be URL encoded. + path = urllib.unquote(state.links.alternate) + entity = Input(self.service, path, kind, state=state) + entities.append(entity) + if 'offset' in kwargs: + entities = entities[kwargs['offset']:] + if 'count' in kwargs: + entities = entities[:kwargs['count']] + if kwargs.get('sort_mode', None) == 'alpha': + sort_field = kwargs.get('sort_field', 'name') + if sort_field == 'name': + f = lambda x: x.name.lower() + else: + f = lambda x: x[sort_field].lower() + entities = sorted(entities, key=f) + if kwargs.get('sort_mode', None) == 'alpha_case': + sort_field = kwargs.get('sort_field', 'name') + if sort_field == 'name': + f = lambda x: x.name + else: + f = lambda x: x[sort_field] + entities = sorted(entities, key=f) + if kwargs.get('sort_dir', 'asc') == 'desc': + entities = list(reversed(entities)) + return entities + + def __iter__(self, **kwargs): + for item in self.iter(**kwargs): + yield item + + def iter(self, **kwargs): + """ Iterates over the collection of inputs. + + :param kwargs: Additional arguments (optional): + + - "count" (``integer``): The maximum number of items to return. + + - "offset" (``integer``): The offset of the first item to return. + + - "search" (``string``): The search query to filter responses. + + - "sort_dir" (``string``): The direction to sort returned items: + "asc" or "desc". + + - "sort_key" (``string``): The field to use for sorting (optional). + + - "sort_mode" (``string``): The collating sequence for sorting + returned items: "auto", "alpha", "alpha_case", or "num". + + :type kwargs: ``dict`` + """ + for item in self.list(**kwargs): + yield item + + def oneshot(self, path, **kwargs): + """ Creates a oneshot data input, which is an upload of a single file + for one-time indexing. + + :param path: The path and filename. + :type path: ``string`` + :param kwargs: Additional arguments (optional). For more about the + available parameters, see `Input parameters `_ on Splunk Developer Portal. + :type kwargs: ``dict`` + """ + self.post('oneshot', name=path, **kwargs) + + +class Job(Entity): + """This class represents a search job.""" + def __init__(self, service, sid, **kwargs): + path = PATH_JOBS + sid + Entity.__init__(self, service, path, skip_refresh=True, **kwargs) + self.sid = sid + + # The Job entry record is returned at the root of the response + def _load_atom_entry(self, response): + return _load_atom(response).entry + + def cancel(self): + """Stops the current search and deletes the results cache. + + :return: The :class:`Job`. + """ + try: + self.post("control", action="cancel") + except HTTPError as he: + if he.status == 404: + # The job has already been cancelled, so + # cancelling it twice is a nop. + pass + else: + raise + return self + + def disable_preview(self): + """Disables preview for this job. + + :return: The :class:`Job`. + """ + self.post("control", action="disablepreview") + return self + + def enable_preview(self): + """Enables preview for this job. + + **Note**: Enabling preview might slow search considerably. + + :return: The :class:`Job`. + """ + self.post("control", action="enablepreview") + return self + + def events(self, **kwargs): + """Returns a streaming handle to this job's events. + + :param kwargs: Additional parameters (optional). For a list of valid + parameters, see `GET search/jobs/{search_id}/events + `_ + in the REST API documentation. + :type kwargs: ``dict`` + + :return: The ``InputStream`` IO handle to this job's events. + """ + kwargs['segmentation'] = kwargs.get('segmentation', 'none') + return self.get("events", **kwargs).body + + def finalize(self): + """Stops the job and provides intermediate results for retrieval. + + :return: The :class:`Job`. + """ + self.post("control", action="finalize") + return self + + def is_done(self): + """Indicates whether this job finished running. + + :return: ``True`` if the job is done, ``False`` if not. + :rtype: ``boolean`` + """ + if not self.is_ready(): + return False + done = (self._state.content['isDone'] == '1') + return done + + def is_ready(self): + """Indicates whether this job is ready for querying. + + :return: ``True`` if the job is ready, ``False`` if not. + :rtype: ``boolean`` + + """ + response = self.get() + if response.status == 204: + return False + self._state = self.read(response) + ready = self._state.content['dispatchState'] not in ['QUEUED', 'PARSING'] + return ready + + @property + def name(self): + """Returns the name of the search job, which is the search ID (SID). + + :return: The search ID. + :rtype: ``string`` + """ + return self.sid + + def pause(self): + """Suspends the current search. + + :return: The :class:`Job`. + """ + self.post("control", action="pause") + return self + + def results(self, **query_params): + """Returns a streaming handle to this job's search results. To get a + nice, Pythonic iterator, pass the handle to :class:`splunklib.results.ResultsReader`, + as in:: + + import splunklib.client as client + import splunklib.results as results + from time import sleep + service = client.connect(...) + job = service.jobs.create("search * | head 5") + while not job.is_done(): + sleep(.2) + rr = results.ResultsReader(job.results()) + for result in rr: + if isinstance(result, results.Message): + # Diagnostic messages may be returned in the results + print '%s: %s' % (result.type, result.message) + elif isinstance(result, dict): + # Normal events are returned as dicts + print result + assert rr.is_preview == False + + Results are not available until the job has finished. If called on + an unfinished job, the result is an empty event set. + + This method makes a single roundtrip + to the server, plus at most two additional round trips if + the ``autologin`` field of :func:`connect` is set to ``True``. + + :param query_params: Additional parameters (optional). For a list of valid + parameters, see `GET search/jobs/{search_id}/results + `_. + :type query_params: ``dict`` + + :return: The ``InputStream`` IO handle to this job's results. + """ + query_params['segmentation'] = query_params.get('segmentation', 'none') + return self.get("results", **query_params).body + + def preview(self, **query_params): + """Returns a streaming handle to this job's preview search results. + + Unlike :class:`splunklib.results.ResultsReader`, which requires a job to + be finished to + return any results, the ``preview`` method returns any results that have + been generated so far, whether the job is running or not. The + returned search results are the raw data from the server. Pass + the handle returned to :class:`splunklib.results.ResultsReader` to get a + nice, Pythonic iterator over objects, as in:: + + import splunklib.client as client + import splunklib.results as results + service = client.connect(...) + job = service.jobs.create("search * | head 5") + rr = results.ResultsReader(job.preview()) + for result in rr: + if isinstance(result, results.Message): + # Diagnostic messages may be returned in the results + print '%s: %s' % (result.type, result.message) + elif isinstance(result, dict): + # Normal events are returned as dicts + print result + if rr.is_preview: + print "Preview of a running search job." + else: + print "Job is finished. Results are final." + + This method makes one roundtrip to the server, plus at most + two more if + the ``autologin`` field of :func:`connect` is set to ``True``. + + :param query_params: Additional parameters (optional). For a list of valid + parameters, see `GET search/jobs/{search_id}/results_preview + `_ + in the REST API documentation. + :type query_params: ``dict`` + + :return: The ``InputStream`` IO handle to this job's preview results. + """ + query_params['segmentation'] = query_params.get('segmentation', 'none') + return self.get("results_preview", **query_params).body + + def searchlog(self, **kwargs): + """Returns a streaming handle to this job's search log. + + :param `kwargs`: Additional parameters (optional). For a list of valid + parameters, see `GET search/jobs/{search_id}/search.log + `_ + in the REST API documentation. + :type kwargs: ``dict`` + + :return: The ``InputStream`` IO handle to this job's search log. + """ + return self.get("search.log", **kwargs).body + + def set_priority(self, value): + """Sets this job's search priority in the range of 0-10. + + Higher numbers indicate higher priority. Unless splunkd is + running as *root*, you can only decrease the priority of a running job. + + :param `value`: The search priority. + :type value: ``integer`` + + :return: The :class:`Job`. + """ + self.post('control', action="setpriority", priority=value) + return self + + def summary(self, **kwargs): + """Returns a streaming handle to this job's summary. + + :param `kwargs`: Additional parameters (optional). For a list of valid + parameters, see `GET search/jobs/{search_id}/summary + `_ + in the REST API documentation. + :type kwargs: ``dict`` + + :return: The ``InputStream`` IO handle to this job's summary. + """ + return self.get("summary", **kwargs).body + + def timeline(self, **kwargs): + """Returns a streaming handle to this job's timeline results. + + :param `kwargs`: Additional timeline arguments (optional). For a list of valid + parameters, see `GET search/jobs/{search_id}/timeline + `_ + in the REST API documentation. + :type kwargs: ``dict`` + + :return: The ``InputStream`` IO handle to this job's timeline. + """ + return self.get("timeline", **kwargs).body + + def touch(self): + """Extends the expiration time of the search to the current time (now) plus + the time-to-live (ttl) value. + + :return: The :class:`Job`. + """ + self.post("control", action="touch") + return self + + def set_ttl(self, value): + """Set the job's time-to-live (ttl) value, which is the time before the + search job expires and is still available. + + :param `value`: The ttl value, in seconds. + :type value: ``integer`` + + :return: The :class:`Job`. + """ + self.post("control", action="setttl", ttl=value) + return self + + def unpause(self): + """Resumes the current search, if paused. + + :return: The :class:`Job`. + """ + self.post("control", action="unpause") + return self + + +class Jobs(Collection): + """This class represents a collection of search jobs. Retrieve this + collection using :meth:`Service.jobs`.""" + def __init__(self, service): + Collection.__init__(self, service, PATH_JOBS, item=Job) + # The count value to say list all the contents of this + # Collection is 0, not -1 as it is on most. + self.null_count = 0 + + def _load_list(self, response): + # Overridden because Job takes a sid instead of a path. + entries = _load_atom_entries(response) + if entries is None: return [] + entities = [] + for entry in entries: + state = _parse_atom_entry(entry) + entity = self.item( + self.service, + entry['content']['sid'], + state=state) + entities.append(entity) + return entities + + def create(self, query, **kwargs): + """ Creates a search using a search query and any additional parameters + you provide. + + :param query: The search query. + :type query: ``string`` + :param kwargs: Additiona parameters (optional). For a list of available + parameters, see `Search job parameters + `_ + on Splunk Developer Portal. + :type kwargs: ``dict`` + + :return: The :class:`Job`. + """ + if kwargs.get("exec_mode", None) == "oneshot": + raise TypeError("Cannot specify exec_mode=oneshot; use the oneshot method instead.") + response = self.post(search=query, **kwargs) + sid = _load_sid(response) + return Job(self.service, sid) + + def export(self, query, **params): + """Runs a search and immediately starts streaming preview events. + This method returns a streaming handle to this job's events as an XML + document from the server. To parse this stream into usable Python objects, + pass the handle to :class:`splunklib.results.ResultsReader`:: + + import splunklib.client as client + import splunklib.results as results + service = client.connect(...) + rr = results.ResultsReader(service.jobs.export("search * | head 5")) + for result in rr: + if isinstance(result, results.Message): + # Diagnostic messages may be returned in the results + print '%s: %s' % (result.type, result.message) + elif isinstance(result, dict): + # Normal events are returned as dicts + print result + assert rr.is_preview == False + + Running an export search is more efficient as it streams the results + directly to you, rather than having to write them out to disk and make + them available later. As soon as results are ready, you will receive + them. + + The ``export`` method makes a single roundtrip to the server (as opposed + to two for :meth:`create` followed by :meth:`preview`), plus at most two + more if the ``autologin`` field of :func:`connect` is set to ``True``. + + :raises `ValueError`: Raised for invalid queries. + :param query: The search query. + :type query: ``string`` + :param params: Additional arguments (optional). For a list of valid + parameters, see `GET search/jobs/export + `_ + in the REST API documentation. + :type params: ``dict`` + + :return: The ``InputStream`` IO handle to raw XML returned from the server. + """ + if "exec_mode" in params: + raise TypeError("Cannot specify an exec_mode to export.") + params['segmentation'] = params.get('segmentation', 'none') + return self.post(path_segment="export", + search=query, + **params).body + + def itemmeta(self): + """There is no metadata available for class:``Jobs``. + + Any call to this method raises a class:``NotSupportedError``. + + :raises: class:``NotSupportedError`` + """ + raise NotSupportedError() + + def oneshot(self, query, **params): + """Run a oneshot search and returns a streaming handle to the results. + + The ``InputStream`` object streams XML fragments from the server. To + parse this stream into usable Python objects, + pass the handle to :class:`splunklib.results.ResultsReader`:: + + import splunklib.client as client + import splunklib.results as results + service = client.connect(...) + rr = results.ResultsReader(service.jobs.oneshot("search * | head 5")) + for result in rr: + if isinstance(result, results.Message): + # Diagnostic messages may be returned in the results + print '%s: %s' % (result.type, result.message) + elif isinstance(result, dict): + # Normal events are returned as dicts + print result + assert rr.is_preview == False + + The ``oneshot`` method makes a single roundtrip to the server (as opposed + to two for :meth:`create` followed by :meth:`results`), plus at most two more + if the ``autologin`` field of :func:`connect` is set to ``True``. + + :raises ValueError: Raised for invalid queries. + + :param query: The search query. + :type query: ``string`` + :param params: Additional arguments (optional): + + - "output_mode": Specifies the output format of the results (XML, + JSON, or CSV). + + - "earliest_time": Specifies the earliest time in the time range to + search. The time string can be a UTC time (with fractional seconds), + a relative time specifier (to now), or a formatted time string. + + - "latest_time": Specifies the latest time in the time range to + search. The time string can be a UTC time (with fractional seconds), + a relative time specifier (to now), or a formatted time string. + + - "rf": Specifies one or more fields to add to the search. + + :type params: ``dict`` + + :return: The ``InputStream`` IO handle to raw XML returned from the server. + """ + if "exec_mode" in params: + raise TypeError("Cannot specify an exec_mode to oneshot.") + params['segmentation'] = params.get('segmentation', 'none') + return self.post(search=query, + exec_mode="oneshot", + **params).body + + +class Loggers(Collection): + """This class represents a collection of service logging categories. + Retrieve this collection using :meth:`Service.loggers`.""" + def __init__(self, service): + Collection.__init__(self, service, PATH_LOGGER) + + def itemmeta(self): + """There is no metadata available for class:``Loggers``. + + Any call to this method raises a class:``NotSupportedError``. + + :raises: class:``NotSupportedError`` + """ + raise NotSupportedError() + + +class Message(Entity): + def __init__(self, service, path, **kwargs): + Entity.__init__(self, service, path, **kwargs) + + @property + def value(self): + """Returns the message value. + + :return: The message value. + :rtype: ``string`` + """ + return self[self.name] + + +class ModularInputKind(Entity): + """This class contains the different types of modular inputs. Retrieve this + collection using :meth:`Service.modular_input_kinds`. + """ + def __contains__(self, name): + args = self.state.content['endpoints']['args'] + if name in args: + return True + else: + return Entity.__contains__(self, name) + + def __getitem__(self, name): + args = self.state.content['endpoint']['args'] + if name in args: + return args['item'] + else: + return Entity.__getitem__(self, name) + + @property + def arguments(self): + """A dictionary of all the arguments supported by this modular input kind. + + The keys in the dictionary are the names of the arguments. The values are + another dictionary giving the metadata about that argument. The possible + keys in that dictionary are ``"title"``, ``"description"``, ``"required_on_create``", + ``"required_on_edit"``, ``"data_type"``. Each value is a string. It should be one + of ``"true"`` or ``"false"`` for ``"required_on_create"`` and ``"required_on_edit"``, + and one of ``"boolean"``, ``"string"``, or ``"number``" for ``"data_type"``. + + :return: A dictionary describing the arguments this modular input kind takes. + :rtype: ``dict`` + """ + return self.state.content['endpoint']['args'] + + def update(self, **kwargs): + """Raises an error. Modular input kinds are read only.""" + raise IllegalOperationException("Modular input kinds cannot be updated via the REST API.") + + +class SavedSearch(Entity): + """This class represents a saved search.""" + def __init__(self, service, path, **kwargs): + Entity.__init__(self, service, path, **kwargs) + + def acknowledge(self): + """Acknowledges the suppression of alerts from this saved search and + resumes alerting. + + :return: The :class:`SavedSearch`. + """ + self.post("acknowledge") + return self + + @property + def alert_count(self): + """Returns the number of alerts fired by this saved search. + + :return: The number of alerts fired by this saved search. + :rtype: ``integer`` + """ + return int(self._state.content.get('triggered_alert_count', 0)) + + def dispatch(self, **kwargs): + """Runs the saved search and returns the resulting search job. + + :param `kwargs`: Additional dispatch arguments (optional). For details, + see the `POST saved/searches/{name}/dispatch + `_ + endpoint in the REST API documentation. + :type kwargs: ``dict`` + :return: The :class:`Job`. + """ + response = self.post("dispatch", **kwargs) + sid = _load_sid(response) + return Job(self.service, sid) + + @property + def fired_alerts(self): + """Returns the collection of fired alerts (a fired alert group) + corresponding to this saved search's alerts. + + :raises IllegalOperationException: Raised when the search is not scheduled. + + :return: A collection of fired alerts. + :rtype: :class:`AlertGroup` + """ + if self['is_scheduled'] == '0': + raise IllegalOperationException('Unscheduled saved searches have no alerts.') + c = Collection( + self.service, + self.service._abspath(PATH_FIRED_ALERTS + self.name, + owner=self._state.access.owner, + app=self._state.access.app, + sharing=self._state.access.sharing), + item=AlertGroup) + return c + + def history(self): + """Returns a list of search jobs corresponding to this saved search. + + :return: A list of :class:`Job` objects. + """ + response = self.get("history") + entries = _load_atom_entries(response) + if entries is None: return [] + jobs = [] + for entry in entries: + job = Job(self.service, entry.title) + jobs.append(job) + return jobs + + def update(self, search=None, **kwargs): + """Updates the server with any changes you've made to the current saved + search along with any additional arguments you specify. + + :param `search`: The search query (optional). + :type search: ``string`` + :param `kwargs`: Additional arguments (optional). For a list of available + parameters, see `Saved search parameters + `_ + on Splunk Developer Portal. + :type kwargs: ``dict`` + + :return: The :class:`SavedSearch`. + """ + # Updates to a saved search *require* that the search string be + # passed, so we pass the current search string if a value wasn't + # provided by the caller. + if search is None: search = self.content.search + Entity.update(self, search=search, **kwargs) + return self + + def scheduled_times(self, earliest_time='now', latest_time='+1h'): + """Returns the times when this search is scheduled to run. + + By default this method returns the times in the next hour. For different + time ranges, set *earliest_time* and *latest_time*. For example, + for all times in the last day use "earliest_time=-1d" and + "latest_time=now". + + :param earliest_time: The earliest time. + :type earliest_time: ``string`` + :param latest_time: The latest time. + :type latest_time: ``string`` + + :return: The list of search times. + """ + response = self.get("scheduled_times", + earliest_time=earliest_time, + latest_time=latest_time) + data = self._load_atom_entry(response) + rec = _parse_atom_entry(data) + times = [datetime.fromtimestamp(int(t)) + for t in rec.content.scheduled_times] + return times + + def suppress(self, expiration): + """Skips any scheduled runs of this search in the next *expiration* + number of seconds. + + :param expiration: The expiration period, in seconds. + :type expiration: ``integer`` + + :return: The :class:`SavedSearch`. + """ + self.post("suppress", expiration=expiration) + return self + + @property + def suppressed(self): + """Returns the number of seconds that this search is blocked from running + (possibly 0). + + :return: The number of seconds. + :rtype: ``integer`` + """ + r = self._run_action("suppress") + if r.suppressed == "1": + return int(r.expiration) + else: + return 0 + + def unsuppress(self): + """Cancels suppression and makes this search run as scheduled. + + :return: The :class:`SavedSearch`. + """ + self.post("suppress", expiration="0") + return self + + +class SavedSearches(Collection): + """This class represents a collection of saved searches. Retrieve this + collection using :meth:`Service.saved_searches`.""" + def __init__(self, service): + Collection.__init__( + self, service, PATH_SAVED_SEARCHES, item=SavedSearch) + + def create(self, name, search, **kwargs): + """ Creates a saved search. + + :param name: The name for the saved search. + :type name: ``string`` + :param search: The search query. + :type search: ``string`` + :param kwargs: Additional arguments (optional). For a list of available + parameters, see `Saved search parameters + `_ + on Splunk Developer Portal. + :type kwargs: ``dict`` + :return: The :class:`SavedSearches` collection. + """ + return Collection.create(self, name, search=search, **kwargs) + + +class Settings(Entity): + """This class represents configuration settings for a Splunk service. + Retrieve this collection using :meth:`Service.settings`.""" + def __init__(self, service, **kwargs): + Entity.__init__(self, service, "/services/server/settings", **kwargs) + + # Updates on the settings endpoint are POSTed to server/settings/settings. + def update(self, **kwargs): + """Updates the settings on the server using the arguments you provide. + + :param kwargs: Additional arguments. For a list of valid arguments, see + `POST server/settings/{name} + `_ + in the REST API documentation. + :type kwargs: ``dict`` + :return: The :class:`Settings` collection. + """ + self.service.post("/services/server/settings/settings", **kwargs) + return self + + +class User(Entity): + """This class represents a Splunk user. + """ + @property + def role_entities(self): + """Returns a list of roles assigned to this user. + + :return: The list of roles. + :rtype: ``list`` + """ + return [self.service.roles[name] for name in self.content.roles] + + +# Splunk automatically lowercases new user names so we need to match that +# behavior here to ensure that the subsequent member lookup works correctly. +class Users(Collection): + """This class represents the collection of Splunk users for this instance of + Splunk. Retrieve this collection using :meth:`Service.users`. + """ + def __init__(self, service): + Collection.__init__(self, service, PATH_USERS, item=User) + + def __getitem__(self, key): + return Collection.__getitem__(self, key.lower()) + + def __contains__(self, name): + return Collection.__contains__(self, name.lower()) + + def create(self, username, password, roles, **params): + """Creates a new user. + + This function makes two roundtrips to the server, plus at most + two more if + the ``autologin`` field of :func:`connect` is set to ``True``. + + :param username: The username. + :type username: ``string`` + :param password: The password. + :type password: ``string`` + :param roles: A single role or list of roles for the user. + :type roles: ``string`` or ``list`` + :param params: Additional arguments (optional). For a list of available + parameters, see `User authentication parameters + `_ + on Splunk Developer Portal. + :type params: ``dict`` + + :return: The new user. + :rtype: :class:`User` + + **Example**:: + + import splunklib.client as client + c = client.connect(...) + users = c.users + boris = users.create("boris", "securepassword", roles="user") + hilda = users.create("hilda", "anotherpassword", roles=["user","power"]) + """ + if not isinstance(username, basestring): + raise ValueError("Invalid username: %s" % str(username)) + username = username.lower() + self.post(name=username, password=password, roles=roles, **params) + # splunkd doesn't return the user in the POST response body, + # so we have to make a second round trip to fetch it. + response = self.get(username) + entry = _load_atom(response, XNAME_ENTRY).entry + state = _parse_atom_entry(entry) + entity = self.item( + self.service, + urllib.unquote(state.links.alternate), + state=state) + return entity + + def delete(self, name): + """ Deletes the user and returns the resulting collection of users. + + :param name: The name of the user to delete. + :type name: ``string`` + + :return: + :rtype: :class:`Users` + """ + return Collection.delete(self, name.lower()) + + +class Role(Entity): + """This class represents a user role. + """ + def grant(self, *capabilities_to_grant): + """Grants additional capabilities to this role. + + :param capabilities_to_grant: Zero or more capabilities to grant this + role. For a list of capabilities, see + `Capabilities `_ + on Splunk Developer Portal. + :type capabilities_to_grant: ``string`` or ``list`` + :return: The :class:`Role`. + + **Example**:: + + service = client.connect(...) + role = service.roles['somerole'] + role.grant('change_own_password', 'search') + """ + possible_capabilities = self.service.capabilities + for capability in capabilities_to_grant: + if capability not in possible_capabilities: + raise NoSuchCapability(capability) + new_capabilities = self['capabilities'] + list(capabilities_to_grant) + self.post(capabilities=new_capabilities) + return self + + def revoke(self, *capabilities_to_revoke): + """Revokes zero or more capabilities from this role. + + :param capabilities_to_revoke: Zero or more capabilities to grant this + role. For a list of capabilities, see + `Capabilities `_ + on Splunk Developer Portal. + :type capabilities_to_revoke: ``string`` or ``list`` + + :return: The :class:`Role`. + + **Example**:: + + service = client.connect(...) + role = service.roles['somerole'] + role.revoke('change_own_password', 'search') + """ + possible_capabilities = self.service.capabilities + for capability in capabilities_to_revoke: + if capability not in possible_capabilities: + raise NoSuchCapability(capability) + old_capabilities = self['capabilities'] + new_capabilities = [] + for c in old_capabilities: + if c not in capabilities_to_revoke: + new_capabilities.append(c) + if new_capabilities == []: + new_capabilities = '' # Empty lists don't get passed in the body, so we have to force an empty argument. + self.post(capabilities=new_capabilities) + return self + + +class Roles(Collection): + """This class represents the collection of roles in the Splunk instance. + Retrieve this collection using :meth:`Service.roles`.""" + def __init__(self, service): + return Collection.__init__(self, service, PATH_ROLES, item=Role) + + def __getitem__(self, key): + return Collection.__getitem__(self, key.lower()) + + def __contains__(self, name): + return Collection.__contains__(self, name.lower()) + + def create(self, name, **params): + """Creates a new role. + + This function makes two roundtrips to the server, plus at most + two more if + the ``autologin`` field of :func:`connect` is set to ``True``. + + :param name: Name for the role. + :type name: ``string`` + :param params: Additional arguments (optional). For a list of available + parameters, see `Roles parameters + `_ + on Splunk Developer Portal. + :type params: ``dict`` + + :return: The new role. + :rtype: :class:`Role` + + **Example**:: + + import splunklib.client as client + c = client.connect(...) + roles = c.roles + paltry = roles.create("paltry", imported_roles="user", defaultApp="search") + """ + if not isinstance(name, basestring): + raise ValueError("Invalid role name: %s" % str(name)) + name = name.lower() + self.post(name=name, **params) + # splunkd doesn't return the user in the POST response body, + # so we have to make a second round trip to fetch it. + response = self.get(name) + entry = _load_atom(response, XNAME_ENTRY).entry + state = _parse_atom_entry(entry) + entity = self.item( + self.service, + urllib.unquote(state.links.alternate), + state=state) + return entity + + def delete(self, name): + """ Deletes the role and returns the resulting collection of roles. + + :param name: The name of the role to delete. + :type name: ``string`` + + :rtype: The :class:`Roles` + """ + return Collection.delete(self, name.lower()) + + +class Application(Entity): + """Represents a locally-installed Splunk app.""" + @property + def setupInfo(self): + """Returns the setup information for the app. + + :return: The setup information. + """ + return self.content.get('eai:setup', None) + + def package(self): + """ Creates a compressed package of the app for archiving.""" + return self._run_action("package") + + def updateInfo(self): + """Returns any update information that is available for the app.""" + return self._run_action("update") + +class KVStoreCollections(Collection): + def __init__(self, service): + Collection.__init__(self, service, 'storage/collections/config', item=KVStoreCollection) + + def create(self, name, indexes = {}, fields = {}, **kwargs): + """Creates a KV Store Collection. + + :param name: name of collection to create + :type name: ``string`` + :param indexes: dictionary of index definitions + :type indexes: ``dict`` + :param fields: dictionary of field definitions + :type fields: ``dict`` + :param kwargs: a dictionary of additional parameters specifying indexes and field definitions + :type kwargs: ``dict`` + + :return: Result of POST request + """ + for k, v in indexes.iteritems(): + if isinstance(v, dict): + v = json.dumps(v) + kwargs['index.' + k] = v + for k, v in fields.iteritems(): + kwargs['field.' + k] = v + return self.post(name=name, **kwargs) + +class KVStoreCollection(Entity): + @property + def data(self): + """Returns data object for this Collection. + + :rtype: :class:`KVStoreData` + """ + return KVStoreCollectionData(self) + + def update_index(self, name, value): + """Changes the definition of a KV Store index. + + :param name: name of index to change + :type name: ``string`` + :param value: new index definition + :type value: ``dict`` or ``string`` + + :return: Result of POST request + """ + kwargs = {} + kwargs['index.' + name] = value if isinstance(value, basestring) else json.dumps(value) + return self.post(**kwargs) + + def update_field(self, name, value): + """Changes the definition of a KV Store field. + + :param name: name of field to change + :type name: ``string`` + :param value: new field definition + :type value: ``string`` + + :return: Result of POST request + """ + kwargs = {} + kwargs['field.' + name] = value + return self.post(**kwargs) + +class KVStoreCollectionData(object): + """This class represents the data endpoint for a KVStoreCollection. + + Retrieve using :meth:`KVStoreCollection.data` + """ + JSON_HEADER = [('Content-Type', 'application/json')] + + def __init__(self, collection): + self.service = collection.service + self.collection = collection + self.owner, self.app, self.sharing = collection._proper_namespace() + self.path = 'storage/collections/data/' + UrlEncoded(self.collection.name) + '/' + + def _get(self, url, **kwargs): + return self.service.get(self.path + url, owner=self.owner, app=self.app, sharing=self.sharing, **kwargs) + + def _post(self, url, **kwargs): + return self.service.post(self.path + url, owner=self.owner, app=self.app, sharing=self.sharing, **kwargs) + + def _delete(self, url, **kwargs): + return self.service.delete(self.path + url, owner=self.owner, app=self.app, sharing=self.sharing, **kwargs) + + def query(self, **query): + """ + Gets the results of query, with optional parameters sort, limit, skip, and fields. + + :param query: Optional parameters. Valid options are sort, limit, skip, and fields + :type query: ``dict`` + + :return: Array of documents retrieved by query. + :rtype: ``array`` + """ + return json.loads(self._get('', **query).body.read()) + + def query_by_id(self, id): + """ + Returns object with _id = id. + + :param id: Value for ID. If not a string will be coerced to string. + :type id: ``string`` + + :return: Document with id + :rtype: ``dict`` + """ + return json.loads(self._get(UrlEncoded(str(id))).body.read()) + + def insert(self, data): + """ + Inserts item into this collection. An _id field will be generated if not assigned in the data. + + :param data: Document to insert + :type data: ``string`` + + :return: _id of inserted object + :rtype: ``dict`` + """ + return json.loads(self._post('', headers=KVStoreCollectionData.JSON_HEADER, body=data).body.read()) + + def delete(self, query=None): + """ + Deletes all data in collection if query is absent. Otherwise, deletes all data matched by query. + + :param query: Query to select documents to delete + :type query: ``string`` + + :return: Result of DELETE request + """ + return self._delete('', **({'query': query}) if query else {}) + + def delete_by_id(self, id): + """ + Deletes document that has _id = id. + + :param id: id of document to delete + :type id: ``string`` + + :return: Result of DELETE request + """ + return self._delete(UrlEncoded(str(id))) + + def update(self, id, data): + """ + Replaces document with _id = id with data. + + :param id: _id of document to update + :type id: ``string`` + :param data: the new document to insert + :type data: ``string`` + + :return: id of replaced document + :rtype: ``dict`` + """ + return json.loads(self._post(UrlEncoded(str(id)), headers=KVStoreCollectionData.JSON_HEADER, body=data).body.read()) + + def batch_find(self, *dbqueries): + """ + Returns array of results from queries dbqueries. + + :param dbqueries: Array of individual queries as dictionaries + :type dbqueries: ``array`` of ``dict`` + + :return: Results of each query + :rtype: ``array`` of ``array`` + """ + if len(dbqueries) < 1: + raise Exception('Must have at least one query.') + + data = json.dumps(dbqueries) + + return json.loads(self._post('batch_find', headers=KVStoreCollectionData.JSON_HEADER, body=data).body.read()) + + def batch_save(self, *documents): + """ + Inserts or updates every document specified in documents. + + :param documents: Array of documents to save as dictionaries + :type documents: ``array`` of ``dict`` + + :return: Results of update operation as overall stats + :rtype: ``dict`` + """ + if len(documents) < 1: + raise Exception('Must have at least one document.') + + data = json.dumps(documents) + + return json.loads(self._post('batch_save', headers=KVStoreCollectionData.JSON_HEADER, body=data).body.read()) diff --git a/package/bin/splunklib/data.py b/package/bin/splunklib/data.py new file mode 100755 index 0000000000..54e0e7eb1c --- /dev/null +++ b/package/bin/splunklib/data.py @@ -0,0 +1,258 @@ +# Copyright 2011-2015 Splunk, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"): you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""The **splunklib.data** module reads the responses from splunkd in Atom Feed +format, which is the format used by most of the REST API. +""" + +from xml.etree.ElementTree import XML + +__all__ = ["load"] + +# LNAME refers to element names without namespaces; XNAME is the same +# name, but with an XML namespace. +LNAME_DICT = "dict" +LNAME_ITEM = "item" +LNAME_KEY = "key" +LNAME_LIST = "list" + +XNAMEF_REST = "{http://dev.splunk.com/ns/rest}%s" +XNAME_DICT = XNAMEF_REST % LNAME_DICT +XNAME_ITEM = XNAMEF_REST % LNAME_ITEM +XNAME_KEY = XNAMEF_REST % LNAME_KEY +XNAME_LIST = XNAMEF_REST % LNAME_LIST + +# Some responses don't use namespaces (eg: search/parse) so we look for +# both the extended and local versions of the following names. + +def isdict(name): + return name == XNAME_DICT or name == LNAME_DICT + +def isitem(name): + return name == XNAME_ITEM or name == LNAME_ITEM + +def iskey(name): + return name == XNAME_KEY or name == LNAME_KEY + +def islist(name): + return name == XNAME_LIST or name == LNAME_LIST + +def hasattrs(element): + return len(element.attrib) > 0 + +def localname(xname): + rcurly = xname.find('}') + return xname if rcurly == -1 else xname[rcurly+1:] + +def load(text, match=None): + """This function reads a string that contains the XML of an Atom Feed, then + returns the + data in a native Python structure (a ``dict`` or ``list``). If you also + provide a tag name or path to match, only the matching sub-elements are + loaded. + + :param text: The XML text to load. + :type text: ``string`` + :param match: A tag name or path to match (optional). + :type match: ``string`` + """ + if text is None: return None + text = text.strip() + if len(text) == 0: return None + nametable = { + 'namespaces': [], + 'names': {} + } + root = XML(text) + items = [root] if match is None else root.findall(match) + count = len(items) + if count == 0: + return None + elif count == 1: + return load_root(items[0], nametable) + else: + return [load_root(item, nametable) for item in items] + +# Load the attributes of the given element. +def load_attrs(element): + if not hasattrs(element): return None + attrs = record() + for key, value in element.attrib.iteritems(): + attrs[key] = value + return attrs + +# Parse a element and return a Python dict +def load_dict(element, nametable = None): + value = record() + children = list(element) + for child in children: + assert iskey(child.tag) + name = child.attrib["name"] + value[name] = load_value(child, nametable) + return value + +# Loads the given elements attrs & value into single merged dict. +def load_elem(element, nametable=None): + name = localname(element.tag) + attrs = load_attrs(element) + value = load_value(element, nametable) + if attrs is None: return name, value + if value is None: return name, attrs + # If value is simple, merge into attrs dict using special key + if isinstance(value, str): + attrs["$text"] = value + return name, attrs + # Both attrs & value are complex, so merge the two dicts, resolving collisions. + collision_keys = [] + for key, val in attrs.iteritems(): + if key in value and key in collision_keys: + value[key].append(val) + elif key in value and key not in collision_keys: + value[key] = [value[key], val] + collision_keys.append(key) + else: + value[key] = val + return name, value + +# Parse a element and return a Python list +def load_list(element, nametable=None): + assert islist(element.tag) + value = [] + children = list(element) + for child in children: + assert isitem(child.tag) + value.append(load_value(child, nametable)) + return value + +# Load the given root element. +def load_root(element, nametable=None): + tag = element.tag + if isdict(tag): return load_dict(element, nametable) + if islist(tag): return load_list(element, nametable) + k, v = load_elem(element, nametable) + return Record.fromkv(k, v) + +# Load the children of the given element. +def load_value(element, nametable=None): + children = list(element) + count = len(children) + + # No children, assume a simple text value + if count == 0: + text = element.text + if text is None: + return None + text = text.strip() + if len(text) == 0: + return None + return text + + # Look for the special case of a single well-known structure + if count == 1: + child = children[0] + tag = child.tag + if isdict(tag): return load_dict(child, nametable) + if islist(tag): return load_list(child, nametable) + + value = record() + for child in children: + name, item = load_elem(child, nametable) + # If we have seen this name before, promote the value to a list + if value.has_key(name): + current = value[name] + if not isinstance(current, list): + value[name] = [current] + value[name].append(item) + else: + value[name] = item + + return value + +# A generic utility that enables "dot" access to dicts +class Record(dict): + """This generic utility class enables dot access to members of a Python + dictionary. + + Any key that is also a valid Python identifier can be retrieved as a field. + So, for an instance of ``Record`` called ``r``, ``r.key`` is equivalent to + ``r['key']``. A key such as ``invalid-key`` or ``invalid.key`` cannot be + retrieved as a field, because ``-`` and ``.`` are not allowed in + identifiers. + + Keys of the form ``a.b.c`` are very natural to write in Python as fields. If + a group of keys shares a prefix ending in ``.``, you can retrieve keys as a + nested dictionary by calling only the prefix. For example, if ``r`` contains + keys ``'foo'``, ``'bar.baz'``, and ``'bar.qux'``, ``r.bar`` returns a record + with the keys ``baz`` and ``qux``. If a key contains multiple ``.``, each + one is placed into a nested dictionary, so you can write ``r.bar.qux`` or + ``r['bar.qux']`` interchangeably. + """ + sep = '.' + + def __call__(self, *args): + if len(args) == 0: return self + return Record((key, self[key]) for key in args) + + def __getattr__(self, name): + try: + return self[name] + except KeyError: + raise AttributeError(name) + + def __delattr__(self, name): + del self[name] + + def __setattr__(self, name, value): + self[name] = value + + @staticmethod + def fromkv(k, v): + result = record() + result[k] = v + return result + + def __getitem__(self, key): + if key in self: + return dict.__getitem__(self, key) + key += self.sep + result = record() + for k,v in self.iteritems(): + if not k.startswith(key): + continue + suffix = k[len(key):] + if '.' in suffix: + ks = suffix.split(self.sep) + z = result + for x in ks[:-1]: + if x not in z: + z[x] = record() + z = z[x] + z[ks[-1]] = v + else: + result[suffix] = v + if len(result) == 0: + raise KeyError("No key or prefix: %s" % key) + return result + + +def record(value=None): + """This function returns a :class:`Record` instance constructed with an + initial value that you provide. + + :param `value`: An initial record value. + :type `value`: ``dict`` + """ + if value is None: value = {} + return Record(value) + diff --git a/package/bin/splunklib/modularinput/__init__.py b/package/bin/splunklib/modularinput/__init__.py new file mode 100755 index 0000000000..ace954a02b --- /dev/null +++ b/package/bin/splunklib/modularinput/__init__.py @@ -0,0 +1,12 @@ +"""The following imports allow these classes to be imported via +the splunklib.modularinput package like so: + +from splunklib.modularinput import * +""" +from .argument import Argument +from .event import Event +from .event_writer import EventWriter +from .input_definition import InputDefinition +from .scheme import Scheme +from .script import Script +from .validation_definition import ValidationDefinition diff --git a/package/bin/splunklib/modularinput/argument.py b/package/bin/splunklib/modularinput/argument.py new file mode 100755 index 0000000000..5eb3f1259e --- /dev/null +++ b/package/bin/splunklib/modularinput/argument.py @@ -0,0 +1,102 @@ +# Copyright 2011-2015 Splunk, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"): you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +try: + import xml.etree.ElementTree as ET +except ImportError: + import xml.etree.cElementTree as ET + +class Argument(object): + """Class representing an argument to a modular input kind. + + ``Argument`` is meant to be used with ``Scheme`` to generate an XML + definition of the modular input kind that Splunk understands. + + ``name`` is the only required parameter for the constructor. + + **Example with least parameters**:: + + arg1 = Argument(name="arg1") + + **Example with all parameters**:: + + arg2 = Argument( + name="arg2", + description="This is an argument with lots of parameters", + validation="is_pos_int('some_name')", + data_type=Argument.data_type_number, + required_on_edit=True, + required_on_create=True + ) + """ + + # Constant values, do not change. + # These should be used for setting the value of an Argument object's data_type field. + data_type_boolean = "BOOLEAN" + data_type_number = "NUMBER" + data_type_string = "STRING" + + def __init__(self, name, description=None, validation=None, + data_type=data_type_string, required_on_edit=False, required_on_create=False, title=None): + """ + :param name: ``string``, identifier for this argument in Splunk. + :param description: ``string``, human-readable description of the argument. + :param validation: ``string`` specifying how the argument should be validated, if using internal validation. + If using external validation, this will be ignored. + :param data_type: ``string``, data type of this field; use the class constants. + "data_type_boolean", "data_type_number", or "data_type_string". + :param required_on_edit: ``Boolean``, whether this arg is required when editing an existing modular input of this kind. + :param required_on_create: ``Boolean``, whether this arg is required when creating a modular input of this kind. + :param title: ``String``, a human-readable title for the argument. + """ + self.name = name + self.description = description + self.validation = validation + self.data_type = data_type + self.required_on_edit = required_on_edit + self.required_on_create = required_on_create + self.title = title + + def add_to_document(self, parent): + """Adds an ``Argument`` object to this ElementTree document. + + Adds an subelement to the parent element, typically + and sets up its subelements with their respective text. + + :param parent: An ``ET.Element`` to be the parent of a new subelement + :returns: An ``ET.Element`` object representing this argument. + """ + arg = ET.SubElement(parent, "arg") + arg.set("name", self.name) + + if self.title is not None: + ET.SubElement(arg, "title").text = self.title + + if self.description is not None: + ET.SubElement(arg, "description").text = self.description + + if self.validation is not None: + ET.SubElement(arg, "validation").text = self.validation + + # add all other subelements to this Argument, represented by (tag, text) + subelements = [ + ("data_type", self.data_type), + ("required_on_edit", self.required_on_edit), + ("required_on_create", self.required_on_create) + ] + + for name, value in subelements: + ET.SubElement(arg, name).text = str(value).lower() + + return arg \ No newline at end of file diff --git a/package/bin/splunklib/modularinput/event.py b/package/bin/splunklib/modularinput/event.py new file mode 100755 index 0000000000..de1d4f19e6 --- /dev/null +++ b/package/bin/splunklib/modularinput/event.py @@ -0,0 +1,107 @@ +# Copyright 2011-2015 Splunk, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"): you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +try: + import xml.etree.cElementTree as ET +except ImportError as ie: + import xml.etree.ElementTree as ET + +class Event(object): + """Represents an event or fragment of an event to be written by this modular input to Splunk. + + To write an input to a stream, call the ``write_to`` function, passing in a stream. + """ + def __init__(self, data=None, stanza=None, time=None, host=None, index=None, source=None, + sourcetype=None, done=True, unbroken=True): + """There are no required parameters for constructing an Event + + **Example with minimal configuration**:: + + my_event = Event( + data="This is a test of my new event.", + stanza="myStanzaName", + time="%.3f" % 1372187084.000 + ) + + **Example with full configuration**:: + + excellent_event = Event( + data="This is a test of my excellent event.", + stanza="excellenceOnly", + time="%.3f" % 1372274622.493, + host="localhost", + index="main", + source="Splunk", + sourcetype="misc", + done=True, + unbroken=True + ) + + :param data: ``string``, the event's text. + :param stanza: ``string``, name of the input this event should be sent to. + :param time: ``float``, time in seconds, including up to 3 decimal places to represent milliseconds. + :param host: ``string``, the event's host, ex: localhost. + :param index: ``string``, the index this event is specified to write to, or None if default index. + :param source: ``string``, the source of this event, or None to have Splunk guess. + :param sourcetype: ``string``, source type currently set on this event, or None to have Splunk guess. + :param done: ``boolean``, is this a complete ``Event``? False if an ``Event`` fragment. + :param unbroken: ``boolean``, Is this event completely encapsulated in this ``Event`` object? + """ + self.data = data + self.done = done + self.host = host + self.index = index + self.source = source + self.sourceType = sourcetype + self.stanza = stanza + self.time = time + self.unbroken = unbroken + + def write_to(self, stream): + """Write an XML representation of self, an ``Event`` object, to the given stream. + + The ``Event`` object will only be written if its data field is defined, + otherwise a ``ValueError`` is raised. + + :param stream: stream to write XML to. + """ + if self.data is None: + raise ValueError("Events must have at least the data field set to be written to XML.") + + event = ET.Element("event") + if self.stanza is not None: + event.set("stanza", self.stanza) + event.set("unbroken", str(int(self.unbroken))) + + # if a time isn't set, let Splunk guess by not creating a