Branch was auto-updated.

.gitmodules

@@ -1,3 +0,0 @@
-[submodule "playbooks"]
-	path = playbooks
-	url = https://github.com/phantomcyber/playbooks.git

bin/generate.py

@@ -230,6 +230,7 @@ def get_deployments(object, deployments):
     for deployment in deployments:
 
         for tag in object['tags'].keys():
+            
             if tag in deployment['tags'].keys():
                 if type(object['tags'][tag]) is str:
                     tag_array = [object['tags'][tag]]
@@ -402,6 +403,9 @@ def prepare_detections(detections, deployments, OUTPUT_PATH):
             if detection['tags']['risk_score']:
                 detection['search'] = detection['search'] + ' | eval risk_score=' + str(detection['tags']['risk_score'])
 
+            if detection['tags']['mitre_attack_id']:
+                detection['search'] = detection['search'] + ' | eval mitre_attack_id=' + detection['tags']['mitre_attack_id'][0]
+
             if detection['type'] == 'Anomaly':
                 detection['search'] = detection['search'] + ' | collect index=signals'
             elif detection['type'] == 'TTP':
@@ -409,6 +413,7 @@ def prepare_detections(detections, deployments, OUTPUT_PATH):
             elif detection['type'] == 'Correlation':
                 detection['search'] = detection['search'] + ' | collect index=alerts'
 
+
         # parse out data_models
         data_model = parse_data_models_from_search(detection['search'])
         if data_model:

bin/jinja2_templates/savedsearches.j2

@@ -84,6 +84,19 @@ action.email.to = {{ detection.deployment.alert_action.email.to }}
 action.email.message.alert = {{ detection.deployment.alert_action.email.message | custom_jinja2_enrichment_filter(detection) }}
 action.email.useNSSubject = 1
 {% endif %}
+{% if detection.deployment.alert_action.slack is defined %}
+action.slack = 1
+action.slack.param.channel = {{ detection.deployment.alert_action.slack.channel | custom_jinja2_enrichment_filter(detection) }}
+action.slack.param.message = {{ detection.deployment.alert_action.slack.message | custom_jinja2_enrichment_filter(detection) }}
+{% endif %}
+{% if detection.deployment.alert_action.phantom is defined %}
+action.sendtophantom = 1
+action.sendtophantom.param._cam_workers = {{ detection.deployment.alert_action.phantom.cam_workers | custom_jinja2_enrichment_filter(detection) }}
+action.sendtophantom.param.label = {{ detection.deployment.alert_action.phantom.label | custom_jinja2_enrichment_filter(detection) }}
+action.sendtophantom.param.phantom_server = {{ detection.deployment.alert_action.phantom.phantom_server | custom_jinja2_enrichment_filter(detection) }}
+action.sendtophantom.param.sensitivity = {{ detection.deployment.alert_action.phantom.sensitivity | custom_jinja2_enrichment_filter(detection) }}
+action.sendtophantom.param.severity = {{ detection.deployment.alert_action.phantom.severity | custom_jinja2_enrichment_filter(detection) }}
+{% endif %}
 {% endif %}
 alert.digest_mode = 1
 {% if detection.disabled is defined %}

detections/cloud/aws_ecr_container_scanning_findings_high.yml

@@ -33,6 +33,8 @@ tags:
   kill_chain_phases:
   - Actions on Objectives
   message: Vulnerabilities with severity high found in image $image$
+  deployments:
+  - Slack Alert
   mitre_attack_id:
   - T1204.003
   nist:

detections/cloud/aws_excessive_security_scanning.yml

@@ -58,7 +58,6 @@ tags:
   - Splunk Enterprise
   - Splunk Enterprise Security
   - Splunk Cloud
-  - Dev Sec Ops Analytics
   required_fields:
   - _time
   - eventName

detections/cloud/circle_ci_disable_security_job.yml

@@ -11,8 +11,9 @@ search: '`circleci` | rename vcs.committer_name as user vcs.subject as commit_me
   workflow_name user commit_message url branch | lookup mandatory_job_for_workflow
   workflow_name OUTPUTNEW job_name AS mandatory_job | search mandatory_job=* | eval
   mandatory_job_executed=if(like(job_names, "%".mandatory_job."%"), 1, 0) | where
-  mandatory_job_executed=0 | rex field=url "(?<repository>[^\/]*\/[^\/]*)$" | `security_content_ctime(firstTime)`
-  | `security_content_ctime(lastTime)` | `circle_ci_disable_security_job_filter`'
+  mandatory_job_executed=0 | eval phase="build" | rex field=url "(?<repository>[^\/]*\/[^\/]*)$" 
+  | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)`
+  | `circle_ci_disable_security_job_filter`'
 how_to_implement: You must index CircleCI logs.
 known_false_positives: unknown
 references: []

detections/cloud/circle_ci_disable_security_step.yml

@@ -11,9 +11,9 @@ search: '`circleci` | rename workflows.job_id AS job_id | join job_id [ | search
   job_id job_name vcs.committer_name vcs.subject vcs.url owners{} | rename vcs.* as
   * , owners{} as user | lookup mandatory_step_for_job job_name OUTPUTNEW step_name
   AS mandatory_step | search mandatory_step=* | eval mandatory_step_executed=if(like(step_names,
-  "%".mandatory_step."%"), 1, 0) | where mandatory_step_executed=0 | rex field=url
-  "(?<repository>[^\/]*\/[^\/]*)$"  | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)`
-  | `circle_ci_disable_security_step_filter`'
+  "%".mandatory_step."%"), 1, 0) | where mandatory_step_executed=0
+  | rex field=url "(?<repository>[^\/]*\/[^\/]*)$" | eval phase="build"  | `security_content_ctime(firstTime)`
+  | `security_content_ctime(lastTime)` | `circle_ci_disable_security_step_filter`'
 how_to_implement: You must index CircleCI logs.
 known_false_positives: unknown
 references: []

detections/cloud/correlation_by_repository_and_risk.yml

@@ -23,6 +23,8 @@ tags:
   kill_chain_phases:
   - Actions on Objectives
   message: Correlation triggered for user $user$
+  deployments:
+  - Slack Alert
   mitre_attack_id:
   - T1204.003
   nist:

detections/cloud/correlation_by_user_and_risk.yml

@@ -23,6 +23,8 @@ tags:
   kill_chain_phases:
   - Actions on Objectives
   message: Correlation triggered for user $user$
+  deployments:
+  - Slack Alert
   mitre_attack_id:
   - T1204.003
   nist:

detections/cloud/github_commit_changes_in_master.yml

@@ -10,11 +10,10 @@ description: This search is to detect a pushed or commit to master or main branc
   Ideally in terms of devsecops the changes made in a branch and do a PR for review.
   of course in some cases admin of the project may did a changes directly to master
   branch
-search: '`github` branches{}.name = main OR branches{}.name = master | eval severity="low"
-  | eval phase="code" |  stats count min(_time) as firstTime max(_time) as lastTime  by
-  commit.author.html_url commit.commit.author.email commit.author.login commit.commit.message
-  repository.pushed_at commit.commit.committer.date, phase, severity | `security_content_ctime(firstTime)`
-  | `security_content_ctime(lastTime)` | `github_commit_changes_in_master_filter`'
+search: '`github` branches{}.name = main OR branches{}.name = master | eval severity="low" | eval phase="code" |  stats count
+  min(_time) as firstTime max(_time) as lastTime  by commit.author.html_url commit.commit.author.email
+  commit.author.login commit.commit.message repository.pushed_at commit.commit.committer.date, phase, severity
+  | eval phase="code" | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `github_commit_changes_in_master_filter`'
 how_to_implement: To successfully implement this search, you need to be ingesting
   logs related to github logs having the fork, commit, push metadata that can be use
   to monitor the changes in a github project.

detections/cloud/github_commit_in_develop.yml

@@ -12,7 +12,7 @@ description: This search is to detect a pushed or commit to develop branch. This
 search: '`github` branches{}.name = main OR branches{}.name = develop |  stats count
   min(_time) as firstTime max(_time) as lastTime  by commit.author.html_url commit.commit.author.email
   commit.author.login commit.commit.message repository.pushed_at commit.commit.committer.date
-  | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `github_commit_in_develop_filter`'
+  | eval phase="code" | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `github_commit_in_develop_filter`'
 how_to_implement: To successfully implement this search, you need to be ingesting
   logs related to github logs having the fork, commit, push metadata that can be use
   to monitor the changes in a github project.

detections/cloud/github_dependabot_alert.yml

@@ -7,11 +7,11 @@ type: Anomaly
 datamodel: []
 description: This search looks for Dependabot Alerts in Github logs.
 search: '`github` alert.id=* action=create | rename repository.full_name as repository,
-  repository.html_url as repository_url sender.login as user | stats min(_time) as
-  firstTime max(_time) as lastTime by action alert.affected_package_name alert.affected_range
-  alert.created_at alert.external_identifier alert.external_reference alert.fixed_in
-  alert.severity repository repository_url user | `security_content_ctime(firstTime)`
-  | `security_content_ctime(lastTime)` | `github_dependabot_alert_filter`'
+  repository.html_url as repository_url sender.login as user | stats min(_time) as firstTime max(_time)
+  as lastTime by action alert.affected_package_name alert.affected_range alert.created_at
+  alert.external_identifier alert.external_reference alert.fixed_in alert.severity
+  repository repository_url user | eval phase="code" | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)`
+  | `github_dependabot_alert_filter`'
 how_to_implement: You must index GitHub logs. You can follow the url in reference
   to onboard GitHub logs.
 known_false_positives: unknown

detections/cloud/github_pull_request_from_unknown_user.yml

@@ -7,11 +7,13 @@ type: Anomaly
 datamodel: []
 description: This search looks for Pull Request from unknown user.
 search: '`github` check_suite.pull_requests{}.id=* | stats count by check_suite.head_commit.author.name
-  repository.full_name check_suite.pull_requests{}.head.ref check_suite.head_commit.message
-  | rename check_suite.head_commit.author.name as user repository.full_name as repository
-  check_suite.pull_requests{}.head.ref as ref_head check_suite.head_commit.message
-  as commit_message | search NOT `github_known_users` | `security_content_ctime(firstTime)`
-  | `security_content_ctime(lastTime)` | `github_pull_request_from_unknown_user_filter`'
+  repository.full_name check_suite.pull_requests{}.head.ref
+  check_suite.head_commit.message | rename check_suite.head_commit.author.name as
+  user repository.full_name as repository check_suite.pull_requests{}.head.ref
+  as ref_head check_suite.head_commit.message as commit_message | search NOT `github_known_users`
+  | eval phase="code"
+  | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` 
+  | `github_pull_request_from_unknown_user_filter`'
 how_to_implement: You must index GitHub logs. You can follow the url in reference
   to onboard GitHub logs.
 known_false_positives: unknown

detections/cloud/gsuite_drive_share_in_external_email.yml

@@ -10,11 +10,11 @@ description: This search is to detect suspicious google drive or google docs fil
   exfitration of data made by an attacker or insider to a targetted machine.
 search: '`gsuite_drive` NOT (email IN("", "null")) | rex field=parameters.owner "[^@]+@(?<src_domain>[^@]+)"
   | rex field=email "[^@]+@(?<dest_domain>[^@]+)" | where src_domain = "internal_test_email.com"
-  and not dest_domain = "internal_test_email.com" | eval phase="plan" | eval severity="low"
-  | stats values(parameters.doc_title) as doc_title, values(parameters.doc_type) as
-  doc_types, values(email) as dst_email_list, values(parameters.visibility) as visibility,
-  count min(_time) as firstTime max(_time) as lastTime by parameters.owner phase severity
-  | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `gsuite_drive_share_in_external_email_filter`'
+  and not dest_domain = "internal_test_email.com" | eval phase="plan" | eval severity="low" | stats values(parameters.doc_title)
+  as doc_title, values(parameters.doc_type) as doc_types, values(email) as dst_email_list,
+  values(parameters.visibility) as visibility, values(parameters.doc_id) as doc_id, count min(_time) as firstTime max(_time)
+  as lastTime by parameters.owner ip_address phase severity  | rename parameters.owner as user ip_address as src_ip | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)`
+  | `gsuite_drive_share_in_external_email_filter`'
 how_to_implement: To successfully implement this search, you need to be ingesting
   logs related to gsuite having the file attachment metadata like file type, file
   extension, source email, destination email, num of attachment and etc.
@@ -25,13 +25,15 @@ references:
 tags:
   analytic_story:
   - DevSecOps
-  confidence: 30
+  confidence: 90
   context:
   - Source:Endpoint
   - Stage:Reconnaissance
   dataset:
   - https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1567.002/gsuite_share_drive/gdrive_share_external.log
-  impact: 30
+  deployments:
+  - Send to Phantom
+  impact: 80
   kill_chain_phases:
   - Exfiltration
   message: suspicious share gdrive from $parameters.owner$ to $email$ namely as $parameters.doc_title$
@@ -60,5 +62,5 @@ tags:
   - parameters.visibility
   - parameters.owner
   - parameters.doc_type
-  risk_score: 9
+  risk_score: 72
   security_domain: endpoint

detections/cloud/gsuite_suspicious_shared_file_name.yml

@@ -17,10 +17,10 @@ search: '`gsuite_drive` parameters.owner_is_team_drive=false "parameters.doc_tit
   "*new order*") parameters.doc_type IN ("document","pdf", "msexcel", "msword", "spreadsheet",
   "presentation") | rex field=parameters.owner "[^@]+@(?<source_domain>[^@]+)" | rex
   field=parameters.target_user "[^@]+@(?<dest_domain>[^@]+)" | where not source_domain="internal_test_email.com"
-  and dest_domain="internal_test_email.com" | eval phase="plan" | eval severity="low"
-  | stats count min(_time) as firstTime max(_time) as lastTime by email parameters.owner
-  parameters.target_user parameters.doc_title parameters.doc_type phase severity |
-  `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `gsuite_suspicious_shared_file_name_filter`'
+  and dest_domain="internal_test_email.com" | eval phase="plan" | eval severity="low" | stats count min(_time) as firstTime
+  max(_time) as lastTime by email parameters.owner parameters.target_user parameters.doc_title
+  parameters.doc_type phase severity | rename parameters.target_user AS user | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)`
+  | `gsuite_suspicious_shared_file_name_filter`'
 how_to_implement: To successfully implement this search, you need to be ingesting
   logs related to gsuite having the file attachment metadata like file type, file
   extension, source email, destination email, num of attachment and etc.
@@ -33,7 +33,7 @@ tags:
   analytic_story:
   - DevSecOps
   automated_detection_testing: passed
-  confidence: 30
+  confidence: 70
   context:
   - Source:Endpoint
   - Stage:Reconnaissance
@@ -68,5 +68,5 @@ tags:
   - parameters.visibility
   - parameters.owner
   - parameters.doc_type
-  risk_score: 9
+  risk_score: 21
   security_domain: endpoint

detections/cloud/kubernetes_nginx_ingress_lfi.yml

@@ -34,6 +34,8 @@ tags:
   kill_chain_phases:
   - Actions on Objectives
   message: Local File Inclusion Attack detected on $host$
+  deployments:
+  - Slack Alert
   mitre_attack_id:
   - T1212
   nist:

detections/cloud/kubernetes_nginx_ingress_rfi.yml

@@ -33,6 +33,8 @@ tags:
   kill_chain_phases:
   - Actions on Objectives
   message: Remote File Inclusion Attack detected on $host$
+  deployments:
+  - Slack Alert
   mitre_attack_id:
   - T1212
   nist:

detections/cloud/kubernetes_scanner_image_pulling.yml

@@ -31,6 +31,8 @@ tags:
   kill_chain_phases:
   - Actions on Objectives
   message: Kubernetes Scanner image pulled on host $host$
+  deployments:
+  - Slack Alert
   mitre_attack_id:
   - T1526
   nist:

detections/endpoint/ssa___detect_dump_lsass_memory_using_comsvcs.yml

@@ -35,6 +35,8 @@ tags:
   - Source:AD
   - Source:Endpoint
   - Stage:Credential Access
+  dataset:
+    - https://media.githubusercontent.com/media/splunk/attack_data/master/datasets/attack_techniques/T1003.001/atomic_red_team/windows-security.log
   impact: 70
   kill_chain_phases:
   - Actions on Objectives

dist/devsecops/bin/app.py

@@ -0,0 +1,112 @@
+# coding=utf-8
+#
+# Copyright © 2011-2015 Splunk, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"): you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+""" Sets the packages path and optionally starts the Python remote debugging client.
+The Python remote debugging client depends on the settings of the variables defined in _pydebug_conf.py.  Set these
+variables in _pydebug_conf.py to enable/disable debugging using either the JetBrains PyCharm or Eclipse PyDev remote
+debug egg which must be copied to your application's bin directory and renamed as _pydebug.egg.
+"""
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+settrace = stoptrace = lambda: NotImplemented
+remote_debugging = None
+
+
+def initialize():
+
+    from os import path
+    from sys import modules, path as python_path
+
+    import platform
+
+    module_dir = path.dirname(path.realpath(__file__))
+    system = platform.system()
+
+    for packages in path.join(module_dir, 'packages'), path.join(path.join(module_dir, 'packages', system)):
+        if not path.isdir(packages):
+            break
+        python_path.insert(0, path.join(packages))
+
+    configuration_file = path.join(module_dir, '_pydebug_conf.py')
+
+    if not path.exists(configuration_file):
+        return
+
+    debug_client = path.join(module_dir, '_pydebug.egg')
+
+    if not path.exists(debug_client):
+        return
+
+    _remote_debugging = {
+        'client_package_location': debug_client,
+        'is_enabled': False,
+        'host': None,
+        'port': 5678,
+        'suspend': True,
+        'stderr_to_server': False,
+        'stdout_to_server': False,
+        'overwrite_prev_trace': False,
+        'patch_multiprocessing': False,
+        'trace_only_current_thread': False}
+
+    exec(compile(open(configuration_file).read(), configuration_file, 'exec'), {'__builtins__': __builtins__}, _remote_debugging)
+    python_path.insert(1, debug_client)
+
+    from splunklib.searchcommands import splunklib_logger as logger
+    import pydevd
+
+    def _settrace():
+        host, port = _remote_debugging['host'], _remote_debugging['port']
+        logger.debug('Connecting to Python debug server at %s:%d', host, port)
+
+        try:
+            pydevd.settrace(
+                host=host,
+                port=port,
+                suspend=_remote_debugging['suspend'],
+                stderrToServer=_remote_debugging['stderr_to_server'],
+                stdoutToServer=_remote_debugging['stdout_to_server'],
+                overwrite_prev_trace=_remote_debugging['overwrite_prev_trace'],
+                patch_multiprocessing=_remote_debugging['patch_multiprocessing'],
+                trace_only_current_thread=_remote_debugging['trace_only_current_thread'])
+        except SystemExit as error:
+            logger.error('Failed to connect to Python debug server at %s:%d: %s', host, port, error)
+        else:
+            logger.debug('Connected to Python debug server at %s:%d', host, port)
+
+    global remote_debugging
+    remote_debugging = _remote_debugging
+
+    global settrace
+    settrace = _settrace
+
+    global stoptrace
+    stoptrace = pydevd.stoptrace
+
+    remote_debugging_is_enabled = _remote_debugging['is_enabled']
+
+    if isinstance(remote_debugging_is_enabled, (list, set, tuple)):
+        app_name = path.splitext(path.basename(modules['__main__'].__file__))[0]
+        remote_debugging_is_enabled = app_name in remote_debugging_is_enabled
+
+    if remote_debugging_is_enabled is True:
+        settrace()
+
+    return
+
+initialize()
+del initialize

dist/devsecops/bin/splunklib/__init__.py

@@ -0,0 +1,20 @@
+# Copyright 2011-2015 Splunk, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"): you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""Python library for Splunk."""
+
+from __future__ import absolute_import
+from splunklib.six.moves import map
+__version_info__ = (1, 6, 16)
+__version__ = ".".join(map(str, __version_info__))

dist/devsecops/bin/splunklib/data.py

@@ -0,0 +1,266 @@
+# Copyright 2011-2015 Splunk, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"): you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""The **splunklib.data** module reads the responses from splunkd in Atom Feed 
+format, which is the format used by most of the REST API.
+"""
+
+from __future__ import absolute_import
+import sys
+from xml.etree.ElementTree import XML
+from splunklib import six
+
+__all__ = ["load"]
+
+# LNAME refers to element names without namespaces; XNAME is the same
+# name, but with an XML namespace.
+LNAME_DICT = "dict"
+LNAME_ITEM = "item"
+LNAME_KEY = "key"
+LNAME_LIST = "list"
+
+XNAMEF_REST = "{http://dev.splunk.com/ns/rest}%s"
+XNAME_DICT = XNAMEF_REST % LNAME_DICT
+XNAME_ITEM = XNAMEF_REST % LNAME_ITEM
+XNAME_KEY = XNAMEF_REST % LNAME_KEY
+XNAME_LIST = XNAMEF_REST % LNAME_LIST
+
+# Some responses don't use namespaces (eg: search/parse) so we look for
+# both the extended and local versions of the following names.
+
+def isdict(name):
+    return name == XNAME_DICT or name == LNAME_DICT
+
+def isitem(name):
+    return name == XNAME_ITEM or name == LNAME_ITEM
+
+def iskey(name):
+    return name == XNAME_KEY or name == LNAME_KEY
+
+def islist(name):
+    return name == XNAME_LIST or name == LNAME_LIST
+
+def hasattrs(element):
+    return len(element.attrib) > 0
+
+def localname(xname):
+    rcurly = xname.find('}')
+    return xname if rcurly == -1 else xname[rcurly+1:]
+
+def load(text, match=None):
+    """This function reads a string that contains the XML of an Atom Feed, then 
+    returns the 
+    data in a native Python structure (a ``dict`` or ``list``). If you also 
+    provide a tag name or path to match, only the matching sub-elements are 
+    loaded.
+
+    :param text: The XML text to load.
+    :type text: ``string``
+    :param match: A tag name or path to match (optional).
+    :type match: ``string``
+    """
+    if text is None: return None
+    text = text.strip()
+    if len(text) == 0: return None
+    nametable = {
+        'namespaces': [],
+        'names': {}
+    }
+
+    # Convert to unicode encoding in only python 2 for xml parser
+    if(sys.version_info < (3, 0, 0) and isinstance(text, unicode)):
+        text = text.encode('utf-8')
+
+    root = XML(text)
+    items = [root] if match is None else root.findall(match)
+    count = len(items)
+    if count == 0: 
+        return None
+    elif count == 1: 
+        return load_root(items[0], nametable)
+    else:
+        return [load_root(item, nametable) for item in items]
+
+# Load the attributes of the given element.
+def load_attrs(element):
+    if not hasattrs(element): return None
+    attrs = record()
+    for key, value in six.iteritems(element.attrib): 
+        attrs[key] = value
+    return attrs
+
+# Parse a <dict> element and return a Python dict
+def load_dict(element, nametable = None):
+    value = record()
+    children = list(element)
+    for child in children:
+        assert iskey(child.tag)
+        name = child.attrib["name"]
+        value[name] = load_value(child, nametable)
+    return value
+
+# Loads the given elements attrs & value into single merged dict.
+def load_elem(element, nametable=None):
+    name = localname(element.tag)
+    attrs = load_attrs(element)
+    value = load_value(element, nametable)
+    if attrs is None: return name, value
+    if value is None: return name, attrs
+    # If value is simple, merge into attrs dict using special key
+    if isinstance(value, six.string_types):
+        attrs["$text"] = value
+        return name, attrs
+    # Both attrs & value are complex, so merge the two dicts, resolving collisions.
+    collision_keys = []
+    for key, val in six.iteritems(attrs):
+        if key in value and key in collision_keys:
+            value[key].append(val)
+        elif key in value and key not in collision_keys:
+            value[key] = [value[key], val]
+            collision_keys.append(key)
+        else:
+            value[key] = val
+    return name, value
+
+# Parse a <list> element and return a Python list
+def load_list(element, nametable=None):
+    assert islist(element.tag)
+    value = []
+    children = list(element)
+    for child in children:
+        assert isitem(child.tag)
+        value.append(load_value(child, nametable))
+    return value
+
+# Load the given root element.
+def load_root(element, nametable=None):
+    tag = element.tag
+    if isdict(tag): return load_dict(element, nametable)
+    if islist(tag): return load_list(element, nametable)
+    k, v = load_elem(element, nametable)
+    return Record.fromkv(k, v)
+
+# Load the children of the given element.
+def load_value(element, nametable=None):
+    children = list(element)
+    count = len(children)
+
+    # No children, assume a simple text value
+    if count == 0:
+        text = element.text
+        if text is None: 
+            return None
+        text = text.strip()
+        if len(text) == 0: 
+            return None
+        return text
+
+    # Look for the special case of a single well-known structure
+    if count == 1:
+        child = children[0]
+        tag = child.tag
+        if isdict(tag): return load_dict(child, nametable)
+        if islist(tag): return load_list(child, nametable)
+
+    value = record()
+    for child in children:
+        name, item = load_elem(child, nametable)
+        # If we have seen this name before, promote the value to a list
+        if name in value:
+            current = value[name]
+            if not isinstance(current, list): 
+                value[name] = [current]
+            value[name].append(item)
+        else:
+            value[name] = item
+
+    return value
+
+# A generic utility that enables "dot" access to dicts
+class Record(dict):
+    """This generic utility class enables dot access to members of a Python 
+    dictionary.
+
+    Any key that is also a valid Python identifier can be retrieved as a field. 
+    So, for an instance of ``Record`` called ``r``, ``r.key`` is equivalent to 
+    ``r['key']``. A key such as ``invalid-key`` or ``invalid.key`` cannot be 
+    retrieved as a field, because ``-`` and ``.`` are not allowed in 
+    identifiers.
+
+    Keys of the form ``a.b.c`` are very natural to write in Python as fields. If 
+    a group of keys shares a prefix ending in ``.``, you can retrieve keys as a 
+    nested dictionary by calling only the prefix. For example, if ``r`` contains
+    keys ``'foo'``, ``'bar.baz'``, and ``'bar.qux'``, ``r.bar`` returns a record
+    with the keys ``baz`` and ``qux``. If a key contains multiple ``.``, each 
+    one is placed into a nested dictionary, so you can write ``r.bar.qux`` or 
+    ``r['bar.qux']`` interchangeably.
+    """
+    sep = '.'
+
+    def __call__(self, *args):
+        if len(args) == 0: return self
+        return Record((key, self[key]) for key in args)
+
+    def __getattr__(self, name):
+        try:
+            return self[name]
+        except KeyError: 
+            raise AttributeError(name)
+
+    def __delattr__(self, name):
+        del self[name]
+
+    def __setattr__(self, name, value):
+        self[name] = value
+
+    @staticmethod
+    def fromkv(k, v):
+        result = record()
+        result[k] = v
+        return result
+
+    def __getitem__(self, key):
+        if key in self:
+            return dict.__getitem__(self, key)
+        key += self.sep
+        result = record()
+        for k,v in six.iteritems(self):
+            if not k.startswith(key):
+                continue
+            suffix = k[len(key):]
+            if '.' in suffix:
+                ks = suffix.split(self.sep)
+                z = result
+                for x in ks[:-1]:
+                    if x not in z:
+                        z[x] = record()
+                    z = z[x]
+                z[ks[-1]] = v
+            else:
+                result[suffix] = v
+        if len(result) == 0:
+            raise KeyError("No key or prefix: %s" % key)
+        return result
+    
+
+def record(value=None): 
+    """This function returns a :class:`Record` instance constructed with an 
+    initial value that you provide.
+    
+    :param `value`: An initial record value.
+    :type `value`: ``dict``
+    """
+    if value is None: value = {}
+    return Record(value)
+

dist/devsecops/bin/splunklib/modularinput/__init__.py

@@ -0,0 +1,12 @@
+"""The following imports allow these classes to be imported via
+the splunklib.modularinput package like so:
+
+from splunklib.modularinput import *
+"""
+from .argument import Argument
+from .event import Event
+from .event_writer import EventWriter
+from .input_definition import InputDefinition
+from .scheme import Scheme
+from .script import Script
+from .validation_definition import ValidationDefinition

dist/devsecops/bin/splunklib/modularinput/argument.py

@@ -0,0 +1,103 @@
+# Copyright 2011-2015 Splunk, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"): you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from __future__ import absolute_import
+try:
+    import xml.etree.ElementTree as ET
+except ImportError:
+    import xml.etree.cElementTree as ET
+
+class Argument(object):
+    """Class representing an argument to a modular input kind.
+
+    ``Argument`` is meant to be used with ``Scheme`` to generate an XML 
+    definition of the modular input kind that Splunk understands.
+
+    ``name`` is the only required parameter for the constructor.
+
+        **Example with least parameters**::
+
+            arg1 = Argument(name="arg1")
+
+        **Example with all parameters**::
+
+            arg2 = Argument(
+                name="arg2",
+                description="This is an argument with lots of parameters",
+                validation="is_pos_int('some_name')",
+                data_type=Argument.data_type_number,
+                required_on_edit=True,
+                required_on_create=True
+            )
+    """
+
+    # Constant values, do not change.
+    # These should be used for setting the value of an Argument object's data_type field.
+    data_type_boolean = "BOOLEAN"
+    data_type_number = "NUMBER"
+    data_type_string = "STRING"
+
+    def __init__(self, name, description=None, validation=None,
+                 data_type=data_type_string, required_on_edit=False, required_on_create=False, title=None):
+        """
+        :param name: ``string``, identifier for this argument in Splunk.
+        :param description: ``string``, human-readable description of the argument.
+        :param validation: ``string`` specifying how the argument should be validated, if using internal validation.
+               If using external validation, this will be ignored.
+        :param data_type: ``string``, data type of this field; use the class constants.
+               "data_type_boolean", "data_type_number", or "data_type_string".
+        :param required_on_edit: ``Boolean``, whether this arg is required when editing an existing modular input of this kind.
+        :param required_on_create: ``Boolean``, whether this arg is required when creating a modular input of this kind.
+        :param title: ``String``, a human-readable title for the argument.
+        """
+        self.name = name
+        self.description = description
+        self.validation = validation
+        self.data_type = data_type
+        self.required_on_edit = required_on_edit
+        self.required_on_create = required_on_create
+        self.title = title
+
+    def add_to_document(self, parent):
+        """Adds an ``Argument`` object to this ElementTree document.
+
+        Adds an <arg> subelement to the parent element, typically <args>
+        and sets up its subelements with their respective text.
+
+        :param parent: An ``ET.Element`` to be the parent of a new <arg> subelement
+        :returns: An ``ET.Element`` object representing this argument.
+        """
+        arg = ET.SubElement(parent, "arg")
+        arg.set("name", self.name)
+
+        if self.title is not None:
+            ET.SubElement(arg, "title").text = self.title
+
+        if self.description is not None:
+            ET.SubElement(arg, "description").text = self.description
+
+        if self.validation is not None:
+            ET.SubElement(arg, "validation").text = self.validation
+
+        # add all other subelements to this Argument, represented by (tag, text)
+        subelements = [
+            ("data_type", self.data_type),
+            ("required_on_edit", self.required_on_edit),
+            ("required_on_create", self.required_on_create)
+        ]
+
+        for name, value in subelements:
+            ET.SubElement(arg, name).text = str(value).lower()
+
+        return arg

dist/devsecops/bin/splunklib/modularinput/event.py

@@ -0,0 +1,114 @@
+# Copyright 2011-2015 Splunk, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"): you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from __future__ import absolute_import
+from io import TextIOBase
+from splunklib.six import ensure_text
+
+try:
+    import xml.etree.cElementTree as ET
+except ImportError as ie:
+    import xml.etree.ElementTree as ET
+
+class Event(object):
+    """Represents an event or fragment of an event to be written by this modular input to Splunk.
+
+    To write an input to a stream, call the ``write_to`` function, passing in a stream.
+    """
+    def __init__(self, data=None, stanza=None, time=None, host=None, index=None, source=None,
+                 sourcetype=None, done=True, unbroken=True):
+        """There are no required parameters for constructing an Event
+
+        **Example with minimal configuration**::
+
+            my_event = Event(
+                data="This is a test of my new event.",
+                stanza="myStanzaName",
+                time="%.3f" % 1372187084.000
+            )
+
+        **Example with full configuration**::
+
+            excellent_event = Event(
+                data="This is a test of my excellent event.",
+                stanza="excellenceOnly",
+                time="%.3f" % 1372274622.493,
+                host="localhost",
+                index="main",
+                source="Splunk",
+                sourcetype="misc",
+                done=True,
+                unbroken=True
+            )
+
+        :param data: ``string``, the event's text.
+        :param stanza: ``string``, name of the input this event should be sent to.
+        :param time: ``float``, time in seconds, including up to 3 decimal places to represent milliseconds.
+        :param host: ``string``, the event's host, ex: localhost.
+        :param index: ``string``, the index this event is specified to write to, or None if default index.
+        :param source: ``string``, the source of this event, or None to have Splunk guess.
+        :param sourcetype: ``string``, source type currently set on this event, or None to have Splunk guess.
+        :param done: ``boolean``, is this a complete ``Event``? False if an ``Event`` fragment.
+        :param unbroken: ``boolean``, Is this event completely encapsulated in this ``Event`` object?
+        """
+        self.data = data
+        self.done = done
+        self.host = host
+        self.index = index
+        self.source = source
+        self.sourceType = sourcetype
+        self.stanza = stanza
+        self.time = time
+        self.unbroken = unbroken
+
+    def write_to(self, stream):
+        """Write an XML representation of self, an ``Event`` object, to the given stream.
+
+        The ``Event`` object will only be written if its data field is defined,
+        otherwise a ``ValueError`` is raised.
+
+        :param stream: stream to write XML to.
+        """
+        if self.data is None:
+            raise ValueError("Events must have at least the data field set to be written to XML.")
+
+        event = ET.Element("event")
+        if self.stanza is not None:
+            event.set("stanza", self.stanza)
+        event.set("unbroken", str(int(self.unbroken)))
+
+        # if a time isn't set, let Splunk guess by not creating a <time> element
+        if self.time is not None:
+            ET.SubElement(event, "time").text = str(self.time)
+
+        # add all other subelements to this Event, represented by (tag, text)
+        subelements = [
+            ("source", self.source),
+            ("sourcetype", self.sourceType),
+            ("index", self.index),
+            ("host", self.host),
+            ("data", self.data)
+        ]
+        for node, value in subelements:
+            if value is not None:
+                ET.SubElement(event, node).text = value
+
+        if self.done:
+            ET.SubElement(event, "done")
+
+        if isinstance(stream, TextIOBase):
+            stream.write(ensure_text(ET.tostring(event)))
+        else:
+            stream.write(ET.tostring(event))
+        stream.flush()

dist/devsecops/bin/splunklib/modularinput/event_writer.py

@@ -0,0 +1,87 @@
+# Copyright 2011-2015 Splunk, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"): you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from __future__ import absolute_import
+import sys
+
+from io import TextIOWrapper, TextIOBase
+from splunklib.six import ensure_str
+from .event import ET
+
+try:
+    from splunklib.six.moves import cStringIO as StringIO
+except ImportError:
+    from splunklib.six import StringIO
+
+class EventWriter(object):
+    """``EventWriter`` writes events and error messages to Splunk from a modular input.
+    Its two important methods are ``writeEvent``, which takes an ``Event`` object,
+    and ``log``, which takes a severity and an error message.
+    """
+
+    # Severities that Splunk understands for log messages from modular inputs.
+    # Do not change these
+    DEBUG = "DEBUG"
+    INFO = "INFO"
+    WARN = "WARN"
+    ERROR = "ERROR"
+    FATAL = "FATAL"
+
+    def __init__(self, output = sys.stdout, error = sys.stderr):
+        """
+        :param output: Where to write the output; defaults to sys.stdout.
+        :param error: Where to write any errors; defaults to sys.stderr.
+        """
+        self._out = output
+        self._err = error
+
+        # has the opening <stream> tag been written yet?
+        self.header_written = False
+
+    def write_event(self, event):
+        """Writes an ``Event`` object to Splunk.
+
+        :param event: An ``Event`` object.
+        """
+
+        if not self.header_written:
+            self._out.write("<stream>")
+            self.header_written = True
+
+        event.write_to(self._out)
+
+    def log(self, severity, message):
+        """Logs messages about the state of this modular input to Splunk.
+        These messages will show up in Splunk's internal logs.
+
+        :param severity: ``string``, severity of message, see severities defined as class constants.
+        :param message: ``string``, message to log.
+        """
+
+        self._err.write("%s %s\n" % (severity, message))
+        self._err.flush()
+
+    def write_xml_document(self, document):
+        """Writes a string representation of an
+        ``ElementTree`` object to the output stream.
+
+        :param document: An ``ElementTree`` object.
+        """
+        self._out.write(ensure_str(ET.tostring(document)))
+        self._out.flush()
+
+    def close(self):
+        """Write the closing </stream> tag to make this XML well formed."""
+        self._out.write("</stream>")
+        self._out.flush()

dist/devsecops/bin/splunklib/modularinput/input_definition.py

@@ -0,0 +1,60 @@
+# Copyright 2011-2015 Splunk, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"): you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from __future__ import absolute_import
+try:
+    import xml.etree.cElementTree as ET
+except ImportError as ie:
+    import xml.etree.ElementTree as ET
+
+from .utils import parse_xml_data
+
+class InputDefinition:
+    """``InputDefinition`` encodes the XML defining inputs that Splunk passes to
+    a modular input script.
+
+     **Example**::
+
+        i = InputDefinition()
+
+    """
+    def __init__ (self):
+        self.metadata = {}
+        self.inputs = {}
+
+    def __eq__(self, other):
+        if not isinstance(other, InputDefinition):
+            return False
+        return self.metadata == other.metadata and self.inputs == other.inputs
+
+    @staticmethod
+    def parse(stream):
+        """Parse a stream containing XML into an ``InputDefinition``.
+
+        :param stream: stream containing XML to parse.
+        :return: definition: an ``InputDefinition`` object.
+        """
+        definition = InputDefinition()
+
+        # parse XML from the stream, then get the root node
+        root = ET.parse(stream).getroot()
+
+        for node in root:
+            if node.tag == "configuration":
+                # get config for each stanza
+                definition.inputs = parse_xml_data(node, "stanza")
+            else:
+                definition.metadata[node.tag] = node.text
+
+        return definition

dist/devsecops/bin/splunklib/modularinput/scheme.py

@@ -0,0 +1,85 @@
+# Copyright 2011-2015 Splunk, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"): you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from __future__ import absolute_import
+try:
+    import xml.etree.cElementTree as ET
+except ImportError:
+    import xml.etree.ElementTree as ET
+
+class Scheme(object):
+    """Class representing the metadata for a modular input kind.
+
+    A ``Scheme`` specifies a title, description, several options of how Splunk should run modular inputs of this
+    kind, and a set of arguments which define a particular modular input's properties.
+
+    The primary use of ``Scheme`` is to abstract away the construction of XML to feed to Splunk.
+    """
+
+    # Constant values, do not change
+    # These should be used for setting the value of a Scheme object's streaming_mode field.
+    streaming_mode_simple = "SIMPLE"
+    streaming_mode_xml = "XML"
+
+    def __init__(self, title):
+        """
+        :param title: ``string`` identifier for this Scheme in Splunk.
+        """
+        self.title = title
+        self.description = None
+        self.use_external_validation = True
+        self.use_single_instance = False
+        self.streaming_mode = Scheme.streaming_mode_xml
+
+        # list of Argument objects, each to be represented by an <arg> tag
+        self.arguments = []
+
+    def add_argument(self, arg):
+        """Add the provided argument, ``arg``, to the ``self.arguments`` list.
+
+        :param arg: An ``Argument`` object to add to ``self.arguments``.
+        """
+        self.arguments.append(arg)
+
+    def to_xml(self):
+        """Creates an ``ET.Element`` representing self, then returns it.
+
+        :returns: an ``ET.Element`` representing this scheme.
+        """
+        root = ET.Element("scheme")
+
+        ET.SubElement(root, "title").text = self.title
+
+        # add a description subelement if it's defined
+        if self.description is not None:
+            ET.SubElement(root, "description").text = self.description
+
+        # add all other subelements to this Scheme, represented by (tag, text)
+        subelements = [
+            ("use_external_validation", self.use_external_validation),
+            ("use_single_instance", self.use_single_instance),
+            ("streaming_mode", self.streaming_mode)
+        ]
+        for name, value in subelements:
+            ET.SubElement(root, name).text = str(value).lower()
+
+        endpoint = ET.SubElement(root, "endpoint")
+
+        args = ET.SubElement(endpoint, "args")
+
+        # add arguments as subelements to the <args> element
+        for arg in self.arguments:
+            arg.add_to_document(args)
+
+        return root

dist/devsecops/bin/splunklib/modularinput/script.py

@@ -0,0 +1,177 @@
+# Copyright 2011-2015 Splunk, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"): you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from __future__ import absolute_import
+from abc import ABCMeta, abstractmethod
+from splunklib.six.moves.urllib.parse import urlsplit
+import sys
+
+from ..client import Service
+from .event_writer import EventWriter
+from .input_definition import InputDefinition
+from .validation_definition import ValidationDefinition
+from splunklib import six
+
+try:
+    import xml.etree.cElementTree as ET
+except ImportError:
+    import xml.etree.ElementTree as ET
+
+
+class Script(six.with_metaclass(ABCMeta, object)):
+    """An abstract base class for implementing modular inputs.
+
+    Subclasses should override ``get_scheme``, ``stream_events``,
+    and optionally ``validate_input`` if the modular input uses
+    external validation.
+
+    The ``run`` function is used to run modular inputs; it typically should
+    not be overridden.
+    """
+
+    def __init__(self):
+        self._input_definition = None
+        self._service = None
+
+    def run(self, args):
+        """Runs this modular input
+
+        :param args: List of command line arguments passed to this script.
+        :returns: An integer to be used as the exit value of this program.
+        """
+
+        # call the run_script function, which handles the specifics of running
+        # a modular input
+        return self.run_script(args, EventWriter(), sys.stdin)
+
+    def run_script(self, args, event_writer, input_stream):
+        """Handles all the specifics of running a modular input
+
+        :param args: List of command line arguments passed to this script.
+        :param event_writer: An ``EventWriter`` object for writing events.
+        :param input_stream: An input stream for reading inputs.
+        :returns: An integer to be used as the exit value of this program.
+        """
+
+        try:
+            if len(args) == 1:
+                # This script is running as an input. Input definitions will be
+                # passed on stdin as XML, and the script will write events on
+                # stdout and log entries on stderr.
+                self._input_definition = InputDefinition.parse(input_stream)
+                self.stream_events(self._input_definition, event_writer)
+                event_writer.close()
+                return 0
+
+            elif str(args[1]).lower() == "--scheme":
+                # Splunk has requested XML specifying the scheme for this
+                # modular input Return it and exit.
+                scheme = self.get_scheme()
+                if scheme is None:
+                    event_writer.log(
+                        EventWriter.FATAL,
+                        "Modular input script returned a null scheme.")
+                    return 1
+                else:
+                    event_writer.write_xml_document(scheme.to_xml())
+                    return 0
+
+            elif args[1].lower() == "--validate-arguments":
+                validation_definition = ValidationDefinition.parse(input_stream)
+                try:
+                    self.validate_input(validation_definition)
+                    return 0
+                except Exception as e:
+                    root = ET.Element("error")
+                    ET.SubElement(root, "message").text = str(e)
+                    event_writer.write_xml_document(root)
+
+                    return 1
+            else:
+                err_string = "ERROR Invalid arguments to modular input script:" + ' '.join(
+                    args)
+                event_writer._err.write(err_string)
+                return 1
+
+        except Exception as e:
+            event_writer.log(EventWriter.ERROR, str(e))
+            return 1
+
+    @property
+    def service(self):
+        """ Returns a Splunk service object for this script invocation.
+
+        The service object is created from the Splunkd URI and session key
+        passed to the command invocation on the modular input stream. It is
+        available as soon as the :code:`Script.stream_events` method is
+        called.
+
+        :return: :class:`splunklib.client.Service`. A value of None is returned,
+            if you call this method before the :code:`Script.stream_events` method
+            is called.
+
+        """
+        if self._service is not None:
+            return self._service
+
+        if self._input_definition is None:
+            return None
+
+        splunkd_uri = self._input_definition.metadata["server_uri"]
+        session_key = self._input_definition.metadata["session_key"]
+
+        splunkd = urlsplit(splunkd_uri, allow_fragments=False)
+
+        self._service = Service(
+            scheme=splunkd.scheme,
+            host=splunkd.hostname,
+            port=splunkd.port,
+            token=session_key,
+        )
+
+        return self._service
+
+    @abstractmethod
+    def get_scheme(self):
+        """The scheme defines the parameters understood by this modular input.
+
+        :return: a ``Scheme`` object representing the parameters for this modular input.
+        """
+
+    def validate_input(self, definition):
+        """Handles external validation for modular input kinds.
+
+        When Splunk calls a modular input script in validation mode, it will
+        pass in an XML document giving information about the Splunk instance (so
+        you can call back into it if needed) and the name and parameters of the
+        proposed input.
+
+        If this function does not throw an exception, the validation is assumed
+        to succeed. Otherwise any errors thrown will be turned into a string and
+        logged back to Splunk.
+
+        The default implementation always passes.
+
+        :param definition: The parameters for the proposed input passed by splunkd.
+        """
+        pass
+
+    @abstractmethod
+    def stream_events(self, inputs, ew):
+        """The method called to stream events into Splunk. It should do all of its output via
+        EventWriter rather than assuming that there is a console attached.
+
+        :param inputs: An ``InputDefinition`` object.
+        :param ew: An object with methods to write events and log messages to Splunk.
+        """

dist/devsecops/bin/splunklib/modularinput/utils.py

@@ -0,0 +1,74 @@
+# Copyright 2011-2015 Splunk, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"): you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# File for utility functions
+
+from __future__ import absolute_import
+from splunklib.six.moves import zip
+def xml_compare(expected, found):
+    """Checks equality of two ``ElementTree`` objects.
+
+    :param expected: An ``ElementTree`` object.
+    :param found: An ``ElementTree`` object.
+    :return: ``Boolean``, whether the two objects are equal.
+    """
+
+    # if comparing the same ET object
+    if expected == found:
+        return True
+
+    # compare element attributes, ignoring order
+    if set(expected.items()) != set(found.items()):
+        return False
+
+    # check for equal number of children
+    expected_children = list(expected)
+    found_children = list(found)
+    if len(expected_children) != len(found_children):
+        return False
+
+    # compare children
+    if not all([xml_compare(a, b) for a, b in zip(expected_children, found_children)]):
+        return False
+
+    # compare elements, if there is no text node, return True
+    if (expected.text is None or expected.text.strip() == "") \
+        and (found.text is None or found.text.strip() == ""):
+        return True
+    else:
+        return expected.tag == found.tag and expected.text == found.text \
+            and expected.attrib == found.attrib
+
+def parse_parameters(param_node):
+    if param_node.tag == "param":
+        return param_node.text
+    elif param_node.tag == "param_list":
+        parameters = []
+        for mvp in param_node:
+            parameters.append(mvp.text)
+        return parameters
+    else:
+        raise ValueError("Invalid configuration scheme, %s tag unexpected." % param_node.tag)
+
+def parse_xml_data(parent_node, child_node_tag):
+    data = {}
+    for child in parent_node:
+        if child.tag == child_node_tag:
+            if child_node_tag == "stanza":
+                data[child.get("name")] = {}
+                for param in child:
+                    data[child.get("name")][param.get("name")] = parse_parameters(param)
+        elif "item" == parent_node.tag:
+            data[child.get("name")] = parse_parameters(child)
+    return data

dist/devsecops/bin/splunklib/modularinput/validation_definition.py

@@ -0,0 +1,86 @@
+# Copyright 2011-2015 Splunk, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"): you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+from __future__ import absolute_import
+try:
+    import xml.etree.cElementTree as ET
+except ImportError as ie:
+    import xml.etree.ElementTree as ET
+
+from .utils import parse_xml_data
+
+
+class ValidationDefinition(object):
+    """This class represents the XML sent by Splunk for external validation of a
+    new modular input.
+
+    **Example**::
+
+        v = ValidationDefinition()
+
+    """
+    def __init__(self):
+        self.metadata = {}
+        self.parameters = {}
+
+    def __eq__(self, other):
+        if not isinstance(other, ValidationDefinition):
+            return False
+        return self.metadata == other.metadata and self.parameters == other.parameters
+
+    @staticmethod
+    def parse(stream):
+        """Creates a ``ValidationDefinition`` from a provided stream containing XML.
+
+        The XML typically will look like this:
+
+        ..  code-block:: xml
+
+            <items>
+               <server_host>myHost</server_host>
+                 <server_uri>https://127.0.0.1:8089</server_uri>
+                 <session_key>123102983109283019283</session_key>
+                 <checkpoint_dir>/opt/splunk/var/lib/splunk/modinputs</checkpoint_dir>
+                 <item name="myScheme">
+                   <param name="param1">value1</param>
+                   <param_list name="param2">
+                     <value>value2</value>
+                     <value>value3</value>
+                     <value>value4</value>
+                   </param_list>
+                 </item>
+            </items>
+
+        :param stream: ``Stream`` containing XML to parse.
+        :return: A ``ValidationDefinition`` object.
+
+        """
+
+        definition = ValidationDefinition()
+
+        # parse XML from the stream, then get the root node
+        root = ET.parse(stream).getroot()
+
+        for node in root:
+            # lone item node
+            if node.tag == "item":
+                # name from item node
+                definition.metadata["name"] = node.get("name")
+                definition.parameters = parse_xml_data(node, "")
+            else:
+                # Store anything else in metadata
+                definition.metadata[node.tag] = node.text
+
+        return definition

dist/devsecops/bin/splunklib/ordereddict.py

@@ -0,0 +1,128 @@
+# Copyright (c) 2009 Raymond Hettinger
+#
+# Permission is hereby granted, free of charge, to any person
+# obtaining a copy of this software and associated documentation files
+# (the "Software"), to deal in the Software without restriction,
+# including without limitation the rights to use, copy, modify, merge,
+# publish, distribute, sublicense, and/or sell copies of the Software,
+# and to permit persons to whom the Software is furnished to do so,
+# subject to the following conditions:
+#
+#     The above copyright notice and this permission notice shall be
+#     included in all copies or substantial portions of the Software.
+#
+#     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+#     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+#     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+#     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+#     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+#     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+#     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+#     OTHER DEALINGS IN THE SOFTWARE.
+
+from UserDict import DictMixin
+
+
+class OrderedDict(dict, DictMixin):
+
+    def __init__(self, *args, **kwds):
+        if len(args) > 1:
+            raise TypeError('expected at most 1 arguments, got %d' % len(args))
+        try:
+            self.__end
+        except AttributeError:
+            self.clear()
+        self.update(*args, **kwds)
+
+    def clear(self):
+        self.__end = end = []
+        end += [None, end, end]         # sentinel node for doubly linked list
+        self.__map = {}                 # key --> [key, prev, next]
+        dict.clear(self)
+
+    def __setitem__(self, key, value):
+        if key not in self:
+            end = self.__end
+            curr = end[1]
+            curr[2] = end[1] = self.__map[key] = [key, curr, end]
+        dict.__setitem__(self, key, value)
+
+    def __delitem__(self, key):
+        dict.__delitem__(self, key)
+        key, prev, next = self.__map.pop(key)
+        prev[2] = next
+        next[1] = prev
+
+    def __iter__(self):
+        end = self.__end
+        curr = end[2]
+        while curr is not end:
+            yield curr[0]
+            curr = curr[2]
+
+    def __reversed__(self):
+        end = self.__end
+        curr = end[1]
+        while curr is not end:
+            yield curr[0]
+            curr = curr[1]
+
+    def popitem(self, last=True):
+        if not self:
+            raise KeyError('dictionary is empty')
+        if last:
+            key = reversed(self).next()
+        else:
+            key = iter(self).next()
+        value = self.pop(key)
+        return key, value
+
+    def __reduce__(self):
+        items = [[k, self[k]] for k in self]
+        tmp = self.__map, self.__end
+        del self.__map, self.__end
+        inst_dict = vars(self).copy()
+        self.__map, self.__end = tmp
+        if inst_dict:
+            return (self.__class__, (items,), inst_dict)
+        return self.__class__, (items,)
+
+    def keys(self):
+        return list(self)
+
+    setdefault = DictMixin.setdefault
+    update = DictMixin.update
+    pop = DictMixin.pop
+    values = DictMixin.values
+    items = DictMixin.items
+    iterkeys = DictMixin.iterkeys
+    itervalues = DictMixin.itervalues
+    iteritems = DictMixin.iteritems
+
+    def __repr__(self):
+        if not self:
+            return '%s()' % (self.__class__.__name__,)
+        return '%s(%r)' % (self.__class__.__name__, self.items())
+
+    def copy(self):
+        return self.__class__(self)
+
+    @classmethod
+    def fromkeys(cls, iterable, value=None):
+        d = cls()
+        for key in iterable:
+            d[key] = value
+        return d
+
+    def __eq__(self, other):
+        if isinstance(other, OrderedDict):
+            if len(self) != len(other):
+                return False
+            for p, q in  zip(self.items(), other.items()):
+                if p != q:
+                    return False
+            return True
+        return dict.__eq__(self, other)
+
+    def __ne__(self, other):
+        return not self == other

dist/devsecops/bin/splunklib/results.py

@@ -0,0 +1,295 @@
+# Copyright 2011-2015 Splunk, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"): you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""The **splunklib.results** module provides a streaming XML reader for Splunk
+search results.
+
+Splunk search results can be returned in a variety of formats including XML,
+JSON, and CSV. To make it easier to stream search results in XML format, they
+are returned as a stream of XML *fragments*, not as a single XML document. This
+module supports incrementally reading one result record at a time from such a
+result stream. This module also provides a friendly iterator-based interface for
+accessing search results while avoiding buffering the result set, which can be
+very large.
+
+To use the reader, instantiate :class:`ResultsReader` on a search result stream
+as follows:::
+
+    reader = ResultsReader(result_stream)
+    for item in reader:
+        print(item)
+    print "Results are a preview: %s" % reader.is_preview
+"""
+
+from __future__ import absolute_import
+
+from io import BytesIO
+
+from splunklib import six
+try:
+    import xml.etree.cElementTree as et
+except:
+    import xml.etree.ElementTree as et
+
+try:
+    from collections import OrderedDict  # must be python 2.7
+except ImportError:
+    from .ordereddict import OrderedDict
+
+try:
+    from splunklib.six.moves import cStringIO as StringIO
+except:
+    from splunklib.six import StringIO
+
+__all__ = [
+    "ResultsReader",
+    "Message"
+]
+
+class Message(object):
+    """This class represents informational messages that Splunk interleaves in the results stream.
+
+    ``Message`` takes two arguments: a string giving the message type (e.g., "DEBUG"), and
+    a string giving the message itself.
+
+    **Example**::
+
+        m = Message("DEBUG", "There's something in that variable...")
+    """
+    def __init__(self, type_, message):
+        self.type = type_
+        self.message = message
+
+    def __repr__(self):
+        return "%s: %s" % (self.type, self.message)
+
+    def __eq__(self, other):
+        return (self.type, self.message) == (other.type, other.message)
+
+    def __hash__(self):
+        return hash((self.type, self.message))
+
+class _ConcatenatedStream(object):
+    """Lazily concatenate zero or more streams into a stream.
+
+    As you read from the concatenated stream, you get characters from
+    each stream passed to ``_ConcatenatedStream``, in order.
+
+    **Example**::
+
+        from StringIO import StringIO
+        s = _ConcatenatedStream(StringIO("abc"), StringIO("def"))
+        assert s.read() == "abcdef"
+    """
+    def __init__(self, *streams):
+        self.streams = list(streams)
+
+    def read(self, n=None):
+        """Read at most *n* characters from this stream.
+
+        If *n* is ``None``, return all available characters.
+        """
+        response = b""
+        while len(self.streams) > 0 and (n is None or n > 0):
+            txt = self.streams[0].read(n)
+            response += txt
+            if n is not None:
+                n -= len(txt)
+            if n is None or n > 0:
+                del self.streams[0]
+        return response
+
+class _XMLDTDFilter(object):
+    """Lazily remove all XML DTDs from a stream.
+
+    All substrings matching the regular expression <?[^>]*> are
+    removed in their entirety from the stream. No regular expressions
+    are used, however, so everything still streams properly.
+
+    **Example**::
+
+        from StringIO import StringIO
+        s = _XMLDTDFilter("<?xml abcd><element><?xml ...></element>")
+        assert s.read() == "<element></element>"
+    """
+    def __init__(self, stream):
+        self.stream = stream
+
+    def read(self, n=None):
+        """Read at most *n* characters from this stream.
+
+        If *n* is ``None``, return all available characters.
+        """
+        response = b""
+        while n is None or n > 0:
+            c = self.stream.read(1)
+            if c == b"":
+                break
+            elif c == b"<":
+                c += self.stream.read(1)
+                if c == b"<?":
+                    while True:
+                        q = self.stream.read(1)
+                        if q == b">":
+                            break
+                else:
+                    response += c
+                    if n is not None:
+                        n -= len(c)
+            else:
+                response += c
+                if n is not None:
+                    n -= 1
+        return response
+
+class ResultsReader(object):
+    """This class returns dictionaries and Splunk messages from an XML results
+    stream.
+
+    ``ResultsReader`` is iterable, and returns a ``dict`` for results, or a
+    :class:`Message` object for Splunk messages. This class has one field,
+    ``is_preview``, which is ``True`` when the results are a preview from a
+    running search, or ``False`` when the results are from a completed search.
+
+    This function has no network activity other than what is implicit in the
+    stream it operates on.
+
+    :param `stream`: The stream to read from (any object that supports
+        ``.read()``).
+
+    **Example**::
+
+        import results
+        response = ... # the body of an HTTP response
+        reader = results.ResultsReader(response)
+        for result in reader:
+            if isinstance(result, dict):
+                print "Result: %s" % result
+            elif isinstance(result, results.Message):
+                print "Message: %s" % result
+        print "is_preview = %s " % reader.is_preview
+    """
+    # Be sure to update the docstrings of client.Jobs.oneshot,
+    # client.Job.results_preview and client.Job.results to match any
+    # changes made to ResultsReader.
+    #
+    # This wouldn't be a class, just the _parse_results function below,
+    # except that you cannot get the current generator inside the
+    # function creating that generator. Thus it's all wrapped up for
+    # the sake of one field.
+    def __init__(self, stream):
+        # The search/jobs/exports endpoint, when run with
+        # earliest_time=rt and latest_time=rt streams a sequence of
+        # XML documents, each containing a result, as opposed to one
+        # results element containing lots of results. Python's XML
+        # parsers are broken, and instead of reading one full document
+        # and returning the stream that follows untouched, they
+        # destroy the stream and throw an error. To get around this,
+        # we remove all the DTD definitions inline, then wrap the
+        # fragments in a fiction <doc> element to make the parser happy.
+        stream = _XMLDTDFilter(stream)
+        stream = _ConcatenatedStream(BytesIO(b"<doc>"), stream, BytesIO(b"</doc>"))
+        self.is_preview = None
+        self._gen = self._parse_results(stream)
+
+    def __iter__(self):
+        return self
+
+    def next(self):
+        return next(self._gen)
+
+    __next__ = next
+
+    def _parse_results(self, stream):
+        """Parse results and messages out of *stream*."""
+        result = None
+        values = None
+        try:
+            for event, elem in et.iterparse(stream, events=('start', 'end')):
+                if elem.tag == 'results' and event == 'start':
+                    # The wrapper element is a <results preview="0|1">. We
+                    # don't care about it except to tell is whether these
+                    # are preview results, or the final results from the
+                    # search.
+                    is_preview = elem.attrib['preview'] == '1'
+                    self.is_preview = is_preview
+                if elem.tag == 'result':
+                    if event == 'start':
+                        result = OrderedDict()
+                    elif event == 'end':
+                        yield result
+                        result = None
+                        elem.clear()
+
+                elif elem.tag == 'field' and result is not None:
+                    # We need the 'result is not None' check because
+                    # 'field' is also the element name in the <meta>
+                    # header that gives field order, which is not what we
+                    # want at all.
+                    if event == 'start':
+                        values = []
+                    elif event == 'end':
+                        field_name = elem.attrib['k']
+                        if len(values) == 1:
+                            result[field_name] = values[0]
+                        else:
+                            result[field_name] = values
+                        # Calling .clear() is necessary to let the
+                        # element be garbage collected. Otherwise
+                        # arbitrarily large results sets will use
+                        # arbitrarily large memory intead of
+                        # streaming.
+                        elem.clear()
+
+                elif elem.tag in ('text', 'v') and event == 'end':
+                    try:
+                        text = "".join(elem.itertext())
+                    except AttributeError:
+                        # Assume we're running in Python < 2.7, before itertext() was added
+                        # So we'll define it here
+
+                        def __itertext(self):
+                          tag = self.tag
+                          if not isinstance(tag, six.string_types) and tag is not None:
+                              return
+                          if self.text:
+                              yield self.text
+                          for e in self:
+                              for s in __itertext(e):
+                                  yield s
+                              if e.tail:
+                                  yield e.tail
+
+                        text = "".join(__itertext(elem))
+                    values.append(text)
+                    elem.clear()
+
+                elif elem.tag == 'msg':
+                    if event == 'start':
+                        msg_type = elem.attrib['type']
+                    elif event == 'end':
+                        text = elem.text if elem.text is not None else ""
+                        yield Message(msg_type, text)
+                        elem.clear()
+        except SyntaxError as pe:
+            # This is here to handle the same incorrect return from
+            # splunk that is described in __init__.
+            if 'no element found' in pe.msg:
+                return
+            else:
+                raise
+
+
+
+

dist/devsecops/bin/splunklib/searchcommands/__init__.py

@@ -0,0 +1,155 @@
+# coding=utf-8
+#
+# Copyright © 2011-2015 Splunk, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"): you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""
+
+.. topic:: Design Notes
+
+  1. Commands are constrained to this ABNF grammar::
+
+        command       = command-name *[wsp option] *[wsp [dquote] field-name [dquote]]
+        command-name  = alpha *( alpha / digit )
+        option        = option-name [wsp] "=" [wsp] option-value
+        option-name   = alpha *( alpha / digit / "_" )
+        option-value  = word / quoted-string
+        word          = 1*( %01-%08 / %0B / %0C / %0E-1F / %21 / %23-%FF ) ; Any character but DQUOTE and WSP
+        quoted-string = dquote *( word / wsp / "\" dquote / dquote dquote ) dquote
+        field-name    = ( "_" / alpha ) *( alpha / digit / "_" / "." / "-" )
+
+     It does not show that :code:`field-name` values may be comma-separated. This is because Splunk strips commas from
+     the command line. A search command will never see them.
+
+  2. Search commands targeting versions of Splunk prior to 6.3 must be statically configured as follows:
+
+     .. code-block:: text
+        :linenos:
+
+        [command_name]
+        filename = command_name.py
+        supports_getinfo = true
+        supports_rawargs = true
+
+     No other static configuration is required or expected and may interfere with command execution.
+
+  3. Commands support dynamic probing for settings.
+
+     Splunk probes for settings dynamically when :code:`supports_getinfo=true`.
+     You must add this line to the commands.conf stanza for each of your search
+     commands.
+
+  4. Commands do not support parsed arguments on the command line.
+
+     Splunk parses arguments when :code:`supports_rawargs=false`. The
+     :code:`SearchCommand` class sets this value unconditionally. You cannot
+     override it.
+
+     **Rationale**
+
+     Splunk parses arguments by stripping quotes, nothing more. This may be useful
+     in some cases, but doesn't work well with our chosen grammar.
+
+  5. Commands consume input headers.
+
+     An input header is provided by Splunk when :code:`enableheader=true`. The
+     :class:`SearchCommand` class sets this value unconditionally. You cannot
+     override it.
+
+  6. Commands produce an output messages header.
+
+     Splunk expects a command to produce an output messages header when
+     :code:`outputheader=true`. The :class:`SearchCommand` class sets this value
+     unconditionally. You cannot override it.
+
+  7. Commands support multi-value fields.
+
+     Multi-value fields are provided and consumed by Splunk when
+     :code:`supports_multivalue=true`. This value is fixed. You cannot override
+     it.
+
+  8. This module represents all fields on the output stream in multi-value
+     format.
+
+     Splunk recognizes two kinds of data: :code:`value` and :code:`list(value)`.
+     The multi-value format represents these data in field pairs. Given field
+     :code:`name` the multi-value format calls for the creation of this pair of
+     fields.
+
+     ================= =========================================================
+     Field name         Field data
+     ================= =========================================================
+     :code:`name`      Value or text from which a list of values was derived.
+
+     :code:`__mv_name` Empty, if :code:`field` represents a :code:`value`;
+                       otherwise, an encoded :code:`list(value)`. Values in the
+                       list are wrapped in dollar signs ($) and separated by
+                       semi-colons (;). Dollar signs ($) within a value are
+                       represented by a pair of dollar signs ($$).
+     ================= =========================================================
+
+     Serializing data in this format enables streaming and reduces a command's
+     memory footprint at the cost of one extra byte of data per field per record
+     and a small amount of extra processing time by the next command in the
+     pipeline.
+
+  9. A :class:`ReportingCommand` must override :meth:`~ReportingCommand.reduce`
+     and may override :meth:`~ReportingCommand.map`. Map/reduce commands on the
+     Splunk processing pipeline are distinguished as this example illustrates.
+
+     **Splunk command**
+
+     .. code-block:: text
+
+         sum total=total_date_hour date_hour
+
+     **Map command line**
+
+     .. code-block:: text
+
+        sum __GETINFO__ __map__ total=total_date_hour date_hour
+        sum __EXECUTE__ __map__ total=total_date_hour date_hour
+
+     **Reduce command line**
+
+     .. code-block:: text
+
+        sum __GETINFO__ total=total_date_hour date_hour
+        sum __EXECUTE__ total=total_date_hour date_hour
+
+     The :code:`__map__` argument is introduced by
+     :meth:`ReportingCommand._execute`. Search command authors cannot influence
+     the contents of the command line in this release.
+
+.. topic:: References
+
+  1. `Search command style guide <http://docs.splunk.com/Documentation/Splunk/6.0/Search/Searchcommandstyleguide>`__
+
+  2. `Commands.conf.spec <http://docs.splunk.com/Documentation/Splunk/5.0.5/Admin/Commandsconf>`_
+
+"""
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from .environment import *
+from .decorators import *
+from .validators import *
+
+from .generating_command import GeneratingCommand
+from .streaming_command import StreamingCommand
+from .eventing_command import EventingCommand
+from .reporting_command import ReportingCommand
+
+from .external_search_command import execute, ExternalSearchCommand
+from .search_command import dispatch, SearchMetric

dist/devsecops/bin/splunklib/searchcommands/decorators.py

@@ -0,0 +1,450 @@
+# coding=utf-8
+#
+# Copyright © 2011-2015 Splunk, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"): you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+from splunklib import six
+
+try:
+    from collections import OrderedDict  # must be python 2.7
+except ImportError:
+    from ..ordereddict import OrderedDict
+
+from inspect import getmembers, isclass, isfunction
+from splunklib.six.moves import map as imap
+
+from .internals import ConfigurationSettingsType, json_encode_string
+from .validators import OptionName
+
+
+class Configuration(object):
+    """ Defines the configuration settings for a search command.
+
+    Documents, validates, and ensures that only relevant configuration settings are applied. Adds a :code:`name` class
+    variable to search command classes that don't have one. The :code:`name` is derived from the name of the class.
+    By convention command class names end with the word "Command". To derive :code:`name` the word "Command" is removed
+    from the end of the class name and then converted to lower case for conformance with the `Search command style guide
+    <http://docs.splunk.com/Documentation/Splunk/latest/Search/Searchcommandstyleguide>`__
+
+    """
+    def __init__(self, o=None, **kwargs):
+        #
+        # The o argument enables the configuration decorator to be used with or without parentheses. For example, it
+        # enables you to write code that looks like this:
+        #
+        #   @Configuration
+        #   class Foo(SearchCommand):
+        #       ...
+        #
+        #   @Configuration()
+        #   class Bar(SearchCommand):
+        #       ...
+        #
+        # Without the o argument, the Python compiler will complain about the first form. With the o argument, both
+        # forms work. The first form provides a value for o: Foo. The second form does does not provide a value for o.
+        # The class or method decorated is not passed to the constructor. A value of None is passed instead.
+        #
+        self.settings = kwargs
+
+    def __call__(self, o):
+
+        if isfunction(o):
+            # We must wait to finalize configuration as the class containing this function is under construction
+            # at the time this call to decorate a member function. This will be handled in the call to
+            # o.ConfigurationSettings.fix_up(o) in the elif clause of this code block.
+            o._settings = self.settings
+        elif isclass(o):
+
+            # Set command name
+
+            name = o.__name__
+            if name.endswith('Command'):
+                name = name[:-len('Command')]
+            o.name = six.text_type(name.lower())
+
+            # Construct ConfigurationSettings instance for the command class
+
+            o.ConfigurationSettings = ConfigurationSettingsType(
+                module=o.__module__ + '.' + o.__name__,
+                name='ConfigurationSettings',
+                bases=(o.ConfigurationSettings,))
+
+            ConfigurationSetting.fix_up(o.ConfigurationSettings, self.settings)
+            o.ConfigurationSettings.fix_up(o)
+            Option.fix_up(o)
+        else:
+            raise TypeError('Incorrect usage: Configuration decorator applied to {0}'.format(type(o), o.__name__))
+
+        return o
+
+
+class ConfigurationSetting(property):
+    """ Generates a :class:`property` representing the named configuration setting
+
+    This is a convenience function designed to reduce the amount of boiler-plate code you must write; most notably for
+    property setters.
+
+    :param name: Configuration setting name.
+    :type name: str or unicode
+
+    :param doc: A documentation string.
+    :type doc: bytes, unicode or NoneType
+
+    :param readonly: If true, specifies that the configuration setting is fixed.
+    :type name: bool or NoneType
+
+    :param value: Configuration setting value.
+
+    :return: A :class:`property` instance representing the configuration setting.
+    :rtype: property
+
+    """
+    def __init__(self, fget=None, fset=None, fdel=None, doc=None, name=None, readonly=None, value=None):
+        property.__init__(self, fget=fget, fset=fset, fdel=fdel, doc=doc)
+        self._readonly = readonly
+        self._value = value
+        self._name = name
+
+    def __call__(self, function):
+        return self.getter(function)
+
+    def deleter(self, function):
+        return self._copy_extra_attributes(property.deleter(self, function))
+
+    def getter(self, function):
+        return self._copy_extra_attributes(property.getter(self, function))
+
+    def setter(self, function):
+        return self._copy_extra_attributes(property.setter(self, function))
+
+    @staticmethod
+    def fix_up(cls, values):
+
+        is_configuration_setting = lambda attribute: isinstance(attribute, ConfigurationSetting)
+        definitions = getmembers(cls, is_configuration_setting)
+        i = 0
+
+        for name, setting in definitions:
+
+            if setting._name is None:
+                setting._name = name = six.text_type(name)
+            else:
+                name = setting._name
+
+            validate, specification = setting._get_specification()
+            backing_field_name = '_' + name
+
+            if setting.fget is None and setting.fset is None and setting.fdel is None:
+
+                value = setting._value
+
+                if setting._readonly or value is not None:
+                    validate(specification, name, value)
+
+                def fget(bfn, value):
+                    return lambda this: getattr(this, bfn, value)
+
+                setting = setting.getter(fget(backing_field_name, value))
+
+                if not setting._readonly:
+
+                    def fset(bfn, validate, specification, name):
+                        return lambda this, value: setattr(this, bfn, validate(specification, name, value))
+
+                    setting = setting.setter(fset(backing_field_name, validate, specification, name))
+
+                setattr(cls, name, setting)
+
+            def is_supported_by_protocol(supporting_protocols):
+
+                def is_supported_by_protocol(version):
+                    return version in supporting_protocols
+
+                return is_supported_by_protocol
+
+            del setting._name, setting._value, setting._readonly
+
+            setting.is_supported_by_protocol = is_supported_by_protocol(specification.supporting_protocols)
+            setting.supporting_protocols = specification.supporting_protocols
+            setting.backing_field_name = backing_field_name
+            definitions[i] = setting
+            setting.name = name
+
+            i += 1
+
+            try:
+                value = values[name]
+            except KeyError:
+                continue
+
+            if setting.fset is None:
+                raise ValueError('The value of configuration setting {} is fixed'.format(name))
+
+            setattr(cls, backing_field_name, validate(specification, name, value))
+            del values[name]
+
+        if len(values) > 0:
+            settings = sorted(list(six.iteritems(values)))
+            settings = imap(lambda n_v: '{}={}'.format(n_v[0], repr(n_v[1])), settings)
+            raise AttributeError('Inapplicable configuration settings: ' + ', '.join(settings))
+
+        cls.configuration_setting_definitions = definitions
+
+    def _copy_extra_attributes(self, other):
+        other._readonly = self._readonly
+        other._value = self._value
+        other._name = self._name
+        return other
+
+    def _get_specification(self):
+
+        name = self._name
+
+        try:
+            specification = ConfigurationSettingsType.specification_matrix[name]
+        except KeyError:
+            raise AttributeError('Unknown configuration setting: {}={}'.format(name, repr(self._value)))
+
+        return ConfigurationSettingsType.validate_configuration_setting, specification
+
+
+class Option(property):
+    """ Represents a search command option.
+
+    Required options must be specified on the search command line.
+
+    **Example:**
+
+    Short form (recommended). When you are satisfied with built-in or custom validation behaviors.
+
+    ..  code-block:: python
+        :linenos:
+
+        from splunklib.searchcommands.decorators import Option
+        from splunklib.searchcommands.validators import Fieldname
+
+        total = Option(
+            doc=''' **Syntax:** **total=***<fieldname>*
+            **Description:** Name of the field that will hold the computed
+            sum''',
+            require=True, validate=Fieldname())
+
+    **Example:**
+
+    Long form. Useful when you wish to manage the option value and its deleter/getter/setter side-effects yourself. You
+    must provide a getter and a setter. If your :code:`Option` requires `destruction <https://docs.python.org/2/reference/datamodel.html#object.__del__>`_ you must
+    also provide a deleter. You must be prepared to accept a value of :const:`None` which indicates that your
+    :code:`Option` is unset.
+
+    ..  code-block:: python
+        :linenos:
+
+        from splunklib.searchcommands import Option
+
+        @Option()
+        def logging_configuration(self):
+            \""" **Syntax:** logging_configuration=<path>
+            **Description:** Loads an alternative logging configuration file for a command invocation. The logging
+            configuration file must be in Python ConfigParser-format. The *<path>* name and all path names specified in
+            configuration are relative to the app root directory.
+
+            \"""
+            return self._logging_configuration
+
+        @logging_configuration.setter
+        def logging_configuration(self, value):
+            if value is not None
+                logging.configure(value)
+                self._logging_configuration = value
+
+        def __init__(self)
+            self._logging_configuration = None
+
+    """
+    def __init__(self, fget=None, fset=None, fdel=None, doc=None, name=None, default=None, require=None, validate=None):
+        property.__init__(self, fget, fset, fdel, doc)
+        self.name = name
+        self.default = default
+        self.validate = validate
+        self.require = bool(require)
+
+    def __call__(self, function):
+        return self.getter(function)
+
+    # region Methods
+
+    def deleter(self, function):
+        return self._copy_extra_attributes(property.deleter(self, function))
+
+    def getter(self, function):
+        return self._copy_extra_attributes(property.getter(self, function))
+
+    def setter(self, function):
+        return self._copy_extra_attributes(property.setter(self, function))
+
+    @classmethod
+    def fix_up(cls, command_class):
+
+        is_option = lambda attribute: isinstance(attribute, Option)
+        definitions = getmembers(command_class, is_option)
+        validate_option_name = OptionName()
+        i = 0
+
+        for name, option in definitions:
+
+            if option.name is None:
+                option.name = name  # no validation required
+            else:
+                validate_option_name(option.name)
+
+            if option.fget is None and option.fset is None and option.fdel is None:
+                backing_field_name = '_' + name
+
+                def fget(bfn):
+                    return lambda this: getattr(this, bfn, None)
+
+                option = option.getter(fget(backing_field_name))
+
+                def fset(bfn, validate):
+                    if validate is None:
+                        return lambda this, value: setattr(this, bfn, value)
+                    return lambda this, value: setattr(this, bfn, validate(value))
+
+                option = option.setter(fset(backing_field_name, option.validate))
+                setattr(command_class, name, option)
+
+            elif option.validate is not None:
+
+                def fset(function, validate):
+                    return lambda this, value: function(this, validate(value))
+
+                option = option.setter(fset(option.fset, option.validate))
+                setattr(command_class, name, option)
+
+            definitions[i] = name, option
+            i += 1
+
+        command_class.option_definitions = definitions
+
+    def _copy_extra_attributes(self, other):
+        other.name = self.name
+        other.default = self.default
+        other.require = self.require
+        other.validate = self.validate
+        return other
+
+    # endregion
+
+    # region Types
+
+    class Item(object):
+        """ Presents an instance/class view over a search command `Option`.
+
+        This class is used by SearchCommand.process to parse and report on option values.
+
+        """
+        def __init__(self, command, option):
+            self._command = command
+            self._option = option
+            self._is_set = False
+            validator = self.validator
+            self._format = six.text_type if validator is None else validator.format
+
+        def __repr__(self):
+            return '(' + repr(self.name) + ', ' + repr(self._format(self.value)) + ')'
+
+        def __str__(self):
+            value = self.value
+            value = 'None' if value is None else json_encode_string(self._format(value))
+            return self.name + '=' + value
+
+        # region Properties
+
+        @property
+        def is_required(self):
+            return bool(self._option.require)
+
+        @property
+        def is_set(self):
+            """ Indicates whether an option value was provided as argument.
+
+            """
+            return self._is_set
+
+        @property
+        def name(self):
+            return self._option.name
+
+        @property
+        def validator(self):
+            return self._option.validate
+
+        @property
+        def value(self):
+            return self._option.__get__(self._command)
+
+        @value.setter
+        def value(self, value):
+            self._option.__set__(self._command, value)
+            self._is_set = True
+
+        # endregion
+
+        # region Methods
+
+        def reset(self):
+            self._option.__set__(self._command, self._option.default)
+            self._is_set = False
+
+        pass
+        # endregion
+
+    class View(OrderedDict):
+        """ Presents an ordered dictionary view of the set of :class:`Option` arguments to a search command.
+
+        This class is used by SearchCommand.process to parse and report on option values.
+
+        """
+        def __init__(self, command):
+            definitions = type(command).option_definitions
+            item_class = Option.Item
+            OrderedDict.__init__(self, ((option.name, item_class(command, option)) for (name, option) in definitions))
+
+        def __repr__(self):
+            text = 'Option.View([' + ','.join(imap(lambda item: repr(item), six.itervalues(self))) + '])'
+            return text
+
+        def __str__(self):
+            text = ' '.join([str(item) for item in six.itervalues(self) if item.is_set])
+            return text
+
+        # region Methods
+
+        def get_missing(self):
+            missing = [item.name for item in six.itervalues(self) if item.is_required and not item.is_set]
+            return missing if len(missing) > 0 else None
+
+        def reset(self):
+            for value in six.itervalues(self):
+                value.reset()
+
+        pass
+        # endregion
+
+    pass
+    # endregion
+
+
+__all__ = ['Configuration', 'Option']

dist/devsecops/bin/splunklib/searchcommands/environment.py

@@ -0,0 +1,123 @@
+# coding=utf-8
+#
+# Copyright © 2011-2015 Splunk, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"): you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from logging import getLogger, root, StreamHandler
+from logging.config import fileConfig
+from os import chdir, environ, path
+from splunklib.six.moves import getcwd
+
+import sys
+
+
+def configure_logging(logger_name, filename=None):
+    """ Configure logging and return the named logger and the location of the logging configuration file loaded.
+
+    This function expects a Splunk app directory structure::
+
+        <app-root>
+            bin
+                ...
+            default
+                ...
+            local
+                ...
+
+    This function looks for a logging configuration file at each of these locations, loading the first, if any,
+    logging configuration file that it finds::
+
+        local/{name}.logging.conf
+        default/{name}.logging.conf
+        local/logging.conf
+        default/logging.conf
+
+    The current working directory is set to *<app-root>* before the logging configuration file is loaded. Hence, paths
+    in the logging configuration file are relative to *<app-root>*. The current directory is reset before return.
+
+    You may short circuit the search for a logging configuration file by providing an alternative file location in
+    `path`. Logging configuration files must be in `ConfigParser format`_.
+
+    #Arguments:
+
+    :param logger_name: Logger name
+    :type logger_name: bytes, unicode
+
+    :param filename: Location of an alternative logging configuration file or `None`.
+    :type filename: bytes, unicode or NoneType
+
+    :returns: The named logger and the location of the logging configuration file loaded.
+    :rtype: tuple
+
+    .. _ConfigParser format: https://docs.python.org/2/library/logging.config.html#configuration-file-format
+
+    """
+    if filename is None:
+        if logger_name is None:
+            probing_paths = [path.join('local', 'logging.conf'), path.join('default', 'logging.conf')]
+        else:
+            probing_paths = [
+                path.join('local', logger_name + '.logging.conf'),
+                path.join('default', logger_name + '.logging.conf'),
+                path.join('local', 'logging.conf'),
+                path.join('default', 'logging.conf')]
+        for relative_path in probing_paths:
+            configuration_file = path.join(app_root, relative_path)
+            if path.exists(configuration_file):
+                filename = configuration_file
+                break
+    elif not path.isabs(filename):
+        found = False
+        for conf in 'local', 'default':
+            configuration_file = path.join(app_root, conf, filename)
+            if path.exists(configuration_file):
+                filename = configuration_file
+                found = True
+                break
+        if not found:
+            raise ValueError('Logging configuration file "{}" not found in local or default directory'.format(filename))
+    elif not path.exists(filename):
+        raise ValueError('Logging configuration file "{}" not found'.format(filename))
+
+    if filename is not None:
+        global _current_logging_configuration_file
+        filename = path.realpath(filename)
+
+        if filename != _current_logging_configuration_file:
+            working_directory = getcwd()
+            chdir(app_root)
+            try:
+                fileConfig(filename, {'SPLUNK_HOME': splunk_home})
+            finally:
+                chdir(working_directory)
+            _current_logging_configuration_file = filename
+
+    if len(root.handlers) == 0:
+        root.addHandler(StreamHandler())
+
+    return None if logger_name is None else getLogger(logger_name), filename
+
+
+_current_logging_configuration_file = None
+
+splunk_home = path.abspath(path.join(getcwd(), environ.get('SPLUNK_HOME', '')))
+app_file = getattr(sys.modules['__main__'], '__file__', sys.executable)
+app_root = path.dirname(path.abspath(path.dirname(app_file)))
+
+splunklib_logger, logging_configuration = configure_logging('splunklib')
+
+
+__all__ = ['app_file', 'app_root', 'logging_configuration', 'splunk_home', 'splunklib_logger']

dist/devsecops/bin/splunklib/searchcommands/eventing_command.py

@@ -0,0 +1,149 @@
+# coding=utf-8
+#
+# Copyright 2011-2015 Splunk, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"): you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from splunklib import six
+from splunklib.six.moves import map as imap
+
+from .decorators import ConfigurationSetting
+from .search_command import SearchCommand
+
+
+class EventingCommand(SearchCommand):
+    """ Applies a transformation to search results as they travel through the events pipeline.
+
+    Eventing commands typically filter, group, order, and/or or augment event records. Examples of eventing commands
+    from Splunk's built-in command set include sort_, dedup_, and cluster_. Each execution of an eventing command
+    should produce a set of event records that is independently usable by downstream processors.
+
+    .. _sort: http://docs.splunk.com/Documentation/Splunk/latest/SearchReference/Sort
+    .. _dedup: http://docs.splunk.com/Documentation/Splunk/latest/SearchReference/Dedup
+    .. _cluster: http://docs.splunk.com/Documentation/Splunk/latest/SearchReference/Cluster
+
+    EventingCommand configuration
+    ==============================
+
+    You can configure your command for operation under Search Command Protocol (SCP) version 1 or 2. SCP 2 requires
+    Splunk 6.3 or later.
+
+    """
+    # region Methods
+
+    def transform(self, records):
+        """ Generator function that processes and yields event records to the Splunk events pipeline.
+
+        You must override this method.
+
+        """
+        raise NotImplementedError('EventingCommand.transform(self, records)')
+
+    def _execute(self, ifile, process):
+        SearchCommand._execute(self, ifile, self.transform)
+
+    # endregion
+
+    class ConfigurationSettings(SearchCommand.ConfigurationSettings):
+        """ Represents the configuration settings that apply to a :class:`EventingCommand`.
+
+        """
+        # region SCP v1/v2 properties
+
+        required_fields = ConfigurationSetting(doc='''
+            List of required fields for this search which back-propagates to the generating search.
+
+            Setting this value enables selected fields mode under SCP 2. Under SCP 1 you must also specify
+            :code:`clear_required_fields=True` to enable selected fields mode. To explicitly select all fields,
+            specify a value of :const:`['*']`. No error is generated if a specified field is missing.
+
+            Default: :const:`None`, which implicitly selects all fields.
+
+            ''')
+
+        # endregion
+
+        # region SCP v1 properties
+
+        clear_required_fields = ConfigurationSetting(doc='''
+            :const:`True`, if required_fields represent the *only* fields required.
+
+            If :const:`False`, required_fields are additive to any fields that may be required by subsequent commands.
+            In most cases, :const:`False` is appropriate for eventing commands.
+
+            Default: :const:`False`
+
+            ''')
+
+        retainsevents = ConfigurationSetting(readonly=True, value=True, doc='''
+            :const:`True`, if the command retains events the way the sort/dedup/cluster commands do.
+
+            If :const:`False`, the command transforms events the way the stats command does.
+
+            Fixed: :const:`True`
+
+            ''')
+
+        # endregion
+
+        # region SCP v2 properties
+
+        maxinputs = ConfigurationSetting(doc='''
+            Specifies the maximum number of events that can be passed to the command for each invocation.
+
+            This limit cannot exceed the value of `maxresultrows` as defined in limits.conf_. Under SCP 1 you must
+            specify this value in commands.conf_.
+
+            Default: The value of `maxresultrows`.
+
+            Supported by: SCP 2
+
+            .. _limits.conf: http://docs.splunk.com/Documentation/Splunk/latest/admin/Limitsconf
+
+            ''')
+
+        type = ConfigurationSetting(readonly=True, value='events', doc='''
+            Command type
+
+            Fixed: :const:`'events'`.
+
+            Supported by: SCP 2
+
+            ''')
+
+        # endregion
+
+        # region Methods
+
+        @classmethod
+        def fix_up(cls, command):
+            """ Verifies :code:`command` class structure.
+
+            """
+            if command.transform == EventingCommand.transform:
+                raise AttributeError('No EventingCommand.transform override')
+            SearchCommand.ConfigurationSettings.fix_up(command)
+
+        # TODO: Stop looking like a dictionary because we don't obey the semantics
+        # N.B.: Does not use Python 2 dict copy semantics
+        def iteritems(self):
+            iteritems = SearchCommand.ConfigurationSettings.iteritems(self)
+            return imap(lambda name_value: (name_value[0], 'events' if name_value[0] == 'type' else name_value[1]), iteritems)
+
+        # N.B.: Does not use Python 3 dict view semantics
+        if not six.PY2:
+            items = iteritems
+
+        # endregion

dist/devsecops/bin/splunklib/searchcommands/external_search_command.py

@@ -0,0 +1,228 @@
+# coding=utf-8
+#
+# Copyright 2011-2015 Splunk, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"): you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from logging import getLogger
+import os
+import sys
+import traceback
+from splunklib import six
+
+if sys.platform == 'win32':
+    from signal import signal, CTRL_BREAK_EVENT, SIGBREAK, SIGINT, SIGTERM
+    from subprocess import Popen
+    import atexit
+
+from . import splunklib_logger as logger
+
+# P1 [ ] TODO: Add ExternalSearchCommand class documentation
+
+
+class ExternalSearchCommand(object):
+    """
+    """
+    def __init__(self, path, argv=None, environ=None):
+
+        if not isinstance(path, (bytes, six.text_type)):
+            raise ValueError('Expected a string value for path, not {}'.format(repr(path)))
+
+        self._logger = getLogger(self.__class__.__name__)
+        self._path = six.text_type(path)
+        self._argv = None
+        self._environ = None
+
+        self.argv = argv
+        self.environ = environ
+
+    # region Properties
+
+    @property
+    def argv(self):
+        return getattr(self, '_argv')
+
+    @argv.setter
+    def argv(self, value):
+        if not (value is None or isinstance(value, (list, tuple))):
+            raise ValueError('Expected a list, tuple or value of None for argv, not {}'.format(repr(value)))
+        self._argv = value
+
+    @property
+    def environ(self):
+        return getattr(self, '_environ')
+
+    @environ.setter
+    def environ(self, value):
+        if not (value is None or isinstance(value, dict)):
+            raise ValueError('Expected a dictionary value for environ, not {}'.format(repr(value)))
+        self._environ = value
+
+    @property
+    def logger(self):
+        return self._logger
+
+    @property
+    def path(self):
+        return self._path
+
+    # endregion
+
+    # region Methods
+
+    def execute(self):
+        # noinspection PyBroadException
+        try:
+            if self._argv is None:
+                self._argv = os.path.splitext(os.path.basename(self._path))[0]
+            self._execute(self._path, self._argv, self._environ)
+        except:
+            error_type, error, tb = sys.exc_info()
+            message = 'Command execution failed: ' + six.text_type(error)
+            self._logger.error(message + '\nTraceback:\n' + ''.join(traceback.format_tb(tb)))
+            sys.exit(1)
+
+    if sys.platform == 'win32':
+
+        @staticmethod
+        def _execute(path, argv=None, environ=None):
+            """ Executes an external search command.
+
+            :param path: Path to the external search command.
+            :type path: unicode
+
+            :param argv: Argument list.
+            :type argv: list or tuple
+                The arguments to the child process should start with the name of the command being run, but this is not
+                enforced. A value of :const:`None` specifies that the base name of path name :param:`path` should be used.
+
+            :param environ: A mapping which is used to define the environment variables for the new process.
+            :type environ: dict or None.
+                This mapping is used instead of the current process’s environment. A value of :const:`None` specifies that
+                the :data:`os.environ` mapping should be used.
+
+            :return: None
+
+            """
+            search_path = os.getenv('PATH') if environ is None else environ.get('PATH')
+            found = ExternalSearchCommand._search_path(path, search_path)
+
+            if found is None:
+                raise ValueError('Cannot find command on path: {}'.format(path))
+
+            path = found
+            logger.debug('starting command="%s", arguments=%s', path, argv)
+
+            def terminate(signal_number, frame):
+                sys.exit('External search command is terminating on receipt of signal={}.'.format(signal_number))
+
+            def terminate_child():
+                if p.pid is not None and p.returncode is None:
+                    logger.debug('terminating command="%s", arguments=%d, pid=%d', path, argv, p.pid)
+                    os.kill(p.pid, CTRL_BREAK_EVENT)
+
+            p = Popen(argv, executable=path, env=environ, stdin=sys.stdin, stdout=sys.stdout, stderr=sys.stderr)
+            atexit.register(terminate_child)
+            signal(SIGBREAK, terminate)
+            signal(SIGINT, terminate)
+            signal(SIGTERM, terminate)
+
+            logger.debug('started command="%s", arguments=%s, pid=%d', path, argv, p.pid)
+            p.wait()
+
+            logger.debug('finished command="%s", arguments=%s, pid=%d, returncode=%d', path, argv, p.pid, p.returncode)
+
+            if p.returncode != 0:
+                sys.exit(p.returncode)
+
+        @staticmethod
+        def _search_path(executable, paths):
+            """ Locates an executable program file.
+
+            :param executable: The name of the executable program to locate.
+            :type executable: unicode
+
+            :param paths: A list of one or more directory paths where executable programs are located.
+            :type paths: unicode
+
+            :return:
+            :rtype: Path to the executable program located or :const:`None`.
+
+            """
+            directory, filename = os.path.split(executable)
+            extension = os.path.splitext(filename)[1].upper()
+            executable_extensions = ExternalSearchCommand._executable_extensions
+
+            if directory:
+                if len(extension) and extension in executable_extensions:
+                    return None
+                for extension in executable_extensions:
+                    path = executable + extension
+                    if os.path.isfile(path):
+                        return path
+                return None
+
+            if not paths:
+                return None
+
+            directories = [directory for directory in paths.split(';') if len(directory)]
+
+            if len(directories) == 0:
+                return None
+
+            if len(extension) and extension in executable_extensions:
+                for directory in directories:
+                    path = os.path.join(directory, executable)
+                    if os.path.isfile(path):
+                        return path
+                return None
+
+            for directory in directories:
+                path_without_extension = os.path.join(directory, executable)
+                for extension in executable_extensions:
+                    path = path_without_extension + extension
+                    if os.path.isfile(path):
+                        return path
+
+            return None
+
+        _executable_extensions = ('.COM', '.EXE')
+    else:
+        @staticmethod
+        def _execute(path, argv, environ):
+            if environ is None:
+                os.execvp(path, argv)
+            else:
+                os.execvpe(path, argv, environ)
+            return
+
+    # endregion
+
+
+def execute(path, argv=None, environ=None, command_class=ExternalSearchCommand):
+    """
+    :param path:
+    :type path: basestring
+    :param argv:
+    :type: argv: list, tuple, or None
+    :param environ:
+    :type environ: dict
+    :param command_class: External search command class to instantiate and execute.
+    :type command_class: type
+    :return:
+    :rtype: None
+    """
+    assert issubclass(command_class, ExternalSearchCommand)
+    command_class(path, argv, environ).execute()

dist/devsecops/bin/splunklib/searchcommands/generating_command.py

@@ -0,0 +1,350 @@
+# coding=utf-8
+#
+# Copyright © 2011-2015 Splunk, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"): you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from .decorators import ConfigurationSetting
+from .search_command import SearchCommand
+
+from splunklib import six
+from splunklib.six.moves import map as imap, filter as ifilter
+
+# P1 [O] TODO: Discuss generates_timeorder in the class-level documentation for GeneratingCommand
+
+
+class GeneratingCommand(SearchCommand):
+    """ Generates events based on command arguments.
+
+    Generating commands receive no input and must be the first command on a pipeline. There are three pipelines:
+    streams, events, and reports. The streams pipeline generates or processes time-ordered event records on an
+    indexer or search head.
+
+    Streaming commands filter, modify, or augment event records and can be applied to subsets of index data in a
+    parallel manner. An example of a streaming command from Splunk's built-in command set is rex_ which extracts and
+    adds fields to event records at search time. Records that pass through the streams pipeline move on to the events
+    pipeline.
+
+    The events pipeline generates or processes records on a search head. Eventing commands typically filter, group,
+    order, or augment event records. Examples of eventing commands from Splunk's built-in command set include sort_,
+    dedup_, and cluster_. Each execution of an eventing command should produce a set of event records that is
+    independently usable by downstream processors. Records that pass through the events pipeline move on to the reports
+    pipeline.
+
+    The reports pipeline also runs on a search head, but yields data structures for presentation, not event records.
+    Examples of streaming from Splunk's built-in command set include chart_, stats_, and contingency_.
+
+    GeneratingCommand configuration
+    ===============================
+
+    Configure your generating command based on the pipeline that it targets. How you configure your command depends on
+    the Search Command Protocol (SCP) version.
+
+    +----------+-------------------------------------+--------------------------------------------+
+    | Pipeline | SCP 1                               | SCP 2                                      |
+    +==========+=====================================+============================================+
+    | streams  | streaming=True[,local=[True|False]] | type='streaming'[,distributed=[true|false] |
+    +----------+-------------------------------------+--------------------------------------------+
+    | events   | retainsevents=True, streaming=False | type='events'                              |
+    +----------+-------------------------------------+--------------------------------------------+
+    | reports  | streaming=False                     | type='reporting'                           |
+    +----------+-------------------------------------+--------------------------------------------+
+
+    Only streaming commands may be distributed to indexers. By default generating commands are configured to run
+    locally in the streams pipeline and will run under either SCP 1 or SCP 2.
+
+    .. code-block:: python
+
+        @Configuration()
+        class StreamingGeneratingCommand(GeneratingCommand)
+            ...
+
+    How you configure your command to run on a different pipeline or in a distributed fashion depends on what SCP
+    protocol versions you wish to support. You must be sure to configure your command consistently for each protocol,
+    if you wish to support both protocol versions correctly.
+
+    .. _chart: http://docs.splunk.com/Documentation/Splunk/latest/SearchReference/Chart
+    .. _cluster: http://docs.splunk.com/Documentation/Splunk/latest/SearchReference/Cluster
+    .. _contingency: http://docs.splunk.com/Documentation/Splunk/latest/SearchReference/Contingency
+    .. _dedup: http://docs.splunk.com/Documentation/Splunk/latest/SearchReference/Dedup
+    .. _rex: http://docs.splunk.com/Documentation/Splunk/latest/SearchReference/Rex
+    .. _sort: http://docs.splunk.com/Documentation/Splunk/latest/SearchReference/Sort
+    .. _stats: http://docs.splunk.com/Documentation/Splunk/latest/SearchReference/Stats
+
+    Distributed Generating command
+    ==============================
+
+    Commands configured like this will run as the first command on search heads and/or indexers on the streams pipeline.
+
+    +----------+---------------------------------------------------+---------------------------------------------------+
+    | Pipeline | SCP 1                                             | SCP 2                                             |
+    +==========+===================================================+===================================================+
+    | streams  | 1. Add this line to your command's stanza in      | 1. Add this configuration setting to your code:   |
+    |          |                                                   |                                                   |
+    |          |    default/commands.conf::                        |    ..  code-block:: python                        |
+    |          |                                                   |                                                   |
+    |          |        local = false                              |        @Configuration(distributed=True)           |
+    |          |                                                   |        class SomeCommand(GeneratingCommand)       |
+    |          |                                                   |            ...                                    |
+    |          | 2. Restart splunk                                 |                                                   |
+    |          |                                                   | 2. You are good to go; no need to restart Splunk  |
+    +----------+---------------------------------------------------+---------------------------------------------------+
+
+    Eventing Generating command
+    ===========================
+
+    Generating commands configured like this will run as the first command on a search head on the events pipeline.
+
+    +----------+---------------------------------------------------+---------------------------------------------------+
+    | Pipeline | SCP 1                                             | SCP 2                                             |
+    +==========+===================================================+===================================================+
+    | events   | You have a choice. Add these configuration        | Add this configuration setting to your command    |
+    |          | settings to your command class:                   | setting to your command class:                    |
+    |          |                                                   |                                                   |
+    |          | .. code-block:: python                            | .. code-block:: python                            |
+    |          |                                                   |                                                   |
+    |          |     @Configuration(                               |     @Configuration(type='events')                 |
+    |          |         retainsevents=True, streaming=False)      |     class SomeCommand(GeneratingCommand)          |
+    |          |     class SomeCommand(GeneratingCommand)          |         ...                                       |
+    |          |         ...                                       |                                                   |
+    |          |                                                   |                                                   |
+    |          | Or add these lines to default/commands.conf:      |                                                   |
+    |          |                                                   |                                                   |
+    |          | ..  code-block:: text                             |                                                   |
+    |          |                                                   |                                                   |
+    |          |     retainsevents = true                          |                                                   |
+    |          |     streaming = false                             |                                                   |
+    +----------+---------------------------------------------------+---------------------------------------------------+
+
+    Configure your command class like this, if you wish to support both protocols:
+
+    ..  code-block:: python
+
+        @Configuration(type='events', retainsevents=True, streaming=False)
+        class SomeCommand(GeneratingCommand)
+            ...
+
+    You might also consider adding these lines to commands.conf instead of adding them to your command class:
+
+    ..  code-block:: python
+
+        retainsevents = false
+        streaming = false
+
+    Reporting Generating command
+    ============================
+
+    Commands configured like this will run as the first command on a search head on the reports pipeline.
+
+    +----------+---------------------------------------------------+---------------------------------------------------+
+    | Pipeline | SCP 1                                             | SCP 2                                             |
+    +==========+===================================================+===================================================+
+    | events   | You have a choice. Add these configuration        | Add this configuration setting to your command    |
+    |          | settings to your command class:                   | setting to your command class:                    |
+    |          |                                                   |                                                   |
+    |          | .. code-block:: python                            | .. code-block:: python                            |
+    |          |                                                   |                                                   |
+    |          |     @Configuration(retainsevents=False)           |     @Configuration(type='reporting')              |
+    |          |     class SomeCommand(GeneratingCommand)          |     class SomeCommand(GeneratingCommand)          |
+    |          |         ...                                       |         ...                                       |
+    |          |                                                   |                                                   |
+    |          | Or add this lines to default/commands.conf:       |                                                   |
+    |          |                                                   |                                                   |
+    |          | .. code-block:: text                              |                                                   |
+    |          |                                                   |                                                   |
+    |          |     retainsevents = false                         |                                                   |
+    |          |     streaming = false                             |                                                   |
+    +----------+---------------------------------------------------+---------------------------------------------------+
+
+    Configure your command class like this, if you wish to support both protocols:
+
+    ..  code-block:: python
+
+        @Configuration(type='reporting', streaming=False)
+        class SomeCommand(GeneratingCommand)
+            ...
+
+    You might also consider adding these lines to commands.conf instead of adding them to your command class:
+
+    ..  code-block:: text
+
+        retainsevents = false
+        streaming = false
+
+    """
+    # region Methods
+
+    def generate(self):
+        """ A generator that yields records to the Splunk processing pipeline
+
+        You must override this method.
+
+        """
+        raise NotImplementedError('GeneratingCommand.generate(self)')
+
+    def _execute(self, ifile, process):
+        """ Execution loop
+
+        :param ifile: Input file object. Unused.
+        :type ifile: file
+
+        :return: `None`.
+
+        """
+        if self._protocol_version == 2:
+            self._execute_v2(ifile, self.generate())
+        else:
+            assert self._protocol_version == 1
+            self._record_writer.write_records(self.generate())
+        self.finish()
+
+    def _execute_chunk_v2(self, process, chunk):
+        count = 0
+        for row in process:
+            self._record_writer.write_record(row)
+            count += 1
+            if count == self._record_writer._maxresultrows:
+                self._finished = False
+                return
+        self._finished = True
+
+    # endregion
+
+    # region Types
+
+    class ConfigurationSettings(SearchCommand.ConfigurationSettings):
+        """ Represents the configuration settings for a :code:`GeneratingCommand` class.
+
+        """
+        # region SCP v1/v2 Properties
+
+        generating = ConfigurationSetting(readonly=True, value=True, doc='''
+            Tells Splunk that this command generates events, but does not process inputs.
+
+            Generating commands must appear at the front of the search pipeline identified by :meth:`type`.
+
+            Fixed: :const:`True`
+
+            Supported by: SCP 1, SCP 2
+
+            ''')
+
+        # endregion
+
+        # region SCP v1 Properties
+
+        generates_timeorder = ConfigurationSetting(doc='''
+            :const:`True`, if the command generates new events.
+
+            Default: :const:`False`
+
+            Supported by: SCP 1
+
+            ''')
+
+        local = ConfigurationSetting(doc='''
+            :const:`True`, if the command should run locally on the search head.
+
+            Default: :const:`False`
+
+            Supported by: SCP 1
+
+            ''')
+
+        retainsevents = ConfigurationSetting(doc='''
+            :const:`True`, if the command retains events the way the sort, dedup, and cluster commands do, or whether it
+            transforms them the way the stats command does.
+
+            Default: :const:`False`
+
+            Supported by: SCP 1
+
+            ''')
+
+        streaming = ConfigurationSetting(doc='''
+            :const:`True`, if the command is streamable.
+
+            Default: :const:`True`
+
+            Supported by: SCP 1
+
+            ''')
+
+        # endregion
+
+        # region SCP v2 Properties
+
+        distributed = ConfigurationSetting(value=False, doc='''
+            True, if this command should be distributed to indexers.
+
+            This value is ignored unless :meth:`type` is equal to :const:`streaming`. It is only this command type that
+            may be distributed.
+
+            Default: :const:`False`
+
+            Supported by: SCP 2
+
+            ''')
+
+        type = ConfigurationSetting(value='streaming', doc='''
+            A command type name.
+
+            ====================  ======================================================================================
+            Value                 Description
+            --------------------  --------------------------------------------------------------------------------------
+            :const:`'events'`     Runs as the first command in the Splunk events pipeline. Cannot be distributed.
+            :const:`'reporting'`  Runs as the first command in the Splunk reports pipeline. Cannot be distributed.
+            :const:`'streaming'`  Runs as the first command in the Splunk streams pipeline. May be distributed.
+            ====================  ======================================================================================
+
+            Default: :const:`'streaming'`
+
+            Supported by: SCP 2
+
+            ''')
+
+        # endregion
+
+        # region Methods
+
+        @classmethod
+        def fix_up(cls, command):
+            """ Verifies :code:`command` class structure.
+
+            """
+            if command.generate == GeneratingCommand.generate:
+                raise AttributeError('No GeneratingCommand.generate override')
+
+        # TODO: Stop looking like a dictionary because we don't obey the semantics
+        # N.B.: Does not use Python 2 dict copy semantics
+        def iteritems(self):
+            iteritems = SearchCommand.ConfigurationSettings.iteritems(self)
+            version = self.command.protocol_version
+            if version == 2:
+                iteritems = ifilter(lambda name_value1: name_value1[0] != 'distributed', iteritems)
+                if not self.distributed and self.type == 'streaming':
+                    iteritems = imap(
+                        lambda name_value: (name_value[0], 'stateful') if name_value[0] == 'type' else (name_value[0], name_value[1]), iteritems)
+            return iteritems
+
+        # N.B.: Does not use Python 3 dict view semantics
+        if not six.PY2:
+            items = iteritems
+
+        pass
+        # endregion
+
+    pass
+    # endregion

dist/devsecops/bin/splunklib/searchcommands/internals.py

@@ -0,0 +1,844 @@
+# coding=utf-8
+#
+# Copyright © 2011-2015 Splunk, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"): you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from __future__ import absolute_import, division, print_function
+
+from io import TextIOWrapper
+from collections import deque, namedtuple
+from splunklib import six
+try:
+    from collections import OrderedDict  # must be python 2.7
+except ImportError:
+    from ..ordereddict import OrderedDict
+from splunklib.six.moves import StringIO
+from itertools import chain
+from splunklib.six.moves import map as imap
+from json import JSONDecoder, JSONEncoder
+from json.encoder import encode_basestring_ascii as json_encode_string
+from splunklib.six.moves import urllib
+
+import csv
+import gzip
+import os
+import re
+import sys
+import warnings
+
+from . import environment
+
+csv.field_size_limit(10485760)  # The default value is 128KB; upping to 10MB. See SPL-12117 for background on this issue
+
+
+def set_binary_mode(fh):
+    """ Helper method to set up binary mode for file handles.
+    Emphasis being sys.stdin, sys.stdout, sys.stderr.
+    For python3, we want to return .buffer
+    For python2+windows we want to set os.O_BINARY
+    """
+    typefile = TextIOWrapper if sys.version_info >= (3, 0) else file
+    # check for file handle
+    if not isinstance(fh, typefile):
+        return fh
+
+    # check for python3 and buffer
+    if sys.version_info >= (3, 0) and hasattr(fh, 'buffer'):
+        return fh.buffer
+    # check for python3
+    elif sys.version_info >= (3, 0):
+        pass
+    # check for windows python2. SPL-175233 -- python3 stdout is already binary
+    elif sys.platform == 'win32':
+        # Work around the fact that on Windows '\n' is mapped to '\r\n'. The typical solution is to simply open files in
+        # binary mode, but stdout is already open, thus this hack. 'CPython' and 'PyPy' work differently. We assume that
+        # all other Python implementations are compatible with 'CPython'. This might or might not be a valid assumption.
+        from platform import python_implementation
+        implementation = python_implementation()
+        if implementation == 'PyPy':
+            return os.fdopen(fh.fileno(), 'wb', 0)
+        else:
+            import msvcrt
+            msvcrt.setmode(fh.fileno(), os.O_BINARY)
+    return fh
+
+
+class CommandLineParser(object):
+    r""" Parses the arguments to a search command.
+
+    A search command line is described by the following syntax.
+
+    **Syntax**::
+
+       command       = command-name *[wsp option] *[wsp [dquote] field-name [dquote]]
+       command-name  = alpha *( alpha / digit )
+       option        = option-name [wsp] "=" [wsp] option-value
+       option-name   = alpha *( alpha / digit / "_" )
+       option-value  = word / quoted-string
+       word          = 1*( %01-%08 / %0B / %0C / %0E-1F / %21 / %23-%FF ) ; Any character but DQUOTE and WSP
+       quoted-string = dquote *( word / wsp / "\" dquote / dquote dquote ) dquote
+       field-name    = ( "_" / alpha ) *( alpha / digit / "_" / "." / "-" )
+
+    **Note:**
+
+    This syntax is constrained to an 8-bit character set.
+
+    **Note:**
+
+    This syntax does not show that `field-name` values may be comma-separated when in fact they can be. This is
+    because Splunk strips commas from the command line. A custom search command will never see them.
+
+    **Example:**
+
+    countmatches fieldname = word_count pattern = \w+ some_text_field
+
+    Option names are mapped to properties in the targeted ``SearchCommand``. It is the responsibility of the property
+    setters to validate the values they receive. Property setters may also produce side effects. For example,
+    setting the built-in `log_level` immediately changes the `log_level`.
+
+    """
+    @classmethod
+    def parse(cls, command, argv):
+        """ Splits an argument list into an options dictionary and a fieldname
+        list.
+
+        The argument list, `argv`, must be of the form::
+
+            *[option]... *[<field-name>]
+
+        Options are validated and assigned to items in `command.options`. Field names are validated and stored in the
+        list of `command.fieldnames`.
+
+        #Arguments:
+
+        :param command: Search command instance.
+        :type command: ``SearchCommand``
+        :param argv: List of search command arguments.
+        :type argv: ``list``
+        :return: ``None``
+
+        #Exceptions:
+
+        ``SyntaxError``: Argument list is incorrectly formed.
+        ``ValueError``: Unrecognized option/field name, or an illegal field value.
+
+        """
+        debug = environment.splunklib_logger.debug
+        command_class = type(command).__name__
+
+        # Prepare
+
+        debug('Parsing %s command line: %r', command_class, argv)
+        command.fieldnames = None
+        command.options.reset()
+        argv = ' '.join(argv)
+
+        command_args = cls._arguments_re.match(argv)
+
+        if command_args is None:
+            raise SyntaxError('Syntax error: {}'.format(argv))
+
+        # Parse options
+
+        for option in cls._options_re.finditer(command_args.group('options')):
+            name, value = option.group('name'), option.group('value')
+            if name not in command.options:
+                raise ValueError(
+                    'Unrecognized {} command option: {}={}'.format(command.name, name, json_encode_string(value)))
+            command.options[name].value = cls.unquote(value)
+
+        missing = command.options.get_missing()
+
+        if missing is not None:
+            if len(missing) > 1:
+                raise ValueError(
+                    'Values for these {} command options are required: {}'.format(command.name, ', '.join(missing)))
+            raise ValueError('A value for {} command option {} is required'.format(command.name, missing[0]))
+
+        # Parse field names
+
+        fieldnames = command_args.group('fieldnames')
+
+        if fieldnames is None:
+            command.fieldnames = []
+        else:
+            command.fieldnames = [cls.unquote(value.group(0)) for value in cls._fieldnames_re.finditer(fieldnames)]
+
+        debug('  %s: %s', command_class, command)
+
+    @classmethod
+    def unquote(cls, string):
+        """ Removes quotes from a quoted string.
+
+        Splunk search command quote rules are applied. The enclosing double-quotes, if present, are removed. Escaped
+        double-quotes ('\"' or '""') are replaced by a single double-quote ('"').
+
+        **NOTE**
+
+        We are not using a json.JSONDecoder because Splunk quote rules are different than JSON quote rules. A
+        json.JSONDecoder does not recognize a pair of double-quotes ('""') as an escaped quote ('"') and will
+        decode single-quoted strings ("'") in addition to double-quoted ('"') strings.
+
+        """
+        if len(string) == 0:
+            return ''
+
+        if string[0] == '"':
+            if len(string) == 1 or string[-1] != '"':
+                raise SyntaxError('Poorly formed string literal: ' + string)
+            string = string[1:-1]
+
+        if len(string) == 0:
+            return ''
+
+        def replace(match):
+            value = match.group(0)
+            if value == '""':
+                return '"'
+            if len(value) < 2:
+                raise SyntaxError('Poorly formed string literal: ' + string)
+            return value[1]
+
+        result = re.sub(cls._escaped_character_re, replace, string)
+        return result
+
+    # region Class variables
+
+    _arguments_re = re.compile(r"""
+        ^\s*
+        (?P<options>     # Match a leading set of name/value pairs
+            (?:
+                (?:(?=\w)[^\d]\w*)                         # name
+                \s*=\s*                                    # =
+                (?:"(?:\\.|""|[^"])*"|(?:\\.|[^\s"])+)\s*  # value
+            )*
+        )\s*
+        (?P<fieldnames>  # Match a trailing set of field names
+            (?:
+                (?:"(?:\\.|""|[^"])*"|(?:\\.|[^\s"])+)\s*
+            )*
+        )\s*$
+        """, re.VERBOSE | re.UNICODE)
+
+    _escaped_character_re = re.compile(r'(\\.|""|[\\"])')
+
+    _fieldnames_re = re.compile(r"""("(?:\\.|""|[^"\\])+"|(?:\\.|[^\s"])+)""")
+
+    _options_re = re.compile(r"""
+        # Captures a set of name/value pairs when used with re.finditer
+        (?P<name>(?:(?=\w)[^\d]\w*))                   # name
+        \s*=\s*                                        # =
+        (?P<value>"(?:\\.|""|[^"])*"|(?:\\.|[^\s"])+)  # value
+        """, re.VERBOSE | re.UNICODE)
+
+    # endregion
+
+
+class ConfigurationSettingsType(type):
+    """ Metaclass for constructing ConfigurationSettings classes.
+
+    Instances of :class:`ConfigurationSettingsType` construct :class:`ConfigurationSettings` classes from classes from
+    a base :class:`ConfigurationSettings` class and a dictionary of configuration settings. The settings in the
+    dictionary are validated against the settings in the base class. You cannot add settings, you can only change their
+    backing-field values and you cannot modify settings without backing-field values. These are considered fixed
+    configuration setting values.
+
+    This is an internal class used in two places:
+
+    + :meth:`decorators.Configuration.__call__`
+
+      Adds a ConfigurationSettings attribute to a :class:`SearchCommand` class.
+
+    + :meth:`reporting_command.ReportingCommand.fix_up`
+
+      Adds a ConfigurationSettings attribute to a :meth:`ReportingCommand.map` method, if there is one.
+
+    """
+    def __new__(mcs, module, name, bases):
+        mcs = super(ConfigurationSettingsType, mcs).__new__(mcs, str(name), bases, {})
+        return mcs
+
+    def __init__(cls, module, name, bases):
+
+        super(ConfigurationSettingsType, cls).__init__(name, bases, None)
+        cls.__module__ = module
+
+    @staticmethod
+    def validate_configuration_setting(specification, name, value):
+        if not isinstance(value, specification.type):
+            if isinstance(specification.type, type):
+                type_names = specification.type.__name__
+            else:
+                type_names = ', '.join(imap(lambda t: t.__name__, specification.type))
+            raise ValueError('Expected {} value, not {}={}'.format(type_names, name, repr(value)))
+        if specification.constraint and not specification.constraint(value):
+            raise ValueError('Illegal value: {}={}'.format(name, repr(value)))
+        return value
+
+    specification = namedtuple(
+        'ConfigurationSettingSpecification', (
+            'type',
+            'constraint',
+            'supporting_protocols'))
+
+    # P1 [ ] TODO: Review ConfigurationSettingsType.specification_matrix for completeness and correctness
+
+    specification_matrix = {
+        'clear_required_fields': specification(
+            type=bool,
+            constraint=None,
+            supporting_protocols=[1]),
+        'distributed': specification(
+            type=bool,
+            constraint=None,
+            supporting_protocols=[2]),
+        'generates_timeorder': specification(
+            type=bool,
+            constraint=None,
+            supporting_protocols=[1]),
+        'generating': specification(
+            type=bool,
+            constraint=None,
+            supporting_protocols=[1, 2]),
+        'local': specification(
+            type=bool,
+            constraint=None,
+            supporting_protocols=[1]),
+        'maxinputs': specification(
+            type=int,
+            constraint=lambda value: 0 <= value <= six.MAXSIZE,
+            supporting_protocols=[2]),
+        'overrides_timeorder': specification(
+            type=bool,
+            constraint=None,
+            supporting_protocols=[1]),
+        'required_fields': specification(
+            type=(list, set, tuple),
+            constraint=None,
+            supporting_protocols=[1, 2]),
+        'requires_preop': specification(
+            type=bool,
+            constraint=None,
+            supporting_protocols=[1]),
+        'retainsevents': specification(
+            type=bool,
+            constraint=None,
+            supporting_protocols=[1]),
+        'run_in_preview': specification(
+            type=bool,
+            constraint=None,
+            supporting_protocols=[2]),
+        'streaming': specification(
+            type=bool,
+            constraint=None,
+            supporting_protocols=[1]),
+        'streaming_preop': specification(
+            type=(bytes, six.text_type),
+            constraint=None,
+            supporting_protocols=[1, 2]),
+        'type': specification(
+            type=(bytes, six.text_type),
+            constraint=lambda value: value in ('events', 'reporting', 'streaming'),
+            supporting_protocols=[2])}
+
+
+class CsvDialect(csv.Dialect):
+    """ Describes the properties of Splunk CSV streams """
+    delimiter = ','
+    quotechar = '"'
+    doublequote = True
+    skipinitialspace = False
+    lineterminator = '\r\n'
+    if sys.version_info >= (3, 0) and sys.platform == 'win32':
+        lineterminator = '\n'
+    quoting = csv.QUOTE_MINIMAL
+
+
+class InputHeader(dict):
+    """ Represents a Splunk input header as a collection of name/value pairs.
+
+    """
+
+    def __str__(self):
+        return '\n'.join([name + ':' + value for name, value in six.iteritems(self)])
+
+    def read(self, ifile):
+        """ Reads an input header from an input file.
+
+        The input header is read as a sequence of *<name>***:***<value>* pairs separated by a newline. The end of the
+        input header is signalled by an empty line or an end-of-file.
+
+        :param ifile: File-like object that supports iteration over lines.
+
+        """
+        name, value = None, None
+
+        for line in ifile:
+            if line == '\n':
+                break
+            item = line.split(':', 1)
+            if len(item) == 2:
+                # start of a new item
+                if name is not None:
+                    self[name] = value[:-1]  # value sans trailing newline
+                name, value = item[0], urllib.parse.unquote(item[1])
+            elif name is not None:
+                # continuation of the current item
+                value += urllib.parse.unquote(line)
+
+        if name is not None:
+            self[name] = value[:-1] if value[-1] == '\n' else value
+
+
+Message = namedtuple('Message', ('type', 'text'))
+
+
+class MetadataDecoder(JSONDecoder):
+
+    def __init__(self):
+        JSONDecoder.__init__(self, object_hook=self._object_hook)
+
+    @staticmethod
+    def _object_hook(dictionary):
+
+        object_view = ObjectView(dictionary)
+        stack = deque()
+        stack.append((None, None, dictionary))
+
+        while len(stack):
+            instance, member_name, dictionary = stack.popleft()
+
+            for name, value in six.iteritems(dictionary):
+                if isinstance(value, dict):
+                    stack.append((dictionary, name, value))
+
+            if instance is not None:
+                instance[member_name] = ObjectView(dictionary)
+
+        return object_view
+
+
+class MetadataEncoder(JSONEncoder):
+
+    def __init__(self):
+        JSONEncoder.__init__(self, separators=MetadataEncoder._separators)
+
+    def default(self, o):
+        return o.__dict__ if isinstance(o, ObjectView) else JSONEncoder.default(self, o)
+
+    _separators = (',', ':')
+
+
+class ObjectView(object):
+
+    def __init__(self, dictionary):
+        self.__dict__ = dictionary
+
+    def __repr__(self):
+        return repr(self.__dict__)
+
+    def __str__(self):
+        return str(self.__dict__)
+
+
+class Recorder(object):
+
+    def __init__(self, path, f):
+        self._recording = gzip.open(path + '.gz', 'wb')
+        self._file = f
+
+    def __getattr__(self, name):
+        return getattr(self._file, name)
+
+    def __iter__(self):
+        for line in self._file:
+            self._recording.write(line)
+            self._recording.flush()
+            yield line
+
+    def read(self, size=None):
+        value = self._file.read() if size is None else self._file.read(size)
+        self._recording.write(value)
+        self._recording.flush()
+        return value
+
+    def readline(self, size=None):
+        value = self._file.readline() if size is None else self._file.readline(size)
+        if len(value) > 0:
+            self._recording.write(value)
+            self._recording.flush()
+        return value
+
+    def record(self, *args):
+        for arg in args:
+            self._recording.write(arg)
+
+    def write(self, text):
+        self._recording.write(text)
+        self._file.write(text)
+        self._recording.flush()
+
+
+class RecordWriter(object):
+
+    def __init__(self, ofile, maxresultrows=None):
+        self._maxresultrows = 50000 if maxresultrows is None else maxresultrows
+
+        self._ofile = set_binary_mode(ofile)
+        self._fieldnames = None
+        self._buffer = StringIO()
+
+        self._writer = csv.writer(self._buffer, dialect=CsvDialect)
+        self._writerow = self._writer.writerow
+        self._finished = False
+        self._flushed = False
+
+        self._inspector = OrderedDict()
+        self._chunk_count = 0
+        self._pending_record_count = 0
+        self._committed_record_count = 0
+
+    @property
+    def is_flushed(self):
+        return self._flushed
+
+    @is_flushed.setter
+    def is_flushed(self, value):
+        self._flushed = True if value else False
+
+    @property
+    def ofile(self):
+        return self._ofile
+
+    @ofile.setter
+    def ofile(self, value):
+        self._ofile = set_binary_mode(value)
+
+    @property
+    def pending_record_count(self):
+        return self._pending_record_count
+
+    @property
+    def _record_count(self):
+        warnings.warn(
+            "_record_count will be deprecated soon. Use pending_record_count instead.",
+             PendingDeprecationWarning
+        )
+        return self.pending_record_count
+
+    @property
+    def committed_record_count(self):
+        return self._committed_record_count
+
+    @property
+    def _total_record_count(self):
+        warnings.warn(
+            "_total_record_count will be deprecated soon. Use committed_record_count instead.",
+             PendingDeprecationWarning
+        )
+        return self.committed_record_count
+
+    def write(self, data):
+        bytes_type = bytes if sys.version_info >= (3, 0) else str
+        if not isinstance(data, bytes_type):
+            data = data.encode('utf-8')
+        self.ofile.write(data)
+
+    def flush(self, finished=None, partial=None):
+        assert finished is None or isinstance(finished, bool)
+        assert partial is None or isinstance(partial, bool)
+        assert not (finished is None and partial is None)
+        assert finished is None or partial is None
+        self._ensure_validity()
+
+    def write_message(self, message_type, message_text, *args, **kwargs):
+        self._ensure_validity()
+        self._inspector.setdefault('messages', []).append((message_type, message_text.format(*args, **kwargs)))
+
+    def write_record(self, record):
+        self._ensure_validity()
+        self._write_record(record)
+
+    def write_records(self, records):
+        self._ensure_validity()
+        write_record = self._write_record
+        for record in records:
+            write_record(record)
+
+    def _clear(self):
+        self._buffer.seek(0)
+        self._buffer.truncate()
+        self._inspector.clear()
+        self._pending_record_count = 0
+
+    def _ensure_validity(self):
+        if self._finished is True:
+            assert self._record_count == 0 and len(self._inspector) == 0
+            raise RuntimeError('I/O operation on closed record writer')
+
+    def _write_record(self, record):
+
+        fieldnames = self._fieldnames
+
+        if fieldnames is None:
+            self._fieldnames = fieldnames = list(record.keys())
+            value_list = imap(lambda fn: (str(fn), str('__mv_') + str(fn)), fieldnames)
+            self._writerow(list(chain.from_iterable(value_list)))
+
+        get_value = record.get
+        values = []
+
+        for fieldname in fieldnames:
+            value = get_value(fieldname, None)
+
+            if value is None:
+                values += (None, None)
+                continue
+
+            value_t = type(value)
+
+            if issubclass(value_t, (list, tuple)):
+
+                if len(value) == 0:
+                    values += (None, None)
+                    continue
+
+                if len(value) > 1:
+                    value_list = value
+                    sv = ''
+                    mv = '$'
+
+                    for value in value_list:
+
+                        if value is None:
+                            sv += '\n'
+                            mv += '$;$'
+                            continue
+
+                        value_t = type(value)
+
+                        if value_t is not bytes:
+
+                            if value_t is bool:
+                                value = str(value.real)
+                            elif value_t is six.text_type:
+                                value = value
+                            elif isinstance(value, six.integer_types) or value_t is float or value_t is complex:
+                                value = str(value)
+                            elif issubclass(value_t, (dict, list, tuple)):
+                                value = str(''.join(RecordWriter._iterencode_json(value, 0)))
+                            else:
+                                value = repr(value).encode('utf-8', errors='backslashreplace')
+
+                        sv += value + '\n'
+                        mv += value.replace('$', '$$') + '$;$'
+
+                    values += (sv[:-1], mv[:-2])
+                    continue
+
+                value = value[0]
+                value_t = type(value)
+
+            if value_t is bool:
+                values += (str(value.real), None)
+                continue
+
+            if value_t is bytes:
+                values += (value, None)
+                continue
+
+            if value_t is six.text_type:
+                if six.PY2:
+                    value = value.encode('utf-8')
+                values += (value, None)
+                continue
+
+            if isinstance(value, six.integer_types) or value_t is float or value_t is complex:
+                values += (str(value), None)
+                continue
+
+            if issubclass(value_t, dict):
+                values += (str(''.join(RecordWriter._iterencode_json(value, 0))), None)
+                continue
+
+            values += (repr(value), None)
+
+        self._writerow(values)
+        self._pending_record_count += 1
+
+        if self.pending_record_count >= self._maxresultrows:
+            self.flush(partial=True)
+
+    try:
+        # noinspection PyUnresolvedReferences
+        from _json import make_encoder
+    except ImportError:
+        # We may be running under PyPy 2.5 which does not include the _json module
+        _iterencode_json = JSONEncoder(separators=(',', ':')).iterencode
+    else:
+        # Creating _iterencode_json this way yields a two-fold performance improvement on Python 2.7.9 and 2.7.10
+        from json.encoder import encode_basestring_ascii
+
+        @staticmethod
+        def _default(o):
+            raise TypeError(repr(o) + ' is not JSON serializable')
+
+        _iterencode_json = make_encoder(
+            {},                       # markers (for detecting circular references)
+            _default,                 # object_encoder
+            encode_basestring_ascii,  # string_encoder
+            None,                     # indent
+            ':', ',',                 # separators
+            False,                    # sort_keys
+            False,                    # skip_keys
+            True                      # allow_nan
+        )
+
+        del make_encoder
+
+
+class RecordWriterV1(RecordWriter):
+
+    def flush(self, finished=None, partial=None):
+
+        RecordWriter.flush(self, finished, partial)  # validates arguments and the state of this instance
+
+        if self.pending_record_count > 0 or (self._chunk_count == 0 and 'messages' in self._inspector):
+
+            messages = self._inspector.get('messages')
+
+            if self._chunk_count == 0:
+
+                # Messages are written to the messages header when we write the first chunk of data
+                # Guarantee: These messages are displayed by splunkweb and the job inspector
+
+                if messages is not None:
+
+                    message_level = RecordWriterV1._message_level.get
+
+                    for level, text in messages:
+                        self.write(message_level(level, level))
+                        self.write('=')
+                        self.write(text)
+                        self.write('\r\n')
+
+                self.write('\r\n')
+
+            elif messages is not None:
+
+                # Messages are written to the messages header when we write subsequent chunks of data
+                # Guarantee: These messages are displayed by splunkweb and the job inspector, if and only if the
+                # command is configured with
+                #
+                #       stderr_dest = message
+                #
+                # stderr_dest is a static configuration setting. This means that it can only be set in commands.conf.
+                # It cannot be set in code.
+
+                stderr = sys.stderr
+
+                for level, text in messages:
+                    print(level, text, file=stderr)
+
+            self.write(self._buffer.getvalue())
+            self._chunk_count += 1
+            self._committed_record_count += self.pending_record_count
+            self._clear()
+
+        self._finished = finished is True
+
+    _message_level = {
+        'DEBUG': 'debug_message',
+        'ERROR': 'error_message',
+        'FATAL': 'error_message',
+        'INFO': 'info_message',
+        'WARN': 'warn_message'
+    }
+
+
+class RecordWriterV2(RecordWriter):
+
+    def flush(self, finished=None, partial=None):
+
+        RecordWriter.flush(self, finished, partial)  # validates arguments and the state of this instance
+
+        if partial or not finished:
+            # Don't flush partial chunks, since the SCP v2 protocol does not
+            # provide a way to send partial chunks yet.
+            return
+
+        if not self.is_flushed:
+            self.write_chunk(finished=True)
+
+    def write_chunk(self, finished=None):
+        inspector = self._inspector
+        self._committed_record_count += self.pending_record_count
+        self._chunk_count += 1
+
+        # TODO: DVPL-6448: splunklib.searchcommands | Add support for partial: true when it is implemented in
+        # ChunkedExternProcessor (See SPL-103525)
+        #
+        # We will need to replace the following block of code with this block:
+        #
+        # metadata = [item for item in (('inspector', inspector), ('finished', finished), ('partial', partial))]
+        #
+        # if partial is True:
+        #     finished = False
+
+        if len(inspector) == 0:
+            inspector = None
+
+        metadata = [item for item in (('inspector', inspector), ('finished', finished))]
+        self._write_chunk(metadata, self._buffer.getvalue())
+        self._clear()
+
+    def write_metadata(self, configuration):
+        self._ensure_validity()
+
+        metadata = chain(six.iteritems(configuration), (('inspector', self._inspector if self._inspector else None),))
+        self._write_chunk(metadata, '')
+        self.write('\n')
+        self._clear()
+
+    def write_metric(self, name, value):
+        self._ensure_validity()
+        self._inspector['metric.' + name] = value
+
+    def _clear(self):
+        super(RecordWriterV2, self)._clear()
+        self._fieldnames = None
+
+    def _write_chunk(self, metadata, body):
+
+        if metadata:
+            metadata = str(''.join(self._iterencode_json(dict([(n, v) for n, v in metadata if v is not None]), 0)))
+            if sys.version_info >= (3, 0):
+                metadata = metadata.encode('utf-8')
+            metadata_length = len(metadata)
+        else:
+            metadata_length = 0
+
+        if sys.version_info >= (3, 0):
+            body = body.encode('utf-8')
+        body_length = len(body)
+
+        if not (metadata_length > 0 or body_length > 0):
+            return
+
+        start_line = 'chunked 1.0,%s,%s\n' % (metadata_length, body_length)
+        self.write(start_line)
+        self.write(metadata)
+        self.write(body)
+        self._ofile.flush()
+        self._flushed = True

dist/devsecops/bin/splunklib/searchcommands/reporting_command.py

@@ -0,0 +1,281 @@
+# coding=utf-8
+#
+# Copyright © 2011-2015 Splunk, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"): you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from itertools import chain
+
+from .internals import ConfigurationSettingsType, json_encode_string
+from .decorators import ConfigurationSetting, Option
+from .streaming_command import StreamingCommand
+from .search_command import SearchCommand
+from .validators import Set
+from splunklib import six
+
+
+class ReportingCommand(SearchCommand):
+    """ Processes search result records and generates a reporting data structure.
+
+    Reporting search commands run as either reduce or map/reduce operations. The reduce part runs on a search head and
+    is responsible for processing a single chunk of search results to produce the command's reporting data structure.
+    The map part is called a streaming preop. It feeds the reduce part with partial results and by default runs on the
+    search head and/or one or more indexers.
+
+    You must implement a :meth:`reduce` method as a generator function that iterates over a set of event records and
+    yields a reporting data structure. You may implement a :meth:`map` method as a generator function that iterates
+    over a set of event records and yields :class:`dict` or :class:`list(dict)` instances.
+
+    ReportingCommand configuration
+    ==============================
+
+    Configure the :meth:`map` operation using a Configuration decorator on your :meth:`map` method. Configure it like
+    you would a :class:`StreamingCommand`. Configure the :meth:`reduce` operation using a Configuration decorator on
+    your :meth:`ReportingCommand` class.
+
+    You can configure your command for operation under Search Command Protocol (SCP) version 1 or 2. SCP 2 requires
+    Splunk 6.3 or later.
+
+    """
+    # region Special methods
+
+    def __init__(self):
+        SearchCommand.__init__(self)
+
+    # endregion
+
+    # region Options
+
+    phase = Option(doc='''
+        **Syntax:** phase=[map|reduce]
+
+        **Description:** Identifies the phase of the current map-reduce operation.
+
+    ''', default='reduce', validate=Set('map', 'reduce'))
+
+    # endregion
+
+    # region Methods
+
+    def map(self, records):
+        """ Override this method to compute partial results.
+
+        :param records:
+        :type records:
+
+        You must override this method, if :code:`requires_preop=True`.
+
+        """
+        return NotImplemented
+
+    def prepare(self):
+
+        phase = self.phase
+
+        if phase == 'map':
+            # noinspection PyUnresolvedReferences
+            self._configuration = self.map.ConfigurationSettings(self)
+            return
+
+        if phase == 'reduce':
+            streaming_preop = chain((self.name, 'phase="map"', str(self._options)), self.fieldnames)
+            self._configuration.streaming_preop = ' '.join(streaming_preop)
+            return
+
+        raise RuntimeError('Unrecognized reporting command phase: {}'.format(json_encode_string(six.text_type(phase))))
+
+    def reduce(self, records):
+        """ Override this method to produce a reporting data structure.
+
+        You must override this method.
+
+        """
+        raise NotImplementedError('reduce(self, records)')
+
+    def _execute(self, ifile, process):
+        SearchCommand._execute(self, ifile, getattr(self, self.phase))
+
+    # endregion
+
+    # region Types
+
+    class ConfigurationSettings(SearchCommand.ConfigurationSettings):
+        """ Represents the configuration settings for a :code:`ReportingCommand`.
+
+        """
+        # region SCP v1/v2 Properties
+
+        required_fields = ConfigurationSetting(doc='''
+            List of required fields for this search which back-propagates to the generating search.
+
+            Setting this value enables selected fields mode under SCP 2. Under SCP 1 you must also specify
+            :code:`clear_required_fields=True` to enable selected fields mode. To explicitly select all fields,
+            specify a value of :const:`['*']`. No error is generated if a specified field is missing.
+
+            Default: :const:`None`, which implicitly selects all fields.
+
+            Supported by: SCP 1, SCP 2
+
+            ''')
+
+        requires_preop = ConfigurationSetting(doc='''
+            Indicates whether :meth:`ReportingCommand.map` is required for proper command execution.
+
+            If :const:`True`, :meth:`ReportingCommand.map` is guaranteed to be called. If :const:`False`, Splunk
+            considers it to be an optimization that may be skipped.
+
+            Default: :const:`False`
+
+            Supported by: SCP 1, SCP 2
+
+            ''')
+
+        streaming_preop = ConfigurationSetting(doc='''
+            Denotes the requested streaming preop search string.
+
+            Computed.
+
+            Supported by: SCP 1, SCP 2
+
+            ''')
+
+        # endregion
+
+        # region SCP v1 Properties
+
+        clear_required_fields = ConfigurationSetting(doc='''
+            :const:`True`, if required_fields represent the *only* fields required.
+
+            If :const:`False`, required_fields are additive to any fields that may be required by subsequent commands.
+            In most cases, :const:`True` is appropriate for reporting commands.
+
+            Default: :const:`True`
+
+            Supported by: SCP 1
+
+            ''')
+
+        retainsevents = ConfigurationSetting(readonly=True, value=False, doc='''
+            Signals that :meth:`ReportingCommand.reduce` transforms _raw events to produce a reporting data structure.
+
+            Fixed: :const:`False`
+
+            Supported by: SCP 1
+
+            ''')
+
+        streaming = ConfigurationSetting(readonly=True, value=False, doc='''
+            Signals that :meth:`ReportingCommand.reduce` runs on the search head.
+
+            Fixed: :const:`False`
+
+            Supported by: SCP 1
+
+            ''')
+
+        # endregion
+
+        # region SCP v2 Properties
+
+        maxinputs = ConfigurationSetting(doc='''
+            Specifies the maximum number of events that can be passed to the command for each invocation.
+
+            This limit cannot exceed the value of `maxresultrows` in limits.conf_. Under SCP 1 you must specify this
+            value in commands.conf_.
+
+            Default: The value of `maxresultrows`.
+
+            Supported by: SCP 2
+
+            .. _limits.conf: http://docs.splunk.com/Documentation/Splunk/latest/admin/Limitsconf
+
+            ''')
+
+        run_in_preview = ConfigurationSetting(doc='''
+            :const:`True`, if this command should be run to generate results for preview; not wait for final output.
+
+            This may be important for commands that have side effects (e.g., outputlookup).
+
+            Default: :const:`True`
+
+            Supported by: SCP 2
+
+            ''')
+
+        type = ConfigurationSetting(readonly=True, value='reporting', doc='''
+            Command type name.
+
+            Fixed: :const:`'reporting'`.
+
+            Supported by: SCP 2
+
+            ''')
+
+        # endregion
+
+        # region Methods
+
+        @classmethod
+        def fix_up(cls, command):
+            """ Verifies :code:`command` class structure and configures the :code:`command.map` method.
+
+            Verifies that :code:`command` derives from :class:`ReportingCommand` and overrides
+            :code:`ReportingCommand.reduce`. It then configures :code:`command.reduce`, if an overriding implementation
+            of :code:`ReportingCommand.reduce` has been provided.
+
+            :param command: :code:`ReportingCommand` class
+
+            Exceptions:
+
+            :code:`TypeError` :code:`command` class is not derived from :code:`ReportingCommand`
+            :code:`AttributeError` No :code:`ReportingCommand.reduce` override
+
+            """
+            if not issubclass(command, ReportingCommand):
+                raise TypeError('{} is not a ReportingCommand'.format( command))
+
+            if command.reduce == ReportingCommand.reduce:
+                raise AttributeError('No ReportingCommand.reduce override')
+
+            if command.map == ReportingCommand.map:
+                cls._requires_preop = False
+                return
+
+            f = vars(command)['map']   # Function backing the map method
+
+            # EXPLANATION OF PREVIOUS STATEMENT: There is no way to add custom attributes to methods. See [Why does
+            # setattr fail on a method](http://stackoverflow.com/questions/7891277/why-does-setattr-fail-on-a-bound-method) for a discussion of this issue.
+
+            try:
+                settings = f._settings
+            except AttributeError:
+                f.ConfigurationSettings = StreamingCommand.ConfigurationSettings
+                return
+
+            # Create new StreamingCommand.ConfigurationSettings class
+
+            module = command.__module__ + '.' + command.__name__ + '.map'
+            name = b'ConfigurationSettings'
+            bases = (StreamingCommand.ConfigurationSettings,)
+
+            f.ConfigurationSettings = ConfigurationSettingsType(module, name, bases)
+            ConfigurationSetting.fix_up(f.ConfigurationSettings, settings)
+            del f._settings
+
+        pass
+        # endregion
+
+    pass
+    # endregion

dist/devsecops/bin/splunklib/searchcommands/streaming_command.py

@@ -0,0 +1,195 @@
+# coding=utf-8
+#
+# Copyright 2011-2015 Splunk, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"): you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from splunklib import six
+from splunklib.six.moves import map as imap, filter as ifilter
+
+from .decorators import ConfigurationSetting
+from .search_command import SearchCommand
+
+
+class StreamingCommand(SearchCommand):
+    """ Applies a transformation to search results as they travel through the streams pipeline.
+
+    Streaming commands typically filter, augment, or update, search result records. Splunk will send them in batches of
+    up to 50,000 records. Hence, a search command must be prepared to be invoked many times during the course of
+    pipeline processing. Each invocation should produce a set of results independently usable by downstream processors.
+
+    By default Splunk may choose to run a streaming command locally on a search head and/or remotely on one or more
+    indexers concurrently. The size and frequency of the search result batches sent to the command will vary based
+    on scheduling considerations.
+
+    StreamingCommand configuration
+    ==============================
+
+    You can configure your command for operation under Search Command Protocol (SCP) version 1 or 2. SCP 2 requires
+    Splunk 6.3 or later.
+
+    """
+    # region Methods
+
+    def stream(self, records):
+        """ Generator function that processes and yields event records to the Splunk stream pipeline.
+
+        You must override this method.
+
+        """
+        raise NotImplementedError('StreamingCommand.stream(self, records)')
+
+    def _execute(self, ifile, process):
+        SearchCommand._execute(self, ifile, self.stream)
+
+    # endregion
+
+    class ConfigurationSettings(SearchCommand.ConfigurationSettings):
+        """ Represents the configuration settings that apply to a :class:`StreamingCommand`.
+
+        """
+        # region SCP v1/v2 properties
+
+        required_fields = ConfigurationSetting(doc='''
+            List of required fields for this search which back-propagates to the generating search.
+
+            Setting this value enables selected fields mode under SCP 2. Under SCP 1 you must also specify
+            :code:`clear_required_fields=True` to enable selected fields mode. To explicitly select all fields,
+            specify a value of :const:`['*']`. No error is generated if a specified field is missing.
+
+            Default: :const:`None`, which implicitly selects all fields.
+
+            Supported by: SCP 1, SCP 2
+
+            ''')
+
+        # endregion
+
+        # region SCP v1 properties
+
+        clear_required_fields = ConfigurationSetting(doc='''
+            :const:`True`, if required_fields represent the *only* fields required.
+
+            If :const:`False`, required_fields are additive to any fields that may be required by subsequent commands.
+            In most cases, :const:`False` is appropriate for streaming commands.
+
+            Default: :const:`False`
+
+            Supported by: SCP 1
+
+            ''')
+
+        local = ConfigurationSetting(doc='''
+            :const:`True`, if the command should run locally on the search head.
+
+            Default: :const:`False`
+
+            Supported by: SCP 1
+
+            ''')
+
+        overrides_timeorder = ConfigurationSetting(doc='''
+            :const:`True`, if the command changes the order of events with respect to time.
+
+            Default: :const:`False`
+
+            Supported by: SCP 1
+
+            ''')
+
+        streaming = ConfigurationSetting(readonly=True, value=True, doc='''
+            Specifies that the command is streamable.
+
+            Fixed: :const:`True`
+
+            Supported by: SCP 1
+
+            ''')
+
+        # endregion
+
+        # region SCP v2 Properties
+
+        distributed = ConfigurationSetting(value=True, doc='''
+            :const:`True`, if this command should be distributed to indexers.
+
+            Under SCP 1 you must either specify `local = False` or include this line in commands.conf_, if this command
+            should be distributed to indexers.
+
+            ..code:
+                local = true
+
+            Default: :const:`True`
+
+            Supported by: SCP 2
+
+            .. commands.conf_: http://docs.splunk.com/Documentation/Splunk/latest/Admin/Commandsconf
+
+            ''')
+
+        maxinputs = ConfigurationSetting(doc='''
+            Specifies the maximum number of events that can be passed to the command for each invocation.
+
+            This limit cannot exceed the value of `maxresultrows` in limits.conf. Under SCP 1 you must specify this
+            value in commands.conf_.
+
+            Default: The value of `maxresultrows`.
+
+            Supported by: SCP 2
+
+            ''')
+
+        type = ConfigurationSetting(readonly=True, value='streaming', doc='''
+            Command type name.
+
+            Fixed: :const:`'streaming'`
+
+            Supported by: SCP 2
+
+            ''')
+
+        # endregion
+
+        # region Methods
+
+        @classmethod
+        def fix_up(cls, command):
+            """ Verifies :code:`command` class structure.
+
+            """
+            if command.stream == StreamingCommand.stream:
+                raise AttributeError('No StreamingCommand.stream override')
+            return
+
+        # TODO: Stop looking like a dictionary because we don't obey the semantics
+        # N.B.: Does not use Python 2 dict copy semantics
+        def iteritems(self):
+            iteritems = SearchCommand.ConfigurationSettings.iteritems(self)
+            version = self.command.protocol_version
+            if version == 1:
+                if self.required_fields is None:
+                    iteritems = ifilter(lambda name_value: name_value[0] != 'clear_required_fields', iteritems)
+            else:
+                iteritems = ifilter(lambda name_value2: name_value2[0] != 'distributed', iteritems)
+                if not self.distributed:
+                    iteritems = imap(
+                        lambda name_value1: (name_value1[0], 'stateful') if name_value1[0] == 'type' else (name_value1[0], name_value1[1]), iteritems)
+            return iteritems
+
+        # N.B.: Does not use Python 3 dict view semantics
+        if not six.PY2:
+            items = iteritems
+
+        # endregion

dist/devsecops/bin/splunklib/searchcommands/validators.py

@@ -0,0 +1,394 @@
+# coding=utf-8
+#
+# Copyright 2011-2015 Splunk, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"): you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from json.encoder import encode_basestring_ascii as json_encode_string
+from collections import namedtuple
+from splunklib.six.moves import StringIO
+from io import open
+import csv
+import os
+import re
+from splunklib import six
+from splunklib.six.moves import getcwd
+
+
+class Validator(object):
+    """ Base class for validators that check and format search command options.
+
+    You must inherit from this class and override :code:`Validator.__call__` and
+    :code:`Validator.format`. :code:`Validator.__call__` should convert the
+    value it receives as argument and then return it or raise a
+    :code:`ValueError`, if the value will not convert.
+
+    :code:`Validator.format` should return a human readable version of the value
+    it receives as argument the same way :code:`str` does.
+
+    """
+    def __call__(self, value):
+        raise NotImplementedError()
+
+    def format(self, value):
+        raise NotImplementedError()
+
+
+class Boolean(Validator):
+    """ Validates Boolean option values.
+
+    """
+    truth_values = {
+        '1': True, '0': False,
+        't': True, 'f': False,
+        'true': True, 'false': False,
+        'y': True, 'n': False,
+        'yes': True, 'no': False
+    }
+
+    def __call__(self, value):
+        if not (value is None or isinstance(value, bool)):
+            value = six.text_type(value).lower()
+            if value not in Boolean.truth_values:
+                raise ValueError('Unrecognized truth value: {0}'.format(value))
+            value = Boolean.truth_values[value]
+        return value
+
+    def format(self, value):
+        return None if value is None else 't' if value else 'f'
+
+
+class Code(Validator):
+    """ Validates code option values.
+
+    This validator compiles an option value into a Python code object that can be executed by :func:`exec` or evaluated
+    by :func:`eval`. The value returned is a :func:`namedtuple` with two members: object, the result of compilation, and
+    source, the original option value.
+
+    """
+    def __init__(self, mode='eval'):
+        """
+        :param mode: Specifies what kind of code must be compiled; it can be :const:`'exec'`, if source consists of a
+            sequence of statements, :const:`'eval'`, if it consists of a single expression, or :const:`'single'` if it
+            consists of a single interactive statement. In the latter case, expression statements that evaluate to
+            something other than :const:`None` will be printed.
+        :type mode: unicode or bytes
+
+        """
+        self._mode = mode
+
+    def __call__(self, value):
+        if value is None:
+            return None
+        try:
+            return Code.object(compile(value, 'string', self._mode), six.text_type(value))
+        except (SyntaxError, TypeError) as error:
+            if six.PY2:
+                message = error.message
+            else:
+                message = str(error)
+
+            six.raise_from(ValueError(message), error)
+
+    def format(self, value):
+        return None if value is None else value.source
+
+    object = namedtuple('Code', ('object', 'source'))
+
+
+class Fieldname(Validator):
+    """ Validates field name option values.
+
+    """
+    pattern = re.compile(r'''[_.a-zA-Z-][_.a-zA-Z0-9-]*$''')
+
+    def __call__(self, value):
+        if value is not None:
+            value = six.text_type(value)
+            if Fieldname.pattern.match(value) is None:
+                raise ValueError('Illegal characters in fieldname: {}'.format(value))
+        return value
+
+    def format(self, value):
+        return value
+
+
+class File(Validator):
+    """ Validates file option values.
+
+    """
+    def __init__(self, mode='rt', buffering=None, directory=None):
+        self.mode = mode
+        self.buffering = buffering
+        self.directory = File._var_run_splunk if directory is None else directory
+
+    def __call__(self, value):
+
+        if value is None:
+            return value
+
+        path = six.text_type(value)
+
+        if not os.path.isabs(path):
+            path = os.path.join(self.directory, path)
+
+        try:
+            value = open(path, self.mode) if self.buffering is None else open(path, self.mode, self.buffering)
+        except IOError as error:
+            raise ValueError('Cannot open {0} with mode={1} and buffering={2}: {3}'.format(
+                value, self.mode, self.buffering, error))
+
+        return value
+
+    def format(self, value):
+        return None if value is None else value.name
+
+    _var_run_splunk = os.path.join(
+        os.environ['SPLUNK_HOME'] if 'SPLUNK_HOME' in os.environ else getcwd(), 'var', 'run', 'splunk')
+
+
+class Integer(Validator):
+    """ Validates integer option values.
+
+    """
+    def __init__(self, minimum=None, maximum=None):
+        if minimum is not None and maximum is not None:
+            def check_range(value):
+                if not (minimum <= value <= maximum):
+                    raise ValueError('Expected integer in the range [{0},{1}], not {2}'.format(minimum, maximum, value))
+                return
+        elif minimum is not None:
+            def check_range(value):
+                if value < minimum:
+                    raise ValueError('Expected integer in the range [{0},+∞], not {1}'.format(minimum, value))
+                return
+        elif maximum is not None:
+            def check_range(value):
+                if value > maximum:
+                    raise ValueError('Expected integer in the range [-∞,{0}], not {1}'.format(maximum, value))
+                return
+        else:
+            def check_range(value):
+                return
+
+        self.check_range = check_range
+        return
+
+    def __call__(self, value):
+        if value is None:
+            return None
+        try:
+            if six.PY2:
+                value = long(value)
+            else:
+                value = int(value)
+        except ValueError:
+            raise ValueError('Expected integer value, not {}'.format(json_encode_string(value)))
+
+        self.check_range(value)
+        return value
+
+    def format(self, value):
+        return None if value is None else six.text_type(int(value))
+
+
+class Duration(Validator):
+    """ Validates duration option values.
+
+    """
+    def __call__(self, value):
+
+        if value is None:
+            return None
+
+        p = value.split(':', 2)
+        result = None
+        _60 = Duration._60
+        _unsigned = Duration._unsigned
+
+        try:
+            if len(p) == 1:
+                result = _unsigned(p[0])
+            if len(p) == 2:
+                result = 60 * _unsigned(p[0]) + _60(p[1])
+            if len(p) == 3:
+                result = 3600 * _unsigned(p[0]) + 60 * _60(p[1]) + _60(p[2])
+        except ValueError:
+            raise ValueError('Invalid duration value: {0}'.format(value))
+
+        return result
+
+    def format(self, value):
+
+        if value is None:
+            return None
+
+        value = int(value)
+
+        s = value % 60
+        m = value // 60 % 60
+        h = value // (60 * 60)
+
+        return '{0:02d}:{1:02d}:{2:02d}'.format(h, m, s)
+
+    _60 = Integer(0, 59)
+    _unsigned = Integer(0)
+
+
+class List(Validator):
+    """ Validates a list of strings
+
+    """
+    class Dialect(csv.Dialect):
+        """ Describes the properties of list option values. """
+        strict = True
+        delimiter = str(',')
+        quotechar = str('"')
+        doublequote = True
+        lineterminator = str('\n')
+        skipinitialspace = True
+        quoting = csv.QUOTE_MINIMAL
+
+    def __init__(self, validator=None):
+        if not (validator is None or isinstance(validator, Validator)):
+            raise ValueError('Expected a Validator instance or None for validator, not {}', repr(validator))
+        self._validator = validator
+
+    def __call__(self, value):
+
+        if value is None or isinstance(value, list):
+            return value
+
+        try:
+            value = next(csv.reader([value], self.Dialect))
+        except csv.Error as error:
+            raise ValueError(error)
+
+        if self._validator is None:
+            return value
+
+        try:
+            for index, item in enumerate(value):
+                value[index] = self._validator(item)
+        except ValueError as error:
+            raise ValueError('Could not convert item {}: {}'.format(index, error))
+
+        return value
+
+    def format(self, value):
+        output = StringIO()
+        writer = csv.writer(output, List.Dialect)
+        writer.writerow(value)
+        value = output.getvalue()
+        return value[:-1]
+
+
+class Map(Validator):
+    """ Validates map option values.
+
+    """
+    def __init__(self, **kwargs):
+        self.membership = kwargs
+
+    def __call__(self, value):
+
+        if value is None:
+            return None
+
+        value = six.text_type(value)
+
+        if value not in self.membership:
+            raise ValueError('Unrecognized value: {0}'.format(value))
+
+        return self.membership[value]
+
+    def format(self, value):
+        return None if value is None else list(self.membership.keys())[list(self.membership.values()).index(value)]
+
+
+class Match(Validator):
+    """ Validates that a value matches a regular expression pattern.
+
+    """
+    def __init__(self, name, pattern, flags=0):
+        self.name = six.text_type(name)
+        self.pattern = re.compile(pattern, flags)
+
+    def __call__(self, value):
+        if value is None:
+            return None
+        value = six.text_type(value)
+        if self.pattern.match(value) is None:
+            raise ValueError('Expected {}, not {}'.format(self.name, json_encode_string(value)))
+        return value
+
+    def format(self, value):
+        return None if value is None else six.text_type(value)
+
+
+class OptionName(Validator):
+    """ Validates option names.
+
+    """
+    pattern = re.compile(r'''(?=\w)[^\d]\w*$''', re.UNICODE)
+
+    def __call__(self, value):
+        if value is not None:
+            value = six.text_type(value)
+            if OptionName.pattern.match(value) is None:
+                raise ValueError('Illegal characters in option name: {}'.format(value))
+        return value
+
+    def format(self, value):
+        return None if value is None else six.text_type(value)
+
+
+class RegularExpression(Validator):
+    """ Validates regular expression option values.
+
+    """
+    def __call__(self, value):
+        if value is None:
+            return None
+        try:
+            value = re.compile(six.text_type(value))
+        except re.error as error:
+            raise ValueError('{}: {}'.format(six.text_type(error).capitalize(), value))
+        return value
+
+    def format(self, value):
+        return None if value is None else value.pattern
+
+
+class Set(Validator):
+    """ Validates set option values.
+
+    """
+    def __init__(self, *args):
+        self.membership = set(args)
+
+    def __call__(self, value):
+        if value is None:
+            return None
+        value = six.text_type(value)
+        if value not in self.membership:
+            raise ValueError('Unrecognized value: {}'.format(value))
+        return value
+
+    def format(self, value):
+        return self.__call__(value)
+
+
+__all__ = ['Boolean', 'Code', 'Duration', 'File', 'Integer', 'List', 'Map', 'RegularExpression', 'Set']

dist/devsecops/bin/splunklib/six.py

@@ -0,0 +1,980 @@
+# Copyright (c) 2010-2020 Benjamin Peterson
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+"""Utilities for writing code that runs on Python 2 and 3"""
+
+from __future__ import absolute_import
+
+import functools
+import itertools
+import operator
+import sys
+import types
+
+__author__ = "Benjamin Peterson <benjamin@python.org>"
+__version__ = "1.14.0"
+
+
+# Useful for very coarse version differentiation.
+PY2 = sys.version_info[0] == 2
+PY3 = sys.version_info[0] == 3
+PY34 = sys.version_info[0:2] >= (3, 4)
+
+if PY3:
+    string_types = str,
+    integer_types = int,
+    class_types = type,
+    text_type = str
+    binary_type = bytes
+
+    MAXSIZE = sys.maxsize
+else:
+    string_types = basestring,
+    integer_types = (int, long)
+    class_types = (type, types.ClassType)
+    text_type = unicode
+    binary_type = str
+
+    if sys.platform.startswith("java"):
+        # Jython always uses 32 bits.
+        MAXSIZE = int((1 << 31) - 1)
+    else:
+        # It's possible to have sizeof(long) != sizeof(Py_ssize_t).
+        class X(object):
+
+            def __len__(self):
+                return 1 << 31
+        try:
+            len(X())
+        except OverflowError:
+            # 32-bit
+            MAXSIZE = int((1 << 31) - 1)
+        else:
+            # 64-bit
+            MAXSIZE = int((1 << 63) - 1)
+        del X
+
+
+def _add_doc(func, doc):
+    """Add documentation to a function."""
+    func.__doc__ = doc
+
+
+def _import_module(name):
+    """Import module, returning the module after the last dot."""
+    __import__(name)
+    return sys.modules[name]
+
+
+class _LazyDescr(object):
+
+    def __init__(self, name):
+        self.name = name
+
+    def __get__(self, obj, tp):
+        result = self._resolve()
+        setattr(obj, self.name, result)  # Invokes __set__.
+        try:
+            # This is a bit ugly, but it avoids running this again by
+            # removing this descriptor.
+            delattr(obj.__class__, self.name)
+        except AttributeError:
+            pass
+        return result
+
+
+class MovedModule(_LazyDescr):
+
+    def __init__(self, name, old, new=None):
+        super(MovedModule, self).__init__(name)
+        if PY3:
+            if new is None:
+                new = name
+            self.mod = new
+        else:
+            self.mod = old
+
+    def _resolve(self):
+        return _import_module(self.mod)
+
+    def __getattr__(self, attr):
+        _module = self._resolve()
+        value = getattr(_module, attr)
+        setattr(self, attr, value)
+        return value
+
+
+class _LazyModule(types.ModuleType):
+
+    def __init__(self, name):
+        super(_LazyModule, self).__init__(name)
+        self.__doc__ = self.__class__.__doc__
+
+    def __dir__(self):
+        attrs = ["__doc__", "__name__"]
+        attrs += [attr.name for attr in self._moved_attributes]
+        return attrs
+
+    # Subclasses should override this
+    _moved_attributes = []
+
+
+class MovedAttribute(_LazyDescr):
+
+    def __init__(self, name, old_mod, new_mod, old_attr=None, new_attr=None):
+        super(MovedAttribute, self).__init__(name)
+        if PY3:
+            if new_mod is None:
+                new_mod = name
+            self.mod = new_mod
+            if new_attr is None:
+                if old_attr is None:
+                    new_attr = name
+                else:
+                    new_attr = old_attr
+            self.attr = new_attr
+        else:
+            self.mod = old_mod
+            if old_attr is None:
+                old_attr = name
+            self.attr = old_attr
+
+    def _resolve(self):
+        module = _import_module(self.mod)
+        return getattr(module, self.attr)
+
+
+class _SixMetaPathImporter(object):
+
+    """
+    A meta path importer to import six.moves and its submodules.
+
+    This class implements a PEP302 finder and loader. It should be compatible
+    with Python 2.5 and all existing versions of Python3
+    """
+
+    def __init__(self, six_module_name):
+        self.name = six_module_name
+        self.known_modules = {}
+
+    def _add_module(self, mod, *fullnames):
+        for fullname in fullnames:
+            self.known_modules[self.name + "." + fullname] = mod
+
+    def _get_module(self, fullname):
+        return self.known_modules[self.name + "." + fullname]
+
+    def find_module(self, fullname, path=None):
+        if fullname in self.known_modules:
+            return self
+        return None
+
+    def __get_module(self, fullname):
+        try:
+            return self.known_modules[fullname]
+        except KeyError:
+            raise ImportError("This loader does not know module " + fullname)
+
+    def load_module(self, fullname):
+        try:
+            # in case of a reload
+            return sys.modules[fullname]
+        except KeyError:
+            pass
+        mod = self.__get_module(fullname)
+        if isinstance(mod, MovedModule):
+            mod = mod._resolve()
+        else:
+            mod.__loader__ = self
+        sys.modules[fullname] = mod
+        return mod
+
+    def is_package(self, fullname):
+        """
+        Return true, if the named module is a package.
+
+        We need this method to get correct spec objects with
+        Python 3.4 (see PEP451)
+        """
+        return hasattr(self.__get_module(fullname), "__path__")
+
+    def get_code(self, fullname):
+        """Return None
+
+        Required, if is_package is implemented"""
+        self.__get_module(fullname)  # eventually raises ImportError
+        return None
+    get_source = get_code  # same as get_code
+
+_importer = _SixMetaPathImporter(__name__)
+
+
+class _MovedItems(_LazyModule):
+
+    """Lazy loading of moved objects"""
+    __path__ = []  # mark as package
+
+
+_moved_attributes = [
+    MovedAttribute("cStringIO", "cStringIO", "io", "StringIO"),
+    MovedAttribute("filter", "itertools", "builtins", "ifilter", "filter"),
+    MovedAttribute("filterfalse", "itertools", "itertools", "ifilterfalse", "filterfalse"),
+    MovedAttribute("input", "__builtin__", "builtins", "raw_input", "input"),
+    MovedAttribute("intern", "__builtin__", "sys"),
+    MovedAttribute("map", "itertools", "builtins", "imap", "map"),
+    MovedAttribute("getcwd", "os", "os", "getcwdu", "getcwd"),
+    MovedAttribute("getcwdb", "os", "os", "getcwd", "getcwdb"),
+    MovedAttribute("getoutput", "commands", "subprocess"),
+    MovedAttribute("range", "__builtin__", "builtins", "xrange", "range"),
+    MovedAttribute("reload_module", "__builtin__", "importlib" if PY34 else "imp", "reload"),
+    MovedAttribute("reduce", "__builtin__", "functools"),
+    MovedAttribute("shlex_quote", "pipes", "shlex", "quote"),
+    MovedAttribute("StringIO", "StringIO", "io"),
+    MovedAttribute("UserDict", "UserDict", "collections"),
+    MovedAttribute("UserList", "UserList", "collections"),
+    MovedAttribute("UserString", "UserString", "collections"),
+    MovedAttribute("xrange", "__builtin__", "builtins", "xrange", "range"),
+    MovedAttribute("zip", "itertools", "builtins", "izip", "zip"),
+    MovedAttribute("zip_longest", "itertools", "itertools", "izip_longest", "zip_longest"),
+    MovedModule("builtins", "__builtin__"),
+    MovedModule("configparser", "ConfigParser"),
+    MovedModule("collections_abc", "collections", "collections.abc" if sys.version_info >= (3, 3) else "collections"),
+    MovedModule("copyreg", "copy_reg"),
+    MovedModule("dbm_gnu", "gdbm", "dbm.gnu"),
+    MovedModule("dbm_ndbm", "dbm", "dbm.ndbm"),
+    MovedModule("_dummy_thread", "dummy_thread", "_dummy_thread" if sys.version_info < (3, 9) else "_thread"),
+    MovedModule("http_cookiejar", "cookielib", "http.cookiejar"),
+    MovedModule("http_cookies", "Cookie", "http.cookies"),
+    MovedModule("html_entities", "htmlentitydefs", "html.entities"),
+    MovedModule("html_parser", "HTMLParser", "html.parser"),
+    MovedModule("http_client", "httplib", "http.client"),
+    MovedModule("email_mime_base", "email.MIMEBase", "email.mime.base"),
+    MovedModule("email_mime_image", "email.MIMEImage", "email.mime.image"),
+    MovedModule("email_mime_multipart", "email.MIMEMultipart", "email.mime.multipart"),
+    MovedModule("email_mime_nonmultipart", "email.MIMENonMultipart", "email.mime.nonmultipart"),
+    MovedModule("email_mime_text", "email.MIMEText", "email.mime.text"),
+    MovedModule("BaseHTTPServer", "BaseHTTPServer", "http.server"),
+    MovedModule("CGIHTTPServer", "CGIHTTPServer", "http.server"),
+    MovedModule("SimpleHTTPServer", "SimpleHTTPServer", "http.server"),
+    MovedModule("cPickle", "cPickle", "pickle"),
+    MovedModule("queue", "Queue"),
+    MovedModule("reprlib", "repr"),
+    MovedModule("socketserver", "SocketServer"),
+    MovedModule("_thread", "thread", "_thread"),
+    MovedModule("tkinter", "Tkinter"),
+    MovedModule("tkinter_dialog", "Dialog", "tkinter.dialog"),
+    MovedModule("tkinter_filedialog", "FileDialog", "tkinter.filedialog"),
+    MovedModule("tkinter_scrolledtext", "ScrolledText", "tkinter.scrolledtext"),
+    MovedModule("tkinter_simpledialog", "SimpleDialog", "tkinter.simpledialog"),
+    MovedModule("tkinter_tix", "Tix", "tkinter.tix"),
+    MovedModule("tkinter_ttk", "ttk", "tkinter.ttk"),
+    MovedModule("tkinter_constants", "Tkconstants", "tkinter.constants"),
+    MovedModule("tkinter_dnd", "Tkdnd", "tkinter.dnd"),
+    MovedModule("tkinter_colorchooser", "tkColorChooser",
+                "tkinter.colorchooser"),
+    MovedModule("tkinter_commondialog", "tkCommonDialog",
+                "tkinter.commondialog"),
+    MovedModule("tkinter_tkfiledialog", "tkFileDialog", "tkinter.filedialog"),
+    MovedModule("tkinter_font", "tkFont", "tkinter.font"),
+    MovedModule("tkinter_messagebox", "tkMessageBox", "tkinter.messagebox"),
+    MovedModule("tkinter_tksimpledialog", "tkSimpleDialog",
+                "tkinter.simpledialog"),
+    MovedModule("urllib_parse", __name__ + ".moves.urllib_parse", "urllib.parse"),
+    MovedModule("urllib_error", __name__ + ".moves.urllib_error", "urllib.error"),
+    MovedModule("urllib", __name__ + ".moves.urllib", __name__ + ".moves.urllib"),
+    MovedModule("urllib_robotparser", "robotparser", "urllib.robotparser"),
+    MovedModule("xmlrpc_client", "xmlrpclib", "xmlrpc.client"),
+    MovedModule("xmlrpc_server", "SimpleXMLRPCServer", "xmlrpc.server"),
+]
+# Add windows specific modules.
+if sys.platform == "win32":
+    _moved_attributes += [
+        MovedModule("winreg", "_winreg"),
+    ]
+
+for attr in _moved_attributes:
+    setattr(_MovedItems, attr.name, attr)
+    if isinstance(attr, MovedModule):
+        _importer._add_module(attr, "moves." + attr.name)
+del attr
+
+_MovedItems._moved_attributes = _moved_attributes
+
+moves = _MovedItems(__name__ + ".moves")
+_importer._add_module(moves, "moves")
+
+
+class Module_six_moves_urllib_parse(_LazyModule):
+
+    """Lazy loading of moved objects in six.moves.urllib_parse"""
+
+
+_urllib_parse_moved_attributes = [
+    MovedAttribute("ParseResult", "urlparse", "urllib.parse"),
+    MovedAttribute("SplitResult", "urlparse", "urllib.parse"),
+    MovedAttribute("parse_qs", "urlparse", "urllib.parse"),
+    MovedAttribute("parse_qsl", "urlparse", "urllib.parse"),
+    MovedAttribute("urldefrag", "urlparse", "urllib.parse"),
+    MovedAttribute("urljoin", "urlparse", "urllib.parse"),
+    MovedAttribute("urlparse", "urlparse", "urllib.parse"),
+    MovedAttribute("urlsplit", "urlparse", "urllib.parse"),
+    MovedAttribute("urlunparse", "urlparse", "urllib.parse"),
+    MovedAttribute("urlunsplit", "urlparse", "urllib.parse"),
+    MovedAttribute("quote", "urllib", "urllib.parse"),
+    MovedAttribute("quote_plus", "urllib", "urllib.parse"),
+    MovedAttribute("unquote", "urllib", "urllib.parse"),
+    MovedAttribute("unquote_plus", "urllib", "urllib.parse"),
+    MovedAttribute("unquote_to_bytes", "urllib", "urllib.parse", "unquote", "unquote_to_bytes"),
+    MovedAttribute("urlencode", "urllib", "urllib.parse"),
+    MovedAttribute("splitquery", "urllib", "urllib.parse"),
+    MovedAttribute("splittag", "urllib", "urllib.parse"),
+    MovedAttribute("splituser", "urllib", "urllib.parse"),
+    MovedAttribute("splitvalue", "urllib", "urllib.parse"),
+    MovedAttribute("uses_fragment", "urlparse", "urllib.parse"),
+    MovedAttribute("uses_netloc", "urlparse", "urllib.parse"),
+    MovedAttribute("uses_params", "urlparse", "urllib.parse"),
+    MovedAttribute("uses_query", "urlparse", "urllib.parse"),
+    MovedAttribute("uses_relative", "urlparse", "urllib.parse"),
+]
+for attr in _urllib_parse_moved_attributes:
+    setattr(Module_six_moves_urllib_parse, attr.name, attr)
+del attr
+
+Module_six_moves_urllib_parse._moved_attributes = _urllib_parse_moved_attributes
+
+_importer._add_module(Module_six_moves_urllib_parse(__name__ + ".moves.urllib_parse"),
+                      "moves.urllib_parse", "moves.urllib.parse")
+
+
+class Module_six_moves_urllib_error(_LazyModule):
+
+    """Lazy loading of moved objects in six.moves.urllib_error"""
+
+
+_urllib_error_moved_attributes = [
+    MovedAttribute("URLError", "urllib2", "urllib.error"),
+    MovedAttribute("HTTPError", "urllib2", "urllib.error"),
+    MovedAttribute("ContentTooShortError", "urllib", "urllib.error"),
+]
+for attr in _urllib_error_moved_attributes:
+    setattr(Module_six_moves_urllib_error, attr.name, attr)
+del attr
+
+Module_six_moves_urllib_error._moved_attributes = _urllib_error_moved_attributes
+
+_importer._add_module(Module_six_moves_urllib_error(__name__ + ".moves.urllib.error"),
+                      "moves.urllib_error", "moves.urllib.error")
+
+
+class Module_six_moves_urllib_request(_LazyModule):
+
+    """Lazy loading of moved objects in six.moves.urllib_request"""
+
+
+_urllib_request_moved_attributes = [
+    MovedAttribute("urlopen", "urllib2", "urllib.request"),
+    MovedAttribute("install_opener", "urllib2", "urllib.request"),
+    MovedAttribute("build_opener", "urllib2", "urllib.request"),
+    MovedAttribute("pathname2url", "urllib", "urllib.request"),
+    MovedAttribute("url2pathname", "urllib", "urllib.request"),
+    MovedAttribute("getproxies", "urllib", "urllib.request"),
+    MovedAttribute("Request", "urllib2", "urllib.request"),
+    MovedAttribute("OpenerDirector", "urllib2", "urllib.request"),
+    MovedAttribute("HTTPDefaultErrorHandler", "urllib2", "urllib.request"),
+    MovedAttribute("HTTPRedirectHandler", "urllib2", "urllib.request"),
+    MovedAttribute("HTTPCookieProcessor", "urllib2", "urllib.request"),
+    MovedAttribute("ProxyHandler", "urllib2", "urllib.request"),
+    MovedAttribute("BaseHandler", "urllib2", "urllib.request"),
+    MovedAttribute("HTTPPasswordMgr", "urllib2", "urllib.request"),
+    MovedAttribute("HTTPPasswordMgrWithDefaultRealm", "urllib2", "urllib.request"),
+    MovedAttribute("AbstractBasicAuthHandler", "urllib2", "urllib.request"),
+    MovedAttribute("HTTPBasicAuthHandler", "urllib2", "urllib.request"),
+    MovedAttribute("ProxyBasicAuthHandler", "urllib2", "urllib.request"),
+    MovedAttribute("AbstractDigestAuthHandler", "urllib2", "urllib.request"),
+    MovedAttribute("HTTPDigestAuthHandler", "urllib2", "urllib.request"),
+    MovedAttribute("ProxyDigestAuthHandler", "urllib2", "urllib.request"),
+    MovedAttribute("HTTPHandler", "urllib2", "urllib.request"),
+    MovedAttribute("HTTPSHandler", "urllib2", "urllib.request"),
+    MovedAttribute("FileHandler", "urllib2", "urllib.request"),
+    MovedAttribute("FTPHandler", "urllib2", "urllib.request"),
+    MovedAttribute("CacheFTPHandler", "urllib2", "urllib.request"),
+    MovedAttribute("UnknownHandler", "urllib2", "urllib.request"),
+    MovedAttribute("HTTPErrorProcessor", "urllib2", "urllib.request"),
+    MovedAttribute("urlretrieve", "urllib", "urllib.request"),
+    MovedAttribute("urlcleanup", "urllib", "urllib.request"),
+    MovedAttribute("URLopener", "urllib", "urllib.request"),
+    MovedAttribute("FancyURLopener", "urllib", "urllib.request"),
+    MovedAttribute("proxy_bypass", "urllib", "urllib.request"),
+    MovedAttribute("parse_http_list", "urllib2", "urllib.request"),
+    MovedAttribute("parse_keqv_list", "urllib2", "urllib.request"),
+]
+for attr in _urllib_request_moved_attributes:
+    setattr(Module_six_moves_urllib_request, attr.name, attr)
+del attr
+
+Module_six_moves_urllib_request._moved_attributes = _urllib_request_moved_attributes
+
+_importer._add_module(Module_six_moves_urllib_request(__name__ + ".moves.urllib.request"),
+                      "moves.urllib_request", "moves.urllib.request")
+
+
+class Module_six_moves_urllib_response(_LazyModule):
+
+    """Lazy loading of moved objects in six.moves.urllib_response"""
+
+
+_urllib_response_moved_attributes = [
+    MovedAttribute("addbase", "urllib", "urllib.response"),
+    MovedAttribute("addclosehook", "urllib", "urllib.response"),
+    MovedAttribute("addinfo", "urllib", "urllib.response"),
+    MovedAttribute("addinfourl", "urllib", "urllib.response"),
+]
+for attr in _urllib_response_moved_attributes:
+    setattr(Module_six_moves_urllib_response, attr.name, attr)
+del attr
+
+Module_six_moves_urllib_response._moved_attributes = _urllib_response_moved_attributes
+
+_importer._add_module(Module_six_moves_urllib_response(__name__ + ".moves.urllib.response"),
+                      "moves.urllib_response", "moves.urllib.response")
+
+
+class Module_six_moves_urllib_robotparser(_LazyModule):
+
+    """Lazy loading of moved objects in six.moves.urllib_robotparser"""
+
+
+_urllib_robotparser_moved_attributes = [
+    MovedAttribute("RobotFileParser", "robotparser", "urllib.robotparser"),
+]
+for attr in _urllib_robotparser_moved_attributes:
+    setattr(Module_six_moves_urllib_robotparser, attr.name, attr)
+del attr
+
+Module_six_moves_urllib_robotparser._moved_attributes = _urllib_robotparser_moved_attributes
+
+_importer._add_module(Module_six_moves_urllib_robotparser(__name__ + ".moves.urllib.robotparser"),
+                      "moves.urllib_robotparser", "moves.urllib.robotparser")
+
+
+class Module_six_moves_urllib(types.ModuleType):
+
+    """Create a six.moves.urllib namespace that resembles the Python 3 namespace"""
+    __path__ = []  # mark as package
+    parse = _importer._get_module("moves.urllib_parse")
+    error = _importer._get_module("moves.urllib_error")
+    request = _importer._get_module("moves.urllib_request")
+    response = _importer._get_module("moves.urllib_response")
+    robotparser = _importer._get_module("moves.urllib_robotparser")
+
+    def __dir__(self):
+        return ['parse', 'error', 'request', 'response', 'robotparser']
+
+_importer._add_module(Module_six_moves_urllib(__name__ + ".moves.urllib"),
+                      "moves.urllib")
+
+
+def add_move(move):
+    """Add an item to six.moves."""
+    setattr(_MovedItems, move.name, move)
+
+
+def remove_move(name):
+    """Remove item from six.moves."""
+    try:
+        delattr(_MovedItems, name)
+    except AttributeError:
+        try:
+            del moves.__dict__[name]
+        except KeyError:
+            raise AttributeError("no such move, %r" % (name,))
+
+
+if PY3:
+    _meth_func = "__func__"
+    _meth_self = "__self__"
+
+    _func_closure = "__closure__"
+    _func_code = "__code__"
+    _func_defaults = "__defaults__"
+    _func_globals = "__globals__"
+else:
+    _meth_func = "im_func"
+    _meth_self = "im_self"
+
+    _func_closure = "func_closure"
+    _func_code = "func_code"
+    _func_defaults = "func_defaults"
+    _func_globals = "func_globals"
+
+
+try:
+    advance_iterator = next
+except NameError:
+    def advance_iterator(it):
+        return it.next()
+next = advance_iterator
+
+
+try:
+    callable = callable
+except NameError:
+    def callable(obj):
+        return any("__call__" in klass.__dict__ for klass in type(obj).__mro__)
+
+
+if PY3:
+    def get_unbound_function(unbound):
+        return unbound
+
+    create_bound_method = types.MethodType
+
+    def create_unbound_method(func, cls):
+        return func
+
+    Iterator = object
+else:
+    def get_unbound_function(unbound):
+        return unbound.im_func
+
+    def create_bound_method(func, obj):
+        return types.MethodType(func, obj, obj.__class__)
+
+    def create_unbound_method(func, cls):
+        return types.MethodType(func, None, cls)
+
+    class Iterator(object):
+
+        def next(self):
+            return type(self).__next__(self)
+
+    callable = callable
+_add_doc(get_unbound_function,
+         """Get the function out of a possibly unbound function""")
+
+
+get_method_function = operator.attrgetter(_meth_func)
+get_method_self = operator.attrgetter(_meth_self)
+get_function_closure = operator.attrgetter(_func_closure)
+get_function_code = operator.attrgetter(_func_code)
+get_function_defaults = operator.attrgetter(_func_defaults)
+get_function_globals = operator.attrgetter(_func_globals)
+
+
+if PY3:
+    def iterkeys(d, **kw):
+        return iter(d.keys(**kw))
+
+    def itervalues(d, **kw):
+        return iter(d.values(**kw))
+
+    def iteritems(d, **kw):
+        return iter(d.items(**kw))
+
+    def iterlists(d, **kw):
+        return iter(d.lists(**kw))
+
+    viewkeys = operator.methodcaller("keys")
+
+    viewvalues = operator.methodcaller("values")
+
+    viewitems = operator.methodcaller("items")
+else:
+    def iterkeys(d, **kw):
+        return d.iterkeys(**kw)
+
+    def itervalues(d, **kw):
+        return d.itervalues(**kw)
+
+    def iteritems(d, **kw):
+        return d.iteritems(**kw)
+
+    def iterlists(d, **kw):
+        return d.iterlists(**kw)
+
+    viewkeys = operator.methodcaller("viewkeys")
+
+    viewvalues = operator.methodcaller("viewvalues")
+
+    viewitems = operator.methodcaller("viewitems")
+
+_add_doc(iterkeys, "Return an iterator over the keys of a dictionary.")
+_add_doc(itervalues, "Return an iterator over the values of a dictionary.")
+_add_doc(iteritems,
+         "Return an iterator over the (key, value) pairs of a dictionary.")
+_add_doc(iterlists,
+         "Return an iterator over the (key, [values]) pairs of a dictionary.")
+
+
+if PY3:
+    def b(s):
+        return s.encode("latin-1")
+
+    def u(s):
+        return s
+    unichr = chr
+    import struct
+    int2byte = struct.Struct(">B").pack
+    del struct
+    byte2int = operator.itemgetter(0)
+    indexbytes = operator.getitem
+    iterbytes = iter
+    import io
+    StringIO = io.StringIO
+    BytesIO = io.BytesIO
+    del io
+    _assertCountEqual = "assertCountEqual"
+    if sys.version_info[1] <= 1:
+        _assertRaisesRegex = "assertRaisesRegexp"
+        _assertRegex = "assertRegexpMatches"
+        _assertNotRegex = "assertNotRegexpMatches"
+    else:
+        _assertRaisesRegex = "assertRaisesRegex"
+        _assertRegex = "assertRegex"
+        _assertNotRegex = "assertNotRegex"
+else:
+    def b(s):
+        return s
+    # Workaround for standalone backslash
+
+    def u(s):
+        return unicode(s.replace(r'\\', r'\\\\'), "unicode_escape")
+    unichr = unichr
+    int2byte = chr
+
+    def byte2int(bs):
+        return ord(bs[0])
+
+    def indexbytes(buf, i):
+        return ord(buf[i])
+    iterbytes = functools.partial(itertools.imap, ord)
+    import StringIO
+    StringIO = BytesIO = StringIO.StringIO
+    _assertCountEqual = "assertItemsEqual"
+    _assertRaisesRegex = "assertRaisesRegexp"
+    _assertRegex = "assertRegexpMatches"
+    _assertNotRegex = "assertNotRegexpMatches"
+_add_doc(b, """Byte literal""")
+_add_doc(u, """Text literal""")
+
+
+def assertCountEqual(self, *args, **kwargs):
+    return getattr(self, _assertCountEqual)(*args, **kwargs)
+
+
+def assertRaisesRegex(self, *args, **kwargs):
+    return getattr(self, _assertRaisesRegex)(*args, **kwargs)
+
+
+def assertRegex(self, *args, **kwargs):
+    return getattr(self, _assertRegex)(*args, **kwargs)
+
+
+def assertNotRegex(self, *args, **kwargs):
+    return getattr(self, _assertNotRegex)(*args, **kwargs)
+
+
+if PY3:
+    exec_ = getattr(moves.builtins, "exec")
+
+    def reraise(tp, value, tb=None):
+        try:
+            if value is None:
+                value = tp()
+            if value.__traceback__ is not tb:
+                raise value.with_traceback(tb)
+            raise value
+        finally:
+            value = None
+            tb = None
+
+else:
+    def exec_(_code_, _globs_=None, _locs_=None):
+        """Execute code in a namespace."""
+        if _globs_ is None:
+            frame = sys._getframe(1)
+            _globs_ = frame.f_globals
+            if _locs_ is None:
+                _locs_ = frame.f_locals
+            del frame
+        elif _locs_ is None:
+            _locs_ = _globs_
+        exec("""exec _code_ in _globs_, _locs_""")
+
+    exec_("""def reraise(tp, value, tb=None):
+    try:
+        raise tp, value, tb
+    finally:
+        tb = None
+""")
+
+
+if sys.version_info[:2] > (3,):
+    exec_("""def raise_from(value, from_value):
+    try:
+        raise value from from_value
+    finally:
+        value = None
+""")
+else:
+    def raise_from(value, from_value):
+        raise value
+
+
+print_ = getattr(moves.builtins, "print", None)
+if print_ is None:
+    def print_(*args, **kwargs):
+        """The new-style print function for Python 2.4 and 2.5."""
+        fp = kwargs.pop("file", sys.stdout)
+        if fp is None:
+            return
+
+        def write(data):
+            if not isinstance(data, basestring):
+                data = str(data)
+            # If the file has an encoding, encode unicode with it.
+            if (isinstance(fp, file) and
+                    isinstance(data, unicode) and
+                    fp.encoding is not None):
+                errors = getattr(fp, "errors", None)
+                if errors is None:
+                    errors = "strict"
+                data = data.encode(fp.encoding, errors)
+            fp.write(data)
+        want_unicode = False
+        sep = kwargs.pop("sep", None)
+        if sep is not None:
+            if isinstance(sep, unicode):
+                want_unicode = True
+            elif not isinstance(sep, str):
+                raise TypeError("sep must be None or a string")
+        end = kwargs.pop("end", None)
+        if end is not None:
+            if isinstance(end, unicode):
+                want_unicode = True
+            elif not isinstance(end, str):
+                raise TypeError("end must be None or a string")
+        if kwargs:
+            raise TypeError("invalid keyword arguments to print()")
+        if not want_unicode:
+            for arg in args:
+                if isinstance(arg, unicode):
+                    want_unicode = True
+                    break
+        if want_unicode:
+            newline = unicode("\n")
+            space = unicode(" ")
+        else:
+            newline = "\n"
+            space = " "
+        if sep is None:
+            sep = space
+        if end is None:
+            end = newline
+        for i, arg in enumerate(args):
+            if i:
+                write(sep)
+            write(arg)
+        write(end)
+if sys.version_info[:2] < (3, 3):
+    _print = print_
+
+    def print_(*args, **kwargs):
+        fp = kwargs.get("file", sys.stdout)
+        flush = kwargs.pop("flush", False)
+        _print(*args, **kwargs)
+        if flush and fp is not None:
+            fp.flush()
+
+_add_doc(reraise, """Reraise an exception.""")
+
+if sys.version_info[0:2] < (3, 4):
+    # This does exactly the same what the :func:`py3:functools.update_wrapper`
+    # function does on Python versions after 3.2. It sets the ``__wrapped__``
+    # attribute on ``wrapper`` object and it doesn't raise an error if any of
+    # the attributes mentioned in ``assigned`` and ``updated`` are missing on
+    # ``wrapped`` object.
+    def _update_wrapper(wrapper, wrapped,
+                        assigned=functools.WRAPPER_ASSIGNMENTS,
+                        updated=functools.WRAPPER_UPDATES):
+        for attr in assigned:
+            try:
+                value = getattr(wrapped, attr)
+            except AttributeError:
+                continue
+            else:
+                setattr(wrapper, attr, value)
+        for attr in updated:
+            getattr(wrapper, attr).update(getattr(wrapped, attr, {}))
+        wrapper.__wrapped__ = wrapped
+        return wrapper
+    _update_wrapper.__doc__ = functools.update_wrapper.__doc__
+
+    def wraps(wrapped, assigned=functools.WRAPPER_ASSIGNMENTS,
+              updated=functools.WRAPPER_UPDATES):
+        return functools.partial(_update_wrapper, wrapped=wrapped,
+                                 assigned=assigned, updated=updated)
+    wraps.__doc__ = functools.wraps.__doc__
+
+else:
+    wraps = functools.wraps
+
+
+def with_metaclass(meta, *bases):
+    """Create a base class with a metaclass."""
+    # This requires a bit of explanation: the basic idea is to make a dummy
+    # metaclass for one level of class instantiation that replaces itself with
+    # the actual metaclass.
+    class metaclass(type):
+
+        def __new__(cls, name, this_bases, d):
+            if sys.version_info[:2] >= (3, 7):
+                # This version introduced PEP 560 that requires a bit
+                # of extra care (we mimic what is done by __build_class__).
+                resolved_bases = types.resolve_bases(bases)
+                if resolved_bases is not bases:
+                    d['__orig_bases__'] = bases
+            else:
+                resolved_bases = bases
+            return meta(name, resolved_bases, d)
+
+        @classmethod
+        def __prepare__(cls, name, this_bases):
+            return meta.__prepare__(name, bases)
+    return type.__new__(metaclass, 'temporary_class', (), {})
+
+
+def add_metaclass(metaclass):
+    """Class decorator for creating a class with a metaclass."""
+    def wrapper(cls):
+        orig_vars = cls.__dict__.copy()
+        slots = orig_vars.get('__slots__')
+        if slots is not None:
+            if isinstance(slots, str):
+                slots = [slots]
+            for slots_var in slots:
+                orig_vars.pop(slots_var)
+        orig_vars.pop('__dict__', None)
+        orig_vars.pop('__weakref__', None)
+        if hasattr(cls, '__qualname__'):
+            orig_vars['__qualname__'] = cls.__qualname__
+        return metaclass(cls.__name__, cls.__bases__, orig_vars)
+    return wrapper
+
+
+def ensure_binary(s, encoding='utf-8', errors='strict'):
+    """Coerce **s** to six.binary_type.
+
+    For Python 2:
+      - `unicode` -> encoded to `str`
+      - `str` -> `str`
+
+    For Python 3:
+      - `str` -> encoded to `bytes`
+      - `bytes` -> `bytes`
+    """
+    if isinstance(s, text_type):
+        return s.encode(encoding, errors)
+    elif isinstance(s, binary_type):
+        return s
+    else:
+        raise TypeError("not expecting type '%s'" % type(s))
+
+
+def ensure_str(s, encoding='utf-8', errors='strict'):
+    """Coerce *s* to `str`.
+
+    For Python 2:
+      - `unicode` -> encoded to `str`
+      - `str` -> `str`
+
+    For Python 3:
+      - `str` -> `str`
+      - `bytes` -> decoded to `str`
+    """
+    if not isinstance(s, (text_type, binary_type)):
+        raise TypeError("not expecting type '%s'" % type(s))
+    if PY2 and isinstance(s, text_type):
+        s = s.encode(encoding, errors)
+    elif PY3 and isinstance(s, binary_type):
+        s = s.decode(encoding, errors)
+    return s
+
+
+def ensure_text(s, encoding='utf-8', errors='strict'):
+    """Coerce *s* to six.text_type.
+
+    For Python 2:
+      - `unicode` -> `unicode`
+      - `str` -> `unicode`
+
+    For Python 3:
+      - `str` -> `str`
+      - `bytes` -> decoded to `str`
+    """
+    if isinstance(s, binary_type):
+        return s.decode(encoding, errors)
+    elif isinstance(s, text_type):
+        return s
+    else:
+        raise TypeError("not expecting type '%s'" % type(s))
+
+
+def python_2_unicode_compatible(klass):
+    """
+    A class decorator that defines __unicode__ and __str__ methods under Python 2.
+    Under Python 3 it does nothing.
+
+    To support Python 2 and 3 with a single code base, define a __str__ method
+    returning text and apply this decorator to the class.
+    """
+    if PY2:
+        if '__str__' not in klass.__dict__:
+            raise ValueError("@python_2_unicode_compatible cannot be applied "
+                             "to %s because it doesn't define __str__()." %
+                             klass.__name__)
+        klass.__unicode__ = klass.__str__
+        klass.__str__ = lambda self: self.__unicode__().encode('utf-8')
+    return klass
+
+
+# Complete the moves implementation.
+# This code is at the end of this module to speed up module loading.
+# Turn this module into a package.
+__path__ = []  # required for PEP 302 and PEP 451
+__package__ = __name__  # see PEP 366 @ReservedAssignment
+if globals().get("__spec__") is not None:
+    __spec__.submodule_search_locations = []  # PEP 451 @UndefinedVariable
+# Remove other six meta path importers, since they cause problems. This can
+# happen if six is removed from sys.modules and then reloaded. (Setuptools does
+# this for some reason.)
+if sys.meta_path:
+    for i, importer in enumerate(sys.meta_path):
+        # Here's some real nastiness: Another "instance" of the six module might
+        # be floating around. Therefore, we can't use isinstance() to check for
+        # the six meta path importer, since the other six instance will have
+        # inserted an importer with different class.
+        if (type(importer).__name__ == "_SixMetaPathImporter" and
+                importer.name == __name__):
+            del sys.meta_path[i]
+            break
+    del i, importer
+# Finally, add the importer to the meta path import hook.
+sys.meta_path.append(_importer)

dist/devsecops/bin/wget.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+# coding=utf-8
+#
+# Copyright 2011-2015 Splunk, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"): you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import app
+import os,sys
+import requests
+
+from splunklib.searchcommands import dispatch, StreamingCommand, Configuration, validators, Option
+from splunklib.searchcommands.validators import Code
+
+@Configuration()
+class WgetCommand(StreamingCommand):
+    """ Call wget from url in data.
+    ##Syntax
+    .. code-block::
+        wget output=<field> <field-list>
+    ##Description
+    The :code:`wget` command calls a url which is present in a field.
+    ##Example
+    tbd
+    """
+
+    output = Option(
+        doc='''
+        **Syntax:** **output=***<output>*
+        **Description:** Name of the field that will hold the return data''',
+        require=True, validate=validators.Fieldname())
+
+    def stream(self, records):
+        self.logger.debug('WgetCommand: %s', self)  # logs command line
+        fieldnames = self.fieldnames
+
+        for record in records:
+            for fieldname in fieldnames:
+                r = requests.get(record[fieldname])
+                record[self.output] = r.json()
+                yield record
+
+dispatch(WgetCommand, sys.argv, sys.stdin, sys.stdout, __name__)

dist/devsecops/default/analytic_stories.conf

@@ -1,6 +1,6 @@
 #############
 # Automatically generated by generator.py in splunk/security_content
-# On Date: 2021-08-27T14:41:52 UTC
+# On Date: 2021-09-13T10:57:27 UTC
 # Author: Splunk Security Research
 # Contact: research@splunk.com
 #############
@@ -14,8 +14,8 @@ modification_date = 2021-08-18
 id = 0ca8c38e-631e-4b81-940c-f9c5450ce41e
 version = 1
 reference = ["https://www.redhat.com/en/topics/devops/what-is-devsecops"]
-detection_searches = ["ESCU - AWS ECR Container Scanning Findings High - Rule", "ESCU - AWS ECR Container Scanning Findings Low Informational Unknown - Rule", "ESCU - AWS ECR Container Scanning Findings Medium - Rule", "ESCU - AWS ECR Container Upload Outside Business Hours - Rule", "ESCU - AWS ECR Container Upload Unknown User - Rule", "ESCU - Kubernetes Nginx Ingress LFI - Rule", "ESCU - Kubernetes Nginx Ingress RFI - Rule", "ESCU - Kubernetes Scanner Image Pulling - Rule"]
-mappings = {"cis20": ["CIS 13"], "kill_chain_phases": ["Actions on Objectives"], "mitre_attack": ["T1204.003", "T1212", "T1526"], "nist": ["DE.CM", "PR.AC", "PR.DS"]}
+detection_searches = ["ESCU - AWS ECR Container Scanning Findings High - Rule", "ESCU - AWS ECR Container Scanning Findings Low Informational Unknown - Rule", "ESCU - AWS ECR Container Scanning Findings Medium - Rule", "ESCU - AWS ECR Container Upload Outside Business Hours - Rule", "ESCU - AWS ECR Container Upload Unknown User - Rule", "ESCU - Circle CI Disable Security Job - Rule", "ESCU - Circle CI Disable Security Step - Rule", "ESCU - Correlation by Repository and Risk - Rule", "ESCU - Correlation by User and Risk - Rule", "ESCU - GitHub Dependabot Alert - Rule", "ESCU - GitHub Pull Request from Unknown User - Rule", "ESCU - Kubernetes Nginx Ingress LFI - Rule", "ESCU - Kubernetes Nginx Ingress RFI - Rule", "ESCU - Kubernetes Scanner Image Pulling - Rule"]
+mappings = {"cis20": ["CIS 13"], "kill_chain_phases": ["Actions on Objectives"], "mitre_attack": ["T1195.001", "T1204.003", "T1212", "T1526", "T1554"], "nist": ["DE.CM", "PR.AC", "PR.DS"]}
 investigative_searches = []
 support_searches = []
 data_models = []

dist/devsecops/default/collections.conf

@@ -1,6 +1,6 @@
 #############
 # Automatically generated by generator.py in splunk/security_content
-# On Date: 2021-08-27T14:41:52 UTC
+# On Date: 2021-09-13T10:57:27 UTC
 # Author: Splunk Security Research
 # Contact: research@splunk.com
 #############

dist/devsecops/default/commands.conf

@@ -0,0 +1,10 @@
+[wget]
+filename = wget.py
+run_in_preview = false
+outputheader = true
+enableheader = true
+requires_srinfo = true
+supports_getinfo = true
+supports_multivalues = true
+supports_rawargs = true
+python.version = python3

dist/devsecops/default/data/ui/nav/default.xml

@@ -2,6 +2,7 @@
   <view name="analytics" default='true' />
   <view name="user_analytics" />
   <view name="repository_analytics" />
+  <view name="dev_sec_ops_analytics" />
   <view name="search" />
   <view name="dashboards" />
 </nav>

dist/devsecops/default/macros.conf

@@ -1,6 +1,6 @@
 #############
 # Automatically generated by generator.py in splunk/security_content
-# On Date: 2021-08-27T14:41:52 UTC
+# On Date: 2021-09-13T10:57:27 UTC
 # Author: Splunk Security Research
 # Contact: research@splunk.com
 #############
@@ -46,6 +46,10 @@ description = This macro limits the output to only domains that are in the brand
 definition = lookup update=true brandMonitoring_lookup domain as urls OUTPUT domain_abuse | search domain_abuse=true
 description = This macro limits the output to only domains that are in the brand monitoring lookup file
 
+[circleci]
+definition = sourcetype=circleci
+description = customer specific splunk configurations(eg- index, source, sourcetype). Replace the macro definition with configurations for your Splunk Environmnent.
+
 [cisco_networks]
 definition = eventtype=cisco_ios
 description = customer specific splunk configurations(eg- index, source, sourcetype). Replace the macro definition with configurations for your Splunk Environmnent.
@@ -110,6 +114,10 @@ description = This limits the query fields to domains that are associated with e
 definition = (query=outlook* AND query=login* AND query=account*)
 description = This limits the query fields to domains that are associated with evilginx masquerading as Outlook
 
+[exchange]
+definition = sourcetype="MSWindows:IIS"
+description = customer specific splunk configurations(eg- index, source, sourcetype). Replace the macro definition with configurations for your Splunk Environmnent.
+
 [f5_bigip_rogue]
 definition = index=netops sourcetype="f5:bigip:rogue"
 description = customer specific splunk configurations(eg- index, source, sourcetype). Replace the macro definition with configurations for your Splunk Environmnent.
@@ -122,6 +130,10 @@ description = This macro is intended to allow_list processes that have been defi
 definition = sourcetype=aws:firehose:json
 description = customer specific splunk configurations(eg- index, source, sourcetype). Replace the macro definition with configurations for your Splunk Environmnent.
 
+[github_known_users]
+definition = user IN (user_names_here)
+description = specify the user allowed to create PRs in Github projects.
+
 [google_gcp_pubnet_message]
 definition = sourcetype="google:gcp:pubsub:message"
 description = customer specific splunk configurations(eg- index, source, sourcetype) for Google GCP. Replace the macro definition with configurations for your Splunk Environmnent.
@@ -291,6 +303,10 @@ description = search data model's summaries only
 definition = (eventName=AuthorizeSecurityGroupIngress OR eventName=CreateSecurityGroup OR eventName=DeleteSecurityGroup OR eventName=DescribeClusterSecurityGroups OR eventName=DescribeDBSecurityGroups OR eventName=DescribeSecurityGroupReferences OR eventName=DescribeSecurityGroups OR eventName=DescribeStaleSecurityGroups OR eventName=RevokeSecurityGroupIngress OR eventName=UpdateSecurityGroupRuleDescriptionsIngress)
 description = This macro is a list of AWS event names associated with security groups
 
+[signals]
+definition = index=signals
+description = customer specific splunk configurations(eg- index, source, sourcetype). Replace the macro definition with configurations for your Splunk Environmnent.
+
 [stream_dns]
 definition = sourcetype=stream:dns
 description = customer specific splunk configurations(eg- index, source, sourcetype). Replace the macro definition with configurations for your Splunk Environmnent.
@@ -363,7 +379,19 @@ description = Update this macro to limit the output results to filter out false
 definition = search *
 description = Update this macro to limit the output results to filter out false positives. 
 
-[aws_excessive_security_scanning_filter]
+[circle_ci_disable_security_job_filter]
+definition = search *
+description = Update this macro to limit the output results to filter out false positives. 
+
+[circle_ci_disable_security_step_filter]
+definition = search *
+description = Update this macro to limit the output results to filter out false positives. 
+
+[correlation_by_repository_and_risk_filter]
+definition = search *
+description = Update this macro to limit the output results to filter out false positives. 
+
+[correlation_by_user_and_risk_filter]
 definition = search *
 description = Update this macro to limit the output results to filter out false positives. 
 
@@ -371,6 +399,22 @@ description = Update this macro to limit the output results to filter out false
 definition = search *
 description = Update this macro to limit the output results to filter out false positives. 
 
+[github_dependabot_alert_filter]
+definition = search *
+description = Update this macro to limit the output results to filter out false positives. 
+
+[github_pull_request_from_unknown_user_filter]
+definition = search *
+description = Update this macro to limit the output results to filter out false positives. 
+
+[github_commit_changes_in_master_filter]
+definition = search *
+description = Update this macro to limit the output results to filter out false positives. 
+
+[github_commit_in_develop_filter]
+definition = search *
+description = Update this macro to limit the output results to filter out false positives. 
+
 [gsuite_drive_share_in_external_email_filter]
 definition = search *
 description = Update this macro to limit the output results to filter out false positives. 

dist/devsecops/default/savedsearches.conf

@@ -1,6 +1,6 @@
 #############
 # Automatically generated by generator.py in splunk/security_content
-# On Date: 2021-08-27T14:41:52 UTC
+# On Date: 2021-09-13T10:57:27 UTC
 # Author: Splunk Security Research
 # Contact: research@splunk.com
 #############
@@ -25,21 +25,19 @@ action.escu.product = ["Splunk Enterprise", "Splunk Enterprise Security", "Splun
 action.escu.providing_technologies = []
 action.escu.analytic_story = ["Dev Sec Ops"]
 action.risk = 1
-action.risk.param._risk_message = Vulnerabilities with severity high found in repository $repositoryName$
-action.risk.param._risk = [{"threat_object_field": "repositoryName", "threat_object_type": "system"}]
+action.risk.param._risk_message = Vulnerabilities with severity high found in image $image$
+action.risk.param._risk = [{"threat_object_field": "image", "threat_object_type": "system"}]
 action.risk.param.verbose = 0
 cron_schedule = 0 * * * *
-dispatch.earliest_time = -70m@m
-dispatch.latest_time = -10m@m
+dispatch.earliest_time = -60m
+dispatch.latest_time = now
 action.correlationsearch.enabled = 1
 action.correlationsearch.label = ESCU - AWS ECR Container Scanning Findings High - Rule
-action.correlationsearch.annotations = {"analytic_story": ["Dev Sec Ops"], "cis20": ["CIS 13"], "confidence": 70, "impact": 70, "kill_chain_phases": ["Actions on Objectives"], "mitre_attack": ["T1204.003"], "nist": ["PR.DS", "PR.AC", "DE.CM"], "observable": [{"name": "repositoryName", "role": ["Victim"], "type": "System"}]}
+action.correlationsearch.annotations = {"analytic_story": ["Dev Sec Ops"], "cis20": ["CIS 13"], "confidence": 100, "impact": 70, "kill_chain_phases": ["Actions on Objectives"], "mitre_attack": ["T1204.003"], "nist": ["PR.DS", "PR.AC", "DE.CM"], "observable": [{"name": "image", "role": ["Victim"], "type": "System"}]}
 schedule_window = auto
-action.notable = 1
-action.notable.param.rule_description = This search looks for AWS CloudTrail events from AWS Elastic Container Service (ECR). You need to activate image scanning in order to get the event DescribeImageScanFindings with the results.
-action.notable.param.rule_title = AWS ECR Container Scanning Findings High
-action.notable.param.security_domain = network
-action.notable.param.severity = high
+action.slack = 1
+action.slack.param.channel = dev_sec_ops_analytics
+action.slack.param.message = Alert AWS ECR Container Scanning Findings High
 alert.digest_mode = 1
 disabled = true
 enableSched = 1
@@ -49,7 +47,7 @@ relation = greater than
 quantity = 0
 realtime_schedule = 0
 is_visible = false
-search = `cloudtrail` eventSource=ecr.amazonaws.com eventName=DescribeImageScanFindings | spath path=responseElements.imageScanFindings.findings{} output=findings | mvexpand findings | spath input=findings| search severity=HIGH | rename name as finding_name, description as finding_description, requestParameters.imageId.imageDigest as imageDigest, requestParameters.repositoryName as repositoryName | eval finding = finding_name.", ".finding_description | eval phase="release" | stats min(_time) as firstTime max(_time) as lastTime by awsRegion, eventName, eventSource, imageDigest, repositoryName, user, userName, src_ip, finding, phase | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `aws_ecr_container_scanning_findings_high_filter` | collect index=findings
+search = `cloudtrail` eventSource=ecr.amazonaws.com eventName=DescribeImageScanFindings | spath path=responseElements.imageScanFindings.findings{} output=findings | mvexpand findings | spath input=findings| search severity=HIGH | rename name as finding_name, description as finding_description, requestParameters.imageId.imageDigest as imageDigest, requestParameters.repositoryName as image | eval finding = finding_name.", ".finding_description | eval phase="release" | eval severity="high" | stats min(_time) as firstTime max(_time) as lastTime by awsRegion, eventName, eventSource, imageDigest, image, user, userName, src_ip, finding, phase, severity | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `aws_ecr_container_scanning_findings_high_filter` | eval risk_score=70 | eval mitre_attack_id=T1204.003 | collect index=alerts
 
 [ESCU - AWS ECR Container Scanning Findings Low Informational Unknown - Rule]
 action.escu = 0
@@ -77,7 +75,7 @@ dispatch.earliest_time = -70m@m
 dispatch.latest_time = -10m@m
 action.correlationsearch.enabled = 1
 action.correlationsearch.label = ESCU - AWS ECR Container Scanning Findings Low Informational Unknown - Rule
-action.correlationsearch.annotations = {"analytic_story": ["Dev Sec Ops"], "cis20": ["CIS 13"], "confidence": 70, "impact": 70, "kill_chain_phases": ["Actions on Objectives"], "mitre_attack": ["T1204.003"], "nist": ["PR.DS", "PR.AC", "DE.CM"], "observable": [{"name": "repositoryName", "role": ["Victim"], "type": "System"}]}
+action.correlationsearch.annotations = {"analytic_story": ["Dev Sec Ops"], "cis20": ["CIS 13"], "confidence": 70, "impact": 10, "kill_chain_phases": ["Actions on Objectives"], "mitre_attack": ["T1204.003"], "nist": ["PR.DS", "PR.AC", "DE.CM"], "observable": [{"name": "repositoryName", "role": ["Victim"], "type": "System"}]}
 schedule_window = auto
 action.notable = 1
 action.notable.param.rule_description = This search looks for AWS CloudTrail events from AWS Elastic Container Service (ECR). You need to activate image scanning in order to get the event DescribeImageScanFindings with the results.
@@ -93,7 +91,7 @@ relation = greater than
 quantity = 0
 realtime_schedule = 0
 is_visible = false
-search = `cloudtrail` eventSource=ecr.amazonaws.com eventName=DescribeImageScanFindings | spath path=responseElements.imageScanFindings.findings{} output=findings | mvexpand findings | spath input=findings| search severity IN (LOW, INFORMATIONAL, UNKNWON) | rename name as finding_name, description as finding_description, requestParameters.imageId.imageDigest as imageDigest, requestParameters.repositoryName as repositoryName | eval finding = finding_name.", ".finding_description | eval phase="release" | stats min(_time) as firstTime max(_time) as lastTime by awsRegion, eventName, eventSource, imageDigest, repositoryName, user, userName, src_ip, finding, phase | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `aws_ecr_container_scanning_findings_low_informational_unknown_filter` | collect index=findings
+search = `cloudtrail` eventSource=ecr.amazonaws.com eventName=DescribeImageScanFindings | spath path=responseElements.imageScanFindings.findings{} output=findings | mvexpand findings | spath input=findings| search severity IN (LOW, INFORMATIONAL, UNKNWON) | rename name as finding_name, description as finding_description, requestParameters.imageId.imageDigest as imageDigest, requestParameters.repositoryName as repositoryName | eval finding = finding_name.", ".finding_description | eval phase="release" | eval severity="low" | stats min(_time) as firstTime max(_time) as lastTime by awsRegion, eventName, eventSource, imageDigest, repositoryName, user, userName, src_ip, finding, phase, severity | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `aws_ecr_container_scanning_findings_low_informational_unknown_filter` | eval risk_score=7 | eval mitre_attack_id=T1204.003
 
 [ESCU - AWS ECR Container Scanning Findings Medium - Rule]
 action.escu = 0
@@ -113,15 +111,15 @@ action.escu.product = ["Splunk Enterprise", "Splunk Enterprise Security", "Splun
 action.escu.providing_technologies = []
 action.escu.analytic_story = ["Dev Sec Ops"]
 action.risk = 1
-action.risk.param._risk_message = Vulnerabilities with severity high found in repository $repositoryName$
-action.risk.param._risk = [{"threat_object_field": "repositoryName", "threat_object_type": "system"}]
+action.risk.param._risk_message = Vulnerabilities with severity high found in image $image$
+action.risk.param._risk = [{"threat_object_field": "image", "threat_object_type": "system"}]
 action.risk.param.verbose = 0
 cron_schedule = 0 * * * *
 dispatch.earliest_time = -70m@m
 dispatch.latest_time = -10m@m
 action.correlationsearch.enabled = 1
 action.correlationsearch.label = ESCU - AWS ECR Container Scanning Findings Medium - Rule
-action.correlationsearch.annotations = {"analytic_story": ["Dev Sec Ops"], "cis20": ["CIS 13"], "confidence": 70, "impact": 70, "kill_chain_phases": ["Actions on Objectives"], "mitre_attack": ["T1204.003"], "nist": ["PR.DS", "PR.AC", "DE.CM"], "observable": [{"name": "repositoryName", "role": ["Victim"], "type": "System"}]}
+action.correlationsearch.annotations = {"analytic_story": ["Dev Sec Ops"], "cis20": ["CIS 13"], "confidence": 70, "impact": 30, "kill_chain_phases": ["Actions on Objectives"], "mitre_attack": ["T1204.003"], "nist": ["PR.DS", "PR.AC", "DE.CM"], "observable": [{"name": "image", "role": ["Victim"], "type": "System"}]}
 schedule_window = auto
 action.notable = 1
 action.notable.param.rule_description = This search looks for AWS CloudTrail events from AWS Elastic Container Service (ECR). You need to activate image scanning in order to get the event DescribeImageScanFindings with the results.
@@ -137,7 +135,7 @@ relation = greater than
 quantity = 0
 realtime_schedule = 0
 is_visible = false
-search = `cloudtrail` eventSource=ecr.amazonaws.com eventName=DescribeImageScanFindings | spath path=responseElements.imageScanFindings.findings{} output=findings | mvexpand findings | spath input=findings| search severity=MEDIUM | rename name as finding_name, description as finding_description, requestParameters.imageId.imageDigest as imageDigest, requestParameters.repositoryName as repositoryName | eval finding = finding_name.", ".finding_description | eval phase="release" | stats min(_time) as firstTime max(_time) as lastTime by awsRegion, eventName, eventSource, imageDigest, repositoryName, user, userName, src_ip, finding, phase | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `aws_ecr_container_scanning_findings_medium_filter` | collect index=findings
+search = `cloudtrail` eventSource=ecr.amazonaws.com eventName=DescribeImageScanFindings | spath path=responseElements.imageScanFindings.findings{} output=findings | mvexpand findings | spath input=findings| search severity=MEDIUM | rename name as finding_name, description as finding_description, requestParameters.imageId.imageDigest as imageDigest, requestParameters.repositoryName as image | eval finding = finding_name.", ".finding_description | eval phase="release" | eval severity="medium" | stats min(_time) as firstTime max(_time) as lastTime by awsRegion, eventName, eventSource, imageDigest, image, user, userName, src_ip, finding, phase, severity | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `aws_ecr_container_scanning_findings_medium_filter` | eval risk_score=21 | eval mitre_attack_id=T1204.003 | collect index=signals
 
 [ESCU - AWS ECR Container Upload Outside Business Hours - Rule]
 action.escu = 0
@@ -181,7 +179,7 @@ relation = greater than
 quantity = 0
 realtime_schedule = 0
 is_visible = false
-search = `cloudtrail` eventSource=ecr.amazonaws.com eventName=PutImage date_hour>=20 OR date_hour<8 NOT (date_wday=saturday OR date_wday=sunday) | rename requestParameters.* as * | eval phase="release" | stats min(_time) as firstTime max(_time) as lastTime by awsRegion, eventName, eventSource, user, userName, src_ip, imageTag, registryId, repositoryName, phase | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `aws_ecr_container_upload_outside_business_hours_filter` | collect index=findings
+search = `cloudtrail` eventSource=ecr.amazonaws.com eventName=PutImage date_hour>=20 OR date_hour<8 NOT (date_wday=saturday OR date_wday=sunday) | rename requestParameters.* as * | rename repositoryName AS image | eval phase="release" | eval severity="medium" | stats min(_time) as firstTime max(_time) as lastTime by awsRegion, eventName, eventSource, user, userName, src_ip, imageTag, registryId, image, phase, severity | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `aws_ecr_container_upload_outside_business_hours_filter` | eval risk_score=49 | eval mitre_attack_id=T1204.003 | collect index=signals
 
 [ESCU - AWS ECR Container Upload Unknown User - Rule]
 action.escu = 0
@@ -225,36 +223,42 @@ relation = greater than
 quantity = 0
 realtime_schedule = 0
 is_visible = false
-search = `cloudtrail` eventSource=ecr.amazonaws.com eventName=PutImage NOT `aws_ecr_users` | rename requestParameters.* as * | eval phase="release" | stats min(_time) as firstTime max(_time) as lastTime by awsRegion, eventName, eventSource, user, userName, src_ip, imageTag, registryId, repositoryName, phase | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `aws_ecr_container_upload_unknown_user_filter` | collect index=findings
+search = `cloudtrail` eventSource=ecr.amazonaws.com eventName=PutImage NOT `aws_ecr_users` | rename requestParameters.* as * | rename repositoryName AS image | eval phase="release" | eval severity="high" | stats min(_time) as firstTime max(_time) as lastTime by awsRegion, eventName, eventSource, user, userName, src_ip, imageTag, registryId, image, phase, severity | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `aws_ecr_container_upload_unknown_user_filter` | eval risk_score=49 | eval mitre_attack_id=T1204.003 | collect index=signals
 
-[ESCU - AWS Excessive Security Scanning - Rule]
+[ESCU - Circle CI Disable Security Job - Rule]
 action.escu = 0
 action.escu.enabled = 1
-description = This search looks for AWS CloudTrail events and analyse the amount of eventNames which starts with Describe by a single user. This indicates that this user scans the configuration of your AWS cloud environment.
-action.escu.mappings = {"cis20": ["CIS 13"], "kill_chain_phases": ["Actions on Objectives"], "mitre_attack": ["T1526"], "nist": ["PR.DS", "PR.AC", "DE.CM"]}
+description = This search looks for disable security job in CircleCI pipeline.
+action.escu.mappings = {"cis20": ["CIS 13"], "kill_chain_phases": ["Actions on Objectives"], "mitre_attack": ["T1554"], "nist": ["PR.DS", "PR.AC", "DE.CM"]}
 action.escu.data_models = []
-action.escu.eli5 = This search looks for AWS CloudTrail events and analyse the amount of eventNames which starts with Describe by a single user. This indicates that this user scans the configuration of your AWS cloud environment.
-action.escu.how_to_implement = You must install splunk AWS add on and Splunk App for AWS. This search works with AWS CloudTrail logs.
-action.escu.known_false_positives = While this search has no known false positives.
-action.escu.creation_date = 2021-04-13
-action.escu.modification_date = 2021-04-13
+action.escu.eli5 = This search looks for disable security job in CircleCI pipeline.
+action.escu.how_to_implement = You must index CircleCI logs.
+action.escu.known_false_positives = unknown
+action.escu.creation_date = 2021-09-02
+action.escu.modification_date = 2021-09-02
 action.escu.confidence = high
-action.escu.full_search_name = ESCU - AWS Excessive Security Scanning - Rule
+action.escu.full_search_name = ESCU - Circle CI Disable Security Job - Rule
 action.escu.search_type = detection
-action.escu.product = ["Splunk Security Analytics for AWS", "Splunk Enterprise", "Splunk Enterprise Security", "Splunk Cloud", "Dev Sec Ops Analytics"]
+action.escu.product = ["Splunk Enterprise", "Splunk Enterprise Security", "Splunk Cloud", "Dev Sec Ops Analytics"]
 action.escu.providing_technologies = []
-action.escu.analytic_story = ["AWS User Monitoring"]
+action.escu.analytic_story = ["Dev Sec Ops"]
 action.risk = 1
-action.risk.param._risk_message = user $user$ has excessive number of api calls $dc_events$ from these IP addresses $src$, violating the threshold of 50,  using the following commands $command$.
-action.risk.param._risk = [{"risk_object_field": "src", "risk_object_type": "system", "risk_score": 18}, {"risk_object_field": "user", "risk_object_type": "user", "risk_score": 18}]
+action.risk.param._risk_message = disable security job $mandatory_job$ in workflow $workflow_name$ from user $user$
+action.risk.param._risk = [{"risk_object_field": "user", "risk_object_type": "user", "risk_score": 72}]
 action.risk.param.verbose = 0
 cron_schedule = 0 * * * *
 dispatch.earliest_time = -70m@m
 dispatch.latest_time = -10m@m
 action.correlationsearch.enabled = 1
-action.correlationsearch.label = ESCU - AWS Excessive Security Scanning - Rule
-action.correlationsearch.annotations = {"analytic_story": ["AWS User Monitoring"], "cis20": ["CIS 13"], "confidence": 60, "context": ["Source:Cloud Data", "Scope:Inbound", "Stage:Recon"], "impact": 30, "kill_chain_phases": ["Actions on Objectives"], "mitre_attack": ["T1526"], "nist": ["PR.DS", "PR.AC", "DE.CM"], "observable": [{"name": "src", "role": ["Attacker"], "type": "IP Address"}, {"name": "user", "role": ["Attacker"], "type": "User"}]}
+action.correlationsearch.label = ESCU - Circle CI Disable Security Job - Rule
+action.correlationsearch.annotations = {"analytic_story": ["Dev Sec Ops"], "cis20": ["CIS 13"], "confidence": 90, "impact": 80, "kill_chain_phases": ["Actions on Objectives"], "mitre_attack": ["T1554"], "nist": ["PR.DS", "PR.AC", "DE.CM"], "observable": [{"name": "user", "role": ["Attacker"], "type": "User"}]}
 schedule_window = auto
+action.notable = 1
+action.notable.param.nes_fields = ['user']
+action.notable.param.rule_description = This search looks for disable security job in CircleCI pipeline.
+action.notable.param.rule_title = Circle CI Disable Security Job
+action.notable.param.security_domain = network
+action.notable.param.severity = high
 alert.digest_mode = 1
 disabled = true
 enableSched = 1
@@ -264,7 +268,136 @@ relation = greater than
 quantity = 0
 realtime_schedule = 0
 is_visible = false
-search = `cloudtrail` eventName=Describe* OR eventName=List* OR eventName=Get*  | stats dc(eventName) as dc_events min(_time) as firstTime max(_time) as lastTime values(eventName) as eventName values(src) as src values(userAgent) as userAgent by user userIdentity.arn | where dc_events > 50 | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)`|`aws_excessive_security_scanning_filter` | collect index=findings
+search = `circleci` | rename vcs.committer_name as user vcs.subject as commit_message vcs.url as url workflows.* as *  | stats values(job_name) as job_names by workflow_id workflow_name user commit_message url branch | lookup mandatory_job_for_workflow workflow_name OUTPUTNEW job_name AS mandatory_job | search mandatory_job=* | eval mandatory_job_executed=if(like(job_names, "%".mandatory_job."%"), 1, 0) | where mandatory_job_executed=0 | eval phase="build" | rex field=url "(?<repository>[^\/]*\/[^\/]*)$" | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `circle_ci_disable_security_job_filter` | eval risk_score=72 | eval mitre_attack_id=T1554 | collect index=signals
+
+[ESCU - Circle CI Disable Security Step - Rule]
+action.escu = 0
+action.escu.enabled = 1
+description = This search looks for disable security step in CircleCI pipeline.
+action.escu.mappings = {"cis20": ["CIS 13"], "kill_chain_phases": ["Actions on Objectives"], "mitre_attack": ["T1554"], "nist": ["PR.DS", "PR.AC", "DE.CM"]}
+action.escu.data_models = []
+action.escu.eli5 = This search looks for disable security step in CircleCI pipeline.
+action.escu.how_to_implement = You must index CircleCI logs.
+action.escu.known_false_positives = unknown
+action.escu.creation_date = 2021-09-01
+action.escu.modification_date = 2021-09-01
+action.escu.confidence = high
+action.escu.full_search_name = ESCU - Circle CI Disable Security Step - Rule
+action.escu.search_type = detection
+action.escu.product = ["Splunk Enterprise", "Splunk Enterprise Security", "Splunk Cloud", "Dev Sec Ops Analytics"]
+action.escu.providing_technologies = []
+action.escu.analytic_story = ["Dev Sec Ops"]
+action.risk = 1
+action.risk.param._risk_message = disable security step $mandatory_step$ in job $job_name$ from user $user$
+action.risk.param._risk = [{"risk_object_field": "user", "risk_object_type": "user", "risk_score": 72}]
+action.risk.param.verbose = 0
+cron_schedule = 0 * * * *
+dispatch.earliest_time = -70m@m
+dispatch.latest_time = -10m@m
+action.correlationsearch.enabled = 1
+action.correlationsearch.label = ESCU - Circle CI Disable Security Step - Rule
+action.correlationsearch.annotations = {"analytic_story": ["Dev Sec Ops"], "cis20": ["CIS 13"], "confidence": 90, "impact": 80, "kill_chain_phases": ["Actions on Objectives"], "mitre_attack": ["T1554"], "nist": ["PR.DS", "PR.AC", "DE.CM"], "observable": [{"name": "user", "role": ["Attacker"], "type": "User"}]}
+schedule_window = auto
+action.notable = 1
+action.notable.param.nes_fields = ['user']
+action.notable.param.rule_description = This search looks for disable security step in CircleCI pipeline.
+action.notable.param.rule_title = Circle CI Disable Security Step
+action.notable.param.security_domain = network
+action.notable.param.severity = high
+alert.digest_mode = 1
+disabled = true
+enableSched = 1
+allow_skew = 100%
+counttype = number of events
+relation = greater than
+quantity = 0
+realtime_schedule = 0
+is_visible = false
+search = `circleci` | rename workflows.job_id AS job_id | join job_id [ | search `circleci` | stats values(name) as step_names count by job_id job_name ] | stats count by step_names job_id job_name vcs.committer_name vcs.subject vcs.url owners{} | rename vcs.* as * , owners{} as user | lookup mandatory_step_for_job job_name OUTPUTNEW step_name AS mandatory_step | search mandatory_step=* | eval mandatory_step_executed=if(like(step_names, "%".mandatory_step."%"), 1, 0) | where mandatory_step_executed=0 | rex field=url "(?<repository>[^\/]*\/[^\/]*)$" | eval phase="build"  | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `circle_ci_disable_security_step_filter` | eval risk_score=72 | eval mitre_attack_id=T1554 | collect index=signals
+
+[ESCU - Correlation by Repository and Risk - Rule]
+action.escu = 0
+action.escu.enabled = 1
+description = This search correlations detections by repository and risk_score
+action.escu.mappings = {"cis20": ["CIS 13"], "kill_chain_phases": ["Actions on Objectives"], "mitre_attack": ["T1204.003"], "nist": ["PR.DS", "PR.AC", "DE.CM"]}
+action.escu.data_models = []
+action.escu.eli5 = This search correlations detections by repository and risk_score
+action.escu.how_to_implement = For Dev Sec Ops POC
+action.escu.known_false_positives = unknown
+action.escu.creation_date = 2021-09-06
+action.escu.modification_date = 2021-09-06
+action.escu.confidence = high
+action.escu.full_search_name = ESCU - Correlation by Repository and Risk - Rule
+action.escu.search_type = detection
+action.escu.product = ["Splunk Enterprise", "Splunk Enterprise Security", "Splunk Cloud", "Dev Sec Ops Analytics"]
+action.escu.providing_technologies = []
+action.escu.analytic_story = ["Dev Sec Ops"]
+action.risk = 1
+action.risk.param._risk_message = Correlation triggered for user $user$
+action.risk.param._risk = [{"risk_object_field": "user", "risk_object_type": "user", "risk_score": 70}]
+action.risk.param.verbose = 0
+cron_schedule = 0 * * * *
+dispatch.earliest_time = -60m
+dispatch.latest_time = now
+action.correlationsearch.enabled = 1
+action.correlationsearch.label = ESCU - Correlation by Repository and Risk - Rule
+action.correlationsearch.annotations = {"analytic_story": ["Dev Sec Ops"], "cis20": ["CIS 13"], "confidence": 100, "impact": 70, "kill_chain_phases": ["Actions on Objectives"], "mitre_attack": ["T1204.003"], "nist": ["PR.DS", "PR.AC", "DE.CM"], "observable": [{"name": "user", "role": ["Victim"], "type": "User"}]}
+schedule_window = auto
+action.slack = 1
+action.slack.param.channel = dev_sec_ops_analytics
+action.slack.param.message = Alert Correlation by Repository and Risk
+alert.digest_mode = 1
+disabled = true
+enableSched = 1
+allow_skew = 100%
+counttype = number of events
+relation = greater than
+quantity = 0
+realtime_schedule = 0
+is_visible = false
+search = `signals` | fillnull | stats sum(risk_score) as risk_score values(source) as signals values(user) as user by repository | sort - risk_score | where risk_score > 80 | `correlation_by_repository_and_risk_filter` | eval risk_score=70 | eval mitre_attack_id=T1204.003 | collect index=alerts
+
+[ESCU - Correlation by User and Risk - Rule]
+action.escu = 0
+action.escu.enabled = 1
+description = This search correlations detections by user and risk_score
+action.escu.mappings = {"cis20": ["CIS 13"], "kill_chain_phases": ["Actions on Objectives"], "mitre_attack": ["T1204.003"], "nist": ["PR.DS", "PR.AC", "DE.CM"]}
+action.escu.data_models = []
+action.escu.eli5 = This search correlations detections by user and risk_score
+action.escu.how_to_implement = For Dev Sec Ops POC
+action.escu.known_false_positives = unknown
+action.escu.creation_date = 2021-09-06
+action.escu.modification_date = 2021-09-06
+action.escu.confidence = high
+action.escu.full_search_name = ESCU - Correlation by User and Risk - Rule
+action.escu.search_type = detection
+action.escu.product = ["Splunk Enterprise", "Splunk Enterprise Security", "Splunk Cloud", "Dev Sec Ops Analytics"]
+action.escu.providing_technologies = []
+action.escu.analytic_story = ["Dev Sec Ops"]
+action.risk = 1
+action.risk.param._risk_message = Correlation triggered for user $user$
+action.risk.param._risk = [{"risk_object_field": "user", "risk_object_type": "user", "risk_score": 70}]
+action.risk.param.verbose = 0
+cron_schedule = 0 * * * *
+dispatch.earliest_time = -60m
+dispatch.latest_time = now
+action.correlationsearch.enabled = 1
+action.correlationsearch.label = ESCU - Correlation by User and Risk - Rule
+action.correlationsearch.annotations = {"analytic_story": ["Dev Sec Ops"], "cis20": ["CIS 13"], "confidence": 100, "impact": 70, "kill_chain_phases": ["Actions on Objectives"], "mitre_attack": ["T1204.003"], "nist": ["PR.DS", "PR.AC", "DE.CM"], "observable": [{"name": "user", "role": ["Victim"], "type": "User"}]}
+schedule_window = auto
+action.slack = 1
+action.slack.param.channel = dev_sec_ops_analytics
+action.slack.param.message = Alert Correlation by User and Risk
+alert.digest_mode = 1
+disabled = true
+enableSched = 1
+allow_skew = 100%
+counttype = number of events
+relation = greater than
+quantity = 0
+realtime_schedule = 0
+is_visible = false
+search = `signals` | fillnull | stats sum(risk_score) as risk_score values(source) as signals values(repository) as repository by user | sort - risk_score | where risk_score > 80 | `correlation_by_user_and_risk_filter` | eval risk_score=70 | eval mitre_attack_id=T1204.003 | collect index=alerts
 
 [ESCU - GSuite Email Suspicious Attachment - Rule]
 action.escu = 0
@@ -285,7 +418,7 @@ action.escu.providing_technologies = []
 action.escu.analytic_story = ["DevSecOps"]
 action.risk = 1
 action.risk.param._risk_message = suspicious email from $source.address$ to $destination{}.address$
-action.risk.param._risk = [{"risk_object_field": "source.address", "risk_object_type": "user", "risk_score": 49}]
+action.risk.param._risk = [{"risk_object_field": "source.address", "risk_object_type": "user", "risk_score": 49}, {"risk_object_field": "destination{}.address", "risk_object_type": "user", "risk_score": 49}]
 action.risk.param.verbose = 0
 cron_schedule = 0 * * * *
 dispatch.earliest_time = -70m@m
@@ -308,7 +441,185 @@ relation = greater than
 quantity = 0
 realtime_schedule = 0
 is_visible = false
-search = `gsuite_gmail` "attachment{}.file_extension_type" IN ("pl", "py", "rb", "sh", "bat", "exe", "dll", "cpl", "com", "js", "vbs", "ps1", "reg","swf", "cmd", "go") | eval phase="plan" | stats count min(_time) as firstTime max(_time) as lastTime values(attachment{}.file_extension_type) as email_attachments, values(attachment{}.sha256) as attachment_sha256, values(payload_size) as payload_size by destination{}.service num_message_attachments  subject destination{}.address source.address phase | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `gsuite_email_suspicious_attachment_filter` | collect index=findings
+search = `gsuite_gmail` "attachment{}.file_extension_type" IN ("pl", "py", "rb", "sh", "bat", "exe", "dll", "cpl", "com", "js", "vbs", "ps1", "reg","swf", "cmd", "go") | eval phase="plan" | eval severity="medium" | stats count min(_time) as firstTime max(_time) as lastTime values(attachment{}.file_extension_type) as email_attachments, values(attachment{}.sha256) as attachment_sha256, values(payload_size) as payload_size by destination{}.service num_message_attachments  subject destination{}.address source.address phase severity | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `gsuite_email_suspicious_attachment_filter` | eval risk_score=49 | eval mitre_attack_id=T1566.001 | collect index=signals
+
+[ESCU - GitHub Dependabot Alert - Rule]
+action.escu = 0
+action.escu.enabled = 1
+description = This search looks for Dependabot Alerts in Github logs.
+action.escu.mappings = {"cis20": ["CIS 13"], "kill_chain_phases": ["Actions on Objectives"], "mitre_attack": ["T1195.001"], "nist": ["PR.DS", "PR.AC", "DE.CM"]}
+action.escu.data_models = []
+action.escu.eli5 = This search looks for Dependabot Alerts in Github logs.
+action.escu.how_to_implement = You must index GitHub logs. You can follow the url in reference to onboard GitHub logs.
+action.escu.known_false_positives = unknown
+action.escu.creation_date = 2021-09-01
+action.escu.modification_date = 2021-09-01
+action.escu.confidence = high
+action.escu.full_search_name = ESCU - GitHub Dependabot Alert - Rule
+action.escu.search_type = detection
+action.escu.product = ["Splunk Enterprise", "Splunk Enterprise Security", "Splunk Cloud", "Dev Sec Ops Analytics"]
+action.escu.providing_technologies = []
+action.escu.analytic_story = ["Dev Sec Ops"]
+action.risk = 1
+action.risk.param._risk_message = Vulnerabilities found in packages used by GitHub repository $repository$
+action.risk.param._risk = [{"threat_object_field": "repository", "threat_object_type": "system"}]
+action.risk.param.verbose = 0
+cron_schedule = 0 * * * *
+dispatch.earliest_time = -70m@m
+dispatch.latest_time = -10m@m
+action.correlationsearch.enabled = 1
+action.correlationsearch.label = ESCU - GitHub Dependabot Alert - Rule
+action.correlationsearch.annotations = {"analytic_story": ["Dev Sec Ops"], "cis20": ["CIS 13"], "confidence": 90, "impact": 30, "kill_chain_phases": ["Actions on Objectives"], "mitre_attack": ["T1195.001"], "nist": ["PR.DS", "PR.AC", "DE.CM"], "observable": [{"name": "repository", "role": ["Victim"], "type": "System"}]}
+schedule_window = auto
+action.notable = 1
+action.notable.param.nes_fields = ['user']
+action.notable.param.rule_description = This search looks for Dependabot Alerts in Github logs.
+action.notable.param.rule_title = GitHub Dependabot Alert
+action.notable.param.security_domain = network
+action.notable.param.severity = high
+alert.digest_mode = 1
+disabled = true
+enableSched = 1
+allow_skew = 100%
+counttype = number of events
+relation = greater than
+quantity = 0
+realtime_schedule = 0
+is_visible = false
+search = `github` alert.id=* action=create | rename repository.full_name as repository, repository.html_url as repository_url sender.login as user | stats min(_time) as firstTime max(_time) as lastTime by action alert.affected_package_name alert.affected_range alert.created_at alert.external_identifier alert.external_reference alert.fixed_in alert.severity repository repository_url user | eval phase="code" | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `github_dependabot_alert_filter` | eval risk_score=27 | eval mitre_attack_id=T1195.001 | collect index=signals
+
+[ESCU - GitHub Pull Request from Unknown User - Rule]
+action.escu = 0
+action.escu.enabled = 1
+description = This search looks for Pull Request from unknown user.
+action.escu.mappings = {"cis20": ["CIS 13"], "kill_chain_phases": ["Actions on Objectives"], "mitre_attack": ["T1195.001"], "nist": ["PR.DS", "PR.AC", "DE.CM"]}
+action.escu.data_models = []
+action.escu.eli5 = This search looks for Pull Request from unknown user.
+action.escu.how_to_implement = You must index GitHub logs. You can follow the url in reference to onboard GitHub logs.
+action.escu.known_false_positives = unknown
+action.escu.creation_date = 2021-09-01
+action.escu.modification_date = 2021-09-01
+action.escu.confidence = high
+action.escu.full_search_name = ESCU - GitHub Pull Request from Unknown User - Rule
+action.escu.search_type = detection
+action.escu.product = ["Splunk Enterprise", "Splunk Enterprise Security", "Splunk Cloud", "Dev Sec Ops Analytics"]
+action.escu.providing_technologies = []
+action.escu.analytic_story = ["Dev Sec Ops"]
+action.risk = 1
+action.risk.param._risk_message = Vulnerabilities found in packages used by GitHub repository $repository$
+action.risk.param._risk = [{"threat_object_field": "repository", "threat_object_type": "system"}]
+action.risk.param.verbose = 0
+cron_schedule = 0 * * * *
+dispatch.earliest_time = -70m@m
+dispatch.latest_time = -10m@m
+action.correlationsearch.enabled = 1
+action.correlationsearch.label = ESCU - GitHub Pull Request from Unknown User - Rule
+action.correlationsearch.annotations = {"analytic_story": ["Dev Sec Ops"], "cis20": ["CIS 13"], "confidence": 90, "impact": 30, "kill_chain_phases": ["Actions on Objectives"], "mitre_attack": ["T1195.001"], "nist": ["PR.DS", "PR.AC", "DE.CM"], "observable": [{"name": "repository", "role": ["Victim"], "type": "System"}]}
+schedule_window = auto
+action.notable = 1
+action.notable.param.nes_fields = ['user']
+action.notable.param.rule_description = This search looks for Pull Request from unknown user.
+action.notable.param.rule_title = GitHub Pull Request from Unknown User
+action.notable.param.security_domain = network
+action.notable.param.severity = high
+alert.digest_mode = 1
+disabled = true
+enableSched = 1
+allow_skew = 100%
+counttype = number of events
+relation = greater than
+quantity = 0
+realtime_schedule = 0
+is_visible = false
+search = `github` check_suite.pull_requests{}.id=* | stats count by check_suite.head_commit.author.name repository.full_name check_suite.pull_requests{}.head.ref check_suite.head_commit.message | rename check_suite.head_commit.author.name as user repository.full_name as repository check_suite.pull_requests{}.head.ref as ref_head check_suite.head_commit.message as commit_message | search NOT `github_known_users` | eval phase="code" | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `github_pull_request_from_unknown_user_filter` | eval risk_score=27 | eval mitre_attack_id=T1195.001 | collect index=signals
+
+[ESCU - Github Commit Changes In Master - Rule]
+action.escu = 0
+action.escu.enabled = 1
+description = This search is to detect a pushed or commit to master or main branch. This is to avoid unwanted modification to master without a review to the changes. Ideally in terms of devsecops the changes made in a branch and do a PR for review. of course in some cases admin of the project may did a changes directly to master branch
+action.escu.mappings = {"kill_chain_phases": ["Exploitation"], "mitre_attack": ["T1199"]}
+action.escu.data_models = []
+action.escu.eli5 = This search is to detect a pushed or commit to master or main branch. This is to avoid unwanted modification to master without a review to the changes. Ideally in terms of devsecops the changes made in a branch and do a PR for review. of course in some cases admin of the project may did a changes directly to master branch
+action.escu.how_to_implement = To successfully implement this search, you need to be ingesting logs related to github logs having the fork, commit, push metadata that can be use to monitor the changes in a github project.
+action.escu.known_false_positives = admin can do changes directly to master branch
+action.escu.creation_date = 2021-08-20
+action.escu.modification_date = 2021-08-20
+action.escu.confidence = high
+action.escu.full_search_name = ESCU - Github Commit Changes In Master - Rule
+action.escu.search_type = detection
+action.escu.product = ["Splunk Enterprise", "Splunk Enterprise Security", "Splunk Cloud", "Dev Sec Ops Analytics"]
+action.escu.providing_technologies = []
+action.escu.analytic_story = ["DevSecOps"]
+action.risk = 1
+action.risk.param._risk_message = suspicious commit by $commit.commit.author.email$ to main branch
+action.risk.param._risk = [{"risk_object_field": "commit.commit.author.email", "risk_object_type": "user", "risk_score": 9}]
+action.risk.param.verbose = 0
+cron_schedule = 0 * * * *
+dispatch.earliest_time = -70m@m
+dispatch.latest_time = -10m@m
+action.correlationsearch.enabled = 1
+action.correlationsearch.label = ESCU - Github Commit Changes In Master - Rule
+action.correlationsearch.annotations = {"analytic_story": ["DevSecOps"], "confidence": 30, "context": ["Source:Endpoint", "Stage:Reconnaissance"], "impact": 30, "kill_chain_phases": ["Exploitation"], "mitre_attack": ["T1199"], "observable": [{"name": "commit.commit.author.email", "role": ["attacker"], "type": "User"}]}
+schedule_window = auto
+action.notable = 1
+action.notable.param.rule_description = This search is to detect a pushed or commit to master or main branch. This is to avoid unwanted modification to master without a review to the changes. Ideally in terms of devsecops the changes made in a branch and do a PR for review. of course in some cases admin of the project may did a changes directly to master branch
+action.notable.param.rule_title = Github Commit Changes In Master
+action.notable.param.security_domain = endpoint
+action.notable.param.severity = high
+alert.digest_mode = 1
+disabled = true
+enableSched = 1
+allow_skew = 100%
+counttype = number of events
+relation = greater than
+quantity = 0
+realtime_schedule = 0
+is_visible = false
+search = `github` branches{}.name = main OR branches{}.name = master | eval severity="low" | eval phase="code" |  stats count min(_time) as firstTime max(_time) as lastTime  by commit.author.html_url commit.commit.author.email commit.author.login commit.commit.message repository.pushed_at commit.commit.committer.date, phase, severity | eval phase="code" | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `github_commit_changes_in_master_filter` | eval risk_score=9 | eval mitre_attack_id=T1199 | collect index=signals
+
+[ESCU - Github Commit In Develop - Rule]
+action.escu = 0
+action.escu.enabled = 1
+description = This search is to detect a pushed or commit to develop branch. This is to avoid unwanted modification to develop without a review to the changes. Ideally in terms of devsecops the changes made in a branch and do a PR for review. of course in some cases admin of the project may did a changes directly to master branch
+action.escu.mappings = {"kill_chain_phases": ["Exploitation"], "mitre_attack": ["T1199"]}
+action.escu.data_models = []
+action.escu.eli5 = This search is to detect a pushed or commit to develop branch. This is to avoid unwanted modification to develop without a review to the changes. Ideally in terms of devsecops the changes made in a branch and do a PR for review. of course in some cases admin of the project may did a changes directly to master branch
+action.escu.how_to_implement = To successfully implement this search, you need to be ingesting logs related to github logs having the fork, commit, push metadata that can be use to monitor the changes in a github project.
+action.escu.known_false_positives = admin can do changes directly to develop branch
+action.escu.creation_date = 2021-09-01
+action.escu.modification_date = 2021-09-01
+action.escu.confidence = high
+action.escu.full_search_name = ESCU - Github Commit In Develop - Rule
+action.escu.search_type = detection
+action.escu.product = ["Splunk Enterprise", "Splunk Enterprise Security", "Splunk Cloud", "Dev Sec Ops Analytics"]
+action.escu.providing_technologies = []
+action.escu.analytic_story = ["DevSecOps"]
+action.risk = 1
+action.risk.param._risk_message = suspicious commit by $commit.commit.author.email$ to develop branch
+action.risk.param._risk = [{"risk_object_field": "commit.commit.author.email", "risk_object_type": "user", "risk_score": 9}]
+action.risk.param.verbose = 0
+cron_schedule = 0 * * * *
+dispatch.earliest_time = -70m@m
+dispatch.latest_time = -10m@m
+action.correlationsearch.enabled = 1
+action.correlationsearch.label = ESCU - Github Commit In Develop - Rule
+action.correlationsearch.annotations = {"analytic_story": ["DevSecOps"], "confidence": 30, "context": ["Source:Endpoint", "Stage:Reconnaissance"], "impact": 30, "kill_chain_phases": ["Exploitation"], "mitre_attack": ["T1199"], "observable": [{"name": "commit.commit.author.email", "role": ["attacker"], "type": "User"}]}
+schedule_window = auto
+action.notable = 1
+action.notable.param.rule_description = This search is to detect a pushed or commit to develop branch. This is to avoid unwanted modification to develop without a review to the changes. Ideally in terms of devsecops the changes made in a branch and do a PR for review. of course in some cases admin of the project may did a changes directly to master branch
+action.notable.param.rule_title = Github Commit In Develop
+action.notable.param.security_domain = endpoint
+action.notable.param.severity = high
+alert.digest_mode = 1
+disabled = true
+enableSched = 1
+allow_skew = 100%
+counttype = number of events
+relation = greater than
+quantity = 0
+realtime_schedule = 0
+is_visible = false
+search = `github` branches{}.name = main OR branches{}.name = develop |  stats count min(_time) as firstTime max(_time) as lastTime  by commit.author.html_url commit.commit.author.email commit.author.login commit.commit.message repository.pushed_at commit.commit.committer.date | eval phase="code" | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `github_commit_in_develop_filter` | eval risk_score=9 | eval mitre_attack_id=T1199 | collect index=signals
 
 [ESCU - Gsuite Drive Share In External Email - Rule]
 action.escu = 0
@@ -329,20 +640,21 @@ action.escu.providing_technologies = []
 action.escu.analytic_story = ["DevSecOps"]
 action.risk = 1
 action.risk.param._risk_message = suspicious share gdrive from $parameters.owner$ to $email$ namely as $parameters.doc_title$
-action.risk.param._risk = [{"risk_object_field": "parameters.owner", "risk_object_type": "user", "risk_score": 9}]
+action.risk.param._risk = [{"risk_object_field": "parameters.owner", "risk_object_type": "user", "risk_score": 9}, {"risk_object_field": "email", "risk_object_type": "user", "risk_score": 9}]
 action.risk.param.verbose = 0
 cron_schedule = 0 * * * *
-dispatch.earliest_time = -70m@m
-dispatch.latest_time = -10m@m
+dispatch.earliest_time = -60m
+dispatch.latest_time = now
 action.correlationsearch.enabled = 1
 action.correlationsearch.label = ESCU - Gsuite Drive Share In External Email - Rule
 action.correlationsearch.annotations = {"analytic_story": ["DevSecOps"], "confidence": 30, "context": ["Source:Endpoint", "Stage:Reconnaissance"], "impact": 30, "kill_chain_phases": ["Exfiltration"], "mitre_attack": ["T1567.002"], "observable": [{"name": "parameters.owner", "role": ["attacker"], "type": "User"}, {"name": "email", "role": ["Victim"], "type": "User"}]}
 schedule_window = auto
-action.notable = 1
-action.notable.param.rule_description = This search is to detect suspicious google drive or google docs files shared outside or externally. This behavior might be a good hunting query to monitor exfitration of data made by an attacker or insider to a targetted machine.
-action.notable.param.rule_title = Gsuite Drive Share In External Email
-action.notable.param.security_domain = endpoint
-action.notable.param.severity = high
+action.sendtophantom = 1
+action.sendtophantom.param._cam_workers = local
+action.sendtophantom.param.label = events
+action.sendtophantom.param.phantom_server = events
+action.sendtophantom.param.sensitivity = amber
+action.sendtophantom.param.severity = medium
 alert.digest_mode = 1
 disabled = true
 enableSched = 1
@@ -352,7 +664,7 @@ relation = greater than
 quantity = 0
 realtime_schedule = 0
 is_visible = false
-search = `gsuite_drive` NOT (email IN("", "null")) | rex field=parameters.owner "[^@]+@(?<src_domain>[^@]+)" | rex field=email "[^@]+@(?<dest_domain>[^@]+)" | where src_domain = "internal_test_email.com" and not dest_domain = "internal_test_email.com" | eval phase="plan" | stats values(parameters.doc_title) as doc_title, values(parameters.doc_type) as doc_types, values(email) as dst_email_list, values(parameters.visibility) as visibility, count min(_time) as firstTime max(_time) as lastTime by parameters.owner phase | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `gsuite_drive_share_in_external_email_filter` | collect index=findings
+search = `gsuite_drive` NOT (email IN("", "null")) | rex field=parameters.owner "[^@]+@(?<src_domain>[^@]+)" | rex field=email "[^@]+@(?<dest_domain>[^@]+)" | where src_domain = "internal_test_email.com" and not dest_domain = "internal_test_email.com" | eval phase="plan" | eval severity="low" | stats values(parameters.doc_title) as doc_title, values(parameters.doc_type) as doc_types, values(email) as dst_email_list, values(parameters.visibility) as visibility, values(parameters.doc_id) as doc_id, count min(_time) as firstTime max(_time) as lastTime by parameters.owner phase severity | rename parameters.owner as user | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `gsuite_drive_share_in_external_email_filter` | eval risk_score=9 | eval mitre_attack_id=T1567.002 | collect index=signals
 
 [ESCU - Gsuite Email Suspicious Subject With Attachment - Rule]
 action.escu = 0
@@ -373,14 +685,14 @@ action.escu.providing_technologies = []
 action.escu.analytic_story = ["DevSecOps"]
 action.risk = 1
 action.risk.param._risk_message = suspicious email from $source.address$ to $destination{}.address$
-action.risk.param._risk = [{"risk_object_field": "source.address", "risk_object_type": "user", "risk_score": 25}]
+action.risk.param._risk = []
 action.risk.param.verbose = 0
 cron_schedule = 0 * * * *
 dispatch.earliest_time = -70m@m
 dispatch.latest_time = -10m@m
 action.correlationsearch.enabled = 1
 action.correlationsearch.label = ESCU - Gsuite Email Suspicious Subject With Attachment - Rule
-action.correlationsearch.annotations = {"analytic_story": ["DevSecOps"], "confidence": 50, "context": ["Source:Endpoint", "Stage:Reconnaissance"], "impact": 50, "kill_chain_phases": ["Exploitation"], "mitre_attack": ["T1566.001"], "observable": [{"name": "source.address", "role": ["attacker"], "type": "User"}, {"name": "destination{}.address", "role": ["Victim"], "type": "User"}]}
+action.correlationsearch.annotations = {"analytic_story": ["DevSecOps"], "confidence": 50, "context": ["Source:Endpoint", "Stage:Reconnaissance"], "impact": 50, "kill_chain_phases": ["Exploitation"], "mitre_attack": ["T1566.001"]}
 schedule_window = auto
 action.notable = 1
 action.notable.param.rule_description = This search is to detect a gsuite email contains suspicious subject having known file type used in spear phishing. This technique is a common and effective entry vector of attacker to compromise a network by luring the user to click or execute the suspicious attachment send from external email account because of the effective social engineering of subject related to delivery, bank and so on. On the other hand this detection may catch a normal email traffic related to legitimate transaction so better to check the email sender, spelling and etc. avoid click link or opening the attachment if you are not expecting this type of e-mail.
@@ -396,7 +708,7 @@ relation = greater than
 quantity = 0
 realtime_schedule = 0
 is_visible = false
-search = `gsuite_gmail` num_message_attachments > 0 subject IN ("*dhl*", "* ups *", "*delivery*", "*parcel*", "*label*", "*invoice*", "*postal*", "* fedex *", "* usps *", "* express *", "*shipment*", "*Banking/Tax*","*shipment*", "*new order*") attachment{}.file_extension_type IN ("doc", "docx", "xls", "xlsx", "ppt", "pptx", "pdf", "zip", "rar", "html","htm","hta") | rex field=source.from_header_address "[^@]+@(?<source_domain>[^@]+)" | rex field=destination{}.address "[^@]+@(?<dest_domain>[^@]+)" | where not source_domain="internal_test_email.com" and dest_domain="internal_test_email.com" | eval phase="plan" | stats count min(_time) as firstTime max(_time) as lastTime values(attachment{}.file_extension_type) as email_attachments, values(attachment{}.sha256) as attachment_sha256, values(payload_size) as payload_size by destination{}.service num_message_attachments  subject destination{}.address source.address plan | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `gsuite_email_suspicious_subject_with_attachment_filter` | collect index=findings
+search = `gsuite_gmail` num_message_attachments > 0 subject IN ("*dhl*", "* ups *", "*delivery*", "*parcel*", "*label*", "*invoice*", "*postal*", "* fedex *", "* usps *", "* express *", "*shipment*", "*Banking/Tax*","*shipment*", "*new order*") attachment{}.file_extension_type IN ("doc", "docx", "xls", "xlsx", "ppt", "pptx", "pdf", "zip", "rar", "html","htm","hta") | rex field=source.from_header_address "[^@]+@(?<source_domain>[^@]+)" | rex field=destination{}.address "[^@]+@(?<dest_domain>[^@]+)" | where not source_domain="internal_test_email.com" and dest_domain="internal_test_email.com" | eval phase="plan" | eval severity="medium" | stats count min(_time) as firstTime max(_time) as lastTime values(attachment{}.file_extension_type) as email_attachments, values(attachment{}.sha256) as attachment_sha256, values(payload_size) as payload_size by destination{}.service num_message_attachments  subject destination{}.address source.address phase severity | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `gsuite_email_suspicious_subject_with_attachment_filter` | eval risk_score=25 | eval mitre_attack_id=T1566.001 | collect index=signals
 
 [ESCU - Gsuite Email With Known Abuse Web Service Link - Rule]
 action.escu = 0
@@ -417,14 +729,14 @@ action.escu.providing_technologies = []
 action.escu.analytic_story = ["DevSecOps"]
 action.risk = 1
 action.risk.param._risk_message = suspicious email from $source.address$ to $destination{}.address$
-action.risk.param._risk = [{"risk_object_field": "source.address", "risk_object_type": "user", "risk_score": 25}]
+action.risk.param._risk = []
 action.risk.param.verbose = 0
 cron_schedule = 0 * * * *
 dispatch.earliest_time = -70m@m
 dispatch.latest_time = -10m@m
 action.correlationsearch.enabled = 1
 action.correlationsearch.label = ESCU - Gsuite Email With Known Abuse Web Service Link - Rule
-action.correlationsearch.annotations = {"analytic_story": ["DevSecOps"], "confidence": 50, "context": ["Source:Endpoint", "Stage:Reconnaissance"], "impact": 50, "kill_chain_phases": ["Exploitation"], "mitre_attack": ["T1566.001"], "observable": [{"name": "source.address", "role": ["attacker"], "type": "User"}, {"name": "destination{}.address", "role": ["Victim"], "type": "User"}]}
+action.correlationsearch.annotations = {"analytic_story": ["DevSecOps"], "confidence": 50, "context": ["Source:Endpoint", "Stage:Reconnaissance"], "impact": 50, "kill_chain_phases": ["Exploitation"], "mitre_attack": ["T1566.001"]}
 schedule_window = auto
 action.notable = 1
 action.notable.param.rule_description = This analytics is to detect a gmail containing a link that are known to be abused by malware or attacker like pastebin, telegram and discord to deliver malicious payload. This event can encounter some normal email traffic within organization and external email that normally using this application and services.
@@ -440,7 +752,7 @@ relation = greater than
 quantity = 0
 realtime_schedule = 0
 is_visible = false
-search = `gsuite_gmail` "link_domain{}" IN ("*pastebin.com*", "*discord*", "*telegram*","t.me") | rex field=source.from_header_address "[^@]+@(?<source_domain>[^@]+)" | rex field=destination{}.address "[^@]+@(?<dest_domain>[^@]+)" | where not source_domain="internal_test_email.com" and dest_domain="internal_test_email.com" | eval phase="plan" |stats values(link_domain{}) as link_domains min(_time) as firstTime max(_time) as lastTime count by is_spam source.address source.from_header_address subject destination{}.address phase | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `gsuite_email_with_known_abuse_web_service_link_filter` | collect index=findings
+search = `gsuite_gmail` "link_domain{}" IN ("*pastebin.com*", "*discord*", "*telegram*","t.me") | rex field=source.from_header_address "[^@]+@(?<source_domain>[^@]+)" | rex field=destination{}.address "[^@]+@(?<dest_domain>[^@]+)" | where not source_domain="internal_test_email.com" and dest_domain="internal_test_email.com" | eval phase="plan" | eval severity="low" |stats values(link_domain{}) as link_domains min(_time) as firstTime max(_time) as lastTime count by is_spam source.address source.from_header_address subject destination{}.address phase severity | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `gsuite_email_with_known_abuse_web_service_link_filter` | eval risk_score=25 | eval mitre_attack_id=T1566.001 | collect index=signals
 
 [ESCU - Gsuite Outbound Email With Attachment To External Domain - Rule]
 action.escu = 0
@@ -461,7 +773,7 @@ action.escu.providing_technologies = []
 action.escu.analytic_story = ["DevSecOps"]
 action.risk = 1
 action.risk.param._risk_message = suspicious email from $source.address$ to $destination{}.address$
-action.risk.param._risk = [{"risk_object_field": "source.address", "risk_object_type": "user", "risk_score": 9}]
+action.risk.param._risk = [{"risk_object_field": "source.address", "risk_object_type": "user", "risk_score": 9}, {"risk_object_field": "destination{}.address", "risk_object_type": "user", "risk_score": 9}]
 action.risk.param.verbose = 0
 cron_schedule = 0 * * * *
 dispatch.earliest_time = -70m@m
@@ -484,7 +796,7 @@ relation = greater than
 quantity = 0
 realtime_schedule = 0
 is_visible = false
-search = `gsuite_gmail` num_message_attachments > 0 | rex field=source.from_header_address "[^@]+@(?<source_domain>[^@]+)" | rex field=destination{}.address "[^@]+@(?<dest_domain>[^@]+)" | where source_domain="internal_test_email.com" and not dest_domain="internal_test_email.com" | eval phase="plan" | stats values(subject) as subject, values(source.from_header_address) as src_domain_list, count as numEvents, dc(source.from_header_address) as numSrcAddresses, min(_time) as firstTime max(_time) as lastTime by dest_domain phase | where numSrcAddresses < 20 |sort - numSrcAddresses | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `gsuite_outbound_email_with_attachment_to_external_domain_filter` | collect index=findings
+search = `gsuite_gmail` num_message_attachments > 0 | rex field=source.from_header_address "[^@]+@(?<source_domain>[^@]+)" | rex field=destination{}.address "[^@]+@(?<dest_domain>[^@]+)" | where source_domain="internal_test_email.com" and not dest_domain="internal_test_email.com" | eval phase="plan" | eval severity="low" | stats values(subject) as subject, values(source.from_header_address) as src_domain_list, count as numEvents, dc(source.from_header_address) as numSrcAddresses, min(_time) as firstTime max(_time) as lastTime by dest_domain phase severity | where numSrcAddresses < 20 |sort - numSrcAddresses | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `gsuite_outbound_email_with_attachment_to_external_domain_filter` | eval risk_score=9 | eval mitre_attack_id=T1048.003 | collect index=signals
 
 [ESCU - Gsuite Suspicious Shared File Name - Rule]
 action.escu = 0
@@ -505,7 +817,7 @@ action.escu.providing_technologies = []
 action.escu.analytic_story = ["DevSecOps"]
 action.risk = 1
 action.risk.param._risk_message = suspicious share gdrive from $parameters.owner$ to $email$ namely as $parameters.doc_title$
-action.risk.param._risk = [{"risk_object_field": "parameters.owner", "risk_object_type": "user", "risk_score": 9}]
+action.risk.param._risk = [{"risk_object_field": "parameters.owner", "risk_object_type": "user", "risk_score": 9}, {"risk_object_field": "email", "risk_object_type": "user", "risk_score": 9}]
 action.risk.param.verbose = 0
 cron_schedule = 0 * * * *
 dispatch.earliest_time = -70m@m
@@ -529,7 +841,7 @@ relation = greater than
 quantity = 0
 realtime_schedule = 0
 is_visible = false
-search = `gsuite_drive` parameters.owner_is_team_drive=false "parameters.doc_title" IN ("*dhl*", "* ups *", "*delivery*", "*parcel*", "*label*", "*invoice*", "*postal*", "*fedex*", "* usps *", "* express *", "*shipment*", "*Banking/Tax*","*shipment*", "*new order*") parameters.doc_type IN ("document","pdf", "msexcel", "msword", "spreadsheet", "presentation") | rex field=parameters.owner "[^@]+@(?<source_domain>[^@]+)" | rex field=parameters.target_user "[^@]+@(?<dest_domain>[^@]+)" | where not source_domain="internal_test_email.com" and dest_domain="internal_test_email.com" | eval phase="plan" | stats count min(_time) as firstTime max(_time) as lastTime by email parameters.owner parameters.target_user parameters.doc_title parameters.doc_type phase | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `gsuite_suspicious_shared_file_name_filter` | collect index=findings
+search = `gsuite_drive` parameters.owner_is_team_drive=false "parameters.doc_title" IN ("*dhl*", "* ups *", "*delivery*", "*parcel*", "*label*", "*invoice*", "*postal*", "*fedex*", "* usps *", "* express *", "*shipment*", "*Banking/Tax*","*shipment*", "*new order*") parameters.doc_type IN ("document","pdf", "msexcel", "msword", "spreadsheet", "presentation") | rex field=parameters.owner "[^@]+@(?<source_domain>[^@]+)" | rex field=parameters.target_user "[^@]+@(?<dest_domain>[^@]+)" | where not source_domain="internal_test_email.com" and dest_domain="internal_test_email.com" | eval phase="plan" | eval severity="low" | stats count min(_time) as firstTime max(_time) as lastTime by email parameters.owner parameters.target_user parameters.doc_title parameters.doc_type phase severity | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `gsuite_suspicious_shared_file_name_filter` | eval risk_score=9 | eval mitre_attack_id=T1566.001 | collect index=signals
 
 [ESCU - Kubernetes Nginx Ingress LFI - Rule]
 action.escu = 0
@@ -553,17 +865,15 @@ action.risk.param._risk_message = Local File Inclusion Attack detected on $host$
 action.risk.param._risk = [{"risk_object_field": "src_ip", "risk_object_type": "system", "risk_score": 49}]
 action.risk.param.verbose = 0
 cron_schedule = 0 * * * *
-dispatch.earliest_time = -70m@m
-dispatch.latest_time = -10m@m
+dispatch.earliest_time = -60m
+dispatch.latest_time = now
 action.correlationsearch.enabled = 1
 action.correlationsearch.label = ESCU - Kubernetes Nginx Ingress LFI - Rule
 action.correlationsearch.annotations = {"analytic_story": ["Dev Sec Ops"], "cis20": ["CIS 13"], "confidence": 70, "impact": 70, "kill_chain_phases": ["Actions on Objectives"], "mitre_attack": ["T1212"], "nist": ["PR.DS", "PR.AC", "DE.CM"], "observable": [{"name": "src_ip", "role": ["Attacker"], "type": "IP Address"}]}
 schedule_window = auto
-action.notable = 1
-action.notable.param.rule_description = This search uses the Kubernetes logs from a nginx ingress controller to detect local file inclusion attacks.
-action.notable.param.rule_title = Kubernetes Nginx Ingress LFI
-action.notable.param.security_domain = network
-action.notable.param.severity = high
+action.slack = 1
+action.slack.param.channel = dev_sec_ops_analytics
+action.slack.param.message = Alert Kubernetes Nginx Ingress LFI
 alert.digest_mode = 1
 disabled = true
 enableSched = 1
@@ -573,7 +883,7 @@ relation = greater than
 quantity = 0
 realtime_schedule = 0
 is_visible = false
-search = `kubernetes_container_controller` | rex field=_raw "^(?<remote_addr>\S+)\s+-\s+-\s+\[(?<time_local>[^\]]*)\]\s\"(?<request>[^\"]*)\"\s(?<status>\S*)\s(?<body_bytes_sent>\S*)\s\"(?<http_referer>[^\"]*)\"\s\"(?<http_user_agent>[^\"]*)\"\s(?<request_length>\S*)\s(?<request_time>\S*)\s\[(?<proxy_upstream_name>[^\]]*)\]\s\[(?<proxy_alternative_upstream_name>[^\]]*)\]\s(?<upstream_addr>\S*)\s(?<upstream_response_length>\S*)\s(?<upstream_response_time>\S*)\s(?<upstream_status>\S*)\s(?<req_id>\S*)" | lookup local_file_inclusion_paths local_file_inclusion_paths AS request OUTPUT lfi_path | search lfi_path=yes | rename remote_addr AS src_ip, upstream_status as status, proxy_upstream_name as proxy | rex field=request "^(?<http_method>\S+)\s(?<url>\S+)\s" | eval phase="operate" | stats count min(_time) as firstTime max(_time) as lastTime by src_ip, status, url, http_method, host, http_user_agent, proxy, phase | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `kubernetes_nginx_ingress_lfi_filter` | collect index=findings
+search = `kubernetes_container_controller` | rex field=_raw "^(?<remote_addr>\S+)\s+-\s+-\s+\[(?<time_local>[^\]]*)\]\s\"(?<request>[^\"]*)\"\s(?<status>\S*)\s(?<body_bytes_sent>\S*)\s\"(?<http_referer>[^\"]*)\"\s\"(?<http_user_agent>[^\"]*)\"\s(?<request_length>\S*)\s(?<request_time>\S*)\s\[(?<proxy_upstream_name>[^\]]*)\]\s\[(?<proxy_alternative_upstream_name>[^\]]*)\]\s(?<upstream_addr>\S*)\s(?<upstream_response_length>\S*)\s(?<upstream_response_time>\S*)\s(?<upstream_status>\S*)\s(?<req_id>\S*)" | lookup local_file_inclusion_paths local_file_inclusion_paths AS request OUTPUT lfi_path | search lfi_path=yes | rename remote_addr AS src_ip, upstream_status as status, proxy_upstream_name as proxy | rex field=request "^(?<http_method>\S+)\s(?<url>\S+)\s" | eval phase="operate" | eval severity="high" | stats count min(_time) as firstTime max(_time) as lastTime by src_ip, status, url, http_method, host, http_user_agent, proxy, phase, severity | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `kubernetes_nginx_ingress_lfi_filter` | eval risk_score=49 | eval mitre_attack_id=T1212 | collect index=alerts
 
 [ESCU - Kubernetes Nginx Ingress RFI - Rule]
 action.escu = 0
@@ -597,17 +907,15 @@ action.risk.param._risk_message = Remote File Inclusion Attack detected on $host
 action.risk.param._risk = [{"risk_object_field": "src_ip", "risk_object_type": "system", "risk_score": 49}]
 action.risk.param.verbose = 0
 cron_schedule = 0 * * * *
-dispatch.earliest_time = -70m@m
-dispatch.latest_time = -10m@m
+dispatch.earliest_time = -60m
+dispatch.latest_time = now
 action.correlationsearch.enabled = 1
 action.correlationsearch.label = ESCU - Kubernetes Nginx Ingress RFI - Rule
 action.correlationsearch.annotations = {"analytic_story": ["Dev Sec Ops"], "cis20": ["CIS 13"], "confidence": 70, "impact": 70, "kill_chain_phases": ["Actions on Objectives"], "mitre_attack": ["T1212"], "nist": ["PR.DS", "PR.AC", "DE.CM"], "observable": [{"name": "src_ip", "role": ["Attacker"], "type": "IP Address"}]}
 schedule_window = auto
-action.notable = 1
-action.notable.param.rule_description = This search uses the Kubernetes logs from a nginx ingress controller to detect remote file inclusion attacks.
-action.notable.param.rule_title = Kubernetes Nginx Ingress RFI
-action.notable.param.security_domain = network
-action.notable.param.severity = high
+action.slack = 1
+action.slack.param.channel = dev_sec_ops_analytics
+action.slack.param.message = Alert Kubernetes Nginx Ingress RFI
 alert.digest_mode = 1
 disabled = true
 enableSched = 1
@@ -617,7 +925,7 @@ relation = greater than
 quantity = 0
 realtime_schedule = 0
 is_visible = false
-search = `kubernetes_container_controller` | rex field=_raw "^(?<remote_addr>\S+)\s+-\s+-\s+\[(?<time_local>[^\]]*)\]\s\"(?<request>[^\"]*)\"\s(?<status>\S*)\s(?<body_bytes_sent>\S*)\s\"(?<http_referer>[^\"]*)\"\s\"(?<http_user_agent>[^\"]*)\"\s(?<request_length>\S*)\s(?<request_time>\S*)\s\[(?<proxy_upstream_name>[^\]]*)\]\s\[(?<proxy_alternative_upstream_name>[^\]]*)\]\s(?<upstream_addr>\S*)\s(?<upstream_response_length>\S*)\s(?<upstream_response_time>\S*)\s(?<upstream_status>\S*)\s(?<req_id>\S*)" | rex field=request "^(?<http_method>\S+)?\s(?<url>\S+)\s" | rex field=url "(?<dest_ip>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})" | search dest_ip=* | rename remote_addr AS src_ip, upstream_status as status, proxy_upstream_name as proxy | eval phase="operate" | stats count min(_time) as firstTime max(_time) as lastTime by src_ip, dest_ip status, url, http_method, host, http_user_agent, proxy, phase | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `kubernetes_nginx_ingress_rfi_filter` | collect index=findings
+search = `kubernetes_container_controller` | rex field=_raw "^(?<remote_addr>\S+)\s+-\s+-\s+\[(?<time_local>[^\]]*)\]\s\"(?<request>[^\"]*)\"\s(?<status>\S*)\s(?<body_bytes_sent>\S*)\s\"(?<http_referer>[^\"]*)\"\s\"(?<http_user_agent>[^\"]*)\"\s(?<request_length>\S*)\s(?<request_time>\S*)\s\[(?<proxy_upstream_name>[^\]]*)\]\s\[(?<proxy_alternative_upstream_name>[^\]]*)\]\s(?<upstream_addr>\S*)\s(?<upstream_response_length>\S*)\s(?<upstream_response_time>\S*)\s(?<upstream_status>\S*)\s(?<req_id>\S*)" | rex field=request "^(?<http_method>\S+)?\s(?<url>\S+)\s" | rex field=url "(?<dest_ip>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})" | search dest_ip=* | rename remote_addr AS src_ip, upstream_status as status, proxy_upstream_name as proxy | eval phase="operate" | eval severity="medium" | stats count min(_time) as firstTime max(_time) as lastTime by src_ip, dest_ip status, url, http_method, host, http_user_agent, proxy, phase, severity | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `kubernetes_nginx_ingress_rfi_filter` | eval risk_score=49 | eval mitre_attack_id=T1212 | collect index=alerts
 
 [ESCU - Kubernetes Scanner Image Pulling - Rule]
 action.escu = 0
@@ -641,17 +949,15 @@ action.risk.param._risk_message = Kubernetes Scanner image pulled on host $host$
 action.risk.param._risk = [{"threat_object_field": "host", "threat_object_type": "entity"}]
 action.risk.param.verbose = 0
 cron_schedule = 0 * * * *
-dispatch.earliest_time = -70m@m
-dispatch.latest_time = -10m@m
+dispatch.earliest_time = -60m
+dispatch.latest_time = now
 action.correlationsearch.enabled = 1
 action.correlationsearch.label = ESCU - Kubernetes Scanner Image Pulling - Rule
-action.correlationsearch.annotations = {"analytic_story": ["Dev Sec Ops"], "cis20": ["CIS 13"], "confidence": 70, "impact": 70, "kill_chain_phases": ["Actions on Objectives"], "mitre_attack": ["T1526"], "nist": ["PR.DS", "PR.AC", "DE.CM"], "observable": [{"name": "host", "type": "Entity"}]}
+action.correlationsearch.annotations = {"analytic_story": ["Dev Sec Ops"], "cis20": ["CIS 13"], "confidence": 90, "impact": 90, "kill_chain_phases": ["Actions on Objectives"], "mitre_attack": ["T1526"], "nist": ["PR.DS", "PR.AC", "DE.CM"], "observable": [{"name": "host", "type": "Entity"}]}
 schedule_window = auto
-action.notable = 1
-action.notable.param.rule_description = This search uses the Kubernetes logs from Splunk Connect from Kubernetes to detect Kubernetes Security Scanner.
-action.notable.param.rule_title = Kubernetes Scanner Image Pulling
-action.notable.param.security_domain = network
-action.notable.param.severity = high
+action.slack = 1
+action.slack.param.channel = dev_sec_ops_analytics
+action.slack.param.message = Alert Kubernetes Scanner Image Pulling
 alert.digest_mode = 1
 disabled = true
 enableSched = 1
@@ -661,7 +967,7 @@ relation = greater than
 quantity = 0
 realtime_schedule = 0
 is_visible = false
-search = `kube_objects_events` object.message IN ("Pulling image *kube-hunter*", "Pulling image *kube-bench*", "Pulling image *kube-recon*", "Pulling image *kube-recon*") | rename object.* AS * | rename involvedObject.* AS * | rename source.host AS host | eval phase="operate" | stats min(_time) as firstTime max(_time) as lastTime count by host, name, namespace, kind, reason, message, phase | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `kubernetes_scanner_image_pulling_filter` | collect index=findings
+search = `kube_objects_events` object.message IN ("Pulling image *kube-hunter*", "Pulling image *kube-bench*", "Pulling image *kube-recon*", "Pulling image *kube-recon*") | rename object.* AS * | rename involvedObject.* AS * | rename source.host AS host | eval phase="operate" | eval severity="high" | stats min(_time) as firstTime max(_time) as lastTime count by host, name, namespace, kind, reason, message, phase, severity | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)` | `kubernetes_scanner_image_pulling_filter` | eval risk_score=81 | eval mitre_attack_id=T1526 | collect index=alerts
 
 ### END ESCU DETECTIONS ###
 

dist/devsecops/default/searchbnf.conf

@@ -0,0 +1,4 @@
+[wget-command]
+syntax = [wget]
+shortdesc = [run wget on a given url]
+usage = public

dist/devsecops/default/transforms.conf

@@ -1,6 +1,6 @@
 #############
 # Automatically generated by generator.py in splunk/security_content
-# On Date: 2021-08-27T14:41:52 UTC
+# On Date: 2021-09-13T10:57:27 UTC
 # Author: Splunk Security Research
 # Contact: research@splunk.com
 #############
@@ -73,6 +73,10 @@ match_type = WILDCARD(dynamic_dns_domains)
 filename = escu_search_id.csv
 # description = A placeholder lookup file to hold information for ESCU Usage dashboard
 
+[images_to_repository]
+filename = images_to_repository.csv
+# description = Mapping images to repositories
+
 [is_suspicious_file_extension_lookup]
 filename = is_suspicious_file_extension_lookup.csv
 # description = A list of suspicious extensions for email attachments
@@ -124,6 +128,14 @@ case_sensitive_match = false
 # description = A list of processes that are not common
 match_type = WILDCARD(process)
 
+[mandatory_job_for_workflow]
+filename = mandatory_job_for_workflow.csv
+# description = A lookup file that will be used to define the mandatory job for workflow
+
+[mandatory_step_for_job]
+filename = mandatory_step_for_job.csv
+# description = A lookup file that will be used to define the mandatory step for job
+
 [network_acl_activity_baseline]
 filename = network_acl_activity_baseline.csv
 # description = A lookup file that will contain the baseline information for number of AWS Network ACL Activity

dist/devsecops/default/use_case_library.conf

@@ -1,6 +1,6 @@
 #############
 # Automatically generated by generator.py in splunk/security_content
-# On Date: 2021-08-27T14:41:52 UTC
+# On Date: 2021-09-13T10:57:27 UTC
 # Author: Splunk Security Research
 # Contact: research@splunk.com
 #############
@@ -14,7 +14,7 @@ version = 1
 references = ["https://www.redhat.com/en/topics/devops/what-is-devsecops"]
 maintainers = [{"company": "Splunk", "email": "-", "name": "Patrick Bareiss"}]
 spec_version = 3
-searches = ["ESCU - AWS ECR Container Scanning Findings High - Rule", "ESCU - AWS ECR Container Scanning Findings Low Informational Unknown - Rule", "ESCU - AWS ECR Container Scanning Findings Medium - Rule", "ESCU - AWS ECR Container Upload Outside Business Hours - Rule", "ESCU - AWS ECR Container Upload Unknown User - Rule", "ESCU - Kubernetes Nginx Ingress LFI - Rule", "ESCU - Kubernetes Nginx Ingress RFI - Rule", "ESCU - Kubernetes Scanner Image Pulling - Rule"]
+searches = ["ESCU - AWS ECR Container Scanning Findings High - Rule", "ESCU - AWS ECR Container Scanning Findings Low Informational Unknown - Rule", "ESCU - AWS ECR Container Scanning Findings Medium - Rule", "ESCU - AWS ECR Container Upload Outside Business Hours - Rule", "ESCU - AWS ECR Container Upload Unknown User - Rule", "ESCU - Circle CI Disable Security Job - Rule", "ESCU - Circle CI Disable Security Step - Rule", "ESCU - Correlation by Repository and Risk - Rule", "ESCU - Correlation by User and Risk - Rule", "ESCU - GitHub Dependabot Alert - Rule", "ESCU - GitHub Pull Request from Unknown User - Rule", "ESCU - Kubernetes Nginx Ingress LFI - Rule", "ESCU - Kubernetes Nginx Ingress RFI - Rule", "ESCU - Kubernetes Scanner Image Pulling - Rule"]
 description = This story is focused around detecting attacks on a DevSecOps lifeccycle which consists of the phases plan, code, build, test, release, deploy, operate and monitor.
 narrative = DevSecOps is a collaborative framework, which thinks about application and infrastructure security from the start. This means that security tools are part of the continuous integration and continuous deployment pipeline. In this analytics story, we focused on detections around the tools used in this framework such as GitHub as a version control system, GDrive for the documentation, CircleCI as the CI/CD pipeline, Kubernetes as the container execution engine and multiple security tools such as Semgrep and Kube-Hunter.
 
@@ -72,14 +72,44 @@ annotations = {"cis20": ["CIS 13"], "kill_chain_phases": ["Actions on Objectives
 known_false_positives = unknown
 providing_technologies = []
 
-[savedsearch://ESCU - AWS Excessive Security Scanning - Rule]
+[savedsearch://ESCU - Circle CI Disable Security Job - Rule]
+type = detection
+asset_type = CircleCI
+confidence = medium
+explanation = This search looks for disable security job in CircleCI pipeline.
+how_to_implement = You must index CircleCI logs.
+annotations = {"cis20": ["CIS 13"], "kill_chain_phases": ["Actions on Objectives"], "mitre_attack": ["T1554"], "nist": ["PR.DS", "PR.AC", "DE.CM"]}
+known_false_positives = unknown
+providing_technologies = []
+
+[savedsearch://ESCU - Circle CI Disable Security Step - Rule]
+type = detection
+asset_type = CircleCI
+confidence = medium
+explanation = This search looks for disable security step in CircleCI pipeline.
+how_to_implement = You must index CircleCI logs.
+annotations = {"cis20": ["CIS 13"], "kill_chain_phases": ["Actions on Objectives"], "mitre_attack": ["T1554"], "nist": ["PR.DS", "PR.AC", "DE.CM"]}
+known_false_positives = unknown
+providing_technologies = []
+
+[savedsearch://ESCU - Correlation by Repository and Risk - Rule]
 type = detection
 asset_type = AWS Account
 confidence = medium
-explanation = This search looks for AWS CloudTrail events and analyse the amount of eventNames which starts with Describe by a single user. This indicates that this user scans the configuration of your AWS cloud environment.
-how_to_implement = You must install splunk AWS add on and Splunk App for AWS. This search works with AWS CloudTrail logs.
-annotations = {"cis20": ["CIS 13"], "kill_chain_phases": ["Actions on Objectives"], "mitre_attack": ["T1526"], "nist": ["PR.DS", "PR.AC", "DE.CM"]}
-known_false_positives = While this search has no known false positives.
+explanation = This search correlations detections by repository and risk_score
+how_to_implement = For Dev Sec Ops POC
+annotations = {"cis20": ["CIS 13"], "kill_chain_phases": ["Actions on Objectives"], "mitre_attack": ["T1204.003"], "nist": ["PR.DS", "PR.AC", "DE.CM"]}
+known_false_positives = unknown
+providing_technologies = []
+
+[savedsearch://ESCU - Correlation by User and Risk - Rule]
+type = detection
+asset_type = AWS Account
+confidence = medium
+explanation = This search correlations detections by user and risk_score
+how_to_implement = For Dev Sec Ops POC
+annotations = {"cis20": ["CIS 13"], "kill_chain_phases": ["Actions on Objectives"], "mitre_attack": ["T1204.003"], "nist": ["PR.DS", "PR.AC", "DE.CM"]}
+known_false_positives = unknown
 providing_technologies = []
 
 [savedsearch://ESCU - GSuite Email Suspicious Attachment - Rule]
@@ -92,6 +122,46 @@ annotations = {"kill_chain_phases": ["Exploitation"], "mitre_attack": ["T1566.00
 known_false_positives = network admin and normal user may send this file attachment as part of their day to day work. having a good protocol in attaching this file type to an e-mail may reduce the risk of having a spear phishing attack.
 providing_technologies = []
 
+[savedsearch://ESCU - GitHub Dependabot Alert - Rule]
+type = detection
+asset_type = GitHub
+confidence = medium
+explanation = This search looks for Dependabot Alerts in Github logs.
+how_to_implement = You must index GitHub logs. You can follow the url in reference to onboard GitHub logs.
+annotations = {"cis20": ["CIS 13"], "kill_chain_phases": ["Actions on Objectives"], "mitre_attack": ["T1195.001"], "nist": ["PR.DS", "PR.AC", "DE.CM"]}
+known_false_positives = unknown
+providing_technologies = []
+
+[savedsearch://ESCU - GitHub Pull Request from Unknown User - Rule]
+type = detection
+asset_type = GitHub
+confidence = medium
+explanation = This search looks for Pull Request from unknown user.
+how_to_implement = You must index GitHub logs. You can follow the url in reference to onboard GitHub logs.
+annotations = {"cis20": ["CIS 13"], "kill_chain_phases": ["Actions on Objectives"], "mitre_attack": ["T1195.001"], "nist": ["PR.DS", "PR.AC", "DE.CM"]}
+known_false_positives = unknown
+providing_technologies = []
+
+[savedsearch://ESCU - Github Commit Changes In Master - Rule]
+type = detection
+asset_type = 
+confidence = medium
+explanation = This search is to detect a pushed or commit to master or main branch. This is to avoid unwanted modification to master without a review to the changes. Ideally in terms of devsecops the changes made in a branch and do a PR for review. of course in some cases admin of the project may did a changes directly to master branch
+how_to_implement = To successfully implement this search, you need to be ingesting logs related to github logs having the fork, commit, push metadata that can be use to monitor the changes in a github project.
+annotations = {"kill_chain_phases": ["Exploitation"], "mitre_attack": ["T1199"]}
+known_false_positives = admin can do changes directly to master branch
+providing_technologies = []
+
+[savedsearch://ESCU - Github Commit In Develop - Rule]
+type = detection
+asset_type = 
+confidence = medium
+explanation = This search is to detect a pushed or commit to develop branch. This is to avoid unwanted modification to develop without a review to the changes. Ideally in terms of devsecops the changes made in a branch and do a PR for review. of course in some cases admin of the project may did a changes directly to master branch
+how_to_implement = To successfully implement this search, you need to be ingesting logs related to github logs having the fork, commit, push metadata that can be use to monitor the changes in a github project.
+annotations = {"kill_chain_phases": ["Exploitation"], "mitre_attack": ["T1199"]}
+known_false_positives = admin can do changes directly to develop branch
+providing_technologies = []
+
 [savedsearch://ESCU - Gsuite Drive Share In External Email - Rule]
 type = detection
 asset_type = 

dist/devsecops/lookups/images_to_repository.csv

@@ -0,0 +1,3 @@
+image, repository
+devsecops/cat_dog_client, splunk/devsecops_poc
+devsecops/cat_dog_server, splunk/devsecops_poc

dist/devsecops/lookups/mandatory_job_for_workflow.csv

@@ -0,0 +1,2 @@
+workflow_name, job_name
+deployment, k8s-security

dist/devsecops/lookups/mandatory_step_for_job.csv

@@ -0,0 +1,2 @@
+job_name, step_name
+k8s-security, Run Kube Hunter

dist/devsecops/metadata/default.meta

@@ -9,6 +9,8 @@ access = read : [ * ], write : [ admin, power ]
 [eventtypes]
 export = system
 
+[savedsearches]
+owner = admin
 
 ### PROPS
 
@@ -33,3 +35,6 @@ export = system
 [viewstates]
 access = read : [ * ], write : [ * ]
 export = system
+
+[searchbnf]
+export = system

spec/deployments.spec.json

@@ -137,6 +137,112 @@
             "rule_description"
           ],
           "type": "object"
+        },
+        "slack": {
+          "$id": "#/properties/alert_action/properties/slack",
+          "additionalProperties": true,
+          "default": {},
+          "description": "By enabling it, a slack message is sent",
+          "examples": [
+            {
+              "channel": "slack_channel",
+              "message": "Alert x triggered"
+            }
+          ],
+          "properties": {
+            "channel": {
+              "$id": "#/properties/alert_action/properties/slack/properties/channel",
+              "default": "",
+              "description": "Slack channel",
+              "examples": [
+                "slack_channel"
+              ],
+              "type": "string"
+            },
+            "message": {
+              "$id": "#/properties/alert_action/properties/slack/properties/message",
+              "default": "",
+              "description": "message",
+              "examples": [
+                "Alert x triggered"
+              ],
+              "type": "string"
+            }
+          },
+          "required": [
+            "channel",
+            "message"
+          ],
+          "type": "object"
+        },
+        "phantom": {
+          "$id": "#/properties/alert_action/properties/phantom",
+          "additionalProperties": true,
+          "default": {},
+          "description": "By enabling it, the event is sent to phantom",
+          "examples": [
+            {
+              "phantom_server": "phantom",
+              "label": "events",
+              "sensitivity": "amber",
+              "severity": "medium",
+              "cam_workers": "local"
+            }
+          ],
+          "properties": {
+            "phantom_server": {
+              "$id": "#/properties/alert_action/properties/phantom/properties/phantom_server",
+              "default": "",
+              "description": "Phantom server",
+              "examples": [
+                "phantom"
+              ],
+              "type": "string"
+            },
+            "label": {
+              "$id": "#/properties/alert_action/properties/phantom/properties/label",
+              "default": "",
+              "description": "label",
+              "examples": [
+                "events"
+              ],
+              "type": "string"
+            },
+            "sensitivity": {
+              "$id": "#/properties/alert_action/properties/phantom/properties/sensitivity",
+              "default": "",
+              "description": "sensitivity",
+              "examples": [
+                "amber"
+              ],
+              "type": "string"
+            },
+            "severity": {
+              "$id": "#/properties/alert_action/properties/phantom/properties/severity",
+              "default": "",
+              "description": "severity",
+              "examples": [
+                "medium"
+              ],
+              "type": "string"
+            },
+            "cam_workers": {
+              "$id": "#/properties/alert_action/properties/phantom/properties/cam_workers",
+              "default": "",
+              "description": "adaptive response worker set",
+              "examples": [
+                "local"
+              ],
+              "type": "string"
+            }
+          },
+          "required": [
+            "phantom_server",
+            "label",
+            "sensitivity",
+            "severity"
+          ],
+          "type": "object"
         }
       },
       "type": "object"