Add rule loader and dependencies

Co-Authored-By: Justin Ibarra <brokensound77@users.noreply.github.com>

detection_rules/__init__.py

@@ -0,0 +1,24 @@
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+
+"""Detection rules."""
+from . import eswrap
+from . import main
+from . import mappings
+from . import misc
+from . import rule_formatter
+from . import rule_loader
+from . import schema
+from . import utils
+
+__all__ = (
+    'eswrap',
+    'mappings',
+    "main",
+    'misc',
+    'rule_formatter',
+    'rule_loader',
+    'schema',
+    'utils',
+)

detection_rules/__main__.py

@@ -0,0 +1,28 @@
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+
+# coding=utf-8
+"""Shell for detection-rules."""
+import os
+
+from .main import root
+
+CURR_DIR = os.path.dirname(os.path.abspath(__file__))
+CLI_DIR = os.path.dirname(CURR_DIR)
+ROOT_DIR = os.path.dirname(CLI_DIR)
+
+BANNER = r"""
+█▀▀▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄   ▄      █▀▀▄ ▄  ▄ ▄   ▄▄▄ ▄▄▄
+█  █ █▄▄  █  █▄▄ █    █   █  █ █ █▀▄ █      █▄▄▀ █  █ █   █▄▄ █▄▄
+█▄▄▀ █▄▄  █  █▄▄ █▄▄  █  ▄█▄ █▄█ █ ▀▄█      █ ▀▄ █▄▄█ █▄▄ █▄▄ ▄▄█
+"""
+
+
+def main():
+    """CLI entry point."""
+    print(BANNER)
+    root(prog_name="detection_rules")
+
+
+main()

detection_rules/attack.py

@@ -0,0 +1,79 @@
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+
+"""Mitre attack info."""
+# from: https://raw.githubusercontent.com/mitre/cti/master/enterprise-attack/enterprise-attack.json
+
+from .utils import load_etc_dump
+
+TACTICS_MAP = {
+    'Initial Access': 'TA0001',
+    'Persistence': 'TA0003',
+    'Privilege Escalation': 'TA0004',
+    'Defense Evasion': 'TA0005',
+    'Credential Access': 'TA0006',
+    'Discovery': 'TA0007',
+    'Lateral Movement': 'TA0008',
+    'Execution': 'TA0002',
+    'Collection': 'TA0009',
+    'Exfiltration': 'TA0011',
+    'Command and Control': 'TA0010',
+    'Impact': 'TA0040'
+}
+TACTICS = list(TACTICS_MAP)
+PLATFORMS = ['Windows', 'macOS', 'Linux']
+
+attack = load_etc_dump('attack.json')
+
+technique_lookup = {}
+
+for item in attack["objects"]:
+    if item["type"] == "attack-pattern" and item["external_references"][0]['source_name'] == 'mitre-attack':
+        technique_id = item['external_references'][0]['external_id']
+        technique_lookup[technique_id] = item
+
+matrix = {tactic: [] for tactic in TACTICS}
+attack_tm = 'ATT&CK\u2122'
+
+
+# Enumerate over the techniques and build the matrix back up
+for technique_id, technique in sorted(technique_lookup.items(), key=lambda kv: kv[1]['name'].lower()):
+    for platform in technique['x_mitre_platforms']:
+        if any(platform.startswith(p) for p in PLATFORMS):
+            break
+    else:
+        continue
+
+    for tactic in technique['kill_chain_phases']:
+        tactic_name = next(t for t in TACTICS if tactic['kill_chain_name'] == 'mitre-attack' and t.lower() == tactic['phase_name'].replace("-", " "))  # noqa: E501
+        matrix[tactic_name].append(technique_id)
+
+for tactic in matrix:
+    matrix[tactic].sort(key=lambda tid: technique_lookup[tid]['name'].lower())
+
+
+TECHNIQUES = {v['name'] for k, v in technique_lookup.items()}
+
+
+def build_threat_map_entry(tactic: str, *technique_ids: str) -> dict:
+    """Build rule threat map from technique IDs."""
+    url_base = 'https://attack.mitre.org/{type}/{id}/'
+    tactic_id = TACTICS_MAP[tactic]
+    entry = {
+        'framework': 'MITRE ATT&CK',
+        'technique': [
+            {
+                'id': tid,
+                'name': technique_lookup[tid]['name'],
+                'reference': url_base.format(type='techniques', id=tid)
+            } for tid in technique_ids
+        ],
+        'tactic': {
+            'id': tactic_id,
+            'name': tactic,
+            'reference': url_base.format(type='tactics', id=tactic_id)
+        }
+    }
+
+    return entry

detection_rules/beats.py

@@ -0,0 +1,160 @@
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+
+"""ECS Schemas management."""
+import os
+
+import kql
+import requests
+import yaml
+
+from .semver import Version
+from .utils import unzip, load_etc_dump, save_etc_dump, get_etc_path
+
+
+def download_latest_beats_schema():
+    """Download additional schemas from ecs releases."""
+    url = 'https://api.github.com/repos/elastic/beats/releases'
+    releases = requests.get(url)
+
+    latest_release = max(releases.json(), key=lambda release: Version(release["tag_name"].lstrip("v")))
+
+    print(f"Downloading beats {latest_release['tag_name']}")
+    response = requests.get(latest_release['zipball_url'])
+
+    print(f"Downloaded {len(response.content) / 1024.0 / 1024.0:.2f} MB release.")
+
+    fs = {}
+    parsed = {}
+
+    with unzip(response.content) as archive:
+        base_directory = archive.namelist()[0]
+
+        for name in archive.namelist():
+            if os.path.basename(name) in ("fields.yml", "fields.common.yml", "config.yml"):
+                contents = archive.read(name)
+
+                # chop off the base directory name
+                key = name[len(base_directory):]
+
+                if key.startswith("x-pack"):
+                    key = key[len("x-pack") + 1:]
+
+                try:
+                    decoded = yaml.safe_load(contents)
+                except yaml.YAMLError:
+                    print(f"Error loading {name}")
+
+                # create a hierarchical structure
+                parsed[key] = decoded
+                branch = fs
+                directory, base_name = os.path.split(key)
+                for limb in directory.split(os.path.sep):
+                    branch = branch.setdefault("folders", {}).setdefault(limb, {})
+
+                branch.setdefault("files", {})[base_name] = decoded
+
+    # remove all non-beat directories
+    fs = {k: v for k, v in fs.get("folders", {}).items() if k.endswith("beat")}
+    print(f"Saving etc/beats_schema/{latest_release['tag_name']}.yml")
+    save_etc_dump(fs, "beats_schemas", latest_release["tag_name"] + ".yml")
+
+
+def _flatten_schema(schema: list, prefix="") -> list:
+    if schema is None:
+        # sometimes we see `fields: null` in the yaml
+        return []
+
+    flattened = []
+    for s in schema:
+        if s.get("type") == "group":
+            flattened.extend(_flatten_schema(s["fields"], prefix=prefix + s["name"] + "."))
+        elif "fields" in s:
+            flattened.extend(_flatten_schema(s["fields"], prefix=prefix))
+        elif "type" in s:
+            s = s.copy()
+            s["name"] = prefix + s["name"]
+            flattened.append(s)
+
+    return flattened
+
+
+def get_field_schema(base_directory, prefix="", include_common=False):
+    base_directory = base_directory.get("folders", {}).get("_meta", {}).get("files", {})
+    flattened = []
+
+    file_names = ("fields.yml", "fields.common.yml") if include_common else ("fields.yml", )
+
+    for name in file_names:
+        if name in base_directory:
+            flattened.extend(_flatten_schema(base_directory[name], prefix=prefix))
+
+    return flattened
+
+
+def get_beats_schema(schema: dict, beat: str, module: str, *datasets: str):
+    if beat not in schema:
+        raise KeyError(f"Unknown beats module {beat}")
+
+    flattened = []
+    beat_dir = schema[beat]
+    flattened.extend(get_field_schema(beat_dir, include_common=True))
+
+    module_dir = beat_dir.get("folders", {}).get("module", {}).get("folders", {}).get(module, {})
+    flattened.extend(get_field_schema(module_dir, include_common=True))
+
+    # if we only have a module then we'll work with what we got
+    if not datasets:
+        datasets = [d for d in module_dir.get("folders", {}) if not d.startswith("_")]
+
+    for dataset in datasets:
+        # replace aws.s3 -> s3
+        if dataset.startswith(module + "."):
+            dataset = dataset[len(module) + 1:]
+
+        dataset_dir = module_dir.get("folders", {}).get(dataset, {})
+        flattened.extend(get_field_schema(dataset_dir, prefix=module + ".", include_common=True))
+
+    return {field["name"]: field for field in sorted(flattened, key=lambda f: f["name"])}
+
+
+SCHEMA = None
+
+
+def read_beats_schema():
+    global SCHEMA
+
+    if SCHEMA is None:
+        beats_schemas = os.listdir(get_etc_path("beats_schemas"))
+        latest = max(beats_schemas, key=lambda b: Version(b.lstrip("v")))
+
+        SCHEMA = load_etc_dump("beats_schemas", latest)
+
+    return SCHEMA
+
+
+def get_schema_for_query(tree: kql.ast, beats: list) -> dict:
+    filtered = {}
+    modules = set()
+    datasets = set()
+
+    # extract out event.module and event.dataset from the query's AST
+    for node in tree:
+        if isinstance(node, kql.ast.FieldComparison) and node.field == kql.ast.Field("event.module"):
+            modules.update(child.value for child in node.value if isinstance(child, kql.ast.String))
+
+        if isinstance(node, kql.ast.FieldComparison) and node.field == kql.ast.Field("event.dataset"):
+            datasets.update(child.value for child in node.value if isinstance(child, kql.ast.String))
+
+    beats_schema = read_beats_schema()
+
+    for beat in beats:
+        # if no modules are specified then grab them all
+        # all_modules = list(beats_schema.get(beat, {}).get("folders", {}).get("module", {}).get("folders", {}))
+        # beat_modules = modules or all_modules
+
+        for module in modules:
+            filtered.update(get_beats_schema(beats_schema, beat, module, *datasets))
+
+    return filtered

detection_rules/ecs.py

@@ -0,0 +1,233 @@
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+
+"""ECS Schemas management."""
+import copy
+import glob
+import os
+import shutil
+
+import requests
+import yaml
+
+from .semver import Version
+from .utils import unzip, load_etc_dump, get_etc_path, cached
+
+ECS_SCHEMAS_DIR = get_etc_path("ecs_schemas")
+
+
+def add_field(schema, name, info):
+    """Nest a dotted field within a dictionary."""
+    if "." not in name:
+        schema[name] = info
+        return
+
+    top, remaining = name.split(".", 1)
+    if not isinstance(schema.get(top), dict):
+        schema[top] = {}
+    add_field(schema, remaining, info)
+
+
+def nest_from_dot(dots, value):
+    """Nest a dotted field and set the inner most value."""
+    fields = dots.split('.')
+
+    if not fields:
+        return {}
+
+    nested = {fields.pop(): value}
+
+    for field in reversed(fields):
+        nested = {field: nested}
+
+    return nested
+
+
+def _recursive_merge(existing, new, depth=0):
+    """Return an existing dict merged into a new one."""
+    for key, value in existing.items():
+        if isinstance(value, dict):
+            if depth == 0:
+                new = copy.deepcopy(new)
+
+            node = new.setdefault(key, {})
+            _recursive_merge(value, node, depth + 1)
+        else:
+            new[key] = value
+
+    return new
+
+
+def get_schema_files():
+    """Get schema files from ecs directory."""
+    return glob.glob(os.path.join(ECS_SCHEMAS_DIR, '*', '*.yml'), recursive=True)
+
+
+def get_schema_map():
+    """Get local schema files by version."""
+    schema_map = {}
+
+    for file_name in get_schema_files():
+        path, name = os.path.split(file_name)
+        name = os.path.splitext(name)[0]
+        version = os.path.basename(path)
+        schema_map.setdefault(version, {})[name] = file_name
+
+    return schema_map
+
+
+@cached
+def get_schemas():
+    """Get local schemas."""
+    schema_map = get_schema_map()
+
+    for version, values in schema_map.items():
+        for name, file_name in values.items():
+            with open(file_name, 'r') as f:
+                schema_map[version][name] = yaml.safe_load(f)
+
+    return schema_map
+
+
+def get_max_version(include_master=False):
+    """Get maximum available schema version."""
+    versions = get_schema_map().keys()
+
+    if include_master and any([v.startswith('master') for v in versions]):
+        return glob.glob(os.path.join(ECS_SCHEMAS_DIR, 'master*'))[0]
+
+    return str(max([Version(v) for v in versions if not v.startswith('master')]))
+
+
+@cached
+def get_schema(version=None, name='ecs_flat'):
+    """Get schema by version."""
+    return get_schemas()[version][name]
+
+
+@cached
+def get_eql_schema(version=None, index_patterns=None):
+    """Return schema in expected format for eql."""
+    schema = get_schema(version, name='ecs_flat')
+    str_types = ('text', 'ip', 'keyword', 'date', 'object', 'geo_point')
+    num_types = ('float', 'integer', 'long')
+    schema = schema.copy()
+
+    def convert_type(t):
+        return 'string' if t in str_types else 'number' if t in num_types else 'boolean'
+
+    converted = {}
+
+    for field, schema_info in schema.items():
+        field_type = schema_info.get('type', '')
+        add_field(converted, field, convert_type(field_type))
+
+    if index_patterns:
+        for index_name in index_patterns:
+            for k, v in flatten(get_index_schema(index_name)).items():
+                add_field(converted, k, convert_type(v))
+
+    return converted
+
+
+def flatten(schema):
+    flattened = {}
+    for k, v in schema.items():
+        if isinstance(v, dict):
+            flattened.update((k + "." + vk, vv) for vk, vv in flatten(v).items())
+        else:
+            flattened[k] = v
+    return flattened
+
+
+@cached
+def get_non_ecs_schema():
+    """Load non-ecs schema."""
+    return load_etc_dump('non-ecs-schema.json')
+
+
+@cached
+def get_index_schema(index_name):
+    return get_non_ecs_schema().get(index_name, {})
+
+
+def flatten_multi_fields(schema):
+    converted = {}
+    for field, info in schema.items():
+        converted[field] = info["type"]
+        for subfield in info.get("multi_fields", []):
+            converted[field + "." + subfield["name"]] = subfield["type"]
+
+    return converted
+
+
+@cached
+def get_kql_schema(version=None, indexes=None, beat_schema=None):
+    """Get schema for KQL."""
+    indexes = indexes or ()
+    converted = flatten_multi_fields(get_schema(version, name='ecs_flat'))
+
+    for index_name in indexes:
+        converted.update(**flatten(get_index_schema(index_name)))
+
+    if isinstance(beat_schema, dict):
+        converted = dict(flatten_multi_fields(beat_schema), **converted)
+
+    return converted
+
+
+def download_schemas(refresh_master=True, refresh_all=False, verbose=True):
+    """Download additional schemas from ecs releases."""
+    existing = [Version(v) for v in get_schema_map()] if not refresh_all else []
+    url = 'https://api.github.com/repos/elastic/ecs/releases'
+    releases = requests.get(url)
+
+    for release in releases.json():
+        version = Version(release.get('tag_name', '').lstrip('v'))
+
+        # we don't ever want beta
+        if not version or version < (1, 0, 1) or version in existing:
+            continue
+
+        schema_dir = os.path.join(ECS_SCHEMAS_DIR, str(version))
+
+        with unzip(requests.get(release['zipball_url']).content) as archive:
+            name_list = archive.namelist()
+            base = name_list[0]
+
+            # members = [m for m in name_list if m.startswith('{}{}/'.format(base, 'use-cases')) and m.endswith('.yml')]
+            members = ['{}generated/ecs/ecs_flat.yml'.format(base), '{}generated/ecs/ecs_nested.yml'.format(base)]
+
+            for member in members:
+                file_name = os.path.basename(member)
+                os.makedirs(schema_dir, exist_ok=True)
+
+                with open(os.path.join(schema_dir, file_name), 'wb') as f:
+                    f.write(archive.read(member))
+
+            if verbose:
+                print('Saved files to {}: \n\t- {}'.format(schema_dir, '\n\t- '.join(members)))
+
+    # handle working master separately
+    if refresh_master:
+        master_ver = requests.get('https://raw.githubusercontent.com/elastic/ecs/master/version')
+        master_ver = Version(master_ver.text.strip())
+        master_schema = requests.get('https://raw.githubusercontent.com/elastic/ecs/master/generated/ecs/ecs_flat.yml')
+        master_schema = yaml.safe_load(master_schema.text)
+
+        # prepend with underscore so that we can differentiate the fact that this is a working master version
+        #   but first clear out any existing masters, since we only ever want 1 at a time
+        existing_master = glob.glob(os.path.join(ECS_SCHEMAS_DIR, 'master_*'))
+        for m in existing_master:
+            shutil.rmtree(m, ignore_errors=True)
+
+        master_dir = os.path.join(ECS_SCHEMAS_DIR, 'master_{}'.format(master_ver))
+        master_file = os.path.join(master_dir, 'ecs_flat.yml')
+        os.makedirs(master_dir, exist_ok=True)
+
+        with open(master_file, 'w') as f:
+            yaml.safe_dump(master_schema, f)
+
+        if verbose:
+            print('Saved files to {}: \n\t- {}'.format(master_dir, 'ecs_flat.yml'))

detection_rules/eswrap.py

@@ -0,0 +1,222 @@
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+
+"""Elasticsearch cli and tmp."""
+import json
+import os
+import time
+
+import click
+from elasticsearch import AuthenticationException, Elasticsearch
+
+from .main import root
+from .misc import set_param_values
+from .utils import normalize_timing_and_sort, unix_time_to_formatted, get_path
+from .rule_loader import get_rule, rta_mappings
+
+COLLECTION_DIR = get_path('collections')
+ERRORS = {
+    'NO_EVENTS': 1,
+    'FAILED_ES_AUTH': 2
+}
+
+
+@root.group('es')
+def es_group():
+    """Helper commands for integrating with Elasticsearch."""
+
+
+def get_es_client(user, password, host=None, cloud_id=None, **kwargs):
+    """Get an auth-validated elsticsearch client."""
+    assert host or cloud_id, 'You must specify a host or cloud-id to authenticate to elasticsearch instance'
+    hosts = [host] if host else host
+
+    client = Elasticsearch(hosts=hosts, cloud_id=cloud_id, http_auth=(user, password), **kwargs)
+    # force login to test auth
+    client.info()
+    return client
+
+
+class Events(object):
+    """Events collected from Elasticsearch."""
+
+    def __init__(self, agent_hostname, events):
+        self.agent_hostname = agent_hostname
+        self.events = self._normalize_event_timing(events)
+
+    @staticmethod
+    def _normalize_event_timing(events):
+        """Normalize event timestamps and sort."""
+        for agent_type, _events in events.items():
+            events[agent_type] = normalize_timing_and_sort(_events)
+
+        return events
+
+    def _get_dump_dir(self, rta_name=None):
+        """Prepare and get the dump path."""
+        if rta_name:
+            dump_dir = get_path('unit_tests', 'data', 'true_positives', rta_name)
+            os.makedirs(dump_dir, exist_ok=True)
+            return dump_dir
+        else:
+            time_str = time.strftime('%Y%m%dT%H%M%SL')
+            dump_dir = os.path.join(COLLECTION_DIR, self.agent_hostname, time_str)
+            os.makedirs(dump_dir, exist_ok=True)
+            return dump_dir
+
+    def evaluate_against_rule_and_update_mapping(self, rule_id, rta_name, verbose=True):
+        """Evaluate a rule against collected events and update mapping."""
+        from .utils import combine_sources, evaluate
+
+        rule = get_rule(rule_id, verbose=False)
+        merged_events = combine_sources(*self.events.values())
+        filtered = evaluate(rule, merged_events)
+
+        if filtered:
+            sources = [e['agent']['type'] for e in filtered]
+            mapping_update = rta_mappings.add_rule_to_mapping_file(rule, len(filtered), rta_name, *sources)
+
+            if verbose:
+                click.echo('Updated rule-mapping file with: \n{}'.format(json.dumps(mapping_update, indent=2)))
+        else:
+            if verbose:
+                click.echo('No updates to rule-mapping file; No matching results')
+
+    def echo_events(self, pager=False, pretty=True):
+        """Print events to stdout."""
+        echo_fn = click.echo_via_pager if pager else click.echo
+        echo_fn(json.dumps(self.events, indent=2 if pretty else None, sort_keys=True))
+
+    def save(self, rta_name=None, dump_dir=None):
+        """Save collected events."""
+        assert self.events, 'Nothing to save. Run Collector.run() method first'
+
+        dump_dir = dump_dir or self._get_dump_dir(rta_name)
+
+        for source, events in self.events.items():
+            path = os.path.join(dump_dir, source + '.jsonl')
+            with open(path, 'w') as f:
+                f.writelines([json.dumps(e, sort_keys=True) + '\n' for e in events])
+                click.echo('{} events saved to: {}'.format(len(events), path))
+
+
+class CollectEvents(object):
+    """Event collector for elastic stack."""
+
+    def __init__(self, client, max_events=3000):
+        self.client = client
+        self.MAX_EVENTS = max_events
+
+    def _build_timestamp_map(self, index_str):
+        """Build a mapping of indexes to timestamp data formats."""
+        mappings = self.client.indices.get_mapping(index=index_str)
+        timestamp_map = {n: m['mappings'].get('properties', {}).get('@timestamp', {}) for n, m in mappings.items()}
+        return timestamp_map
+
+    def _get_current_time(self, agent_hostname, index_str):
+        """Get timestamp of most recent event."""
+        # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-date-format.html
+        timestamp_map = self._build_timestamp_map(index_str)
+
+        last_event = self._search_window(agent_hostname, index_str, start_time='now-1m', size=1, sort='@timestamp:desc')
+        last_event = last_event['hits']['hits'][0]
+
+        index = last_event['_index']
+        timestamp = last_event['_source']['@timestamp']
+        event_date_format = timestamp_map[index].get('format', '').split('||')
+
+        # there are many native supported date formats and even custom data formats, but most, including beats use the
+        # default `strict_date_optional_time`. It would be difficult to try to account for all possible formats, so this
+        # will work on the default and unix time.
+        if set(event_date_format) & {'epoch_millis', 'epoch_second'}:
+            timestamp = unix_time_to_formatted(timestamp)
+
+        return timestamp
+
+    def _search_window(self, agent_hostname, index_str, start_time, end_time='now', size=None, sort='@timestamp:asc',
+                       **match):
+        """Collect all events within a time window and parse by source."""
+        match = match.copy()
+        match.update({"agent.hostname": agent_hostname})
+        body = {"query": {"bool": {"filter": [
+            {"match": {"agent.hostname": agent_hostname}},
+            {"range": {"@timestamp": {"gt": start_time, "lte": end_time, "format": "strict_date_optional_time"}}}]
+        }}}
+
+        if match:
+            body['query']['bool']['filter'].extend([{'match': {k: v}} for k, v in match.items()])
+
+        return self.client.search(index=index_str, body=body, size=size or self.MAX_EVENTS, sort=sort)
+
+    @staticmethod
+    def _group_events_by_type(events):
+        """Group events by agent.type."""
+        event_by_type = {}
+
+        for event in events['hits']['hits']:
+            event_by_type.setdefault(event['_source']['agent']['type'], []).append(event['_source'])
+
+        return event_by_type
+
+    def run(self, agent_hostname, indexes, verbose=True, **match):
+        """Collect the events."""
+        index_str = ','.join(indexes)
+        start_time = self._get_current_time(agent_hostname, index_str)
+
+        if verbose:
+            click.echo('Setting start of event capture to: {}'.format(click.style(start_time, fg='yellow')))
+
+        click.pause('Press any key once detonation is complete ...')
+        time.sleep(5)
+        events = self._group_events_by_type(self._search_window(agent_hostname, index_str, start_time, **match))
+
+        return Events(agent_hostname, events)
+
+
+@es_group.command('collect-events')
+@click.argument('agent-hostname')
+@click.option('--host', callback=set_param_values, expose_value=True)
+@click.option('--cloud-id', callback=set_param_values, expose_value=True)
+@click.option('--user', '-u', callback=set_param_values, expose_value=True, hide_input=False)
+@click.option('--password', '-p', callback=set_param_values, expose_value=True, hide_input=True)
+@click.option('--index', '-i', multiple=True, help='Index(es) to search against (default: all indexes)')
+@click.option('--agent-type', '-a', help='Restrict results to a source type (agent.type) ex: auditbeat')
+@click.option('--rta-name', '-r', help='Name of RTA in order to save events directly to unit tests data directory')
+@click.option('--rule-id', help='Updates rule mapping in rule-mapping.yml file (requires --rta-name)')
+@click.option('--view-events', is_flag=True, help='Print events after saving')
+def collect_events(agent_hostname, host, cloud_id, user, password, index, agent_type, rta_name, rule_id, view_events):
+    """Collect events from Elasticsearch."""
+    match = {'agent.type': agent_type} if agent_type else {}
+
+    try:
+        client = get_es_client(host=host, use_ssl=True, cloud_id=cloud_id, user=user, password=password)
+    except AuthenticationException:
+        click.secho('Failed authentication for {}'.format(host or cloud_id), fg='red', err=True)
+        return ERRORS['FAILED_ES_AUTH']
+
+    try:
+        collector = CollectEvents(client)
+        events = collector.run(agent_hostname, index, **match)
+        events.save(rta_name)
+    except AssertionError:
+        click.secho('No events collected! Verify events are streaming and that the agent-hostname is correct',
+                    err=True, fg='red')
+        return ERRORS['NO_EVENTS']
+
+    if rta_name and rule_id:
+        events.evaluate_against_rule_and_update_mapping(rule_id, rta_name)
+
+    if view_events and events.events:
+        events.echo_events(pager=True)
+
+    return events
+
+
+@es_group.command('normalize-data')
+@click.argument('events-file', type=click.File('r'))
+def normalize_file(events_file):
+    """Normalize Elasticsearch data timestamps and sort."""
+    file_name = os.path.splitext(os.path.basename(events_file.name))[0]
+    events = Events('_', {file_name: [json.loads(e) for e in events_file.readlines()]})
+    events.save(dump_dir=os.path.dirname(events_file.name))

detection_rules/main.py

@@ -0,0 +1,352 @@
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+
+"""CLI commands for detection_rules."""
+import glob
+import io
+import json
+import os
+
+import click
+import jsonschema
+import pytoml
+from eql import load_dump
+
+from .misc import nested_set
+from . import rule_loader
+from .packaging import PACKAGE_FILE, Package, manage_versions
+from .rule import RULE_TYPE_OPTIONS, Rule
+from .rule_formatter import toml_write
+from .utils import get_path, clear_caches
+
+
+RULES_DIR = get_path('rules')
+
+
+@click.group('detection-rules', context_settings={'help_option_names': ['-h', '--help']})
+def root():
+    """Commands for detection-rules repository."""
+
+
+@root.command('create-rule')
+@click.argument('path', type=click.Path(dir_okay=False))
+@click.option('--config', '-c', type=click.Path(exists=True, dir_okay=False), help='Rule or config file')
+@click.option('--required-only', is_flag=True, help='Only prompt for required fields')
+@click.option('--rule-type', '-t', type=click.Choice(RULE_TYPE_OPTIONS), help='Type of rule to create')
+def create_rule(path, config, required_only, rule_type):
+    """Create a detection rule."""
+    config = load_dump(config) if config else {}
+    try:
+        return Rule.build(path, rule_type=rule_type, required_only=required_only, save=True, **config)
+    finally:
+        rule_loader.reset()
+
+
+@root.command('load-from-file')
+@click.argument('infile', type=click.Path(dir_okay=False, exists=True), nargs=-1, required=False)
+@click.option('--directory', '-d', type=click.Path(file_okay=False, exists=True), help='Load files from a directory')
+def load_from_file(infile, directory):
+    """Load rules from file(s)."""
+    if infile:
+        for rule_file in infile:
+            rule_path = os.path.join(RULES_DIR, os.path.basename(rule_file))
+            rule = Rule(rule_path, load_dump(rule_file))
+            rule.save(as_rule=True, verbose=True)
+    elif directory:
+        for rule_file in glob.glob(os.path.join(directory, '**', '*.*'), recursive=True):
+            try:
+                rule_path = os.path.join(RULES_DIR, os.path.basename(rule_file))
+                rule = Rule(rule_path, load_dump(rule_file))
+                rule.save(as_rule=True, verbose=True)
+            except ValueError:
+                click.echo('Unable to load file: {}'.format(rule_file))
+    else:
+        click.echo('No files specified!')
+
+
+@root.command('toml-lint')
+@click.option('--rule-file', '-f', type=click.File('r'), help='Optionally specify a specific rule file only')
+def toml_lint(rule_file):
+    """Cleanup files with some simple toml formatting."""
+    if rule_file:
+        contents = pytoml.load(rule_file)
+        rule = Rule(path=rule_file.name, contents=contents)
+
+        # removed unneeded defaults
+        for field in rule_loader.find_unneeded_defaults(rule):
+            rule.contents.pop(field, None)
+
+        rule.save(as_rule=True)
+    else:
+        for rule in rule_loader.load_rules().values():
+
+            # removed unneeded defaults
+            for field in rule_loader.find_unneeded_defaults(rule):
+                rule.contents.pop(field, None)
+
+            rule.save(as_rule=True)
+
+    rule_loader.reset()
+    click.echo('Toml file linting complete')
+
+
+@root.command('mass-update')
+@click.argument('query')
+@click.option('--field', type=(str, str), multiple=True,
+              help='Use rule-search to retrieve a subset of rules and modify values '
+                   '(ex: --field management.ecs_version 1.1.1).\n'
+                   'Note this is limited to string fields only. Nested fields should use dot notation.')
+@click.pass_context
+def mass_update(ctx, query, field):
+    """Update multiple rules based on eql results."""
+    results = ctx.invoke(search_rules, query=query, verbose=False)
+    rules = [rule_loader.get_rule(r['rule_id']) for r in results]
+
+    for rule in rules:
+        for key, value in field:
+            nested_set(rule.contents, key, value)
+
+        rule.validate(as_rule=True)
+        rule.save()
+
+    ctx.invoke(search_rules, query=query, columns=[k[0].split('.')[-1] for k in field])
+
+    return
+
+
+@root.command('view-rule')
+@click.argument('rule-id', required=False)
+@click.option('--rule-file', '-f', type=click.Path(dir_okay=False), help='Optionally view a rule from a specified file')
+@click.option('--as-api/--as-rule', default=True, help='Print the rule in final api or rule format')
+@click.option('--optimize/--no-optimize', default=False, help='When viewing in api format, include optimizations')
+def view_rule(rule_id, rule_file, as_api, optimize):
+    """View an internal rule or specified rule file."""
+    if rule_id:
+        rule = rule_loader.get_rule(rule_id, verbose=False)
+    elif rule_file:
+        rule = Rule(rule_file, load_dump(rule_file))
+    else:
+        click.secho('Unknown rule!', fg='red')
+        return
+
+    if not rule:
+        click.secho('Unknown format!', fg='red')
+        return
+
+    if optimize and as_api:
+        rule.tune()
+
+    click.echo(toml_write(rule.rule_format()) if not as_api else json.dumps(rule.contents, indent=2, sort_keys=True))
+
+    return rule
+
+
+@root.command('validate-rule')
+@click.argument('rule-id', required=False)
+@click.option('--rule-name', '-n')
+@click.option('--path', '-p', type=click.Path(dir_okay=False))
+def validate_rule(rule_id, rule_name, path):
+    """Check if a rule staged in rules dir validates against a schema."""
+    rule = rule_loader.get_rule(rule_id, rule_name, path, verbose=False)
+
+    if not rule:
+        return click.secho('Rule not found!', fg='red')
+
+    try:
+        rule.validate(as_rule=True)
+    except jsonschema.ValidationError as e:
+        click.echo(e)
+
+    click.echo('Rule validation successful')
+
+    return rule
+
+
+license_header = """
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+""".strip()
+
+
+@root.command('license-check')
+@click.pass_context
+def license_check(ctx):
+    """Check that all code files contain a valid license."""
+
+    failed = False
+
+    for path in glob.glob(get_path("**", "*.py"), recursive=True):
+        if path.startswith(get_path("env", "")):
+            continue
+
+        relative_path = os.path.relpath(path)
+
+        with io.open(path, "rt", encoding="utf-8") as f:
+            contents = f.read()
+
+            # skip over shebang lines
+            if contents.startswith("#!/"):
+                _, _, contents = contents.partition("\n")
+
+            if not contents.lstrip("\r\n").startswith(license_header):
+                if not failed:
+                    click.echo("Missing license headers for:", err=True)
+
+                failed = True
+                click.echo(relative_path, err=True)
+
+    ctx.exit(int(failed))
+
+
+@root.command('validate-all')
+@click.option('--fail/--no-fail', default=True, help='Fail on first failure or process through all printing errors.')
+def validate_all(fail):
+    """Check if all rules validates against a schema."""
+    rule_loader.load_rules(verbose=True, error=fail)
+    click.echo('Rule validation successful')
+
+
+@root.command('rule-search')
+@click.argument('query', required=False)
+@click.option('--columns', '-c', multiple=True, help='Specify columns to add the table')
+@click.option('--language', type=click.Choice(["eql", "kql"]), default="kql")
+def search_rules(query, columns, language, verbose=True):
+    """Use KQL to find matching rules."""
+    from kql import get_evaluator
+    from eql.table import Table
+    from eql.build import get_engine
+    from eql import parse_query
+    from eql.pipes import CountPipe
+
+    flattened_rules = []
+
+    for file_name, rule_doc in rule_loader.load_rule_files().items():
+        flat = {"file": os.path.relpath(file_name)}
+        flat.update(rule_doc)
+        flat.update(rule_doc["metadata"])
+        flat.update(rule_doc["rule"])
+        attacks = [threat for threat in rule_doc["rule"].get("threat", []) if threat["framework"] == "MITRE ATT&CK"]
+        techniques = [t["id"] for threat in attacks for t in threat.get("technique", [])]
+        tactics = [threat["tactic"]["name"] for threat in attacks]
+        flat.update(techniques=techniques, tactics=tactics)
+        flattened_rules.append(flat)
+
+    flattened_rules.sort(key=lambda dct: dct["name"])
+
+    if language == "kql":
+        evaluator = get_evaluator(query) if query else lambda x: True
+        filtered = list(filter(evaluator, flattened_rules))
+    elif language == "eql":
+        parsed = parse_query(query, implied_any=True, implied_base=True)
+        evaluator = get_engine(parsed)
+        filtered = [result.events[0].data for result in evaluator(flattened_rules)]
+
+        if not columns and any(isinstance(pipe, CountPipe) for pipe in parsed.pipes):
+            columns = ["key", "count", "percent"]
+
+    if columns:
+        columns = ",".join(columns).split(",")
+    else:
+        columns = ["rule_id", "file", "name"]
+
+    table = Table.from_list(columns, filtered)
+
+    if verbose:
+        click.echo(table)
+
+    return filtered
+
+
+@root.command('build-release')
+@click.argument('config-file', type=click.Path(exists=True, dir_okay=False), required=False, default=PACKAGE_FILE)
+@click.option('--update-version-lock', '-u', is_flag=True,
+              help='Save version.lock.json file with updated rule versions in the package')
+def build_release(config_file, update_version_lock):
+    """Assemble all the rules into Kibana-ready release files."""
+    config = load_dump(config_file)['package']
+    click.echo('[+] Building package {}'.format(config.get('name')))
+    package = Package.from_config(config, update_version_lock=update_version_lock)
+    package.save()
+    package.get_package_hash(verbose=True)
+    click.echo('- {} rules included'.format(len(package.rules)))
+
+
+@root.command('update-lock-versions')
+@click.argument('rule-ids', nargs=-1, required=True)
+def update_lock_versions(rule_ids):
+    """Update rule hashes in version.lock.json file without bumping version."""
+    from .packaging import manage_versions
+
+    if not click.confirm('Are you sure you want to update hashes without a version bump?'):
+        return
+
+    rules = [r for r in rule_loader.load_rules(verbose=False).values() if r.id in rule_ids]
+    changed, new = manage_versions(rules, exclude_version_update=True, add_new=False, save_changes=True)
+
+    if not changed:
+        click.echo('No hashes updated')
+
+    return changed
+
+
+@root.command('kibana-diff')
+@click.option('--rule-id', '-r', multiple=True, help='Optionally specify rule ID')
+@click.option('--branch', '-b', default='master', help='Specify the kibana branch to diff against')
+def kibana_diff(rule_id, branch):
+    """Diff rules against their version represented in kibana if exists."""
+    from .misc import get_kibana_rules
+
+    if rule_id:
+        rules = [r for r in rule_loader.load_rules(verbose=False).values() if r.id in rule_id]
+    else:
+        rules = [r for r in rule_loader.load_rules(verbose=False).values() if r.metadata['maturity'] == 'production']
+
+    # add versions to the rules
+    manage_versions(rules, verbose=False)
+
+    rule_paths = [os.path.basename(r.path) for r in rules]
+    try:
+        original_gh_rules = get_kibana_rules(*rule_paths, branch=branch).values()
+    except ValueError as e:
+        click.secho(e.args[0], fg='red', err=True)
+        return
+
+    gh_rule_versions = {r['rule_id']: r.pop('version') for r in original_gh_rules}
+    rule_versions = {r.id: r.contents.pop('version') for r in rules}
+
+    gh_rules = {r['rule_id']: Rule('_', r) for r in original_gh_rules}
+
+    rule_ids = [r.id for r in rules]
+    gh_rule_ids = [r.id for r in gh_rules.values()]
+
+    missing_rules = [r for r in gh_rules.values() if r.id in list(set(gh_rule_ids).difference(set(rule_ids)))]
+
+    diff = {
+        'missing_from_kibana': [],
+        'diff': [],
+        'missing_from_rules': ['{} - {}'.format(r.id, r.name) for r in missing_rules]
+    }
+    for rule in rules:
+        if rule.id not in gh_rule_ids:
+            diff['missing_from_kibana'].append('{} - {}'.format(rule.id, rule.name))
+            continue
+
+        gh_rule = gh_rules[rule.id]
+
+        if rule.get_hash() != gh_rule.get_hash():
+            diff['diff'].append('versions - repo: {}, kibana: {} -> {} - {}'.format(
+                rule_versions[rule.id], gh_rule_versions[rule.id], rule.id, rule.name))
+
+    click.echo(json.dumps(diff, indent=2, sort_keys=True))
+
+
+@root.command("test")
+@click.pass_context
+def test_rules(ctx):
+    """Run unit tests over all of the rules."""
+    import pytest
+
+    clear_caches()
+    ctx.exit(pytest.main(["-v"]))

detection_rules/mappings.py

@@ -0,0 +1,75 @@
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+
+"""RTA to rule mappings."""
+import os
+from collections import defaultdict
+
+from .schema import validate_rta_mapping
+from .utils import load_etc_dump, save_etc_dump, get_path
+
+
+RTA_DIR = get_path("rta")
+
+
+class RtaMappings(object):
+    """Rta-mapping helper class."""
+
+    def __init__(self):
+        """Rta-mapping validation and prep."""
+        self.mapping = load_etc_dump('rule-mapping.yml')  # type: dict
+        self.validate()
+
+        self._rta_mapping = defaultdict(list)
+        self._remote_rta_mapping = {}
+        self._rule_mappings = {}
+
+    def validate(self):
+        """Validate mapping against schema."""
+        for k, v in self.mapping.items():
+            validate_rta_mapping(v)
+
+    def add_rule_to_mapping_file(self, rule, rta_name, count=0, *sources):
+        """Insert a rule mapping into the mapping file."""
+        mapping = self.mapping
+        rule_map = {
+            'count': count,
+            'rta_name': rta_name,
+            'rule_name': rule.name,
+        }
+
+        if sources:
+            rule_map['sources'] = list(sources)
+
+        mapping[rule.id] = rule_map
+        self.mapping = dict(sorted(mapping.items()))
+        save_etc_dump(self.mapping, 'rule-mapping.yml')
+        return rule_map
+
+    def get_rta_mapping(self):
+        """Build the rule<-->rta mapping based off the mapping file."""
+        if not self._rta_mapping:
+            self._rta_mapping = {rule_id: data['rta'] for rule_id, data in self.mapping.items()}
+
+        return self._rta_mapping
+
+    def get_rta_files(self, rta_list=None, rule_ids=None):
+        """Get the full paths to RTA files, given a list of names or rule ids."""
+        full_rta_mapping = self.get_rta_mapping()
+        rta_files = set()
+        rta_list = set(rta_list or [])
+
+        if rule_ids:
+            for rule_id, rta_map in full_rta_mapping.items():
+                if rule_id in rule_ids:
+                    rta_list.update(rta_map)
+
+        for rta_name in rta_list:
+            # rip off the extension and add .py
+            rta_name, _ = os.path.splitext(os.path.basename(rta_name))
+            rta_path = os.path.abspath(os.path.join(RTA_DIR, rta_name + ".py"))
+            if os.path.exists(rta_path):
+                rta_files.add(rta_path)
+
+        return list(sorted(rta_files))

detection_rules/misc.py

@@ -0,0 +1,197 @@
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+
+"""Misc support."""
+import json
+import os
+import re
+import time
+import uuid
+
+import click
+import requests
+
+from .utils import ROOT_DIR
+
+_CONFIG = {}
+
+
+def nested_get(_dict, dot_key, default=None):
+    """Get a nested field from a nested dict with dot notation."""
+    if _dict is None or dot_key is None:
+        return default
+    elif '.' in dot_key and isinstance(_dict, dict):
+        dot_key = dot_key.split('.')
+        this_key = dot_key.pop(0)
+        return nested_get(_dict.get(this_key, default), '.'.join(dot_key), default)
+    else:
+        return _dict.get(dot_key, default)
+
+
+def nested_set(_dict, dot_key, value):
+    """Set a nested field from a a key in dot notation."""
+    for key in dot_key.split('.')[:-1]:
+        _dict = _dict.setdefault(key, {})
+
+    if isinstance(_dict, dict):
+        _dict[dot_key[-1]] = value
+    else:
+        raise ValueError('dict cannot set a value to a non-dict for {}'.format(dot_key))
+
+
+def schema_prompt(name, value=None, required=False, **options):
+    """Interactively prompt based on schema requirements."""
+    name = str(name)
+    field_type = options.get('type')
+    pattern = options.get('pattern')
+    enum = options.get('enum', [])
+    minimum = options.get('minimum')
+    maximum = options.get('maximum')
+    min_item = options.get('min_items', 0)
+    max_items = options.get('max_items', 9999)
+
+    default = options.get('default')
+    if default is not None and str(default).lower() in ('true', 'false'):
+        default = str(default).lower()
+
+    if 'date' in name:
+        default = time.strftime('%Y/%m/%d')
+
+    if name == 'rule_id':
+        default = str(uuid.uuid4())
+
+    def _check_type(_val):
+        if field_type in ('number', 'integer') and not str(_val).isdigit():
+            print('Number expected but got: {}'.format(_val))
+            return False
+        if pattern and (not re.match(pattern, _val) or len(re.match(pattern, _val).group(0)) != len(_val)):
+            print('{} did not match pattern: {}!'.format(_val, pattern))
+            return False
+        if enum and _val not in enum:
+            print('{} not in valid options: {}'.format(_val, ', '.join(enum)))
+            return False
+        if minimum and (type(_val) == int and int(_val) < minimum):
+            print('{} is less than the minimum: {}'.format(str(_val), str(minimum)))
+            return False
+        if maximum and (type(_val) == int and int(_val) > maximum):
+            print('{} is greater than the maximum: {}'.format(str(_val), str(maximum)))
+            return False
+        if field_type == 'boolean' and _val.lower() not in ('true', 'false'):
+            print('Boolean expected but got: {}'.format(str(_val)))
+            return False
+        return True
+
+    def _convert_type(_val):
+        if field_type == 'boolean' and not type(_val) == bool:
+            _val = True if _val.lower() == 'true' else False
+        return int(_val) if field_type in ('number', 'integer') else _val
+
+    prompt = '{name}{default}{required}{multi}'.format(
+        name=name,
+        default=' [{}] ("n/a" to leave blank) '.format(default) if default else '',
+        required=' (required) ' if required else '',
+        multi=' (multi, comma separated) ' if field_type == 'array' else '').strip() + ': '
+
+    while True:
+        result = value or input(prompt) or default
+        if result == 'n/a':
+            result = None
+
+        if not result:
+            if required:
+                value = None
+                continue
+            else:
+                return
+
+        if field_type == 'array':
+            result_list = result.split(',')
+
+            if not (min_item < len(result_list) < max_items):
+                if required:
+                    value = None
+                    break
+                else:
+                    return []
+
+            for value in result_list:
+                if not _check_type(value):
+                    if required:
+                        value = None
+                        break
+                    else:
+                        return []
+            return [_convert_type(r) for r in result_list]
+        else:
+            if _check_type(result):
+                return _convert_type(result)
+            elif required:
+                value = None
+                continue
+            return
+
+
+def get_kibana_rules_map(branch='master'):
+    """Get list of available rules from the Kibana repo and return a list of URLs."""
+    r = requests.get('https://api.github.com/repos/elastic/kibana/branches?per_page=1000')
+    branch_names = [b['name'] for b in r.json()]
+    if branch not in branch_names:
+        raise ValueError('branch "{}" does not exist in kibana'.format(branch))
+
+    url = ('https://api.github.com/repos/elastic/kibana/contents/x-pack/{legacy}plugins/siem/server/lib/'
+           'detection_engine/rules/prepackaged_rules?ref={branch}')
+
+    gh_rules = requests.get(url.format(legacy='', branch=branch)).json()
+
+    # pre-7.8 the siem was under the legacy directory
+    if isinstance(gh_rules, dict) and gh_rules.get('message', '') == 'Not Found':
+        gh_rules = requests.get(url.format(legacy='legacy/', branch=branch)).json()
+
+    return {os.path.splitext(r['name'])[0]: r['download_url'] for r in gh_rules if r['name'].endswith('.json')}
+
+
+def get_kibana_rules(*rule_paths, branch='master', verbose=True):
+    """Retrieve prepackaged rules from kibana repo."""
+    if verbose:
+        click.echo('Downloading rules from {} branch in kibana repo...'.format(branch))
+
+    if rule_paths:
+        rule_paths = [os.path.splitext(os.path.basename(p))[0] for p in rule_paths]
+        return {n: requests.get(r).json() for n, r in get_kibana_rules_map(branch).items() if n in rule_paths}
+    else:
+        return {n: requests.get(r).json() for n, r in get_kibana_rules_map(branch).items()}
+
+
+def parse_config():
+    """Parse a default config file."""
+    global _CONFIG
+
+    if not _CONFIG:
+        config_file = os.path.join(ROOT_DIR, '.siem-rules-cfg.json')
+
+        if os.path.exists(config_file):
+            with open(config_file) as f:
+                _CONFIG = json.load(f)
+
+            click.secho('Loaded config file: {}'.format(config_file), fg='yellow')
+
+    return _CONFIG
+
+
+def set_param_values(ctx, param, value):
+    """Get value for defined key."""
+    key = param.name
+    config = parse_config()
+    env_key = 'SR_' + key
+    prompt = True if param.hide_input is not False else False
+
+    if value:
+        return value
+    elif os.environ.get(env_key):
+        return os.environ[env_key]
+    elif config.get(key):
+        return config[key]
+    elif prompt:
+        return click.prompt(key, default=param.default if not param.default else None, hide_input=param.hide_input,
+                            show_default=True if param.default else False)

detection_rules/packaging.py

@@ -0,0 +1,252 @@
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+
+"""Packaging and preparation for releases."""
+import base64
+import hashlib
+import json
+import os
+import shutil
+from collections import OrderedDict
+
+import click
+
+from . import rule_loader
+from .rule import Rule  # noqa: F401
+from .utils import get_path, get_etc_path
+
+RELEASE_DIR = get_path("releases")
+PACKAGE_FILE = get_etc_path('packages.yml')
+RULE_VERSIONS = get_etc_path('version.lock.json')
+
+
+def filter_rule(rule, config_filter):  # type: (Rule,dict) -> bool  # rule.contents (not api), filter_dict -> match
+    """Filter a rule based off metadata and a package configuration."""
+    flat_rule = rule.flattened_contents
+    for key, values in config_filter.items():
+        if key not in flat_rule:
+            return False
+
+        values = set([v.lower() if isinstance(v, str) else v for v in values])
+        rule_value = flat_rule[key]
+
+        if isinstance(rule_value, list):
+            rule_values = {v.lower() if isinstance(v, str) else v for v in rule_value}
+        else:
+            rule_values = {rule_value.lower() if isinstance(rule_value, str) else rule_value}
+
+        if len(rule_values & values) == 0:
+            return False
+
+    return True
+
+
+def manage_versions(rules, current_versions=None, exclude_version_update=False, add_new=True, save_changes=False,
+                    verbose=True):
+    # type: (list, dict, bool, bool, bool, bool) -> [list, list]
+    """Update the contents of the version.lock file and optionally save changes."""
+    new_rules = {}
+    changed_rules = []
+
+    if current_versions is None:
+        with open(RULE_VERSIONS, 'r') as f:
+            current_versions = json.load(f)
+
+    for rule in rules:
+        # it is a new rule, so add it if specified, and add an initial version to the rule
+        if rule.id not in current_versions:
+            new_rules[rule.id] = {'rule_name': rule.name, 'version': 1, 'sha256': rule.get_hash()}
+            rule.contents['version'] = 1
+        else:
+            version_lock_info = current_versions.get(rule.id)
+            version = version_lock_info['version']
+            rule_hash = rule.get_hash()
+
+            # if it has been updated, then we need to bump the version info and optionally save the changes later
+            if rule_hash != version_lock_info['sha256']:
+                rule.contents['version'] = version + 1
+
+                if not exclude_version_update:
+                    version_lock_info['version'] = rule.contents['version']
+
+                version_lock_info.update(sha256=rule_hash, rule_name=rule.name)
+                changed_rules.append(rule.id)
+            else:
+                rule.contents['version'] = version
+
+    # update the document with the new rules
+    if new_rules or changed_rules:
+        if verbose:
+            click.echo('Rule hash changes detected!')
+
+        if save_changes:
+            current_versions.update(new_rules if add_new else {})
+            current_versions = OrderedDict(sorted(current_versions.items(), key=lambda x: x[1]['rule_name']))
+
+            with open(RULE_VERSIONS, 'w') as f:
+                json.dump(current_versions, f, indent=2, sort_keys=True)
+
+                if verbose:
+                    click.echo('Updated version.lock.json file with:')
+        else:
+            if verbose:
+                click.echo('run `build-release --update-version-lock` to update the version.lock.json file')
+
+        if verbose:
+            if changed_rules:
+                click.echo(' - {} changed rule version(s)'.format(len(changed_rules)))
+            if new_rules:
+                click.echo(' - {} new rule version addition(s)'.format(len(new_rules)))
+
+    return changed_rules, new_rules.keys()
+
+
+class Package(object):
+    """Packaging object for siem rules and releases."""
+
+    def __init__(self, rules, name, tune=False, release=False, current_versions=None, min_version=None,
+                 max_version=None, update_version_lock=False):
+        """Initialize a package."""
+        self.rules = [r.copy() for r in rules]  # type: list[Rule]
+        self.name = name
+        self.release = release
+
+        self.changed_rules, self.new_rules = self._add_versions(current_versions, update_version_lock)
+
+        if min_version or max_version:
+            self.rules = [r for r in self.rules
+                          if (min_version or 0) <= r.contents['version'] <= (max_version or r.contents['version'])]
+
+        if tune:
+            for rule in rules:
+                rule.tune()
+
+    def _add_versions(self, current_versions, update_versions_lock=False):
+        """Add versions to rules at load time."""
+        return manage_versions(self.rules, current_versions=current_versions, save_changes=update_versions_lock)
+
+    def save_release_files(self, directory, changed_rules, new_rules):
+        """Release a package."""
+        # TODO:
+        # xslx of mitre coverage
+        # release notes
+
+        with open(os.path.join(directory, '{}-summary.txt'.format(self.name)), 'w') as f:
+            f.write(self.generate_summary(changed_rules, new_rules))
+        with open(os.path.join(directory, '{}-consolidated.json'.format(self.name)), 'w') as f:
+            json.dump(json.loads(self.get_consolidated()), f, sort_keys=True, indent=2)
+
+    def get_consolidated(self, as_api=True):
+        """Get a consolidated package of the rules in a single file."""
+        full_package = []
+        for rule in self.rules:
+            full_package.append(rule.contents if as_api else rule.rule_format())
+
+        return json.dumps(full_package, sort_keys=True)
+
+    def save(self, verbose=True):
+        """Save a package and all artifacts."""
+        save_dir = os.path.join(RELEASE_DIR, self.name)
+        rules_dir = os.path.join(save_dir, 'rules')
+        extras_dir = os.path.join(save_dir, 'extras')
+
+        # remove anything that existed before
+        shutil.rmtree(save_dir, ignore_errors=True)
+        os.makedirs(rules_dir, exist_ok=True)
+        os.makedirs(extras_dir, exist_ok=True)
+
+        for rule in self.rules:
+            rule.save(new_path=os.path.join(rules_dir, os.path.basename(rule.path)))
+
+        if self.release:
+            self.save_release_files(extras_dir, self.changed_rules, self.new_rules)
+
+            # zip all rules only and place in extras
+            shutil.make_archive(os.path.join(extras_dir, self.name), 'zip', root_dir=os.path.dirname(rules_dir),
+                                base_dir=os.path.basename(rules_dir))
+
+            # zip everything and place in release root
+            shutil.make_archive(os.path.join(save_dir, '{}-all'.format(self.name)), 'zip',
+                                root_dir=os.path.dirname(extras_dir), base_dir=os.path.basename(extras_dir))
+
+        if verbose:
+            click.echo('Package saved to: {}'.format(save_dir))
+
+    def from_github(self):
+        """Retrieve previously released and staged packages."""
+
+    def get_package_hash(self, as_api=True, verbose=True):
+        """Get hash of package contents."""
+        contents = base64.b64encode(self.get_consolidated(as_api=as_api).encode('utf-8'))
+        sha256 = hashlib.sha256(contents).hexdigest()
+
+        if verbose:
+            click.echo('- sha256: {}'.format(sha256))
+
+        return sha256
+
+    @classmethod
+    def from_config(cls, config=None, update_version_lock=False):  # type: (dict, bool) -> Package
+        """Load a rules package given a config."""
+        all_rules = rule_loader.load_rules(verbose=False).values()
+        config = config or {}
+        rule_filter = config.pop('filter', {})
+        min_version = config.pop('min_version', None)
+        max_version = config.pop('max_version', None)
+
+        rules = filter(lambda rule: filter_rule(rule, rule_filter), all_rules)
+        update = config.pop('update', {})
+        package = cls(rules, min_version=min_version, max_version=max_version, update_version_lock=update_version_lock,
+                      **config)
+
+        # Allow for some fields to be overwritten
+        if update.get('data', {}):
+            for rule in package.rules:
+                for sub_dict, values in update.items():
+                    rule.contents[sub_dict].update(values)
+
+        return package
+
+    def generate_summary(self, changed_rules, new_rules):
+        """Generate stats on package."""
+        ecs_versions = set()
+        indices = set()
+        changed = []
+        new = []
+
+        for rule in self.rules:
+            ecs_versions.update(rule.ecs_version)
+            indices.update(rule.contents.get('index', ''))
+
+            if rule.id in changed_rules:
+                changed.append('{} (v{})'.format(rule.name, rule.contents.get('version')))
+            elif rule.id in new_rules:
+                new.append('{} (v{})'.format(rule.name, rule.contents.get('version')))
+
+        total = 'Total Rules: {}'.format(len(self.rules))
+        sha256 = 'Package Hash: {}'.format(self.get_package_hash(verbose=False))
+        ecs_versions = 'ECS Versions: {}'.format(', '.join(ecs_versions))
+        indices = 'Included Indexes: {}'.format(', '.join(indices))
+        new_rules = 'New Rules: \n{}'.format('\n'.join(' - ' + s for s in sorted(new)) if new else 'N/A')
+        modified_rules = 'Modified Rules: \n{}'.format('\n'.join(' - ' + s for s in sorted(changed)) if new else 'N/A')
+        return '\n'.join([total, sha256, ecs_versions, indices, new_rules, modified_rules])
+
+    def generate_mitre(self):
+        """Create an excel file based on mitre coverage."""
+        # mapping with highlights of covered cells - links to pivot table with technique id selected
+
+    def reconcile_changes(self):
+        """Parse and generate changes since previous release based on changed.toml file."""
+        # at packaging, generate flat changes file to standard, based on consolidated and deduped interpretation of
+        #   changed.toml and clear out changes.toml
+        # - all based on api_format only
+        # see packages.yml - can update management.changed = True:
+        #   until released in package, then added with filter and changed to False
+
+    def generate_change_notes(self):
+        """Generate change release notes."""
+
+    def bump_versions(self, save_changes=False, current_versions=None):
+        """Bump the versions of all production rules included in a release and optionally save changes."""
+        return manage_versions(self.rules, current_versions=current_versions, save_changes=save_changes)

detection_rules/rule.py

@@ -0,0 +1,357 @@
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+"""Rule object."""
+import base64
+import copy
+import hashlib
+import json
+import os
+
+import click
+import kql
+
+from . import ecs, beats
+from .attack import TACTICS, build_threat_map_entry, technique_lookup
+from .rule_formatter import nested_normalize, toml_write
+from .schema import metadata_schema, schema_validate, get_schema
+from .utils import get_path, clear_caches, cached
+
+
+RULES_DIR = get_path("rules")
+RULE_TYPE_OPTIONS = ['machine_learning', 'query', 'saved_id']
+_META_SCHEMA_REQ_DEFAULTS = {}
+
+
+class Rule(object):
+    """Rule class containing all the information about a rule."""
+
+    def __init__(self, path, contents, tune=False):
+        """Create a Rule from a toml management format."""
+        self.path = os.path.realpath(path)
+        self.contents = contents.get('rule', contents)
+        self.metadata = self.set_metadata(contents.get('metadata', contents))
+
+        self.formatted_rule = copy.deepcopy(self.contents).get('query', None)
+
+        self.validate()
+        self.unoptimized_query = self.contents.get('query')
+
+        if tune:
+            self.tune_rule = True
+            self.tune()
+
+        self._original_hash = self.get_hash()
+
+    def __str__(self):
+        return 'name={}, path={}, query={}'.format(self.name, self.path, self.query)
+
+    def __repr__(self):
+        return '{}(path={}, contents={}, tune={})'.format(type(self).__name__, repr(self.path), repr(self.contents),
+                                                          repr(self.tune_rule))
+
+    def __eq__(self, other):
+        if type(self) == type(other):
+            return self.get_hash() == other.get_hash()
+        return False
+
+    def copy(self):
+        return Rule(path=self.path, contents={'rule': self.contents.copy(), 'metadata': self.metadata.copy()})
+
+    @property
+    def id(self):
+        return self.contents.get("rule_id")
+
+    @property
+    def name(self):
+        return self.contents.get("name")
+
+    @property
+    def query(self):
+        return self.contents.get('query')
+
+    @property
+    def parsed_kql(self):
+        if self.query and self.contents['language'] == 'kuery':
+            return kql.parse(self.query)
+
+    @property
+    def filters(self):
+        return self.contents.get('filters')
+
+    @property
+    def ecs_version(self):
+        return sorted(self.metadata.get('ecs_version', []))
+
+    @property
+    def flattened_contents(self):
+        return dict(self.contents, **self.metadata)
+
+    @property
+    def type(self):
+        return self.contents.get('type')
+
+    def to_eql(self):
+        if self.query and self.contents['language'] == 'kuery':
+            return kql.to_eql(self.query)
+
+    @staticmethod
+    @cached
+    def get_meta_schema_required_defaults():
+        """Get the default values for required properties in the metadata schema."""
+        required = [v for v in metadata_schema['required']]
+        properties = {k: v for k, v in metadata_schema['properties'].items() if k in required}
+        return {k: v.get('default') or [v['items']['default']] for k, v in properties.items()}
+
+    def set_metadata(self, contents):
+        """Parse metadata fields and set missing required fields to the default values."""
+        metadata = {k: v for k, v in contents.items() if k in metadata_schema['properties']}
+        defaults = self.get_meta_schema_required_defaults().copy()
+        defaults.update(metadata)
+        return defaults
+
+    def rule_format(self, formatted_query=True):
+        """Get the contents in rule format."""
+        contents = self.contents.copy()
+        if formatted_query:
+            if self.formatted_rule:
+                contents['query'] = self.formatted_rule
+        return {'metadata': self.metadata, 'rule': contents}
+
+    def normalize(self, indent=2):
+        """Normalize the (api only) contents and return a serialized dump of it."""
+        return json.dumps(nested_normalize(self.contents), sort_keys=True, indent=indent)
+
+    def tune(self):
+        """Tune query by including applicable fields derived from metadata."""
+        # if not self.query:
+        #     return
+        #
+        # self.unoptimized_query = self.contents.get('query')
+        #
+        # if not hasattr(self.parsed_query, 'terms'):
+        #     # can prepend here if we want
+        #     return
+        #
+        # # TODO: This is error prone and absolutely can/should be better done with a custom walker to:
+        # #  - find these fields
+        # #  - move them to the front/highest precedence
+        # #  - dedup+update them with these values from metadata
+        # #  I am going to leave it for now as a good mechanism for testing the theory and since it only impacts at
+        # #    "package" time and will open an issue in the meantime
+        #
+        # # add os version
+        # # many os ecs fields - will optimize later
+        # # if not any(str(term.left) == '' for term in parsed_query.terms) and self.metadata.get('os_type_list'):
+        # #     self.contents['query'] = ':({}) and '.format(' or '.join(self.metadata['_os_type_list'])) + self.query
+        #
+        # # add ecs version
+        # # handle these better with eql2kql
+        # compares = [str(term.left) == 'ecs.version' for term in self.parsed_query.terms
+        #             if isinstance(term, Comparison)]
+        # in_sets = [str(term.expression) == 'ecs.version' for term in self.parsed_query.terms
+        #            if isinstance(term, InSet)]
+        #
+        # if any(in_sets):
+        #     pass
+        # elif any(compares):
+        #     pass
+        # elif not (any(compares) or any(in_sets)):
+        #     ecs_query = ' or '.join(self.metadata['ecs_version'])
+        #     self.contents['query'] = 'ecs.version:({}) and '.format(ecs_query) + self.query
+
+    def untune(self):
+        """Restore query to pre-tuned state."""
+        # self.contents['query'] = self.unoptimized_query
+
+    def get_path(self):
+        """Wrapper around getting path."""
+        if not self.path:
+            raise ValueError('path not set for rule: \n\t{}'.format(self))
+
+        return self.path
+
+    def needs_save(self):
+        """Determines if the rule was changed from original or was never saved."""
+        return self._original_hash != self.get_hash()
+
+    @classmethod  # TODO
+    def from_eql_rule(cls, path, contents, validate=False):
+        """Create a rule from loaded rule (toml) contents."""
+        # if validate:
+        #     jsonschema.validate(contents, rule_schema)
+
+        return cls(path, contents)
+
+    def bump_version(self):
+        """Bump the version of the rule."""
+        self.contents['version'] += 1
+
+    def validate(self, as_rule=False, versioned=False):
+        """Validate against a rule schema, query schema, and linting."""
+        self.normalize()
+
+        if as_rule:
+            schema_validate(self.rule_format(), as_rule=True)
+        else:
+            schema_validate(self.contents, versioned=versioned)
+
+        if self.query and self.contents['language'] == 'kuery':
+            # validate against all specified schemas or the latest if none specified
+            ecs_versions = self.metadata.get('ecs_version')
+
+            indexes = self.contents.get("index", [])
+            beat_types = [index.split("-")[0] for index in indexes if "beat-*" in index]
+            beat_schema = beats.get_schema_for_query(self.parsed_kql, beat_types) if beat_types else None
+
+            if not ecs_versions:
+                kql.parse(self.query, schema=ecs.get_kql_schema(indexes=indexes, beat_schema=beat_schema))
+            else:
+                for version in ecs_versions:
+                    try:
+                        schema = ecs.get_kql_schema(version=version, indexes=indexes, beat_schema=beat_schema)
+                    except KeyError:
+                        raise KeyError(
+                            'Unknown ecs schema version: {} in rule {}.\n'
+                            'Do you need to update schemas?'.format(version, self.name))
+
+                    try:
+                        kql.parse(self.query, schema=schema)
+                    except kql.KqlParseError as exc:
+                        message = exc.error_msg
+                        trailer = None
+                        if "Unknown field" in message and beat_types:
+                            trailer = "\nTry adding event.module and event.dataset to specify beats module"
+
+                        raise kql.KqlParseError(exc.error_msg, exc.line, exc.column, exc.source,
+                                                len(exc.caret.lstrip()), trailer=trailer)
+
+    def save(self, new_path=None, as_rule=False, verbose=False):
+        """Save as pretty toml rule file as toml."""
+        path, _ = os.path.splitext(new_path or self.get_path())
+        path += '.toml' if as_rule else '.json'
+
+        if as_rule:
+            toml_write(self.rule_format(), path)
+        else:
+            with open(path, 'w', newline='\n') as f:
+                json.dump(self.contents, f, sort_keys=True, indent=2)
+                f.write('\n')
+
+        if verbose:
+            print('Rule {} saved to {}'.format(self.name, path))
+
+    def get_hash(self):
+        """Get a standardized hash of a rule to consistently check for changes."""
+        contents = base64.b64encode(json.dumps(self.contents, sort_keys=True).encode('utf-8'))
+        return hashlib.sha256(contents).hexdigest()
+
+    @classmethod
+    def build(cls, path=None, rule_type=None, required_only=True, save=True, **kwargs):
+        """Build a rule from data and prompts."""
+        from .misc import schema_prompt
+        # from .rule_loader import rta_mappings
+
+        kwargs = copy.deepcopy(kwargs)
+
+        while rule_type not in RULE_TYPE_OPTIONS:
+            rule_type = click.prompt('Rule type ({})'.format(', '.join(RULE_TYPE_OPTIONS)))
+
+        schema = get_schema(rule_type)
+        props = schema['properties']
+        opt_reqs = schema.get('required', [])
+        contents = {}
+        skipped = []
+
+        for name, options in props.items():
+
+            if name == 'type':
+                contents[name] = rule_type
+                continue
+
+            # these are set at package release time
+            if name == 'version':
+                continue
+
+            if required_only and name not in opt_reqs:
+                continue
+
+            # build this from technique ID
+            if name == 'threat':
+                threat_map = []
+
+                while click.confirm('add mitre tactic?'):
+                    tactic = schema_prompt('mitre tactic name', type='string', enum=TACTICS, required=True)
+                    technique_ids = schema_prompt(f'technique IDs for {tactic}', type='array', required=True,
+                                                  enum=list(technique_lookup))
+                    threat_map.append(build_threat_map_entry(tactic, *technique_ids))
+
+                if len(threat_map) > 0:
+                    contents[name] = threat_map
+                continue
+
+            if kwargs.get(name):
+                contents[name] = schema_prompt(kwargs.pop(name))
+                continue
+
+            result = schema_prompt(name, required=name in opt_reqs, **options)
+
+            if result:
+                if name not in opt_reqs and result == options.get('default', ''):
+                    skipped.append(name)
+                    continue
+
+                contents[name] = result
+
+        metadata = {}
+        ecs_version = schema_prompt('ecs_version', required=False, value=None,
+                                    **metadata_schema['properties']['ecs_version'])
+        if ecs_version:
+            metadata['ecs_version'] = ecs_version
+
+        # validate before creating
+        schema_validate(contents)
+
+        suggested_path = os.path.join(RULES_DIR, contents['name'])  # TODO: UPDATE BASED ON RULE STRUCTURE
+        path = os.path.realpath(path or input('File path for rule [{}]: '.format(suggested_path)) or suggested_path)
+
+        rule = None
+
+        try:
+            rule = cls(path, {'rule': contents, 'metadata': metadata})
+        except kql.KqlParseError as e:
+            if e.error_msg == 'Unknown field':
+                warning = ('If using a non-ECS field, you must update "ecs{}.non-ecs-schema.json" under `beats` or '
+                           '`legacy-endgame` (Non-ECS fields should be used minimally).'.format(os.path.sep))
+                click.secho(e.args[0], fg='red', err=True)
+                click.secho(warning, fg='yellow', err=True)
+                click.pause()
+
+            # if failing due to a query, loop until resolved or terminated
+            while True:
+                try:
+                    contents['query'] = click.edit(contents['query'], extension='.eql')
+                    rule = cls(path, {'rule': contents, 'metadata': metadata})
+                except kql.KqlParseError as e:
+                    click.secho(e.args[0], fg='red', err=True)
+                    click.pause()
+
+                    if e.error_msg.startswith("Unknown field"):
+                        # get the latest schema for schema errors
+                        clear_caches()
+                        ecs.get_kql_schema(indexes=contents.get("index", []))
+                    continue
+
+                break
+
+        if save:
+            rule.save(verbose=True, as_rule=True)
+
+        if skipped:
+            print('Did not set the following values because they are un-required when set to the default value')
+            print(' - {}'.format('\n - '.join(skipped)))
+
+        # rta_mappings.add_rule_to_mapping_file(rule)
+        click.echo('Placeholder added to rule-mapping.yml')
+
+        return rule

detection_rules/rule_formatter.py

@@ -0,0 +1,193 @@
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+
+"""Helper functions for managing rules in the repository."""
+import copy
+import io
+import textwrap
+from collections import OrderedDict
+
+import toml
+
+from .schema import NONFORMATTED_FIELDS
+
+SQ = "'"
+DQ = '"'
+TRIPLE_SQ = SQ * 3
+TRIPLE_DQ = DQ * 3
+
+
+def cleanup_whitespace(val):
+    if isinstance(val, str):
+        return " ".join(line.strip() for line in val.strip().splitlines())
+    return val
+
+
+def nested_normalize(d, skip_cleanup=False):
+    if isinstance(d, str):
+        return d if skip_cleanup else cleanup_whitespace(d)
+    elif isinstance(d, list):
+        return [nested_normalize(val) for val in d]
+    elif isinstance(d, dict):
+        for k, v in d.items():
+            if k == 'query':
+                # TODO: the linter still needs some work, but once up to par, uncomment to implement - kql.lint(v)
+                d.update({k: nested_normalize(v)})
+            elif k in NONFORMATTED_FIELDS:
+                # let these maintain newlines and whitespace for markdown support
+                d.update({k: nested_normalize(v, skip_cleanup=True)})
+            else:
+                d.update({k: nested_normalize(v)})
+        return d
+    else:
+        return d
+
+
+def wrap_text(v, block_indent=0, join=False):
+    """Block and indent a blob of text."""
+    v = ' '.join(v.split())
+    lines = textwrap.wrap(v, initial_indent=' ' * block_indent, subsequent_indent=' ' * block_indent, width=120,
+                          break_long_words=False, break_on_hyphens=False)
+    lines = [line + '\n' for line in lines]
+    return lines if not join else ''.join(lines)
+
+
+class NonformattedField(str):
+    """Non-formatting class."""
+
+
+class RuleTomlEncoder(toml.TomlEncoder):
+    """Generate a pretty form of toml."""
+
+    def __init__(self, _dict=dict, preserve=False):
+        """Create the encoder but override some default functions."""
+        super(RuleTomlEncoder, self).__init__(_dict, preserve)
+        self._old_dump_str = toml.TomlEncoder().dump_funcs[str]
+        self._old_dump_list = toml.TomlEncoder().dump_funcs[list]
+        self.dump_funcs[str] = self.dump_str
+        self.dump_funcs[type(u"")] = self.dump_str
+        self.dump_funcs[list] = self.dump_list
+        self.dump_funcs[NonformattedField] = self.dump_str
+
+    def dump_str(self, v):
+        """Change the TOML representation to multi-line or single quote when logical."""
+        initial_newline = ['\n']
+
+        if isinstance(v, NonformattedField):
+            # first line break is not forced like other multiline string dumps
+            lines = v.splitlines(True)
+            initial_newline = []
+
+        else:
+            lines = wrap_text(v)
+
+        multiline = len(lines) > 1
+        raw = (multiline or (DQ in v and SQ not in v)) and TRIPLE_DQ not in v
+
+        if multiline:
+            if raw:
+                return "".join([TRIPLE_DQ] + initial_newline + lines + [TRIPLE_DQ])
+            else:
+                return "\n".join([TRIPLE_SQ] + [self._old_dump_str(line)[1:-1] for line in lines] + [TRIPLE_SQ])
+        elif raw:
+            return u"'{:s}'".format(lines[0])
+        return self._old_dump_str(v)
+
+    def _dump_flat_list(self, v):
+        """A slightly tweaked version of original dump_list, removing trailing commas."""
+        if not v:
+            return "[]"
+
+        retval = "[" + str(self.dump_value(v[0])) + ","
+        for u in v[1:]:
+            retval += " " + str(self.dump_value(u)) + ","
+        retval = retval.rstrip(',') + "]"
+        return retval
+
+    def dump_list(self, v):
+        """Dump a list more cleanly."""
+        if all([isinstance(d, str) for d in v]) and sum(len(d) + 3 for d in v) > 100:
+            dump = []
+            for item in v:
+                if len(item) > (120 - 4 - 3 - 3) and ' ' in item:
+                    dump.append('    """\n{}    """'.format(wrap_text(item, block_indent=4, join=True)))
+                else:
+                    dump.append(' ' * 4 + self.dump_value(item))
+            return '[\n{},\n]'.format(',\n'.join(dump))
+        return self._dump_flat_list(v)
+
+
+def toml_write(rule_contents, outfile=None):
+    """Write rule in TOML."""
+    def write(text, nl=True):
+        if outfile:
+            outfile.write(text)
+            if nl:
+                outfile.write(u"\n")
+        else:
+            print(text, end='' if not nl else '\n')
+
+    encoder = RuleTomlEncoder()
+    contents = copy.deepcopy(rule_contents)
+    needs_close = False
+
+    def _do_write(_data, _contents):
+        query = None
+
+        if _data == 'rule':
+            # - We want to avoid the encoder for the query and instead use kql-lint.
+            # - Linting is done in rule.normalize() which is also called in rule.validate().
+            # - Until lint has tabbing, this is going to result in all queries being flattened with no wrapping,
+            #     but will at least purge extraneous white space
+            query = contents['rule'].pop('query', '').strip()
+
+            tags = contents['rule'].get("tags", [])
+
+            if tags and isinstance(tags, list):
+                contents['rule']["tags"] = list(sorted(set(tags)))
+
+        top = OrderedDict()
+        bottom = OrderedDict()
+
+        for k in sorted(list(_contents)):
+            v = _contents.pop(k)
+
+            if isinstance(v, dict):
+                bottom[k] = OrderedDict(sorted(v.items()))
+            elif isinstance(v, list):
+                if any([isinstance(value, (dict, list)) for value in v]):
+                    bottom[k] = v
+                else:
+                    top[k] = v
+            elif k in NONFORMATTED_FIELDS:
+                top[k] = NonformattedField(v)
+            else:
+                top[k] = v
+
+        if query:
+            top.update({'query': "XXxXX"})
+
+        top.update(bottom)
+        top = toml.dumps(OrderedDict({data: top}), encoder=encoder)
+
+        # we want to preserve the query format, but want to modify it in the context of encoded dump
+        if query:
+            formatted_query = "\nquery = '''\n{}\n'''{}".format(query, '\n\n' if bottom else '')
+            top = top.replace('query = "XXxXX"', formatted_query)
+
+        write(top)
+
+    try:
+
+        if outfile and not isinstance(outfile, io.IOBase):
+            needs_close = True
+            outfile = open(outfile, 'w')
+
+        for data in ('metadata', 'rule'):
+            _contents = contents.get(data, {})
+            _do_write(data, _contents)
+
+    finally:
+        if needs_close:
+            outfile.close()

detection_rules/rule_loader.py

@@ -0,0 +1,192 @@
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+
+"""Load rule metadata transform between rule and api formats."""
+import functools
+import glob
+import io
+import os
+import re
+from collections import OrderedDict
+
+import click
+import pytoml
+
+from .mappings import RtaMappings
+from .rule import RULES_DIR, Rule
+from .schema import get_schema
+from .utils import get_path, cached
+
+
+RTA_DIR = get_path("rta")
+FILE_PATTERN = r'^([a-z0-9_])+\.(json|toml)$'
+
+
+def mock_loader(f):
+    """Mock rule loader."""
+    @functools.wraps(f)
+    def wrapped(*args, **kwargs):
+        try:
+            return f(*args, **kwargs)
+        finally:
+            load_rules.clear()
+
+    return wrapped
+
+
+def reset():
+    """Clear all rule caches."""
+    load_rule_files.clear()
+    load_rules.clear()
+    get_rule.clear()
+    filter_rules.clear()
+
+
+@cached
+def load_rule_files(verbose=True):
+    """Load the rule YAML files, but without parsing the EQL query portion."""
+    file_lookup = {}  # type: dict[str, dict]
+
+    if verbose:
+        print("Loading rules from {}".format(RULES_DIR))
+
+    for rule_file in sorted(glob.glob(os.path.join(RULES_DIR, '**', '*.toml'), recursive=True)):
+        try:
+            # use pytoml instead of toml because of annoying bugs
+            # https://github.com/uiri/toml/issues/152
+            # might also be worth looking at https://github.com/sdispater/tomlkit
+            with io.open(rule_file, "r", encoding="utf-8") as f:
+                file_lookup[rule_file] = pytoml.load(f)
+        except Exception:
+            print(u"Error loading {}".format(rule_file))
+            raise
+
+    if verbose:
+        print("Loaded {} rules".format(len(file_lookup)))
+    return file_lookup
+
+
+@cached
+def load_rules(file_lookup=None, verbose=True, error=True):
+    """Load all the rules from toml files."""
+    file_lookup = file_lookup or load_rule_files(verbose=verbose)
+
+    failed = False
+    rules = []  # type: list[Rule]
+    errors = []
+    queries = []
+    rule_ids = set()
+    rule_names = set()
+
+    for rule_file, rule_contents in file_lookup.items():
+        try:
+            rule = Rule(rule_file, rule_contents)
+
+            if rule.id in rule_ids:
+                raise KeyError("Rule has duplicate ID to {}".format(next(r for r in rules if r.id == rule.id).path))
+
+            if rule.name in rule_names:
+                raise KeyError("Rule has duplicate name to {}".format(
+                    next(r for r in rules if r.name == rule.name).path))
+
+            if rule.parsed_kql:
+                if rule.parsed_kql in queries:
+                    raise KeyError("Rule has duplicate query with {}".format(
+                        next(r for r in rules if r.parsed_kql == rule.parsed_kql).path))
+
+                queries.append(rule.parsed_kql)
+
+            if not re.match(FILE_PATTERN, os.path.basename(rule.path)):
+                raise ValueError(f"Rule {rule.path} does not meet rule name standard of {FILE_PATTERN}")
+
+            rules.append(rule)
+            rule_ids.add(rule.id)
+            rule_names.add(rule.name)
+
+        except Exception as e:
+            failed = True
+            err_msg = "Invalid rule file in {}\n{}".format(rule_file, click.style(e.args[0], fg='red'))
+            errors.append(err_msg)
+            if error:
+                print(err_msg)
+                raise e
+
+    if failed:
+        if verbose:
+            for e in errors:
+                print(e)
+
+    return OrderedDict([(rule.id, rule) for rule in sorted(rules, key=lambda r: r.name)])
+
+
+@cached
+def get_rule(rule_id=None, rule_name=None, file_name=None, verbose=True):
+    """Get a rule based on its id."""
+    rules_lookup = load_rules(verbose=verbose)
+    if rule_id is not None:
+        return rules_lookup.get(rule_id)
+
+    for rule in rules_lookup.values():  # type: Rule
+        if rule.name == rule_name:
+            return rule
+        elif rule.path == file_name:
+            return rule
+
+
+def get_rule_name(rule_id, verbose=True):
+    """Get the name of a rule given the rule id."""
+    rule = get_rule(rule_id, verbose=verbose)
+    if rule:
+        return rule.name
+
+
+def get_file_name(rule_id, verbose=True):
+    """Get the file path that corresponds to a rule."""
+    rule = get_rule(rule_id, verbose=verbose)
+    if rule:
+        return rule.path
+
+
+def get_rule_contents(rule_id, verbose=True):
+    """Get the full contents for a rule_id."""
+    rule = get_rule(rule_id, verbose=verbose)
+    if rule:
+        return rule.contents
+
+
+@cached
+def filter_rules(rules, metadata_field, value):
+    """Filter rules based on the metadata."""
+    return [rule for rule in rules if rule.metadata.get(metadata_field, {}) == value]
+
+
+def get_production_rules():
+    """Get rules with a maturity of production."""
+    return filter_rules(load_rules().values(), 'maturity', 'production')
+
+
+def find_unneeded_defaults(rule):
+    """Remove values that are not required in the schema which are set with default values."""
+    schema = get_schema(rule.contents['type'])
+    props = schema['properties']
+    unrequired_defaults = [p for p in props if p not in schema['required'] and props[p].get('default')]
+    default_matches = {p: rule.contents[p] for p in unrequired_defaults
+                       if rule.contents.get(p) and rule.contents[p] == props[p]['default']}
+    return default_matches
+
+
+rta_mappings = RtaMappings()
+
+
+__all__ = (
+    "load_rules",
+    "get_file_name",
+    "get_production_rules",
+    "get_rule",
+    "filter_rules",
+    "get_rule_name",
+    "get_rule_contents",
+    "reset",
+    "rta_mappings"
+)

detection_rules/schema.py

@@ -0,0 +1,238 @@
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+
+"""Definitions for rule metadata and schemas."""
+import time
+
+import jsl
+import jsonschema
+
+from . import ecs
+from .attack import TACTICS, TACTICS_MAP, TECHNIQUES, technique_lookup
+
+UUID_PATTERN = r'[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}'
+DATE_PATTERN = r'\d{4}/\d{2}/\d{2}'
+VERSION_PATTERN = r'\d+\.\d+\.\d+'
+RULE_LEVELS = ['recommended', 'aggressive']
+MATURITY_LEVELS = ['development', 'testing', 'staged', 'production', 'deprecated']
+OS_OPTIONS = ['windows', 'linux', 'macos', 'solaris']  # need to verify with ecs
+INTERVAL_PATTERN = r'\d+[mshd]'
+MITRE_URL_PATTERN = r'https://attack.mitre.org/{type}/T[A-Z0-9]+/'
+
+NONFORMATTED_FIELDS = 'note',
+
+
+# kibana/.../siem/server/lib/detection_engine/routes/schemas/add_prepackaged_rules_schema.ts
+#                           /detection_engine/routes/schemas/schemas.ts
+# rule_id is required here
+# output_index is not allowed (and instead the space index must be used)
+# immutable defaults to true instead of to false and if it is there can only be true
+# enabled defaults to false instead of true
+# version is a required field that must exist
+
+MACHINE_LEARNING = 'machine_learning'
+SAVED_QUERY = 'saved_query'
+QUERY = 'query'
+
+
+class FilterMetadata(jsl.Document):
+    """Base class for siem rule meta filters."""
+
+    negate = jsl.BooleanField()
+    type = jsl.StringField()
+    key = jsl.StringField()
+    value = jsl.StringField()
+    disabled = jsl.BooleanField()
+    indexRefName = jsl.StringField()
+    alias = jsl.StringField()  # null acceptable
+    params = jsl.DictField(properties={'query': jsl.StringField()})
+
+
+class FilterQuery(jsl.Document):
+    """Base class for siem rule query filters."""
+
+    match = jsl.DictField({
+        'event.action': jsl.DictField(properties={
+            'query': jsl.StringField(),
+            'type': jsl.StringField()
+        })
+    })
+
+
+class FilterState(jsl.Document):
+    """Base class for siem rule $state filters."""
+
+    store = jsl.StringField()
+
+
+class FilterExists(jsl.Document):
+    """Base class for siem rule $state filters."""
+
+    field = jsl.StringField()
+
+
+class Filters(jsl.Document):
+    """Schema for filters"""
+
+    exists = jsl.DocumentField(FilterExists)
+    meta = jsl.DocumentField(FilterMetadata)
+    state = jsl.DocumentField(FilterState, name='$state')
+    query = jsl.DocumentField(FilterQuery)
+
+
+class ThreatTactic(jsl.Document):
+    """Threat tactics."""
+
+    id = jsl.StringField(enum=TACTICS_MAP.values())
+    name = jsl.StringField(enum=TACTICS)
+    reference = jsl.StringField(MITRE_URL_PATTERN.format(type='tactics'))
+
+
+class ThreatTechnique(jsl.Document):
+    """Threat tactics."""
+
+    id = jsl.StringField(enum=list(technique_lookup))
+    name = jsl.StringField(enum=TECHNIQUES)
+    reference = jsl.StringField(MITRE_URL_PATTERN.format(type='techniques'))
+
+
+class Threat(jsl.Document):
+    """Threat framework mapping such as MITRE ATT&CK."""
+
+    framework = jsl.StringField(default='MITRE ATT&CK', required=True)
+    tactic = jsl.DocumentField(ThreatTactic, required=True)
+    technique = jsl.ArrayField(jsl.DocumentField(ThreatTechnique), required=True)
+
+
+class SiemRuleApiSchema(jsl.Document):
+    """Schema for siem rule in API format."""
+
+    actions = jsl.ArrayField(required=False)
+    author = jsl.ArrayField(jsl.StringField(default="Elastic"), required=True, min_items=1)
+    description = jsl.StringField(required=True)
+    # api defaults to false if blank
+    enabled = jsl.BooleanField(default=False, required=False)
+    exceptions_list = jsl.ArrayField(required=False)
+    # _ required since `from` is a reserved word in python
+    from_ = jsl.StringField(required=False, default='now-6m', name='from')
+    false_positives = jsl.ArrayField(jsl.StringField(), required=False)
+    filters = jsl.ArrayField(jsl.DocumentField(Filters))
+    interval = jsl.StringField(pattern=INTERVAL_PATTERN, default='5m', required=False)
+    license = jsl.StringField(required=True, default="Elastic License")
+    max_signals = jsl.IntField(minimum=1, required=False, default=100)  # cap a max?
+    meta = jsl.DictField(required=False)
+    name = jsl.StringField(required=True)
+    note = jsl.StringField(required=False)
+    # output_index = jsl.StringField(required=False)  # this is NOT allowed!
+    references = jsl.ArrayField(jsl.StringField(), required=False)
+    risk_score = jsl.IntField(minimum=0, maximum=100, required=True, default=21)
+    rule_id = jsl.StringField(pattern=UUID_PATTERN, required=True)
+    severity = jsl.StringField(enum=['low', 'medium', 'high', 'critical'], default='low', required=True)
+    # saved_id - type must be 'saved_query' to allow this or else it is forbidden
+    tags = jsl.ArrayField(jsl.StringField(), required=False)
+    throttle = jsl.StringField(required=False)
+    timeline_id = jsl.StringField(required=False)
+    timeline_title = jsl.StringField(required=False)
+    to = jsl.StringField(required=False, default='now')
+    # require this to be always validated with a role
+    # type = jsl.StringField(enum=[MACHINE_LEARNING, QUERY, SAVED_QUERY], required=True)
+    threat = jsl.ArrayField(jsl.DocumentField(Threat), required=False, min_items=1)
+
+    with jsl.Scope(MACHINE_LEARNING) as ml_scope:
+        ml_scope.anomaly_threshold = jsl.IntField(required=True, minimum=0)
+        ml_scope.machine_learning_job_id = jsl.StringField(required=True)
+        ml_scope.type = jsl.StringField(enum=[MACHINE_LEARNING], required=True, default=MACHINE_LEARNING)
+
+    with jsl.Scope(QUERY) as query_scope:
+        query_scope.index = jsl.ArrayField(jsl.StringField(), required=False)
+        # this is not required per the API but we will enforce it here
+        query_scope.language = jsl.StringField(enum=['kuery', 'lucene'], required=True, default='kuery')
+        query_scope.query = jsl.StringField(required=True)
+        query_scope.type = jsl.StringField(enum=[QUERY], required=True, default=QUERY)
+
+    with jsl.Scope(SAVED_QUERY) as saved_id_scope:
+        saved_id_scope.index = jsl.ArrayField(jsl.StringField(), required=False)
+        saved_id_scope.saved_id = jsl.StringField(required=True)
+        saved_id_scope.type = jsl.StringField(enum=[SAVED_QUERY], required=True, default=SAVED_QUERY)
+
+
+class VersionedApiSchema(SiemRuleApiSchema):
+    """Schema for siem rule in API format with version."""
+
+    version = jsl.IntField(minimum=1, default=1, required=True)
+
+
+class SiemRuleTomlMetadata(jsl.Document):
+    """Schema for siem rule toml metadata."""
+
+    creation_date = jsl.StringField(required=True, pattern=DATE_PATTERN, default=time.strftime('%Y/%m/%d'))
+
+    # added to query with rule.optimize()
+    # rule validated against each ecs schema contained
+    ecs_version = jsl.ArrayField(
+        jsl.StringField(pattern=VERSION_PATTERN, required=True, default=ecs.get_max_version()), required=True)
+    maturity = jsl.StringField(enum=MATURITY_LEVELS, default='development', required=True)
+
+    # if present, add to query
+    os_type_list = jsl.ArrayField(jsl.StringField(enum=OS_OPTIONS), required=False)
+    related_endpoint_rules = jsl.ArrayField(jsl.ArrayField(jsl.StringField(), min_items=2, max_items=2),
+                                            required=False)
+    updated_date = jsl.StringField(required=True, pattern=DATE_PATTERN, default=time.strftime('%Y/%m/%d'))
+
+
+class SiemRuleTomlSchema(jsl.Document):
+    """Schema for siem rule in management toml format."""
+
+    metadata = jsl.DocumentField(SiemRuleTomlMetadata)
+    rule = jsl.DocumentField(SiemRuleApiSchema)
+
+
+class Package(jsl.Document):
+    """Schema for siem rule staging."""
+
+
+class MappingCount(jsl.Document):
+    """Mapping count schema."""
+
+    count = jsl.IntField(minimum=0, required=True)
+    rta_name = jsl.StringField(pattern=r'[a-zA-Z-_]+', required=True)
+    rule_name = jsl.StringField(required=True)
+    sources = jsl.ArrayField(jsl.StringField(), min_items=1)
+
+
+cached_schemas = {}
+
+
+def get_schema(role, as_rule=False, versioned=False):
+    """Get applicable schema by role type and rule format."""
+    if (role, as_rule, versioned) not in cached_schemas:
+        if versioned:
+            cls = VersionedApiSchema
+        else:
+            cls = SiemRuleTomlSchema if as_rule else SiemRuleApiSchema
+
+        cached_schemas[(role, as_rule, versioned)] = cls.get_schema(ordered=True, role=role)
+
+    return cached_schemas[(role, as_rule, versioned)]
+
+
+def schema_validate(contents, as_rule=False, versioned=False):
+    """Validate against all schemas until first hit."""
+    assert isinstance(contents, dict)
+    role = contents.get('rule', {}).get('type') if as_rule else contents.get('type')
+
+    if not role:
+        raise ValueError('Missing rule type!')
+
+    return jsonschema.validate(contents, get_schema(role, as_rule, versioned))
+
+
+metadata_schema = SiemRuleTomlMetadata.get_schema(ordered=True)
+package_schema = Package.get_schema(ordered=True)
+mapping_schema = MappingCount.get_schema(ordered=True)
+
+
+def validate_rta_mapping(mapping):
+    """Validate the RTA mapping."""
+    jsonschema.validate(mapping, mapping_schema)

detection_rules/semver.py

@@ -0,0 +1,25 @@
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+
+"""Helper functionality for comparing semantic versions."""
+import re
+
+
+class Version(tuple):
+
+    def __new__(cls, version):
+        if not isinstance(version, (int, list, tuple)):
+            version = tuple(int(a) if a.isdigit() else a for a in re.split(r'[.-]', version))
+
+        return tuple.__new__(cls, version)
+
+    def bump(self):
+        """Increment the version."""
+        versions = list(self)
+        versions[-1] += 1
+        return Version(versions)
+
+    def __str__(self):
+        """Convert back to a string."""
+        return ".".join(str(dig) for dig in self)

detection_rules/utils.py

@@ -0,0 +1,186 @@
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+
+"""Util functions."""
+import contextlib
+import functools
+import gzip
+import io
+import json
+import os
+import time
+import zipfile
+from datetime import datetime
+
+import kql
+
+import eql.utils
+from eql.utils import stream_json_lines
+
+CURR_DIR = os.path.dirname(os.path.abspath(__file__))
+ROOT_DIR = os.path.dirname(CURR_DIR)
+ETC_DIR = os.path.join(ROOT_DIR, "etc")
+
+
+def get_json_iter(f):
+    """Get an iterator over a JSON file."""
+    first = f.read(2)
+    f.seek(0)
+
+    if first[0] == '[' or first == "{\n":
+        return json.load(f)
+    else:
+        data = list(stream_json_lines(f))
+    return data
+
+
+def get_path(*paths):
+    """Get a file by relative path."""
+    return os.path.join(ROOT_DIR, *paths)
+
+
+def get_etc_path(*paths):
+    """Load a file from the etc/ folder."""
+    return os.path.join(ETC_DIR, *paths)
+
+
+def get_etc_file(name, mode="r"):
+    """Load a file from the etc/ folder."""
+    with open(get_etc_path(name), mode) as f:
+        return f.read()
+
+
+def load_etc_dump(*path):
+    """Load a json/yml/toml file from the etc/ folder."""
+    return eql.utils.load_dump(get_etc_path(*path))
+
+
+def save_etc_dump(contents, *path):
+    """Load a json/yml/toml file from the etc/ folder."""
+    return eql.utils.save_dump(contents, get_etc_path(*path))
+
+
+def get_ecs_fields(endgame_field):
+    ecs_mapping = load_etc_dump('ecs_mappings.json')
+    return ecs_mapping.get(endgame_field)
+
+
+def save_gzip(contents):
+    gz_file = io.BytesIO()
+
+    with gzip.GzipFile(mode="w", fileobj=gz_file) as f:
+        if not isinstance(contents, bytes):
+            contents = contents.encode("utf8")
+        f.write(contents)
+
+    return gz_file.getvalue()
+
+
+@contextlib.contextmanager
+def unzip(contents):  # type: (bytes) -> zipfile.ZipFile
+    """Get zipped contents."""
+    zipped = io.BytesIO(contents)
+    archive = zipfile.ZipFile(zipped, mode="r")
+
+    try:
+        yield archive
+
+    finally:
+        archive.close()
+
+
+def unzip_and_save(contents, path, member=None, verbose=True):
+    """Save unzipped from raw zipped contents."""
+    with unzip(contents) as archive:
+
+        if member:
+            archive.extract(member, path)
+        else:
+            archive.extractall(path)
+
+        if verbose:
+            name_list = archive.namelist()[member] if not member else archive.namelist()
+            print('Saved files to {}: \n\t- {}'.format(path, '\n\t- '.join(name_list)))
+
+
+def event_sort(events, timestamp='@timestamp', date_format='%Y-%m-%dT%H:%M:%S.%f%z', asc=True):
+    """Sort events from elasticsearch by timestamp."""
+    def _event_sort(event):
+        t = event[timestamp]
+        return (time.mktime(time.strptime(t, date_format)) + int(t.split('.')[-1][:-1]) / 1000) * 1000
+
+    return sorted(events, key=_event_sort, reverse=not asc)
+
+
+def combine_sources(*sources):  # type: (list[list]) -> list
+    """Combine lists of events from multiple sources."""
+    combined = []
+    for source in sources:
+        combined.extend(source.copy())
+
+    return event_sort(combined)
+
+
+def evaluate(rule, events):
+    """Evaluate a query against events."""
+    evaluator = kql.get_evaluator(kql.parse(rule.query))
+    filtered = list(filter(evaluator, events))
+    return filtered
+
+
+def unix_time_to_formatted(timestamp):  # type: (int|str) -> str
+    """Converts unix time in seconds or milliseconds to the default format."""
+    if isinstance(timestamp, (int, float)):
+        if timestamp > 2 ** 32:
+            timestamp = round(timestamp / 1000, 3)
+
+        return datetime.utcfromtimestamp(timestamp).strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z'
+
+
+def normalize_timing_and_sort(events, timestamp='@timestamp', asc=True):
+    """Normalize timestamp formats and sort events."""
+    for event in events:
+        _timestamp = event[timestamp]
+        if not isinstance(_timestamp, str):
+            event[timestamp] = unix_time_to_formatted(_timestamp)
+
+    return event_sort(events, timestamp=timestamp, asc=asc)
+
+
+def freeze(obj):
+    """Helper function to make mutable objects immutable and hashable."""
+    if isinstance(obj, (list, tuple)):
+        return tuple(freeze(o) for o in obj)
+    elif isinstance(obj, dict):
+        return freeze(list(sorted(obj.items())))
+    else:
+        return obj
+
+
+_cache = {}
+
+
+def cached(f):
+    """Helper function to memoize functions."""
+    func_key = id(f)
+
+    @functools.wraps(f)
+    def wrapped(*args, **kwargs):
+        _cache.setdefault(func_key, {})
+        cache_key = freeze(args), freeze(kwargs)
+
+        if cache_key not in _cache[func_key]:
+            _cache[func_key][cache_key] = f(*args, **kwargs)
+
+        return _cache[func_key][cache_key]
+
+    def clear():
+        _cache.pop(func_key, None)
+
+    wrapped.clear = clear
+    return wrapped
+
+
+def clear_caches():
+    _cache.clear()

etc/ecs_mappings.json

@@ -0,0 +1,216 @@
+{
+  "channel_name": [
+    "winlog.channel"
+  ],
+  "command_line": [
+    "process.command_line",
+    "process.args"
+  ],
+  "destination_address": [
+    "destination.ip"
+  ],
+  "destination_port": [
+    "destination.port",
+    "server.port"
+  ],
+  "effective_gid": [
+    "user.group.id"
+  ],
+  "effective_group_name": [
+    "user.group.name"
+  ],
+  "effective_uid": [
+    "user.id"
+  ],
+  "effective_user_name": [
+    "user.name"
+  ],
+  "endpoint.core_os": [
+    "host.os.platform"
+  ],
+  "endpoint.hostname": [
+    "host.hostname"
+  ],
+  "endpoint.ip_address": [
+    "host.ip"
+  ],
+  "endpoint.mac_address": [
+    "host.mac"
+  ],
+  "endpoint.name": [
+    "host.name"
+  ],
+  "endpoint.operating_system": [
+    "host.os.full"
+  ],
+  "event_id": [
+    "winlog.event_id"
+  ],
+  "event_message": [
+    "winlog.message",
+    "winlog.event_data.*"
+  ],
+  "eventlog_user_sid": [
+    "winlog.user.sid"
+  ],
+  "file_name": [
+    "file.name",
+    "file.extension"
+  ],
+  "file_path": [
+    "file.path"
+  ],
+  "fileid": [
+    "file.inode"
+  ],
+  "http_request": [
+    "http.request.method",
+    "http.request.referrer",
+    "http.version",
+    "http.*"
+  ],
+  "imphash": [
+    "file.hash.imphash"
+  ],
+  "in_packet_count": [
+    "source.packets",
+    "destination.packets"
+  ],
+  "ip_address": [
+    "source.ip"
+  ],
+  "logon_type": [
+    "winlog.event_data.LogonType"
+  ],
+  "machine_id": [
+    "host.id"
+  ],
+  "md5": [
+    "process.hash.md5",
+    "file.hash.md5"
+  ],
+  "opcode": [
+    "winlog.opcode"
+  ],
+  "out_packet_count": [
+    "source.packets",
+    "destination.packets"
+  ],
+  "parent_pid": [
+    "process.parent.id"
+  ],
+  "parent_process_name": [
+    "process.parent.name"
+  ],
+  "parent_process_path": [
+    "process.parent.executable"
+  ],
+  "pid": [
+    "process.pid"
+  ],
+  "ppid": [
+    "process.parent.pid"
+  ],
+  "process_name": [
+    "process.name"
+  ],
+  "process_path": [
+    "process.executable"
+  ],
+  "protocol": [
+    "network.transport"
+  ],
+  "provider_guid": [
+    "winlog.provider_guid"
+  ],
+  "provider_name": [
+    "winlog.provider_name"
+  ],
+  "query_name": [
+    "dns.question.name"
+  ],
+  "query_options": [
+    "dns.flags"
+  ],
+  "query_results": [
+    "dns.answers.data"
+  ],
+  "query_status": [
+    "dns.response_code"
+  ],
+  "query_type": [
+    "dns.type"
+  ],
+  "severity": [
+    "event.severity"
+  ],
+  "sha1": [
+    "process.hash.sha1",
+    "file.hash.sha1"
+  ],
+  "sha256": [
+    "process.hash.sha256",
+    "file.hash.sha256"
+  ],
+  "source_address": [
+    "source.address",
+    "source.ip",
+    "client.address"
+  ],
+  "source_port": [
+    "source.port",
+    "client.port"
+  ],
+  "source_process_name": [
+    "process.name"
+  ],
+  "source_process_path": [
+    "process.executable"
+  ],
+  "subject_domain_name": [
+    "winlog.event_data.SubjectDomainName"
+  ],
+  "subject_logon_id": [
+    "winlog.event_data.SubjectLogonId"
+  ],
+  "subject_user_name": [
+    "winlog.event_data.UserName"
+  ],
+  "subject_user_sid": [
+    "winlog.event_data.UserSid"
+  ],
+  "target_domain_name": [
+    "winlog.event_data.TargetDomainName",
+    "user.domain"
+  ],
+  "target_logon_id": [
+    "winlog.event_data.TargetLogonId"
+  ],
+  "target_user_name": [
+    "user.name"
+  ],
+  "task": [
+    "winlog.task"
+  ],
+  "tid": [
+    "process.thread.id"
+  ],
+  "timestamp": [
+    "@timestamp"
+  ],
+  "total_in_bytes": [
+    "destination.bytes"
+  ],
+  "total_out_bytes": [
+    "source.bytes"
+  ],
+  "user_domain": [
+    "user.domain"
+  ],
+  "user_name": [
+    "user.name"
+  ],
+  "user_sid": [
+    "user.identifer"
+  ]
+}

etc/non-ecs-schema.json

@@ -0,0 +1,13 @@
+{
+  "endgame-*": {
+    "endgame": {
+      "metadata": {
+        "type": "keyword"
+      },
+      "event_subtype_full": "keyword"
+    }
+  },
+  "winlogbeat-*": {
+    "winlog.event_data.OriginalFileName": "keyword"
+  }
+}

etc/packages.yml

@@ -0,0 +1,17 @@
+---
+package:
+  name: "7.9"
+  tune: false
+  release: true
+  # as_eql: true
+  filter:
+    ecs_version:
+    - 1.4.0
+    - 1.5.0
+    maturity:
+    - production
+    # need to add to schema - updated rules get this set to true when changed
+    #   and reset when added to a package
+    # changed: true - maybe
+  # update:
+    # severity: low

etc/rule-mapping.yml

@@ -0,0 +1,2 @@
+---
+{}

etc/test_toml.json

@@ -0,0 +1,119 @@
+[
+  {
+    "metadata": {
+      "creation_date": "2020/02/26",
+      "ecs_version": [
+        "1.4.0"
+      ],
+      "maturity": "development",
+      "updated_date": "2020/02/26"
+    },
+    "rule": {
+      "description": "This rule detects network events that may indicate the use of SSH traffic from the Internet. SSH is commonly used by\nsystem administrators to remotely control a system using the command line shell. If it is exposed to the Internet, it\nshould be done with strong security controls as it is frequently targeted and exploited by threat actors as an initial\naccess or back-door vector.\n",
+      "false_positives": [
+        "    SSH connections may be made directly to Internet destinations in order to access Linux cloud server instances but\n    such connections are usually made only by engineers. In such cases, only SSH gateways, bastions or jump servers may\n    be expected Internet destinations and can be exempted from this rule. SSH may be required by some work-flows such as\n    remote access and support for specialized software products and servers. Such work-flows are usually known and not\n    unexpected. Usage that is unfamiliar to server or network owners can be unexpected and suspicious.\n    "
+      ],
+      "index": [
+        "filebeat-*"
+      ],
+      "language": "kuery",
+      "max_signals": 100,
+      "name": "SSH (Secure Shell) to the Internet",
+      "risk_score": 21,
+      "rule_id": "6f1500bc-62d7-4eb9-8601-7485e87da2f4",
+      "severity": "low",
+      "tags": [
+        "Elastic",
+        "Network"
+      ],
+      "type": "query",
+      "version": 2,
+      "query": "network.transport: tcp and destination.port:22 and (\n    network.direction: outbound or (\n        source.ip: (10.0.0.0/8 or 172.16.0.0/12 or 192.168.0.0/16) and\n        not destination.ip: (10.0.0.0/8 or 172.16.0.0/12 or 192.168.0.0/16)\n    )\n)\n",
+      "threat": [
+        {
+          "framework": "MITRE ATT&CK",
+          "technique": [
+            {
+              "id": "T1043",
+              "name": "Commonly Used Port",
+              "reference": "https://attack.mitre.org/techniques/T1043/"
+            }
+          ],
+          "tactic": {
+            "id": "TA0011",
+            "name": "Command and Control",
+            "reference": "https://attack.mitre.org/tactics/TA0011/"
+          }
+        }
+      ]
+    }
+  },
+  {
+    "metadata": {
+      "field": "value"
+    },
+    "rule": {
+      "field2": "value2"
+    }
+  },
+  {
+    "metadata": {
+      "just": "some",
+      "flat": "fields",
+      "for": "testing",
+      "and": [
+        [
+          "nested",
+          "fields"
+        ],
+        [
+          "too",
+          "!"
+        ]
+      ]
+    },
+    "rule": {
+      "first": "1st",
+      "second": "2nd",
+      "third": "3rd",
+      "fourth": 4,
+      "fifth": [
+        1,
+        2,
+        3
+      ],
+      "list": [
+        {
+          "one": [
+            2,
+            3
+          ]
+        },
+        {
+          "two": {
+            "three": 4
+          }
+        }
+      ],
+      "more_data": {
+        "one": {
+          "two": {
+            "three": {
+              "four": {
+                "five": [
+                  [
+                    111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111,
+                    2222222222222222222222222222222222222222222222222222222222222222222222222222222222222222,
+                    333333333333333333333333333333333333333333333333333333333333333333
+                  ],
+                  [[4], [5], [6]],
+                  [["seven"], ["nine"], ["eleven"], [12, 13, 14]]
+                ]
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+]

etc/version.lock.json

@@ -0,0 +1,727 @@
+{
+  "0022d47d-39c7-4f69-a232-4fe9dc7a3acd": {
+    "rule_name": "System Shells via Services",
+    "sha256": "39089b35d9aa4d3d8b9e595e74dc11548e4a0609e120ed58fc38941baca8cd5e",
+    "version": 2
+  },
+  "041d4d41-9589-43e2-ba13-5680af75ebc2": {
+    "rule_name": "Potential DNS Tunneling via Iodine",
+    "sha256": "b4ffc6e7d9017294f8d07909cee42f866d5764b1d8bd5d9ae018f7e2a15e600e",
+    "version": 2
+  },
+  "05e5a668-7b51-4a67-93ab-e9af405c9ef3": {
+    "rule_name": "Interactive Terminal Spawned via Perl",
+    "sha256": "f57a4c89c80964ccf26e8d78aca95f82e70958925cafa412a32399af9d7a2b20",
+    "version": 1
+  },
+  "06dceabf-adca-48af-ac79-ffdf4c3b1e9a": {
+    "rule_name": "Potential Evasion via Filter Manager",
+    "sha256": "a07971ec80d45ced9da4aea88a0601b6a8f041afb86d39b627cfd420cd97227f",
+    "version": 2
+  },
+  "08d5d7e2-740f-44d8-aeda-e41f4263efaf": {
+    "rule_name": "TCP Port 8000 Activity to the Internet",
+    "sha256": "0b1f0b1e073fe5119cc1b5c24a1c29d1e5fff4006cde69aa3dffae2e7cdd7fb8",
+    "version": 3
+  },
+  "0a97b20f-4144-49ea-be32-b540ecc445de": {
+    "rule_name": "Malware - Detected - Elastic Endpoint",
+    "sha256": "60247a5e9bfd70eb9e86f0ce55895501cc14cb659a2c73f3c2a93b38cb6c4f52",
+    "version": 2
+  },
+  "0b29cab4-dbbd-4a3f-9e8e-1287c7c11ae5": {
+    "rule_name": "Anomalous Windows Process Creation",
+    "sha256": "fb14a53c48d6663ac266c0636b3364f4f8c9ed9fff10e7e6ff37267406e43dde",
+    "version": 1
+  },
+  "0d69150b-96f8-467c-a86d-a67a3378ce77": {
+    "rule_name": "Nping Process Activity",
+    "sha256": "368b19e0dafeefeaa308320ca31b368f4b1ebcd55e3b73a4844f0523d34c365e",
+    "version": 2
+  },
+  "0e79980b-4250-4a50-a509-69294c14e84b": {
+    "rule_name": "MsBuild Making Network Connections",
+    "sha256": "cfa69218bdfdfdf32807bf699d622df0b11fba0dae582842fd5a410d15168864",
+    "version": 2
+  },
+  "0f616aee-8161-4120-857e-742366f5eeb3": {
+    "rule_name": "PowerShell spawning Cmd",
+    "sha256": "99a17ac0cd39f66214f8ec357b8287019dcce014b088380af6369a3823ff0a72",
+    "version": 2
+  },
+  "120559c6-5e24-49f4-9e30-8ffe697df6b9": {
+    "rule_name": "User Discovery via Whoami",
+    "sha256": "516a989374daf3dff00c50cf25c5752bc6529f4654dfaeac14d468918a27a99b",
+    "version": 2
+  },
+  "125417b8-d3df-479f-8418-12d7e034fee3": {
+    "rule_name": "Attempt to Disable IPTables or Firewall",
+    "sha256": "e44bf0572d1c16dc6c3afb5fd7099af64775de2e2bc02f5657de1ad9835a7302",
+    "version": 1
+  },
+  "139c7458-566a-410c-a5cd-f80238d6a5cd": {
+    "rule_name": "SQL Traffic to the Internet",
+    "sha256": "165e687f11971e3a90f512eae1f7cd66b809dbf41574e310042102cb893ce160",
+    "version": 3
+  },
+  "143cb236-0956-4f42-a706-814bcaa0cf5a": {
+    "rule_name": "RPC (Remote Procedure Call) from the Internet",
+    "sha256": "54bcc73de76c78b3bd35e493bd4ac3d7758ea5c691f7d5503f36163a438a005f",
+    "version": 3
+  },
+  "1781d055-5c66-4adf-9c59-fc0fa58336a5": {
+    "rule_name": "Unusual Windows Username",
+    "sha256": "fc01b465045b17167db21385546bc1c3c6c6d00e2dcc077703ef41b87536c95b",
+    "version": 1
+  },
+  "1781d055-5c66-4adf-9c71-fc0fa58338c7": {
+    "rule_name": "Unusual Windows Service",
+    "sha256": "7fb46c159b57cf544b91a13e6e7632acb6001dd7fda33e1de92e1529e530aa72",
+    "version": 1
+  },
+  "1781d055-5c66-4adf-9d60-fc0fa58337b6": {
+    "rule_name": "Suspicious Powershell Script",
+    "sha256": "d688f99e0bb99ac74c6bde09e1e90726eee185b92fb0a119e163bd31f45905d7",
+    "version": 1
+  },
+  "1781d055-5c66-4adf-9d82-fc0fa58449c8": {
+    "rule_name": "Unusual Windows User Privilege Elevation Activity",
+    "sha256": "afdfda57d393812c5bac293dac5069a851209e7bb83fb528af21394881d2ba66",
+    "version": 1
+  },
+  "1781d055-5c66-4adf-9e93-fc0fa69550c9": {
+    "rule_name": "Unusual Windows Remote User",
+    "sha256": "cb9c569b6a9375c526c32bea068c3d8609f88fedab3fce24b89baf05c756d084",
+    "version": 1
+  },
+  "17e68559-b274-4948-ad0b-f8415bb31126": {
+    "rule_name": "Unusual Network Destination Domain Name",
+    "sha256": "71a7ab62d3d11c3dabd0eaa0ebcb656f5926e07601d6e35e4ccf73bccf4a5308",
+    "version": 1
+  },
+  "1aa9181a-492b-4c01-8b16-fa0735786b2b": {
+    "rule_name": "User Account Creation",
+    "sha256": "bd894910fcaa18d7682c39203ab04b8efbf6bd36df137014b50a9cd6b2a8fe54",
+    "version": 2
+  },
+  "1b21abcc-4d9f-4b08-a7f5-316f5f94b973": {
+    "rule_name": "Connection to Internal Network via Telnet",
+    "sha256": "be1434347fc362c5e025579b0fd1b7776d003c084fd58be2b916c6bce97d8d20",
+    "version": 1
+  },
+  "2003cdc8-8d83-4aa5-b132-1f9a8eb48514": {
+    "rule_name": "Exploit - Detected - Elastic Endpoint",
+    "sha256": "7d7e9fdb6626ccb0dbb1fc1f1485b23901223f3af87c173a49372233037c6876",
+    "version": 2
+  },
+  "231876e7-4d1f-4d63-a47c-47dd1acdc1cb": {
+    "rule_name": "Potential Shell via Web Server",
+    "sha256": "124c52f61e77e57f3a90a5b533e814d477e78eb64a8bd951a5c67696a4e92155",
+    "version": 3
+  },
+  "2856446a-34e6-435b-9fb5-f8f040bfa7ed": {
+    "rule_name": "Net command via SYSTEM account",
+    "sha256": "eb00b6196dac7a17f5797b814c4a61737a00f4cbcfbe36f4068de27ba5c6d40b",
+    "version": 1
+  },
+  "2863ffeb-bf77-44dd-b7a5-93ef94b72036": {
+    "rule_name": "Exploit - Prevented - Elastic Endpoint",
+    "sha256": "d010930d010bb22e3d5090f8fdef2fa9ec625d6d8f5d19614a06a9cbc1ab7869",
+    "version": 2
+  },
+  "2bf78aa2-9c56-48de-b139-f169bf99cf86": {
+    "rule_name": "Adobe Hijack Persistence",
+    "sha256": "d75866b9396e39d5b24a7075ad3da22bb731415543ba986f92f12bcaae41fc51",
+    "version": 2
+  },
+  "2d8043ed-5bda-4caf-801c-c1feb7410504": {
+    "rule_name": "Enumeration of Kernel Modules",
+    "sha256": "ffbc06dd83e7a71513185c0793862b5a3535aab407969318726ecc8198a48827",
+    "version": 1
+  },
+  "2f8a1226-5720-437d-9c20-e0029deb6194": {
+    "rule_name": "Attempt to Disable Syslog Service",
+    "sha256": "bbe27e711309f490218ea0cd982daa31562a888f92899334d667bd08c1884dae",
+    "version": 1
+  },
+  "31b4c719-f2b4-41f6-a9bd-fce93c2eaf62": {
+    "rule_name": "Bypass UAC via Event Viewer",
+    "sha256": "f37d188cbd09fa5e1c1203d77f22f30ae87f5a8874e420a9b911a29bbde0f32a",
+    "version": 1
+  },
+  "32923416-763a-4531-bb35-f33b9232ecdb": {
+    "rule_name": "RPC (Remote Procedure Call) to the Internet",
+    "sha256": "442c2027c43d3542f008452f789257629c30b9b0bf46fdf4bf60a7ef9383d9b7",
+    "version": 3
+  },
+  "32f4675e-6c49-4ace-80f9-97c9259dca2e": {
+    "rule_name": "Suspicious MS Outlook Child Process",
+    "sha256": "ea60210c4757f8b73e87ea7d3b3c96fda5eb51f7b6e0011393bd3ef48a282238",
+    "version": 2
+  },
+  "34fde489-94b0-4500-a76f-b8a157cf9269": {
+    "rule_name": "Telnet Port Activity",
+    "sha256": "561e4cc5f9ac4bd683de676c3ddc7b3cea3c9245b4b3d024976750052b7c9539",
+    "version": 2
+  },
+  "35df0dd8-092d-4a83-88c1-5151a804f31b": {
+    "rule_name": "Unusual Parent-Child Relationship",
+    "sha256": "26da1776418ca4f784295a501983859639d0edca40f266fceb98f18d5e5ae873",
+    "version": 2
+  },
+  "3838e0e3-1850-4850-a411-2e8c5ba40ba8": {
+    "rule_name": "Network Connection via Certutil",
+    "sha256": "61c10e5bfbf59f40256d91f5aed9bb63b95ddf1f3aa5e8fabb5a66e2ee4b11dc",
+    "version": 1
+  },
+  "3a86e085-094c-412d-97ff-2439731e59cb": {
+    "rule_name": "Setgid Bit Set via chmod",
+    "sha256": "ace4e4dd54e8193f6f9864c3202907d0026d32ccef3ef0e07f48cc030cbf86c5",
+    "version": 1
+  },
+  "3ad49c61-7adc-42c1-b788-732eda2f5abf": {
+    "rule_name": "VNC (Virtual Network Computing) to the Internet",
+    "sha256": "0d2a9bb546e3d7efa14722afdb1a0d9465a6523dc186298a4ea53acae919b4d5",
+    "version": 3
+  },
+  "3b382770-efbb-44f4-beed-f5e0a051b895": {
+    "rule_name": "Malware - Prevented - Elastic Endpoint",
+    "sha256": "9cb592e8da5f94d4c3676f09b065f521b2d30c2e9301a98517e756f7b2cbfbf8",
+    "version": 2
+  },
+  "3c7e32e6-6104-46d9-a06e-da0f8b5795a0": {
+    "rule_name": "Unusual Linux Network Port Activity",
+    "sha256": "e8c3cb8f40b5f6ef0975a62443f9504c84842f669c26b91657506c5286c8ebd4",
+    "version": 1
+  },
+  "4330272b-9724-4bc6-a3ca-f1532b81e5c2": {
+    "rule_name": "Unusual Login Activity",
+    "sha256": "1b7e41fe98f0e26118b8628a1325a59002b48b787d93ab96a2d158ecd004e368",
+    "version": 1
+  },
+  "43303fd4-4839-4e48-b2b2-803ab060758d": {
+    "rule_name": "Web Application Suspicious Activity: No User Agent",
+    "sha256": "dd2f91dbccd0af4d0a576013c193844643e84a54aa6373fbce114dc1dfb25dc3",
+    "version": 2
+  },
+  "445a342e-03fb-42d0-8656-0367eb2dead5": {
+    "rule_name": "Unusual Windows Path Activity",
+    "sha256": "c40b729895a99fd1f7a8514a420f2f0eabe57f408bed85a2cc86178acc9b9cef",
+    "version": 1
+  },
+  "453f659e-0429-40b1-bfdb-b6957286e04b": {
+    "rule_name": "Permission Theft - Prevented - Elastic Endpoint",
+    "sha256": "2015029af5a5a0330f929920e99f93bd06e9d0a2f1ccf5b7b7c30c953f7ca340",
+    "version": 2
+  },
+  "4630d948-40d4-4cef-ac69-4002e29bc3db": {
+    "rule_name": "Adding Hidden File Attribute via Attrib",
+    "sha256": "b3347c68deb04c69d524eaf1d092ca1cd26a18c32eb7d57a72b2d19f994a1a44",
+    "version": 2
+  },
+  "46f804f5-b289-43d6-a881-9387cf594f75": {
+    "rule_name": "Unusual Process For a Linux Host",
+    "sha256": "e0e4ab88394545469f2d5eee21909298fb6d8b5c0e2fa4ef35331d1fd7ac9f23",
+    "version": 1
+  },
+  "47f09343-8d1f-4bb5-8bb0-00c9d18f5010": {
+    "rule_name": "Execution via Regsvcs/Regasm",
+    "sha256": "f33bc5f0b3e49b2e47b15376fd2a68d1de7d14204a05fe17a1551291f5603673",
+    "version": 1
+  },
+  "4b438734-3793-4fda-bd42-ceeada0be8f9": {
+    "rule_name": "Disable Windows Firewall Rules via Netsh",
+    "sha256": "50f4b863d2184bb0c86fa854e7f009ac21a0c0919deb80c41cbfc2c55ab264e2",
+    "version": 2
+  },
+  "52aaab7b-b51c-441a-89ce-4387b3aea886": {
+    "rule_name": "Unusual Network Connection via RunDLL32",
+    "sha256": "d85fea308f2fcf2b3a99f38acc2800b7c397a310deb5d2042ccb023d9e491c0f",
+    "version": 3
+  },
+  "52afbdc5-db15-485e-bc24-f5707f820c4b": {
+    "rule_name": "Unusual Linux Network Activity",
+    "sha256": "c564c6efa5f18c2c696f39d0c6befcfdf6a48220cb287746c98c89d38d9a9301",
+    "version": 1
+  },
+  "52afbdc5-db15-485e-bc35-f5707f820c4c": {
+    "rule_name": "Unusual Linux Web Activity",
+    "sha256": "ff6fd99b9f6141b59784df526e92405db1b15197adca3c16b13f8f193a3389e3",
+    "version": 1
+  },
+  "52afbdc5-db15-596e-bc35-f5707f820c4b": {
+    "rule_name": "Unusual Linux Network Service",
+    "sha256": "04797144ac125fbff20df8d63db471c237f3ee6319f2e75b4607a0d3201d88c8",
+    "version": 1
+  },
+  "53a26770-9cbd-40c5-8b57-61d01a325e14": {
+    "rule_name": "Suspicious PDF Reader Child Process",
+    "sha256": "75442e7eb596645909961fe13ee945137ef2c3ab5207f5aa6d7b204216ec335b",
+    "version": 1
+  },
+  "55d551c6-333b-4665-ab7e-5d14a59715ce": {
+    "rule_name": "PsExec Network Connection",
+    "sha256": "aa7366d7de1e17ceb977e79a1bb88f56cf5451ddadc12d7e349d93a9f6ada328",
+    "version": 2
+  },
+  "56557cde-d923-4b88-adee-c61b3f3b5dc3": {
+    "rule_name": "Windows CryptoAPI Spoofing Vulnerability (CVE-2020-0601 - CurveBall)",
+    "sha256": "90375f3a66c917be3c0951978877e9886ba55a6c2a7378ec45547952bf391c28",
+    "version": 1
+  },
+  "5700cb81-df44-46aa-a5d7-337798f53eb8": {
+    "rule_name": "VNC (Virtual Network Computing) from the Internet",
+    "sha256": "8803236e7bdb9aec8904a7bb13d47b3bd3c0ba7dc2cfff1c27439466d0170691",
+    "version": 3
+  },
+  "571afc56-5ed9-465d-a2a9-045f099f6e7e": {
+    "rule_name": "Credential Dumping - Detected - Elastic Endpoint",
+    "sha256": "5c3fac8543701612b433b719a609135a1e95c4081bbbbdbdeb0203c18e0a7b77",
+    "version": 2
+  },
+  "581add16-df76-42bb-af8e-c979bfb39a59": {
+    "rule_name": "Deleting Backup Catalogs with Wbadmin",
+    "sha256": "3bbb66c55707077ad2ef97fc1d95cff0527e95a49f83b84b82fc688e61017760",
+    "version": 2
+  },
+  "5b03c9fb-9945-4d2f-9568-fd690fee3fba": {
+    "rule_name": "Virtual Machine Fingerprinting",
+    "sha256": "2ded9ffeea250303a503fac66aaefca610fa0fc2f96cf6917b4676b41437c6fb",
+    "version": 1
+  },
+  "610949a1-312f-4e04-bb55-3a79b8c95267": {
+    "rule_name": "Unusual Process Network Connection",
+    "sha256": "eb6f8ca3e9a957d3573fc09527b54c4c316eaed783f5bad2d6057875e934d43d",
+    "version": 2
+  },
+  "61c31c14-507f-4627-8c31-072556b89a9c": {
+    "rule_name": "Mknod Process Activity",
+    "sha256": "7e94855d83fa70497f23ac1ac0996234b462102bc82f06e6c442c6ac437075ed",
+    "version": 2
+  },
+  "63e65ec3-43b1-45b0-8f2d-45b34291dc44": {
+    "rule_name": "Network Connection via Signed Binary",
+    "sha256": "d2e4829910b24fb980060b8e3adb7ce138fe2de8b47193c5801bd6fb28d4a339",
+    "version": 2
+  },
+  "647fc812-7996-4795-8869-9c4ea595fe88": {
+    "rule_name": "Anomalous Process For a Linux Population",
+    "sha256": "b6f615b8c0851c51a8a2cd76d70f32c13d3ec6868a1435b3b99ad29e7b211c55",
+    "version": 1
+  },
+  "67a9beba-830d-4035-bfe8-40b7e28f8ac4": {
+    "rule_name": "SMTP to the Internet",
+    "sha256": "7b9fdb9bb74d1bae58a5e0d2913992a48f694d895c651a879d1be6a215fb6822",
+    "version": 3
+  },
+  "69c251fb-a5d6-4035-b5ec-40438bd829ff": {
+    "rule_name": "Modification of Boot Configuration",
+    "sha256": "aca262dc656d8203cd46799960df4e1de99bfcc6660953e6ba10fe25145e9237",
+    "version": 1
+  },
+  "6d448b96-c922-4adb-b51c-b767f1ea5b76": {
+    "rule_name": "Unusual Process For a Windows Host",
+    "sha256": "c7e849caff0ec964d5ad2d1e7523abadba7a887846bbea3270bc900d3231da2e",
+    "version": 1
+  },
+  "6e40d56f-5c0e-4ac6-aece-bee96645b172": {
+    "rule_name": "Anomalous Process For a Windows Population",
+    "sha256": "da3a7b5c5addf4b3e760a505232b8048c8a865d9fa8b5b09df1bc3f5da942956",
+    "version": 1
+  },
+  "6ea71ff0-9e95-475b-9506-2580d1ce6154": {
+    "rule_name": "DNS Activity to the Internet",
+    "sha256": "267fe9694bab5833b3a374a799162c4cd8ba8e25777232a023e22f53e27e9260",
+    "version": 3
+  },
+  "6f1500bc-62d7-4eb9-8601-7485e87da2f4": {
+    "rule_name": "SSH (Secure Shell) to the Internet",
+    "sha256": "164c2588d9a22e77c2830a79e0b5f72df6050ee724d4113a3bb75668faa6a8d7",
+    "version": 3
+  },
+  "7405ddf1-6c8e-41ce-818f-48bea6bcaed8": {
+    "rule_name": "Potential Modification of Accessibility Binaries",
+    "sha256": "bbc8ce5e399e0d06157aff9e07a922bbf3d9428adb0e602274443225d27e13d3",
+    "version": 2
+  },
+  "746edc4c-c54c-49c6-97a1-651223819448": {
+    "rule_name": "Unusual DNS Activity",
+    "sha256": "1a2b60f6f140c9da40b5a57790581a31d9d022d23d9ae4988cfe074241d98d4a",
+    "version": 1
+  },
+  "75ee75d8-c180-481c-ba88-ee50129a6aef": {
+    "rule_name": "Web Application Suspicious Activity: Unauthorized Method",
+    "sha256": "a59eebb16d201dcb8ef6d461222854017ddeb42f13709fbd86cf664eb176443c",
+    "version": 2
+  },
+  "77a3c3df-8ec4-4da4-b758-878f551dee69": {
+    "rule_name": "Adversary Behavior - Detected - Elastic Endpoint",
+    "sha256": "642078d517a828b397afeaf66b572336c03b072695a1be761941da5a0da82ab2",
+    "version": 2
+  },
+  "7a137d76-ce3d-48e2-947d-2747796a78c0": {
+    "rule_name": "Network Sniffing via Tcpdump",
+    "sha256": "70194d4c327e916fa7cbd73fd373d5282732e1604ed2f1f9e80e005746b2ebc7",
+    "version": 2
+  },
+  "7d2c38d7-ede7-4bdf-b140-445906e6c540": {
+    "rule_name": "Tor Activity to the Internet",
+    "sha256": "149e45da1494348a5a1ebb5802b048d5efa9d4fd5ff588f8ac0c32d416a1220c",
+    "version": 3
+  },
+  "80c52164-c82a-402c-9964-852533d58be1": {
+    "rule_name": "Process Injection - Detected - Elastic Endpoint",
+    "sha256": "ebc218c788a01666499b9ccd9126099fad2c1262f34d1fbf904cbc2ff179a2d0",
+    "version": 2
+  },
+  "81cc58f5-8062-49a2-ba84-5cc4b4d31c40": {
+    "rule_name": "Persistence via Kernel Module Modification",
+    "sha256": "0be3f6a38c6fe9504aec5239d9dcf6962b2bde90eb07f5f6f1a682da2fbf8b52",
+    "version": 2
+  },
+  "87ec6396-9ac4-4706-bcf0-2ebb22002f43": {
+    "rule_name": "FTP (File Transfer Protocol) Activity to the Internet",
+    "sha256": "82a95329040bb9a03fc93ae26ead52d063732e01e55fc91a50ea51bd60febfb6",
+    "version": 3
+  },
+  "89f9a4b0-9f8f-4ee0-8823-c4751a6d6696": {
+    "rule_name": "Command Prompt Network Connection",
+    "sha256": "0117b0bffd43900d7a93110cd44c4b786cb30d62832dd9a7594e1a95d00428e2",
+    "version": 2
+  },
+  "8a1b0278-0f9a-487d-96bd-d4833298e87a": {
+    "rule_name": "Setuid Bit Set via chmod",
+    "sha256": "6fb4352bf42cc1842367ccfd3077c0ab58b47cc855253b70cdb5283e451067da",
+    "version": 1
+  },
+  "8c1bdde8-4204-45c0-9e0c-c85ca3902488": {
+    "rule_name": "RDP (Remote Desktop Protocol) from the Internet",
+    "sha256": "aa661ef6bef1c2951cde1a90dab2dd8ea17e01838b9ee69872963950dd5f76a9",
+    "version": 3
+  },
+  "8cb4f625-7743-4dfb-ae1b-ad92be9df7bd": {
+    "rule_name": "Ransomware - Detected - Elastic Endpoint",
+    "sha256": "3ccc4f8e13efe9a61b5624bed9ce6407cbe1bb6919e36f1e375cdffebe54da7b",
+    "version": 2
+  },
+  "90169566-2260-4824-b8e4-8615c3b4ed52": {
+    "rule_name": "Hping Process Activity",
+    "sha256": "c2df3568f4994b77e0b62a787cb3ff25a0c064ea75a849875344555e84da23c9",
+    "version": 2
+  },
+  "91f02f01-969f-4167-8d77-07827ac4cee0": {
+    "rule_name": "Unusual Web User Agent",
+    "sha256": "12b16eb9930172fcbf4ddec9b03ce0dc2bf5effe5e132e7a338f9ef4d34aece7",
+    "version": 1
+  },
+  "91f02f01-969f-4167-8f55-07827ac3acc9": {
+    "rule_name": "Unusual Web Request",
+    "sha256": "2c2c81f80ebe5fa568d94f6b44778a3617d2c98a60db66ff89b7734fe0f58227",
+    "version": 1
+  },
+  "91f02f01-969f-4167-8f66-07827ac3bdd9": {
+    "rule_name": "DNS Tunneling",
+    "sha256": "cc489289aea78cec83cf6baafd052523a12a5de52b7fd051de469de7aedb11e1",
+    "version": 1
+  },
+  "931e25a5-0f5e-4ae0-ba0d-9e94eff7e3a4": {
+    "rule_name": "Sudoers File Modification",
+    "sha256": "992014dda37755b93706823224d6c773881d207f2e95bf28ae9c8b142a7ab08d",
+    "version": 1
+  },
+  "97f22dab-84e8-409d-955e-dacd1d31670b": {
+    "rule_name": "Base64 Encoding/Decoding Activity",
+    "sha256": "3a99fc1237f79b736e78e2504a7f03a3c051127d56280d18e7942bc28458e0f8",
+    "version": 1
+  },
+  "990838aa-a953-4f3e-b3cb-6ddf7584de9e": {
+    "rule_name": "Process Injection - Prevented - Elastic Endpoint",
+    "sha256": "2c548340fbb54eff9e1b152ee566003019a76115e631d639787f29d932c707c0",
+    "version": 2
+  },
+  "9d110cb3-5f4b-4c9a-b9f5-53f0a1707ae1": {
+    "rule_name": "Trusted Developer Application Usage",
+    "sha256": "c50856866464349948eb885d9d4b377a46b8a7470c6f32673c3d5e61ff0242b3",
+    "version": 2
+  },
+  "9d110cb3-5f4b-4c9a-b9f5-53f0a1707ae2": {
+    "rule_name": "Microsoft Build Engine Started by a Script Process",
+    "sha256": "ac0023e83e5909cd445a89e0d4ab90bb7b6b30a351e62e88c166a7679a935777",
+    "version": 1
+  },
+  "9d110cb3-5f4b-4c9a-b9f5-53f0a1707ae3": {
+    "rule_name": "Microsoft Build Engine Started by a System Process",
+    "sha256": "1b36179b1f136fb76beaff305ce0c197b0f004809f7d87db6152af0b530279e9",
+    "version": 1
+  },
+  "9d110cb3-5f4b-4c9a-b9f5-53f0a1707ae4": {
+    "rule_name": "Microsoft Build Engine Using an Alternate Name",
+    "sha256": "adef27d5671cae0ca7e338d1831deb5be547ca7e2bcc3f5c325d51e378062b15",
+    "version": 1
+  },
+  "9d110cb3-5f4b-4c9a-b9f5-53f0a1707ae5": {
+    "rule_name": "Microsoft Build Engine Loading Windows Credential Libraries",
+    "sha256": "7ea5a0d0ea0780b698bc9007712ebc10b6cf49e8c5622c73700bda45ecba141a",
+    "version": 1
+  },
+  "9d110cb3-5f4b-4c9a-b9f5-53f0a1707ae6": {
+    "rule_name": "Microsoft Build Engine Started an Unusual Process",
+    "sha256": "40f6a7c2ade30ed37caf7db9f4f86b589bb220935a04d1bd9cacb4595999d0de",
+    "version": 1
+  },
+  "9d110cb3-5f4b-4c9a-b9f5-53f0a1707ae9": {
+    "rule_name": "Process Injection by the Microsoft Build Engine",
+    "sha256": "3e1bd6080789278af1262594434fb9df61bf980a196edef457066754ddb9b428",
+    "version": 1
+  },
+  "9f9a2a82-93a8-4b1a-8778-1780895626d4": {
+    "rule_name": "File Permission Modification in Writable Directory",
+    "sha256": "2791ea7aab403f784e99b1a00253127bc01b533c071ae239de2d951df5744226",
+    "version": 1
+  },
+  "a1329140-8de3-4445-9f87-908fb6d824f4": {
+    "rule_name": "File Deletion via Shred",
+    "sha256": "670fee406466143d95c3d010a8f318d9551a33ceeb3b18edb862aa50517d153e",
+    "version": 1
+  },
+  "a4ec1382-4557-452b-89ba-e413b22ed4b8": {
+    "rule_name": "Network Connection via Mshta",
+    "sha256": "0d5aa3558796ad1cfa586b0c0d1a99b2ba6d0cd6baeae3a1f31af0dd384a5859",
+    "version": 2
+  },
+  "a624863f-a70d-417f-a7d2-7a404638d47f": {
+    "rule_name": "Suspicious MS Office Child Process",
+    "sha256": "97f3907a36f237b97c3812f4574501cc28397821d814246ef551cab92809bf20",
+    "version": 2
+  },
+  "a87a4e42-1d82-44bd-b0bf-d9b7f91fb89e": {
+    "rule_name": "Web Application Suspicious Activity: POST Request Declined",
+    "sha256": "fa702d3f843d3b28f1d7be714e638126bdf36a51086d6f46c0931dcdff59637a",
+    "version": 2
+  },
+  "a9198571-b135-4a76-b055-e3e5a476fd83": {
+    "rule_name": "Hex Encoding/Decoding Activity",
+    "sha256": "f41d31dc2d8ac8a4da2ea71dc3a2eda04b2195581612b5f40f5ed7d4b23c1939",
+    "version": 1
+  },
+  "a9cb3641-ff4b-4cdc-a063-b4b8d02a67c7": {
+    "rule_name": "IPSEC NAT Traversal Port Activity",
+    "sha256": "bfbecd958d7ac30b7a1ce42d1222a026e48e6d5b5b56580dc6063537a66e872b",
+    "version": 2
+  },
+  "ad0e5e75-dd89-4875-8d0a-dfdc1828b5f3": {
+    "rule_name": "Proxy Port Activity to the Internet",
+    "sha256": "c8dde04e8b38da965110f057e8b7a7198cd6c1fdaf122919861a3bf44b715108",
+    "version": 3
+  },
+  "adb961e0-cb74-42a0-af9e-29fc41f88f5f": {
+    "rule_name": "Netcat Network Activity",
+    "sha256": "05e16bd1d644813e92367add6a0e06dd94e5d5bb13deee5ae76e2b4e1c5e6bf8",
+    "version": 2
+  },
+  "afcce5ad-65de-4ed2-8516-5e093d3ac99a": {
+    "rule_name": "Local Scheduled Task Commands",
+    "sha256": "c6cdd952bda0851019bfb4442e8f981491c254b0442fc875baf088eee4d1b4b6",
+    "version": 2
+  },
+  "b29ee2be-bf99-446c-ab1a-2dc0183394b8": {
+    "rule_name": "Network Connection via Compiled HTML File",
+    "sha256": "75b96e48cffe957398885c16d19668180a9c8a3f34393533298f1a57f73540ef",
+    "version": 2
+  },
+  "b347b919-665f-4aac-b9e8-68369bf2340c": {
+    "rule_name": "Unusual Linux Username",
+    "sha256": "ba2d648dd481c9efc13942569f8c5b1bac7f3110b10ab10e52d548b0501cddae",
+    "version": 1
+  },
+  "b5ea4bfe-a1b2-421f-9d47-22a75a6f2921": {
+    "rule_name": "Volume Shadow Copy Deletion via VssAdmin",
+    "sha256": "7ca6f8f1c070f5ece4dd1a4e11ce37658fb18141bdd7995482ab65cee2ede3bc",
+    "version": 2
+  },
+  "b86afe07-0d98-4738-b15d-8d7465f95ff5": {
+    "rule_name": "Network Connection via MsXsl",
+    "sha256": "f1713a791e10f0c22d6ef141ac00027fe0c25c74e5c170917c82039d82df95e4",
+    "version": 1
+  },
+  "ba342eb2-583c-439f-b04d-1fdd7c1417cc": {
+    "rule_name": "Unusual Windows Network Activity",
+    "sha256": "b3020cd50fdd4071cd24ba4864a6f71571c7be9dfbc04a05675fc5f82cd2b765",
+    "version": 1
+  },
+  "c0be5f31-e180-48ed-aa08-96b36899d48f": {
+    "rule_name": "Credential Manipulation - Detected - Elastic Endpoint",
+    "sha256": "ff22df35f9c904de4bd9ea09ad6930326a56135cd21a69d9b159ffc80d1f1eb3",
+    "version": 2
+  },
+  "c3167e1b-f73c-41be-b60b-87f4df707fe3": {
+    "rule_name": "Permission Theft - Detected - Elastic Endpoint",
+    "sha256": "14615d66138ce8a151124e86105e476cbc86391f48b9d623e5d89d3fedfd5244",
+    "version": 2
+  },
+  "c5dc3223-13a2-44a2-946c-e9dc0aa0449c": {
+    "rule_name": "Microsoft Build Engine Started by an Office Application",
+    "sha256": "b576737d3673fc434f53908d325f1e19780acf43f29e5ca403d233a76e49157f",
+    "version": 1
+  },
+  "c6474c34-4953-447a-903e-9fcb7b6661aa": {
+    "rule_name": "IRC (Internet Relay Chat) Protocol Activity to the Internet",
+    "sha256": "2b1aea0e382da26a9095637412779ea07ba1c600085708fd8533cb908cb4246b",
+    "version": 3
+  },
+  "c82b2bd8-d701-420c-ba43-f11a155b681a": {
+    "rule_name": "SMB (Windows File Sharing) Activity to the Internet",
+    "sha256": "5aa93865218f76c6c0e0a221e6525c3a25990e004dcd5a130abcb2693e799521",
+    "version": 3
+  },
+  "c82c7d8f-fb9e-4874-a4bd-fd9e3f9becf1": {
+    "rule_name": "Direct Outbound SMB Connection",
+    "sha256": "043f27706d54abb341bc27bfe0e0ab28cb301040d67f45e454b69ed91c413acd",
+    "version": 2
+  },
+  "c87fca17-b3a9-4e83-b545-f30746c53920": {
+    "rule_name": "Nmap Process Activity",
+    "sha256": "2dd6d209fe4baeb9f4de665c48ae09886f0ebedac1f5348e8cd9670ad3cba231",
+    "version": 2
+  },
+  "c9e38e64-3f4c-4bf3-ad48-0e61a60ea1fa": {
+    "rule_name": "Credential Manipulation - Prevented - Elastic Endpoint",
+    "sha256": "4de5fba453fc6e77905f049de58e3d4e949c02b8a7c5664824d6de0fafa98aaf",
+    "version": 2
+  },
+  "cc16f774-59f9-462d-8b98-d27ccd4519ec": {
+    "rule_name": "Process Discovery via Tasklist",
+    "sha256": "69c100cf7e526df7fe98c60f5dfdec39b5e283b70b8258a214d48f549136b3ee",
+    "version": 2
+  },
+  "cd4d5754-07e1-41d4-b9a5-ef4ea6a0a126": {
+    "rule_name": "Socat Process Activity",
+    "sha256": "bb51f973a0c732418f775ff5034494ac78d917dbe268104196275c056c2c6eee",
+    "version": 2
+  },
+  "cd66a5af-e34b-4bb0-8931-57d0a043f2ef": {
+    "rule_name": "Kernel Module Removal",
+    "sha256": "3d0384a5dfcab595d6e37e51bb57c14999240e81e677c28a72980729d3843e5f",
+    "version": 1
+  },
+  "d2053495-8fe7-4168-b3df-dad844046be3": {
+    "rule_name": "PPTP (Point to Point Tunneling Protocol) Activity",
+    "sha256": "1d1338c7e5a451124c5457b9f951ecb4fea2d25f66e1e0a42fd5bd42901fb5c4",
+    "version": 2
+  },
+  "d331bbe2-6db4-4941-80a5-8270db72eb61": {
+    "rule_name": "Clearing Windows Event Logs",
+    "sha256": "f7c0075c089b3dc58718cf914f458d603e8334259676255fa410666ed4838619",
+    "version": 2
+  },
+  "d49cc73f-7a16-4def-89ce-9fc7127d7820": {
+    "rule_name": "Web Application Suspicious Activity: sqlmap User Agent",
+    "sha256": "2afadbc58b81aa3f157bc2a6e2336c2cdc01c4f7cf1b2e49ccda73115e1416e1",
+    "version": 2
+  },
+  "d6450d4e-81c6-46a3-bd94-079886318ed5": {
+    "rule_name": "Strace Process Activity",
+    "sha256": "208748780ef08f6f5518d19ddc02b27ca464e0a2283e3bd597d1adb209faedf6",
+    "version": 2
+  },
+  "d76b02ef-fc95-4001-9297-01cb7412232f": {
+    "rule_name": "Interactive Terminal Spawned via Python",
+    "sha256": "8b733d719fb36d87ffc7c4061b5745a309c9aa9f4486b916839612c5f2842d78",
+    "version": 1
+  },
+  "d7e62693-aab9-4f66-a21a-3d79ecdd603d": {
+    "rule_name": "SMTP on Port 26/TCP",
+    "sha256": "2e069990afc7e2595fbb44fbf7409fa9689e06fe1bf4e38c5eb6b63b7e668278",
+    "version": 2
+  },
+  "db8c33a8-03cd-4988-9e2c-d0a4863adb13": {
+    "rule_name": "Credential Dumping - Prevented - Elastic Endpoint",
+    "sha256": "00d2b15422187a2e7d8cb886d61b5af82e900690fce4d231b81a6c88e71329b2",
+    "version": 2
+  },
+  "dc9c1f74-dac3-48e3-b47f-eb79db358f57": {
+    "rule_name": "Volume Shadow Copy Deletion via WMIC",
+    "sha256": "47c0662835bb960faa810a241cec8a0e3a5e16dea5c5b24a2391bc46a41ccbb7",
+    "version": 2
+  },
+  "debff20a-46bc-4a4d-bae5-5cdd14222795": {
+    "rule_name": "Base16 or Base32 Encoding/Decoding Activity",
+    "sha256": "12c5ab3282cc98896c2fdc6019cacda579fde5f1c9b155ea12a5bf89308e6771",
+    "version": 1
+  },
+  "df959768-b0c9-4d45-988c-5606a2be8e5a": {
+    "rule_name": "Unusual Process Execution - Temp",
+    "sha256": "ab00dfd7fb69715948b4b0b71cbd2aa4f01da00124717860ee62245023c94201",
+    "version": 2
+  },
+  "e19e64ee-130e-4c07-961f-8a339f0b8362": {
+    "rule_name": "Connection to External Network via Telnet",
+    "sha256": "b8febd6d9d552e61b554f6a3c60eda8224b8c7c6a270cdb5540b7bcb013eacf9",
+    "version": 1
+  },
+  "e3343ab9-4245-4715-b344-e11c56b0a47f": {
+    "rule_name": "Process Activity via Compiled HTML File",
+    "sha256": "ad3d7159bb5aef8d8658ab0c89416a2b0e94b8e775f4a65da241352f3a198508",
+    "version": 2
+  },
+  "e3c5d5cb-41d5-4206-805c-f30561eae3ac": {
+    "rule_name": "Ransomware - Prevented - Elastic Endpoint",
+    "sha256": "c9af59f7a05da04fffa3d96c38a40b342820cb3bee6c800ee9df883882d538ec",
+    "version": 2
+  },
+  "e56993d2-759c-4120-984c-9ec9bb940fd5": {
+    "rule_name": "RDP (Remote Desktop Protocol) to the Internet",
+    "sha256": "f56dd354749071664740a90eaaa0b989322cd84d851b0b1488d1c9a6c6a18f7e",
+    "version": 3
+  },
+  "e8571d5f-bea1-46c2-9f56-998de2d3ed95": {
+    "rule_name": "Local Service Commands",
+    "sha256": "bf7942d8947c958f37d4e71bfe3ffbfa274316bae269c280383be9f9777314ba",
+    "version": 2
+  },
+  "ea0784f0-a4d7-4fea-ae86-4baaf27a6f17": {
+    "rule_name": "SSH (Secure Shell) from the Internet",
+    "sha256": "cc73cfef9c8a52df72988a68e4c672f5b2836557e5505eca35867724e53c1960",
+    "version": 3
+  },
+  "eb9eb8ba-a983-41d9-9c93-a1c05112ca5e": {
+    "rule_name": "Potential Disabling of SELinux",
+    "sha256": "20096fe4e38033d51625a5d76c64b63cb2b3aa9087cfd527013f99a9621a8a37",
+    "version": 1
+  },
+  "ef862985-3f13-4262-a686-5f357bbb9bc2": {
+    "rule_name": "Whoami Process Activity",
+    "sha256": "25c739bb74073b6d0cf882b7f2771dfbb2dc85f27a64414f34f2544332c305ec",
+    "version": 2
+  },
+  "f545ff26-3c94-4fd0-bd33-3c7f95a3a0fc": {
+    "rule_name": "Windows Script Executing PowerShell",
+    "sha256": "4c6602cef669e1229027ac71756ae75547d9250e04b526e7d1706cf4cca6655d",
+    "version": 2
+  },
+  "f675872f-6d85-40a3-b502-c0d2ef101e92": {
+    "rule_name": "Delete Volume USN Journal with Fsutil",
+    "sha256": "bebe50d02b82af53fc18f68f84e4e4e23ea720312aa2427254627f3fb82caa6b",
+    "version": 2
+  },
+  "fb02b8d3-71ee-4af1-bacd-215d23f17efa": {
+    "rule_name": "Network Connection via Regsvr",
+    "sha256": "1ea67c38378949c4946d6704ded6c3d6f1a0c7b890e45045b2e6a2264bc92fc5",
+    "version": 2
+  },
+  "fd4a992d-6130-4802-9ff8-829b89ae801f": {
+    "rule_name": "Potential Application Shimming via Sdbinst",
+    "sha256": "d3a22bf6b97ce616591989d43fca5248d4c14ee87c9c5dd572b3f3a7ac1120a1",
+    "version": 2
+  },
+  "fd70c98a-c410-42dc-a2e3-761c71848acf": {
+    "rule_name": "Encoding or Decoding Files via CertUtil",
+    "sha256": "811d196a3ef110b9651feedc5bd363205c4eb0317305f1b5aafd2884a7369967",
+    "version": 2
+  },
+  "fd7a6052-58fa-4397-93c3-4795249ccfa2": {
+    "rule_name": "Svchost spawning Cmd",
+    "sha256": "07aef064be12522287511146bf3cef378006cd6d6785bfe022972bee00ce2d1e",
+    "version": 2
+  }
+}

requirements.txt

@@ -0,0 +1,19 @@
+jsl==0.2.4
+jsonschema==3.2.0
+pytoml
+toml==0.10.0
+requests==2.22.0
+Click==7.0
+PyYAML==5.1.2
+
+eql~=0.9
+elasticsearch~=7.5.1
+
+dataclasses-json~=0.4.2
+
+# test deps
+pyflakes==2.2.0
+flake8==3.8.1
+pep8-naming==0.7.0
+pytest>=3.6
+jsonschema==3.2.0