admin/SpecterOps-Nemesis
1
0
Fork 0
mirror of https://github.com/SpecterOps/Nemesis synced 2026-06-08 12:36:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #32 from SpecterOps/yara_mods

Browse Source 
More Yara mods
This commit is contained in:
Will
2024-01-18 00:08:24 -08:00
committed by GitHub
parent 467fc2727c 342ca28606
commit 53777ef87e
7 changed files with 1496 additions and 1358 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/dashboard/dashboard/pages/3_File_Viewer.py
+30 -1
View File
@@ -436,7 +436,36 @@ def build_page(username: str):
st.divider()
elif chosen_tab == "yara_matches":
st.json(es_result["yaraMatches"]["yaraMatches"])
matches = es_result["yaraMatches"]["yaraMatches"]
for match in matches:
rule_file = match["ruleFile"]
rule_name = match["ruleName"]
rule_description = match["ruleDescription"] if "ruleDescription" in match else ""
rule_text = match["ruleText"] if "ruleText" in match else ""
st.subheader(f"{rule_name} ({rule_file})", divider="red")
if rule_description:
st.markdown(f"**Description:** $~~~~~~$ {rule_description}")
if match["ruleStringMatches"]:
string_matches = []
for rule_string_match in match["ruleStringMatches"]:
identifier = rule_string_match["identifier"]
for rule_string_match_instance in rule_string_match["yaraStringMatchInstances"]:
matched_string = rule_string_match_instance["matchedString"]
offset = rule_string_match_instance["offset"]
length = rule_string_match_instance["length"]
string_matches.append((identifier, matched_string, offset, length))
if len(string_matches) > 0:
string_data = []
st.markdown("**String Matches:**")
for string_match in string_matches:
string_data.append([string_match[0], string_match[1], string_match[2], string_match[3]])
df = pd.DataFrame(string_data, columns=['Identifier', 'Matched String', 'Offset', 'Length'])
st.table(df)
if rule_text:
with st.expander(f"###### Rule Definition"):
st.code(rule_text, language="yaml")
elif chosen_tab == "elasticsearch_info":
if es_results != {}:
cmd/enrichment/enrichment/tasks/file_processor.py
+44 -21
View File
@@ -18,6 +18,7 @@ import enrichment.lib.canaries as canary_helpers
import enrichment.lib.helpers as helpers
import nemesiscommon.constants as constants
import nemesispb.nemesis_pb2 as pb
import plyara
import structlog
import yara
from binaryornot.check import is_binary
@@ -29,6 +30,7 @@ from nemesiscommon.nemesis_tempfile import TempFile
from nemesiscommon.services.alerter import AlerterInterface
from nemesiscommon.storage import StorageInterface
from nemesiscommon.tasking import TaskInterface
from plyara import utils as plyara_utils
from prometheus_async import aio
from prometheus_client import Summary
from sumy.nlp.tokenizers import Tokenizer
@@ -165,8 +167,25 @@ class FileProcessor(TaskInterface):
except:
continue
# compile all the rules
self.yara_rules = yara.compile(filepaths=yara_files)
# save off the rule definitions in a readable way
self.yara_rule_definitions = {}
yara_rule_definitions_raw = list()
parser = plyara.Plyara()
for file_path in yara_file_paths:
with open(file_path, 'r') as fh:
try:
parsed_yara_rules = parser.parse_string(fh.read())
yara_rule_definitions_raw += parsed_yara_rules
except Exception as e:
logger.error(f"Error parsing yara file '{file_path}' : {e}")
parser.clear()
pass
for rule_def in yara_rule_definitions_raw:
self.yara_rule_definitions[rule_def['rule_name']] = plyara_utils.rebuild_yara_rule(rule_def)
async def run(self) -> None:
await logger.ainfo("Starting the File Processor")
@@ -245,7 +264,7 @@ class FileProcessor(TaskInterface):
file_data.hashes.CopyFrom(file_hashes)
else:
enrichments_failure.append(constants.E_FILE_HASHES)
await logger.aerror("Hash enrichment: Failed to hash file")
await logger.aerror(f"Hash enrichment: Failed to hash file: {file_path_on_disk}")
# now get its magic type from the first 2048 bytes using python-magic
file_magic_type = helpers.get_magic_type(file_path_on_disk)
@@ -637,7 +656,7 @@ class FileProcessor(TaskInterface):
text=text,
)
# run Yara on the file
# run Yara OPSEC rules on the file
yara_matches = await self.yara_opsec_scan(file_path_on_disk)
if isinstance(yara_matches, pb.Error):
enrichments_failure.append(constants.E_YARA_SCAN)
@@ -647,7 +666,7 @@ class FileProcessor(TaskInterface):
if yara_matches.yara_matches_present and len(yara_matches.yara_matches) > 0:
file_data.yara_matches.CopyFrom(yara_matches)
alert_rules = [t.rule_title for t in yara_matches.yara_matches if t.rule_title not in constants.EXCLUDED_YARA_RULES]
alert_rules = [t.rule_name for t in yara_matches.yara_matches if t.rule_name not in constants.EXCLUDED_YARA_RULES]
if alert_rules:
rule_matches_str = ", ".join(alert_rules)
@@ -922,25 +941,29 @@ class FileProcessor(TaskInterface):
yara_matches = pb.YaraMatches()
def mycallback(data):
if(data["matches"]):
yara_matches.yara_matches_present = True
yara_match = pb.YaraMatches.YaraMatch()
yara_match.rule_file = data["namespace"]
yara_match.rule_title = data["rule"]
if "name" in data["meta"]:
yara_match.rule_name = data["meta"]["name"]
if "description" in data["meta"]:
yara_match.rule_description = data["meta"]["description"]
yara_match.rule_title = data["rule"]
if "strings" in data:
yara_match.strings.extend([f"{m}" for m in data["strings"]])
yara_matches.yara_matches.extend([yara_match])
return yara.CALLBACK_CONTINUE
if os.path.exists(file_path):
try:
self.yara_rules.match(file_path, callback=mycallback, which_callbacks=yara.CALLBACK_MATCHES)
for match in self.yara_rules.match(file_path):
yara_matches.yara_matches_present = True
yara_match = pb.YaraMatches.YaraMatch()
yara_match.rule_file = match.namespace
yara_match.rule_name = match.rule
if yara_match.rule_name in self.yara_rule_definitions:
yara_match.rule_text = self.yara_rule_definitions[yara_match.rule_name]
if hasattr(match, 'meta') and "description" in match.meta:
yara_match.rule_description = match.meta["description"]
if hasattr(match, 'strings'):
for yara_string in match.strings:
yara_string_match = pb.YaraMatches.YaraStringMatch()
yara_string_match.identifier = yara_string.identifier
for instance in yara_string.instances:
yara_string_match_instance = pb.YaraMatches.YaraStringMatchInstance()
yara_string_match_instance.matched_string = f"{instance}"
yara_string_match_instance.offset = instance.offset
yara_string_match_instance.length = instance.matched_length
yara_string_match.yara_string_match_instances.extend([yara_string_match_instance])
yara_match.rule_string_matches.extend([yara_string_match])
yara_matches.yara_matches.extend([yara_match])
except Exception as e:
yara_matches = helpers.nemesis_error(f"yara_scan_file error for {file_path} : {e}")
await logger.aexception(e, message="yara_scan_file error", file_path=file_path)
@@ -949,7 +972,7 @@ class FileProcessor(TaskInterface):
# try to download the file from the nemesis API
file_uuid = uuid.UUID(file_path)
with await self.storage.download(file_uuid) as temp_file:
self.yara_rules.match(temp_file.name, callback=mycallback, which_callbacks=yara.CALLBACK_MATCHES)
self.yara_rules.match(temp_file.name)
except Exception as e:
yara_matches = helpers.nemesis_error(f"yara_scan_file error for {file_path} : {e}")
await logger.aexception(e, message="yara_scan_file error", file_path=file_path)
cmd/enrichment/poetry.lock
Generated
+1313 -1277
View File
File diff suppressed because it is too large Load Diff
cmd/enrichment/pyproject.toml
+1
View File
@@ -32,6 +32,7 @@ python-magic = "^0.4.27"
pefile = "^2022.5.30"
dnfile = "^0.12.0"
yara-python = "^4.2.3"
plyara = "^2.1.1"
sumy = "^0.10.0"
nltk = "^3.7"
olefile = "^0.46"
packages/protobufs/nemesis.proto
+13 -4
View File
@@ -1475,12 +1475,21 @@ message NoseyParker {
message YaraMatches {
message YaraStringMatchInstance {
string matched_string = 1;
uint64 offset = 2;
uint64 length = 3;
}
message YaraStringMatch {
string identifier = 1;
repeated YaraStringMatchInstance yara_string_match_instances = 2;
}
message YaraMatch {
string rule_file = 1;
string rule_title = 2;
string rule_name = 3;
string rule_description = 4;
repeated string strings = 5;
string rule_name = 2;
string rule_description = 3;
string rule_text = 4;
repeated YaraStringMatch rule_string_matches = 5;
}
bool yara_matches_present = 1;
repeated YaraMatch yara_matches = 2;
packages/python/nemesispb/nemesispb/nemesis_pb2.py
+52 -48
View File
File diff suppressed because one or more lines are too long
packages/python/nemesispb/nemesispb/nemesis_pb2.pyi
+43 -7
View File
@@ -3883,31 +3883,67 @@ global___NoseyParker = NoseyParker
class YaraMatches(google.protobuf.message.Message):
DESCRIPTOR: google.protobuf.descriptor.Descriptor
@typing_extensions.final
class YaraStringMatchInstance(google.protobuf.message.Message):
DESCRIPTOR: google.protobuf.descriptor.Descriptor
MATCHED_STRING_FIELD_NUMBER: builtins.int
OFFSET_FIELD_NUMBER: builtins.int
LENGTH_FIELD_NUMBER: builtins.int
matched_string: builtins.str
offset: builtins.int
length: builtins.int
def __init__(
self,
*,
matched_string: builtins.str = ...,
offset: builtins.int = ...,
length: builtins.int = ...,
) -> None: ...
def ClearField(self, field_name: typing_extensions.Literal["length", b"length", "matched_string", b"matched_string", "offset", b"offset"]) -> None: ...
@typing_extensions.final
class YaraStringMatch(google.protobuf.message.Message):
DESCRIPTOR: google.protobuf.descriptor.Descriptor
IDENTIFIER_FIELD_NUMBER: builtins.int
YARA_STRING_MATCH_INSTANCES_FIELD_NUMBER: builtins.int
identifier: builtins.str
@property
def yara_string_match_instances(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___YaraMatches.YaraStringMatchInstance]: ...
def __init__(
self,
*,
identifier: builtins.str = ...,
yara_string_match_instances: collections.abc.Iterable[global___YaraMatches.YaraStringMatchInstance] | None = ...,
) -> None: ...
def ClearField(self, field_name: typing_extensions.Literal["identifier", b"identifier", "yara_string_match_instances", b"yara_string_match_instances"]) -> None: ...
@typing_extensions.final
class YaraMatch(google.protobuf.message.Message):
DESCRIPTOR: google.protobuf.descriptor.Descriptor
RULE_FILE_FIELD_NUMBER: builtins.int
RULE_TITLE_FIELD_NUMBER: builtins.int
RULE_NAME_FIELD_NUMBER: builtins.int
RULE_DESCRIPTION_FIELD_NUMBER: builtins.int
STRINGS_FIELD_NUMBER: builtins.int
RULE_TEXT_FIELD_NUMBER: builtins.int
RULE_STRING_MATCHES_FIELD_NUMBER: builtins.int
rule_file: builtins.str
rule_title: builtins.str
rule_name: builtins.str
rule_description: builtins.str
rule_text: builtins.str
@property
def strings(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: ...
def rule_string_matches(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___YaraMatches.YaraStringMatch]: ...
def __init__(
self,
*,
rule_file: builtins.str = ...,
rule_title: builtins.str = ...,
rule_name: builtins.str = ...,
rule_description: builtins.str = ...,
strings: collections.abc.Iterable[builtins.str] | None = ...,
rule_text: builtins.str = ...,
rule_string_matches: collections.abc.Iterable[global___YaraMatches.YaraStringMatch] | None = ...,
) -> None: ...
def ClearField(self, field_name: typing_extensions.Literal["rule_description", b"rule_description", "rule_file", b"rule_file", "rule_name", b"rule_name", "rule_title", b"rule_title", "strings", b"strings"]) -> None: ...
def ClearField(self, field_name: typing_extensions.Literal["rule_description", b"rule_description", "rule_file", b"rule_file", "rule_name", b"rule_name", "rule_string_matches", b"rule_string_matches", "rule_text", b"rule_text"]) -> None: ...
YARA_MATCHES_PRESENT_FIELD_NUMBER: builtins.int
YARA_MATCHES_FIELD_NUMBER: builtins.int