mirror of
https://github.com/SpecterOps/Nemesis
synced 2026-06-08 12:36:42 +00:00
Merge pull request #7 from SpecterOps/dashboard_file_viewer
Dashboard file viewer
This commit is contained in:
Will
-1
@@ -44,7 +44,6 @@ if st.session_state["authentication_status"]:
|
||||
st.write(templates.number_of_results(total_hits, results["took"] / 1000), unsafe_allow_html=True)
|
||||
|
||||
for i in range(num_results):
|
||||
# http://172.16.111.187:8080/api/download/bd48a461-a1a5-42a9-8e87-07ce62408303?name=nosey_parker_google.config
|
||||
object_id = results["hits"]["hits"][i]["_source"]["objectId"]
|
||||
file_name = results["hits"]["hits"][i]["_source"]["name"]
|
||||
download_url = f"{NEMESIS_HTTP_SERVER}/api/download/{object_id}?name={file_name}"
|
||||
@@ -1,8 +1,8 @@
|
||||
# Standard Libraries
|
||||
import datetime
|
||||
import ntpath
|
||||
import os
|
||||
import re
|
||||
import ntpath
|
||||
import urllib.parse
|
||||
from typing import List
|
||||
|
||||
@@ -341,7 +341,7 @@ if st.session_state["authentication_status"]:
|
||||
# replace - with _ since streamlit doesn't support -'s in session state
|
||||
unique_db_id = file["unique_db_id"].replace("-", "_")
|
||||
|
||||
kibana_link = f"{NEMESIS_HTTP_SERVER}/kibana/app/discover#/?_a=(filters:!((query:(match_phrase:(objectId:'{object_id}')))),index:'26360ae8-a518-4dac-b499-ef682d3f6bac')&_g=(time:(from:now-1y%2Fd,to:now))"
|
||||
dashboard_link = f"{NEMESIS_HTTP_SERVER}/dashboard/File_Viewer?object_id={object_id}"
|
||||
|
||||
url_enc_file_name = urllib.parse.quote(file["name"])
|
||||
|
||||
@@ -382,9 +382,9 @@ if st.session_state["authentication_status"]:
|
||||
with mui.Tooltip(title="Download the file"):
|
||||
with html.span:
|
||||
mui.IconButton(mui.icon.Download, href=base_file_url)
|
||||
with mui.Tooltip(title="View the file in Kibana"):
|
||||
with mui.Tooltip(title="View file details in Nemesis"):
|
||||
with html.span:
|
||||
mui.IconButton(mui.icon.Search, href=kibana_link, target="_blank")
|
||||
mui.IconButton(mui.icon.Search, href=dashboard_link, target="_blank")
|
||||
with mui.Tooltip(title="View the file in browser"):
|
||||
with html.span:
|
||||
mui.IconButton(mui.icon.TextSnippet, href=view_download_url, target="_blank")
|
||||
@@ -0,0 +1,369 @@
|
||||
# Standard Libraries
|
||||
import base64
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import urllib.parse
|
||||
|
||||
# 3rd Party Libraries
|
||||
import extra_streamlit_components as stx
|
||||
import requests
|
||||
import streamlit as st
|
||||
import templates
|
||||
import utils
|
||||
from annotated_text import annotated_text, annotation
|
||||
from streamlit_cookies_manager import CookieManager
|
||||
from streamlit_elements import (dashboard, editor, elements, html, lazy, mui,
|
||||
sync)
|
||||
|
||||
POSTGRES_CONNECTION_URI = os.environ.get("POSTGRES_CONNECTION_URI") or ""
|
||||
DB_ITERATION_SIZE = os.environ.get("DB_ITERATION_SIZE") or "1000"
|
||||
NEMESIS_HTTP_SERVER = os.environ.get("NEMESIS_HTTP_SERVER")
|
||||
PAGE_SIZE = 8
|
||||
|
||||
|
||||
global sources, projects
|
||||
sources = []
|
||||
projects = []
|
||||
|
||||
current_user = utils.header()
|
||||
|
||||
# should be defined in ./packages/python/nemesiscommon/nemesiscommon/contents.py - E_TAG_*
|
||||
filter_tags = [
|
||||
"contains_dpapi",
|
||||
"noseyparker_results",
|
||||
"parsed_creds",
|
||||
"encrypted",
|
||||
"deserialization",
|
||||
"cmd_execution",
|
||||
"remoting",
|
||||
"yara_matches",
|
||||
"file_canary",
|
||||
]
|
||||
|
||||
if st.session_state["authentication_status"]:
|
||||
cookies = CookieManager()
|
||||
if not cookies.ready():
|
||||
st.stop()
|
||||
|
||||
object_id = ""
|
||||
if "object_id" not in st.session_state:
|
||||
st.session_state.object_id = None
|
||||
|
||||
triage_pattern = re.compile(r"^triage_(?P<db_id>[0-9a-f]{8}_[0-9a-f]{4}_[0-9a-f]{4}_[0-9a-f]{4}_[0-9a-f]{12})_(?P<triage_value>.*)")
|
||||
notes_pattern = re.compile(r"^file_notes_(?P<db_id>[0-9a-f]{8}_[0-9a-f]{4}_[0-9a-f]{4}_[0-9a-f]{4}_[0-9a-f]{12})$")
|
||||
|
||||
for state in st.session_state:
|
||||
triage_matches = triage_pattern.search(state)
|
||||
if triage_matches:
|
||||
db_id = triage_matches.group("db_id").replace("_", "-")
|
||||
triage_value = triage_matches.group("triage_value")
|
||||
utils.update_triage_table(db_id, "file_data_enriched", current_user, triage_value)
|
||||
del st.session_state[state]
|
||||
else:
|
||||
notes_matches = notes_pattern.search(state)
|
||||
if notes_matches:
|
||||
db_id = notes_matches.group("db_id").replace("_", "-")
|
||||
utils.update_notes_table(db_id, "file_data_enriched", current_user, st.session_state[state].target.value)
|
||||
del st.session_state[state]
|
||||
|
||||
set_search_params = {}
|
||||
para = st.experimental_get_query_params()
|
||||
|
||||
for key in para.keys():
|
||||
match key:
|
||||
case "object_id":
|
||||
st.session_state.object_id = para["object_id"][0]
|
||||
object_id = st.session_state.object_id
|
||||
|
||||
if not st.session_state.object_id:
|
||||
object_id = st.text_input("Enter file object_id")
|
||||
|
||||
if object_id != "":
|
||||
|
||||
if object_id and not re.match(r"^[0-9a-fA-F]{8}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{12}$", object_id):
|
||||
st.error(f"object_id '{object_id}' is not a valid UUID", icon="🚨")
|
||||
|
||||
elif object_id:
|
||||
file = utils.get_file_information(object_id)
|
||||
|
||||
if not((file is None) or len(file) == 0):
|
||||
object_id = file["object_id"]
|
||||
extension = pathlib.Path(file["name"]).suffix.strip(".").lower()
|
||||
|
||||
# replace - with _ since streamlit doesn't support -'s in session state
|
||||
unique_db_id = file["unique_db_id"].replace("-", "_")
|
||||
|
||||
kibana_link = f"{NEMESIS_HTTP_SERVER}/kibana/app/discover#/?_a=(filters:!((query:(match_phrase:(objectId:'{object_id}')))),index:'26360ae8-a518-4dac-b499-ef682d3f6bac')&_g=(time:(from:now-1y%2Fd,to:now))"
|
||||
|
||||
url_enc_file_name = urllib.parse.quote(file["name"])
|
||||
download_url = f"http://enrichment-webapi:9910/download/{object_id}"
|
||||
pdf_download_url = ""
|
||||
extracted_source_download_url = ""
|
||||
is_ascii = False
|
||||
public_download_url = f"{NEMESIS_HTTP_SERVER}/api/download/{object_id}?name={url_enc_file_name}&action=download"
|
||||
|
||||
if "magic_type" in file and file["magic_type"] and ("ASCII text" in file["magic_type"] or "Unicode text" in file["magic_type"]):
|
||||
is_ascii = True
|
||||
if "converted_pdf_id" in file and file["converted_pdf_id"] != "00000000-0000-0000-0000-000000000000":
|
||||
pdf_download_url = f"http://enrichment-webapi:9910/download/{file['converted_pdf_id']}"
|
||||
if "extracted_source_id" in file and file["extracted_source_id"] != "00000000-0000-0000-0000-000000000000":
|
||||
extracted_source_download_url = f"http://enrichment-webapi:9910/download/{file['extracted_source_id']}"
|
||||
if file["name"].endswith(".pdf"):
|
||||
pdf_download_url = download_url
|
||||
|
||||
tabs = [
|
||||
stx.TabBarItemData(id="basic_file_info", title="Basic File Info", description="Basic File Information"),
|
||||
]
|
||||
|
||||
es_results = utils.elastic_file_search(object_id)
|
||||
has_np_results = False
|
||||
if es_results and es_results["hits"]["total"]["value"] == 1:
|
||||
if "noseyparker" in es_results["hits"]["hits"][0]["_source"]:
|
||||
tabs.append(stx.TabBarItemData(id="noseyparker_results", title="Noseyparker Results", description="Noseyparker Results"))
|
||||
|
||||
tabs.append(stx.TabBarItemData(id="elasticsearch_info", title="Elasticsearch Info", description="Elasticsearch Information Dump"))
|
||||
|
||||
chosen_tab = stx.tab_bar(
|
||||
data=tabs,
|
||||
default="basic_file_info",
|
||||
)
|
||||
|
||||
if chosen_tab == "basic_file_info":
|
||||
layout = [
|
||||
# Grid layout parameters: element_identifier, x_pos, y_pos, width, height, [item properties...]
|
||||
dashboard.Item("1", 0, 0, 10, 2.5, isDraggable=False, isResizable=False, sx={"height": "100%"}),
|
||||
dashboard.Item("2", 0, 0, 10, 5, isDraggable=False, isResizable=False, sx={"height": "100%"}),
|
||||
]
|
||||
with elements("dashboard"):
|
||||
with dashboard.Grid(layout=layout):
|
||||
with mui.Card(
|
||||
key="1",
|
||||
sx={
|
||||
"display": "flex",
|
||||
"flexDirection": "column",
|
||||
"borderRadius": 2,
|
||||
"overflow": "auto",
|
||||
"overflowY": "auto",
|
||||
"m": "10",
|
||||
"gap": "10px",
|
||||
},
|
||||
padding=1,
|
||||
elevation=1,
|
||||
spacing=10,
|
||||
):
|
||||
with mui.AppBar(position="sticky", variant="h7", sx={"minHeight": 32}):
|
||||
with mui.Toolbar(variant="dense", sx={"minHeight": 48, "height": 48}):
|
||||
mui.Typography(file["name"])
|
||||
with mui.Tooltip(title="Download the file"):
|
||||
with html.span:
|
||||
mui.IconButton(mui.icon.Download, href=public_download_url)
|
||||
with mui.Tooltip(title="View the file in Kibana"):
|
||||
with html.span:
|
||||
mui.IconButton(mui.icon.Search, href=kibana_link, target="_blank")
|
||||
if extracted_source_download_url:
|
||||
with mui.Tooltip(title="Download the extracted source code"):
|
||||
with html.span:
|
||||
mui.IconButton(mui.icon.Code, href=extracted_source_download_url, target="_blank")
|
||||
|
||||
mui.Box(sx={"flexGrow": 1})
|
||||
|
||||
with mui.Tooltip(title="Mark file as useful"):
|
||||
with html.span:
|
||||
mui.IconButton(mui.icon.ThumbUpOffAlt, onClick=sync(f"triage_{unique_db_id}_useful"))
|
||||
with mui.Tooltip(title="Mark file as not useful"):
|
||||
with html.span:
|
||||
mui.IconButton(mui.icon.ThumbDownOffAlt, onClick=sync(f"triage_{unique_db_id}_notuseful"))
|
||||
with mui.Tooltip(title="Mark file as needing additional investigation"):
|
||||
with html.span:
|
||||
mui.IconButton(mui.icon.QuestionMark, onClick=sync(f"triage_{unique_db_id}_unknown"))
|
||||
if file["triage"]:
|
||||
with html.span:
|
||||
mui.Typography("triage")
|
||||
|
||||
# Information table
|
||||
with mui.CardContent(sx={"flex": 1}):
|
||||
with mui.TableContainer(sx={"maxHeight": 200}):
|
||||
with mui.Table(size="small", overflowX="hidden", whiteSpace="nowrap"):
|
||||
with mui.TableBody():
|
||||
identifier_style = {
|
||||
"fontWeight": "bold",
|
||||
"borderRight": "1px solid",
|
||||
"whiteSpace": "nowrap",
|
||||
"padding": "0px 5px 0px 0px",
|
||||
}
|
||||
|
||||
with mui.TableRow(hover=True, padding="none"):
|
||||
mui.TableCell("Path", size="small", sx=identifier_style)
|
||||
mui.TableCell(file["path"], width="100%")
|
||||
with mui.TableRow(hover=True, padding="none"):
|
||||
if file['source']:
|
||||
mui.TableCell("Source / Timestamp", size="small", sx=identifier_style)
|
||||
mui.TableCell(f"{file['source']} @ {file['timestamp']}", size="small")
|
||||
else:
|
||||
mui.TableCell("Timestamp", size="small", sx=identifier_style)
|
||||
mui.TableCell(f"{file['timestamp']}", size="small")
|
||||
with mui.TableRow(hover=True, padding="none"):
|
||||
mui.TableCell("Size", sx=identifier_style)
|
||||
mui.TableCell(f"{file['size']}")
|
||||
with mui.TableRow(hover=True, padding="none"):
|
||||
mui.TableCell("SHA1 hash", sx=identifier_style)
|
||||
mui.TableCell(file["sha1"])
|
||||
with mui.TableRow(hover=True, padding="none"):
|
||||
mui.TableCell("Magic Type", sx=identifier_style)
|
||||
mui.TableCell(file["magic_type"])
|
||||
if file["tags"]:
|
||||
with mui.TableRow(hover=True, padding="none"):
|
||||
mui.TableCell("Tags", sx=identifier_style)
|
||||
with mui.TableCell():
|
||||
# Tags
|
||||
for tag in file["tags"]:
|
||||
mui.Chip(label=tag, color="primary")
|
||||
# Notes
|
||||
mui.Typography("Comments:")
|
||||
with mui.Box(sx={"flexGrow": 1}):
|
||||
end = mui.IconButton(mui.icon.Save, onClick=sync())
|
||||
|
||||
mui.TextField(
|
||||
# label="Input Any Notes Here",
|
||||
key=f"file_notes_{unique_db_id}",
|
||||
defaultValue=file["notes"],
|
||||
variant="outlined",
|
||||
margin="none",
|
||||
multiline=True,
|
||||
onChange=lazy(sync(f"file_notes_{unique_db_id}")),
|
||||
fullWidth=True,
|
||||
sx={"flexGrow": 1},
|
||||
InputProps={"endAdornment": end},
|
||||
)
|
||||
|
||||
if is_ascii:
|
||||
# Monaco editor display for ascii files
|
||||
with mui.Card(
|
||||
key="2",
|
||||
sx={
|
||||
"display": "flex",
|
||||
"flexDirection": "column",
|
||||
"borderRadius": 2,
|
||||
"overflow": "auto",
|
||||
"overflowY": "auto",
|
||||
"m": "10",
|
||||
"gap": "10px",
|
||||
},
|
||||
padding=1,
|
||||
elevation=1,
|
||||
spacing=10,
|
||||
):
|
||||
response = requests.get(download_url)
|
||||
if response.status_code != 200:
|
||||
st.error(f"Error retrieving text data from {download_url}, status code: {response.status_code}", icon="🚨")
|
||||
else:
|
||||
with mui.Card(
|
||||
sx={
|
||||
"display": "flex",
|
||||
"overflow": "auto",
|
||||
"overflowY": "auto",
|
||||
},
|
||||
padding=1,
|
||||
elevation=1,
|
||||
spacing=10,
|
||||
):
|
||||
editor.Monaco(
|
||||
height="64vh",
|
||||
defaultValue=response.content.decode('utf-8'),
|
||||
language=utils.map_extension_to_monaco_language(extension)
|
||||
)
|
||||
elif pdf_download_url:
|
||||
# Inline PDF file file display, if PDF is present
|
||||
with mui.Card(
|
||||
key="2",
|
||||
sx={
|
||||
"display": "flex",
|
||||
"flexDirection": "column",
|
||||
"borderRadius": 2,
|
||||
"overflow": "auto",
|
||||
"overflowY": "auto",
|
||||
"m": "10",
|
||||
"gap": "10px",
|
||||
},
|
||||
padding=1,
|
||||
elevation=1,
|
||||
spacing=10,
|
||||
):
|
||||
try:
|
||||
response = requests.get(pdf_download_url)
|
||||
if response.status_code != 200:
|
||||
st.error(f"Error retrieving PDF data from {pdf_download_url}, status code: {response.status_code}", icon="🚨")
|
||||
else:
|
||||
with mui.Card(
|
||||
sx={
|
||||
"display": "flex",
|
||||
"overflow": "auto",
|
||||
"overflowY": "auto",
|
||||
},
|
||||
padding=1,
|
||||
elevation=1,
|
||||
spacing=10,
|
||||
):
|
||||
b64_data = base64.b64encode(response.content).decode('utf-8')
|
||||
html.iframe(
|
||||
src=f"data:application/pdf;base64,{b64_data}",
|
||||
height=785,
|
||||
width="100%",
|
||||
type="application/pdf"
|
||||
)
|
||||
except Exception as e:
|
||||
st.error(f"Error retrieving PDF data from {pdf_download_url} : {e}", icon="🚨")
|
||||
|
||||
elif chosen_tab == "noseyparker_results":
|
||||
if es_results != {}:
|
||||
total_hits = es_results["hits"]["total"]["value"]
|
||||
num_results = len(es_results["hits"]["hits"])
|
||||
if total_hits > 0:
|
||||
for i in range(num_results):
|
||||
object_id = es_results["hits"]["hits"][i]["_source"]["objectId"]
|
||||
file_name = es_results["hits"]["hits"][i]["_source"]["name"]
|
||||
download_url = f"{NEMESIS_HTTP_SERVER}/api/download/{object_id}?name={file_name}"
|
||||
kibana_link = f"{NEMESIS_HTTP_SERVER}/kibana/app/discover#/?_a=(filters:!((query:(match_phrase:(objectId:'{object_id}')))),index:'26360ae8-a518-4dac-b499-ef682d3f6bac')&_g=(time:(from:now-1y%2Fd,to:now))"
|
||||
path = es_results["hits"]["hits"][i]["_source"]["path"]
|
||||
sha1 = es_results["hits"]["hits"][i]["_source"]["hashes"]["sha1"]
|
||||
source = ""
|
||||
if "metadata" in es_results["hits"]["hits"][i]["_source"] and "source" in es_results["hits"]["hits"][i]["_source"]["metadata"]:
|
||||
source = es_results["hits"]["hits"][i]["_source"]["metadata"]["source"]
|
||||
|
||||
if source:
|
||||
expander_text = f"{source} : **{path}** (SHA1: {sha1})"
|
||||
else:
|
||||
expander_text = f"**{path}** (SHA1: {sha1})"
|
||||
|
||||
for ruleMatch in es_results["hits"]["hits"][i]["_source"]["noseyparker"]["ruleMatches"]:
|
||||
for match in ruleMatch["matches"]:
|
||||
if "matching" in match["snippet"]:
|
||||
rule_name = match["ruleName"]
|
||||
|
||||
if "before" in match["snippet"]:
|
||||
before = match["snippet"]["before"].replace("\n\t", " ")
|
||||
else:
|
||||
before = ""
|
||||
|
||||
matching = match["snippet"]["matching"]
|
||||
|
||||
if "after" in match["snippet"]:
|
||||
after = match["snippet"]["after"].replace("\n\t", " ")
|
||||
else:
|
||||
after = ""
|
||||
|
||||
st.write(f"<b>Rule</b>: {rule_name}", unsafe_allow_html=True)
|
||||
annotated_text(annotation(before, "context", color="#8ef"), annotation(matching, "match"), annotation(after, "context", color="#8ef"))
|
||||
st.divider()
|
||||
|
||||
elif chosen_tab == "elasticsearch_info":
|
||||
if es_results != {}:
|
||||
total_hits = es_results["hits"]["total"]["value"]
|
||||
if total_hits == 0:
|
||||
st.warning("No results found in Elasticsearch!")
|
||||
elif total_hits == 1:
|
||||
st.subheader("Elasticsearch Data")
|
||||
st.json(es_results["hits"]["hits"][0])
|
||||
else:
|
||||
st.warning("Too many results found in Elasticsearch!")
|
||||
@@ -474,6 +474,66 @@ def postgres_count_masterkeys(show_all=True, show_dec=True, key_type=""):
|
||||
return -1
|
||||
|
||||
|
||||
def get_file_information(object_id: str):
|
||||
"""Gets information from Postgres about a specific file."""
|
||||
|
||||
if not re.match(r"^[0-9a-fA-F]{8}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{12}$", object_id):
|
||||
raise Exception(f"object_id '{object_id}' supplied to get_file_information() is not a proper UUID")
|
||||
|
||||
try:
|
||||
with engine.connect() as conn:
|
||||
params = {
|
||||
"object_id": object_id,
|
||||
}
|
||||
|
||||
query = """
|
||||
SELECT file_data_enriched.project_id as project_id,
|
||||
file_data_enriched.source as source,
|
||||
file_data_enriched.timestamp as timestamp,
|
||||
file_data_enriched.unique_db_id::varchar,
|
||||
file_data_enriched.agent_id as agent_id,
|
||||
file_data_enriched.object_id::varchar as object_id,
|
||||
file_data_enriched.path as path,
|
||||
file_data_enriched.name as name,
|
||||
file_data_enriched.size as size,
|
||||
file_data_enriched.md5 as md5,
|
||||
file_data_enriched.sha1 as sha1,
|
||||
file_data_enriched.sha256 as sha256,
|
||||
file_data_enriched.nemesis_file_type as nemesis_file_type,
|
||||
file_data_enriched.magic_type as magic_type,
|
||||
file_data_enriched.converted_pdf_id::varchar as converted_pdf_id,
|
||||
file_data_enriched.extracted_plaintext_id::varchar as extracted_plaintext_id,
|
||||
file_data_enriched.extracted_source_id::varchar as extracted_source_id,
|
||||
file_data_enriched.tags as tags,
|
||||
file_data_enriched.originating_object_id as originating_object_id,
|
||||
triage.value as triage,
|
||||
triage.unique_db_id as triage_unique_db_id,
|
||||
notes.value as notes
|
||||
|
||||
FROM file_data_enriched
|
||||
|
||||
LEFT JOIN triage
|
||||
ON file_data_enriched.unique_db_id = triage.unique_db_id
|
||||
|
||||
LEFT JOIN notes
|
||||
ON file_data_enriched.unique_db_id = notes.unique_db_id
|
||||
|
||||
WHERE object_id = :object_id
|
||||
"""
|
||||
|
||||
df = pd.read_sql_query(sql_text(query), conn, params=params)
|
||||
if len(df) == 0:
|
||||
st.error(f"object_id '{object_id}' not found")
|
||||
elif len(df) == 1:
|
||||
return df.iloc[0]
|
||||
else:
|
||||
st.error(f"Too many results for object_id '{object_id}' : {len(df)}", icon="🚨")
|
||||
|
||||
except Exception as e:
|
||||
st.error(f"Error retrieving `file_data_enriched` from the database: {e}", icon="🚨")
|
||||
return None
|
||||
|
||||
|
||||
def postgres_file_search(
|
||||
start: datetime.datetime,
|
||||
end: datetime.datetime,
|
||||
@@ -699,7 +759,6 @@ def authenticate_header() -> str:
|
||||
#
|
||||
######################################################
|
||||
|
||||
|
||||
def simplify_es_text_result(result: dict) -> dict:
|
||||
"""Simplifies an elastic result into the three parts we want to use."""
|
||||
res = result["_source"]
|
||||
@@ -744,6 +803,23 @@ def get_elastic_total_indexed_documents(index_name="file_data_plaintext", query=
|
||||
return 0
|
||||
|
||||
|
||||
def elastic_file_search(object_id: str) -> dict:
|
||||
"""
|
||||
Searches the 'file_data_enriched' index in Elasticsearch for
|
||||
the matching document, returning all fields.
|
||||
"""
|
||||
try:
|
||||
es_client = wait_for_elasticsearch()
|
||||
query = {"term": {"objectId.keyword": object_id}}
|
||||
return es_client.search(index="file_data_enriched", query=query)
|
||||
except Exception as e:
|
||||
if "index_not_found_exception" in f"{e}":
|
||||
st.error("Elastic index 'file_data_enriched' doesn't yet exist!", icon="🚨")
|
||||
else:
|
||||
st.error(f"Exception querying Elastic: {e}", icon="🚨")
|
||||
return {}
|
||||
|
||||
|
||||
def elastic_text_search(search_term: str, from_i: int, size: int) -> dict:
|
||||
"""
|
||||
Searches the 'file_data_plaintext' index in Elasticsearch for
|
||||
@@ -831,6 +907,60 @@ def elastic_np_search(from_i: int, size: int) -> dict:
|
||||
#
|
||||
######################################################
|
||||
|
||||
def map_extension_to_monaco_language(extension: str) -> str:
|
||||
"""
|
||||
Maps a file extension to a source code language for Monaco.
|
||||
|
||||
Ref: https://microsoft.github.io/monaco-editor/
|
||||
"""
|
||||
|
||||
language_mappings = {
|
||||
"bat": "batch",
|
||||
"c": "c",
|
||||
"cpp": "cpp",
|
||||
"cs": "csharp",
|
||||
"css": "css",
|
||||
"cypher": "cypher",
|
||||
"dockerfile": "dockerfile",
|
||||
"fs": "fsharp",
|
||||
"go": "go",
|
||||
"gql": "graphql",
|
||||
"graphql": "graphql",
|
||||
"html": "html",
|
||||
"ini": "ini",
|
||||
"java": "java",
|
||||
"js": "javascript",
|
||||
"lua": "lua",
|
||||
"md": "markdown",
|
||||
"mysql": "mysql",
|
||||
"php": "php",
|
||||
"php3": "php",
|
||||
"php4": "php",
|
||||
"php5": "php",
|
||||
"pl": "perl",
|
||||
"ps1": "powershell",
|
||||
"psd1": "powershell",
|
||||
"psm1": "powershell",
|
||||
"proto": "proto",
|
||||
"py": "python",
|
||||
"r": "r",
|
||||
"rb": "ruby",
|
||||
"rs": "rust",
|
||||
"shell": "shell",
|
||||
"sh": "shell",
|
||||
"sql": "sql",
|
||||
"swift": "swift",
|
||||
"ts": "typescript",
|
||||
"ts": "typescript",
|
||||
"vb": "vb",
|
||||
"wgsl": "wgsl",
|
||||
"xml": "xml",
|
||||
"yaml": "yaml",
|
||||
"json": "json"
|
||||
}
|
||||
return language_mappings.get(extension.lower(), "plaintext")
|
||||
|
||||
|
||||
def is_valid_chromium_file_path(file_path: str) -> bool:
|
||||
"""Returns true if the supplied path is a valid Chromium file path."""
|
||||
|
||||
|
||||
@@ -421,7 +421,7 @@ class Container(containers.DeclarativeContainer):
|
||||
#
|
||||
# Services
|
||||
#
|
||||
alerter_service = providers.Factory(NemesisAlerter, outputq_alert, config.public_kibana_url)
|
||||
alerter_service = providers.Factory(NemesisAlerter, outputq_alert, config.public_nemesis_url)
|
||||
|
||||
elasticsearch_client = providers.Factory(
|
||||
AsyncElasticsearch,
|
||||
|
||||
@@ -907,7 +907,7 @@ def is_pe_extension(file_path: str) -> bool:
|
||||
|
||||
def is_source_code(file_path: str) -> bool:
|
||||
"""Returns True if the supplied file_path matches a number of supported source code file extensions."""
|
||||
source_code_regex = "^.*\\.(aspx|c|cpp|cs|go|groovy|java|jsp|js|lua|php|php3" "|php4|php5|ps1|psd1|psm1|py|rb|rs|sql|sh|swift|vb|vbs)$"
|
||||
source_code_regex = "^.*\\.(aspx|c|cpp|cs|go|groovy|java|jsp|js|lua|php|php3|php4|php5|ps1|psd1|psm1|py|rb|rs|sql|sh|swift|vb|vbs)$"
|
||||
return re.match(source_code_regex, file_path, re.IGNORECASE) is not None
|
||||
|
||||
|
||||
|
||||
@@ -53,6 +53,7 @@ class EnrichmentSettings(FileProcessingService): # type: ignore
|
||||
elasticsearch_password: str
|
||||
elasticsearch_url: HttpUrlWithSlash
|
||||
web_api_url: HttpUrlWithSlash
|
||||
public_nemesis_url: HttpUrlWithSlash
|
||||
public_kibana_url: HttpUrlWithSlash
|
||||
slack_webhook_url: Optional[str]
|
||||
slack_username: str
|
||||
|
||||
@@ -9,13 +9,12 @@ import nemesispb.nemesis_pb2 as pb
|
||||
import passwordcracker.settings as settings
|
||||
from dependency_injector import containers, providers
|
||||
from nemesiscommon.constants import NemesisQueue
|
||||
from nemesiscommon.messaging_rabbitmq import (
|
||||
NemesisRabbitMQConsumer,
|
||||
NemesisRabbitMQProducer,
|
||||
)
|
||||
from nemesiscommon.messaging_rabbitmq import (NemesisRabbitMQConsumer,
|
||||
NemesisRabbitMQProducer)
|
||||
from nemesiscommon.services.alerter import NemesisAlerter
|
||||
from nemesiscommon.tasking import TaskDispatcher
|
||||
from passwordcracker.services.john_the_ripper_cracker import JohnTheRipperCracker
|
||||
from passwordcracker.services.john_the_ripper_cracker import \
|
||||
JohnTheRipperCracker
|
||||
from passwordcracker.settings import PasswordCrackerSettings
|
||||
from passwordcracker.tasks.password_cracker import PasswordCracker
|
||||
|
||||
@@ -76,7 +75,7 @@ class Container(containers.DeclarativeContainer):
|
||||
alerting_service = providers.Factory(
|
||||
NemesisAlerter,
|
||||
outputq_alert,
|
||||
config.public_kibana_url,
|
||||
config.public_nemesis_url,
|
||||
)
|
||||
cracker_service = providers.Factory(
|
||||
JohnTheRipperCracker,
|
||||
|
||||
@@ -13,7 +13,7 @@ class CrackWordlistSize(IntEnum):
|
||||
|
||||
class PasswordCrackerSettings(NemesisServiceSettings): # type: ignore
|
||||
rabbitmq_connection_uri: AnyUrl
|
||||
public_kibana_url: HttpUrlWithSlash
|
||||
public_nemesis_url: HttpUrlWithSlash
|
||||
data_download_dir: str
|
||||
crack_wordlist_top_words: CrackWordlistSize # either 10000 or 100000 for now
|
||||
|
||||
|
||||
@@ -80,6 +80,8 @@ spec:
|
||||
configMapKeyRef:
|
||||
name: operation-config
|
||||
key: nemesis-http-server
|
||||
- name: PUBLIC_NEMESIS_URL
|
||||
value: "$(NEMESIS_HTTP_SERVER)/dashboard/"
|
||||
- name: PUBLIC_KIBANA_URL
|
||||
value: "$(NEMESIS_HTTP_SERVER)/kibana/"
|
||||
- name: WEB_API_URL
|
||||
|
||||
@@ -46,8 +46,8 @@ spec:
|
||||
value: "True"
|
||||
- name: PROMETHEUS_PORT
|
||||
value: "9090"
|
||||
- name: PUBLIC_KIBANA_URL
|
||||
value: "$(NEMESIS_HTTP_SERVER)/kibana/"
|
||||
- name: PUBLIC_NEMESIS_URL
|
||||
value: "$(NEMESIS_HTTP_SERVER)/dashboard/"
|
||||
- name: RABBITMQ_CONNECTION_URI
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
|
||||
@@ -28,11 +28,11 @@ class AlerterInterface:
|
||||
|
||||
class NemesisAlerter(AlerterInterface):
|
||||
alert_queue: NemesisRabbitMQProducer
|
||||
kibana_url: str
|
||||
nemesis_url: str
|
||||
|
||||
def __init__(self, alert_queue: NemesisRabbitMQProducer, kibana_url: str):
|
||||
def __init__(self, alert_queue: NemesisRabbitMQProducer, nemesis_url: str):
|
||||
self.alert_queue = alert_queue
|
||||
self.kibana_url = kibana_url
|
||||
self.nemesis_url = nemesis_url
|
||||
|
||||
async def alert(self, text: str) -> None:
|
||||
alert_msg = pb.Alert()
|
||||
@@ -48,15 +48,15 @@ class NemesisAlerter(AlerterInterface):
|
||||
header = f"*{title}*\n" if title else ""
|
||||
text = f"\n{text}" if text else ""
|
||||
|
||||
full_kibana_url = f"{self.kibana_url}app/discover#/?_a=(filters:!((query:(match_phrase:(objectId:'{file_data.object_id}')))),index:'26360ae8-a518-4dac-b499-ef682d3f6bac')&_g=(time:(from:now-1y%2Fd,to:now))"
|
||||
kibana_footer = f"\n<{full_kibana_url}|*File in Kibana*>"
|
||||
full_nemesis_url = f"{self.nemesis_url}File_Viewer?object_id={file_data.object_id}"
|
||||
nemesis_footer = f"\n<{full_nemesis_url}|*View File in Nemesis*>"
|
||||
|
||||
try:
|
||||
metadata_dict = MessageToDict(metadata, preserving_proto_field_name=True)
|
||||
timestamp = metadata_dict["timestamp"]
|
||||
agent_type = metadata_dict["agent_type"]
|
||||
agent_id = metadata_dict["agent_id"]
|
||||
message = f"{header}*File:* {file_name}\n*SHA1:* {sha1_hash}\n*Downloaded:* {timestamp}\n*Agent:* {agent_id} (type: {agent_type}){text}{kibana_footer}"
|
||||
message = f"{header}*File:* {file_name}\n*SHA1:* {sha1_hash}\n*Downloaded:* {timestamp}\n*Agent:* {agent_id} (type: {agent_type}){text}{nemesis_footer}"
|
||||
|
||||
await self.alert(text=message)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user