From e1562f96b658a849294964fdbff8f46a08fc6682 Mon Sep 17 00:00:00 2001 From: HarmJ0y Date: Tue, 5 Sep 2023 21:39:02 -0700 Subject: [PATCH 1/3] Start of File Viewer page for the Dashboard -Start of File Viewer page for the Dashboard -Alerting currently not working correctly --- .../{5_Credentials.py => 11_Credentials.py} | 0 .../{6_NoseyParker.py => 13_NoseyParker.py} | 1 - .../pages/{7_Hashes.py => 15_Hashes.py} | 0 .../pages/{2_Files.py => 1_Files.py} | 8 +- .../dashboard/pages/3_File_Viewer.py | 343 ++++++++++++++++++ ...ocument Search.py => 5_Document Search.py} | 0 .../pages/{3_Chromium.py => 7_Chromium.py} | 0 .../pages/{4_DPAPI.py => 9_DPAPI.py} | 0 cmd/dashboard/dashboard/utils.py | 132 ++++++- cmd/enrichment/enrichment/containers.py | 2 +- cmd/enrichment/enrichment/lib/helpers.py | 2 +- cmd/enrichment/enrichment/settings.py | 1 + .../passwordcracker/containers.py | 11 +- .../passwordcracker/settings.py | 2 +- kubernetes/enrichment/deployment.yaml | 2 + kubernetes/passwordcracker/deployment.yaml | 4 +- .../nemesiscommon/services/alerter.py | 12 +- 17 files changed, 497 insertions(+), 23 deletions(-) rename cmd/dashboard/dashboard/pages/{5_Credentials.py => 11_Credentials.py} (100%) rename cmd/dashboard/dashboard/pages/{6_NoseyParker.py => 13_NoseyParker.py} (97%) rename cmd/dashboard/dashboard/pages/{7_Hashes.py => 15_Hashes.py} (100%) rename cmd/dashboard/dashboard/pages/{2_Files.py => 1_Files.py} (99%) create mode 100644 cmd/dashboard/dashboard/pages/3_File_Viewer.py rename cmd/dashboard/dashboard/pages/{1_Document Search.py => 5_Document Search.py} (100%) rename cmd/dashboard/dashboard/pages/{3_Chromium.py => 7_Chromium.py} (100%) rename cmd/dashboard/dashboard/pages/{4_DPAPI.py => 9_DPAPI.py} (100%) diff --git a/cmd/dashboard/dashboard/pages/5_Credentials.py b/cmd/dashboard/dashboard/pages/11_Credentials.py similarity index 100% rename from cmd/dashboard/dashboard/pages/5_Credentials.py rename to cmd/dashboard/dashboard/pages/11_Credentials.py diff --git a/cmd/dashboard/dashboard/pages/6_NoseyParker.py b/cmd/dashboard/dashboard/pages/13_NoseyParker.py similarity index 97% rename from cmd/dashboard/dashboard/pages/6_NoseyParker.py rename to cmd/dashboard/dashboard/pages/13_NoseyParker.py index c24f2be..d0ea670 100644 --- a/cmd/dashboard/dashboard/pages/6_NoseyParker.py +++ b/cmd/dashboard/dashboard/pages/13_NoseyParker.py @@ -44,7 +44,6 @@ if st.session_state["authentication_status"]: st.write(templates.number_of_results(total_hits, results["took"] / 1000), unsafe_allow_html=True) for i in range(num_results): - # http://172.16.111.187:8080/api/download/bd48a461-a1a5-42a9-8e87-07ce62408303?name=nosey_parker_google.config object_id = results["hits"]["hits"][i]["_source"]["objectId"] file_name = results["hits"]["hits"][i]["_source"]["name"] download_url = f"{NEMESIS_HTTP_SERVER}/api/download/{object_id}?name={file_name}" diff --git a/cmd/dashboard/dashboard/pages/7_Hashes.py b/cmd/dashboard/dashboard/pages/15_Hashes.py similarity index 100% rename from cmd/dashboard/dashboard/pages/7_Hashes.py rename to cmd/dashboard/dashboard/pages/15_Hashes.py diff --git a/cmd/dashboard/dashboard/pages/2_Files.py b/cmd/dashboard/dashboard/pages/1_Files.py similarity index 99% rename from cmd/dashboard/dashboard/pages/2_Files.py rename to cmd/dashboard/dashboard/pages/1_Files.py index faad3c9..9d2835d 100644 --- a/cmd/dashboard/dashboard/pages/2_Files.py +++ b/cmd/dashboard/dashboard/pages/1_Files.py @@ -1,8 +1,8 @@ # Standard Libraries import datetime +import ntpath import os import re -import ntpath import urllib.parse from typing import List @@ -341,7 +341,7 @@ if st.session_state["authentication_status"]: # replace - with _ since streamlit doesn't support -'s in session state unique_db_id = file["unique_db_id"].replace("-", "_") - kibana_link = f"{NEMESIS_HTTP_SERVER}/kibana/app/discover#/?_a=(filters:!((query:(match_phrase:(objectId:'{object_id}')))),index:'26360ae8-a518-4dac-b499-ef682d3f6bac')&_g=(time:(from:now-1y%2Fd,to:now))" + dashboard_link = f"{NEMESIS_HTTP_SERVER}/dashboard/File_Viewer?object_id={object_id}" url_enc_file_name = urllib.parse.quote(file["name"]) @@ -382,9 +382,9 @@ if st.session_state["authentication_status"]: with mui.Tooltip(title="Download the file"): with html.span: mui.IconButton(mui.icon.Download, href=base_file_url) - with mui.Tooltip(title="View the file in Kibana"): + with mui.Tooltip(title="View file details in Nemesis"): with html.span: - mui.IconButton(mui.icon.Search, href=kibana_link, target="_blank") + mui.IconButton(mui.icon.Search, href=dashboard_link, target="_blank") with mui.Tooltip(title="View the file in browser"): with html.span: mui.IconButton(mui.icon.TextSnippet, href=view_download_url, target="_blank") diff --git a/cmd/dashboard/dashboard/pages/3_File_Viewer.py b/cmd/dashboard/dashboard/pages/3_File_Viewer.py new file mode 100644 index 0000000..250ef51 --- /dev/null +++ b/cmd/dashboard/dashboard/pages/3_File_Viewer.py @@ -0,0 +1,343 @@ +# Standard Libraries +import base64 +import os +import pathlib +import re +import urllib.parse + +# 3rd Party Libraries +import extra_streamlit_components as stx +import requests +import streamlit as st +import templates +import utils +from annotated_text import annotated_text, annotation +from streamlit_cookies_manager import CookieManager +from streamlit_elements import (dashboard, editor, elements, html, lazy, mui, + sync) + +POSTGRES_CONNECTION_URI = os.environ.get("POSTGRES_CONNECTION_URI") or "" +DB_ITERATION_SIZE = os.environ.get("DB_ITERATION_SIZE") or "1000" +NEMESIS_HTTP_SERVER = os.environ.get("NEMESIS_HTTP_SERVER") +PAGE_SIZE = 8 + + +global sources, projects +sources = [] +projects = [] + +current_user = utils.header() + +# should be defined in ./packages/python/nemesiscommon/nemesiscommon/contents.py - E_TAG_* +filter_tags = [ + "contains_dpapi", + "noseyparker_results", + "parsed_creds", + "encrypted", + "deserialization", + "cmd_execution", + "remoting", + "yara_matches", + "file_canary", +] + +if st.session_state["authentication_status"]: + cookies = CookieManager() + if not cookies.ready(): + st.stop() + + object_id = "" + if "object_id" not in st.session_state: + st.session_state.object_id = None + + triage_pattern = re.compile(r"^triage_(?P[0-9a-f]{8}_[0-9a-f]{4}_[0-9a-f]{4}_[0-9a-f]{4}_[0-9a-f]{12})_(?P.*)") + notes_pattern = re.compile(r"^file_notes_(?P[0-9a-f]{8}_[0-9a-f]{4}_[0-9a-f]{4}_[0-9a-f]{4}_[0-9a-f]{12})$") + + for state in st.session_state: + triage_matches = triage_pattern.search(state) + if triage_matches: + db_id = triage_matches.group("db_id").replace("_", "-") + triage_value = triage_matches.group("triage_value") + utils.update_triage_table(db_id, "file_data_enriched", current_user, triage_value) + del st.session_state[state] + else: + notes_matches = notes_pattern.search(state) + if notes_matches: + db_id = notes_matches.group("db_id").replace("_", "-") + utils.update_notes_table(db_id, "file_data_enriched", current_user, st.session_state[state].target.value) + del st.session_state[state] + + set_search_params = {} + para = st.experimental_get_query_params() + + for key in para.keys(): + match key: + case "object_id": + st.session_state.object_id = para["object_id"][0] + object_id = st.session_state.object_id + + if not st.session_state.object_id: + object_id = st.text_input("Enter file object_id") + + if object_id != "": + + if object_id and not re.match(r"^[0-9a-fA-F]{8}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{12}$", object_id): + st.error(f"object_id '{object_id}' is not a valid UUID", icon="🚨") + + elif object_id: + file = utils.get_file_information(object_id) + + if not((file is None) or len(file) == 0): + object_id = file["object_id"] + extension = pathlib.Path(file["name"]).suffix.strip(".").lower() + + # replace - with _ since streamlit doesn't support -'s in session state + unique_db_id = file["unique_db_id"].replace("-", "_") + + kibana_link = f"{NEMESIS_HTTP_SERVER}/kibana/app/discover#/?_a=(filters:!((query:(match_phrase:(objectId:'{object_id}')))),index:'26360ae8-a518-4dac-b499-ef682d3f6bac')&_g=(time:(from:now-1y%2Fd,to:now))" + + url_enc_file_name = urllib.parse.quote(file["name"]) + download_url = f"http://enrichment-webapi:9910/download/{object_id}" + pdf_download_url = "" + extracted_source_download_url = "" + is_ascii = False + public_download_url = f"{NEMESIS_HTTP_SERVER}/api/download/{object_id}?name={url_enc_file_name}&action=download" + + if "magic_type" in file and file["magic_type"] and ("ASCII text" in file["magic_type"] or "Unicode text" in file["magic_type"]): + is_ascii = True + if "converted_pdf_id" in file and file["converted_pdf_id"] != "00000000-0000-0000-0000-000000000000": + pdf_download_url = f"http://enrichment-webapi:9910/download/{file['converted_pdf_id']}" + if "extracted_source_id" in file and file["extracted_source_id"] != "00000000-0000-0000-0000-000000000000": + extracted_source_download_url = f"http://enrichment-webapi:9910/download/{file['extracted_source_id']}" + if file["name"].endswith(".pdf"): + pdf_download_url = download_url + + tabs = [ + stx.TabBarItemData(id="basic_file_info", title="Basic File Info", description="Basic File Information"), + stx.TabBarItemData(id="elasticsearch_info", title="Elasticsearch Info", description="Elasticsearch Information Dump") + ] + + es_results = utils.elastic_file_search(object_id) + has_np_results = False + if es_results and es_results["hits"]["total"]["value"] == 1: + if "noseyparker" in es_results["hits"]["hits"][0]["_source"]: + tabs.append(stx.TabBarItemData(id="noseyparker_results", title="Noseyparker Results", description="Noseyparker Results")) + + if pdf_download_url: + tabs.append(stx.TabBarItemData(id="pdf_viewer", title="PDF Viewer", description="PDF View of the File")) + if is_ascii: + tabs.append(stx.TabBarItemData(id="file_view", title="File View", description="View of File Data")) + + chosen_tab = stx.tab_bar( + data=tabs, + default="basic_file_info", + ) + + if chosen_tab == "basic_file_info": + layout = [ + # Grid layout parameters: element_identifier, x_pos, y_pos, width, height, [item properties...] + dashboard.Item("1", 0, 0, 10, 2.5, isDraggable=False, isResizable=False, sx={"height": "100%"}), + ] + with elements("dashboard"): + with dashboard.Grid(layout=layout): + with mui.Card( + key="1", + sx={ + "display": "flex", + "flexDirection": "column", + "borderRadius": 2, + "overflow": "auto", + "overflowY": "auto", + "m": "10", + "gap": "10px", + }, + padding=1, + elevation=1, + spacing=10, + ): + with mui.AppBar(position="sticky", variant="h7", sx={"minHeight": 32}): + with mui.Toolbar(variant="dense", sx={"minHeight": 48, "height": 48}): + mui.Typography(file["name"]) + with mui.Tooltip(title="Download the file"): + with html.span: + mui.IconButton(mui.icon.Download, href=public_download_url) + with mui.Tooltip(title="View the file in Kibana"): + with html.span: + mui.IconButton(mui.icon.Search, href=kibana_link, target="_blank") + if extracted_source_download_url: + with mui.Tooltip(title="Download the extracted source code"): + with html.span: + mui.IconButton(mui.icon.Code, href=extracted_source_download_url, target="_blank") + + mui.Box(sx={"flexGrow": 1}) + + with mui.Tooltip(title="Mark file as useful"): + with html.span: + mui.IconButton(mui.icon.ThumbUpOffAlt, onClick=sync(f"triage_{unique_db_id}_useful")) + with mui.Tooltip(title="Mark file as not useful"): + with html.span: + mui.IconButton(mui.icon.ThumbDownOffAlt, onClick=sync(f"triage_{unique_db_id}_notuseful")) + with mui.Tooltip(title="Mark file as needing additional investigation"): + with html.span: + mui.IconButton(mui.icon.QuestionMark, onClick=sync(f"triage_{unique_db_id}_unknown")) + if file["triage"]: + with html.span: + mui.Typography("triage") + + # Information table + with mui.CardContent(sx={"flex": 1}): + with mui.TableContainer(sx={"maxHeight": 200}): + with mui.Table(size="small", overflowX="hidden", whiteSpace="nowrap"): + with mui.TableBody(): + identifier_style = { + "fontWeight": "bold", + "borderRight": "1px solid", + "whiteSpace": "nowrap", + "padding": "0px 5px 0px 0px", + } + + with mui.TableRow(hover=True, padding="none"): + mui.TableCell("Path", size="small", sx=identifier_style) + mui.TableCell(file["path"], width="100%") + with mui.TableRow(hover=True, padding="none"): + if file['source']: + mui.TableCell("Source / Timestamp", size="small", sx=identifier_style) + mui.TableCell(f"{file['source']} @ {file['timestamp']}", size="small") + else: + mui.TableCell("Timestamp", size="small", sx=identifier_style) + mui.TableCell(f"{file['timestamp']}", size="small") + with mui.TableRow(hover=True, padding="none"): + mui.TableCell("Size", sx=identifier_style) + mui.TableCell(f"{file['size']}") + with mui.TableRow(hover=True, padding="none"): + mui.TableCell("SHA1 hash", sx=identifier_style) + mui.TableCell(file["sha1"]) + with mui.TableRow(hover=True, padding="none"): + mui.TableCell("Magic Type", sx=identifier_style) + mui.TableCell(file["magic_type"]) + if file["tags"]: + with mui.TableRow(hover=True, padding="none"): + mui.TableCell("Tags", sx=identifier_style) + with mui.TableCell(): + # Tags + for tag in file["tags"]: + mui.Chip(label=tag, color="primary") + # Notes + mui.Typography("Comments:") + with mui.Box(sx={"flexGrow": 1}): + end = mui.IconButton(mui.icon.Save, onClick=sync()) + + mui.TextField( + # label="Input Any Notes Here", + key=f"file_notes_{unique_db_id}", + defaultValue=file["notes"], + variant="outlined", + margin="none", + multiline=True, + onChange=lazy(sync(f"file_notes_{unique_db_id}")), + fullWidth=True, + sx={"flexGrow": 1}, + InputProps={"endAdornment": end}, + ) + + elif chosen_tab == "noseyparker_results": + if es_results != {}: + total_hits = es_results["hits"]["total"]["value"] + num_results = len(es_results["hits"]["hits"]) + if total_hits > 0: + for i in range(num_results): + object_id = es_results["hits"]["hits"][i]["_source"]["objectId"] + file_name = es_results["hits"]["hits"][i]["_source"]["name"] + download_url = f"{NEMESIS_HTTP_SERVER}/api/download/{object_id}?name={file_name}" + kibana_link = f"{NEMESIS_HTTP_SERVER}/kibana/app/discover#/?_a=(filters:!((query:(match_phrase:(objectId:'{object_id}')))),index:'26360ae8-a518-4dac-b499-ef682d3f6bac')&_g=(time:(from:now-1y%2Fd,to:now))" + path = es_results["hits"]["hits"][i]["_source"]["path"] + sha1 = es_results["hits"]["hits"][i]["_source"]["hashes"]["sha1"] + source = "" + if "metadata" in es_results["hits"]["hits"][i]["_source"] and "source" in es_results["hits"]["hits"][i]["_source"]["metadata"]: + source = es_results["hits"]["hits"][i]["_source"]["metadata"]["source"] + + if source: + expander_text = f"{source} : **{path}** (SHA1: {sha1})" + else: + expander_text = f"**{path}** (SHA1: {sha1})" + + for ruleMatch in es_results["hits"]["hits"][i]["_source"]["noseyparker"]["ruleMatches"]: + for match in ruleMatch["matches"]: + if "matching" in match["snippet"]: + rule_name = match["ruleName"] + + if "before" in match["snippet"]: + before = match["snippet"]["before"].replace("\n\t", " ") + else: + before = "" + + matching = match["snippet"]["matching"] + + if "after" in match["snippet"]: + after = match["snippet"]["after"].replace("\n\t", " ") + else: + after = "" + + st.write(f"Rule: {rule_name}", unsafe_allow_html=True) + annotated_text(annotation(before, "context", color="#8ef"), annotation(matching, "match"), annotation(after, "context", color="#8ef")) + st.divider() + + elif chosen_tab == "file_view": + with elements("file_view"): + response = requests.get(download_url) + if response.status_code != 200: + st.error(f"Error retrieving text data from {download_url}, status code: {response.status_code}", icon="🚨") + else: + with mui.Card( + sx={ + "display": "flex", + "overflow": "auto", + "overflowY": "auto", + }, + padding=1, + elevation=1, + spacing=10, + ): + editor.Monaco( + height=600, + defaultValue=response.content.decode('utf-8'), + language=utils.map_extension_to_monaco_language(extension) + ) + + elif chosen_tab == "pdf_viewer": + with elements("pdf_viewer"): + try: + response = requests.get(pdf_download_url) + if response.status_code != 200: + st.error(f"Error retrieving PDF data from {pdf_download_url}, status code: {response.status_code}", icon="🚨") + else: + with mui.Card( + sx={ + "display": "flex", + "overflow": "auto", + "overflowY": "auto", + }, + padding=1, + elevation=1, + spacing=10, + ): + b64_data = base64.b64encode(response.content).decode('utf-8') + html.iframe( + src=f"data:application/pdf;base64,{b64_data}", + height="700", + width="100%", + type="application/pdf" + ) + except Exception as e: + st.error(f"Error retrieving PDF data from {pdf_download_url} : {e}", icon="🚨") + + elif chosen_tab == "elasticsearch_info": + if es_results != {}: + total_hits = es_results["hits"]["total"]["value"] + if total_hits == 0: + st.warning("No results found in Elasticsearch!") + elif total_hits == 1: + st.subheader("Elasticsearch Data") + st.json(es_results["hits"]["hits"][0]) + else: + st.warning("Too many results found in Elasticsearch!") diff --git a/cmd/dashboard/dashboard/pages/1_Document Search.py b/cmd/dashboard/dashboard/pages/5_Document Search.py similarity index 100% rename from cmd/dashboard/dashboard/pages/1_Document Search.py rename to cmd/dashboard/dashboard/pages/5_Document Search.py diff --git a/cmd/dashboard/dashboard/pages/3_Chromium.py b/cmd/dashboard/dashboard/pages/7_Chromium.py similarity index 100% rename from cmd/dashboard/dashboard/pages/3_Chromium.py rename to cmd/dashboard/dashboard/pages/7_Chromium.py diff --git a/cmd/dashboard/dashboard/pages/4_DPAPI.py b/cmd/dashboard/dashboard/pages/9_DPAPI.py similarity index 100% rename from cmd/dashboard/dashboard/pages/4_DPAPI.py rename to cmd/dashboard/dashboard/pages/9_DPAPI.py diff --git a/cmd/dashboard/dashboard/utils.py b/cmd/dashboard/dashboard/utils.py index 5d63501..7a849df 100644 --- a/cmd/dashboard/dashboard/utils.py +++ b/cmd/dashboard/dashboard/utils.py @@ -474,6 +474,66 @@ def postgres_count_masterkeys(show_all=True, show_dec=True, key_type=""): return -1 +def get_file_information(object_id: str): + """Gets information from Postgres about a specific file.""" + + if not re.match(r"^[0-9a-fA-F]{8}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{12}$", object_id): + raise Exception(f"object_id '{object_id}' supplied to get_file_information() is not a proper UUID") + + try: + with engine.connect() as conn: + params = { + "object_id": object_id, + } + + query = """ + SELECT file_data_enriched.project_id as project_id, + file_data_enriched.source as source, + file_data_enriched.timestamp as timestamp, + file_data_enriched.unique_db_id::varchar, + file_data_enriched.agent_id as agent_id, + file_data_enriched.object_id::varchar as object_id, + file_data_enriched.path as path, + file_data_enriched.name as name, + file_data_enriched.size as size, + file_data_enriched.md5 as md5, + file_data_enriched.sha1 as sha1, + file_data_enriched.sha256 as sha256, + file_data_enriched.nemesis_file_type as nemesis_file_type, + file_data_enriched.magic_type as magic_type, + file_data_enriched.converted_pdf_id::varchar as converted_pdf_id, + file_data_enriched.extracted_plaintext_id::varchar as extracted_plaintext_id, + file_data_enriched.extracted_source_id::varchar as extracted_source_id, + file_data_enriched.tags as tags, + file_data_enriched.originating_object_id as originating_object_id, + triage.value as triage, + triage.unique_db_id as triage_unique_db_id, + notes.value as notes + + FROM file_data_enriched + + LEFT JOIN triage + ON file_data_enriched.unique_db_id = triage.unique_db_id + + LEFT JOIN notes + ON file_data_enriched.unique_db_id = notes.unique_db_id + + WHERE object_id = :object_id + """ + + df = pd.read_sql_query(sql_text(query), conn, params=params) + if len(df) == 0: + st.error(f"object_id '{object_id}' not found") + elif len(df) == 1: + return df.iloc[0] + else: + st.error(f"Too many results for object_id '{object_id}' : {len(df)}", icon="🚨") + + except Exception as e: + st.error(f"Error retrieving `file_data_enriched` from the database: {e}", icon="🚨") + return None + + def postgres_file_search( start: datetime.datetime, end: datetime.datetime, @@ -699,7 +759,6 @@ def authenticate_header() -> str: # ###################################################### - def simplify_es_text_result(result: dict) -> dict: """Simplifies an elastic result into the three parts we want to use.""" res = result["_source"] @@ -744,6 +803,23 @@ def get_elastic_total_indexed_documents(index_name="file_data_plaintext", query= return 0 +def elastic_file_search(object_id: str) -> dict: + """ + Searches the 'file_data_enriched' index in Elasticsearch for + the matching document, returning all fields. + """ + try: + es_client = wait_for_elasticsearch() + query = {"term": {"objectId.keyword": object_id}} + return es_client.search(index="file_data_enriched", query=query) + except Exception as e: + if "index_not_found_exception" in f"{e}": + st.error("Elastic index 'file_data_enriched' doesn't yet exist!", icon="🚨") + else: + st.error(f"Exception querying Elastic: {e}", icon="🚨") + return {} + + def elastic_text_search(search_term: str, from_i: int, size: int) -> dict: """ Searches the 'file_data_plaintext' index in Elasticsearch for @@ -831,6 +907,60 @@ def elastic_np_search(from_i: int, size: int) -> dict: # ###################################################### +def map_extension_to_monaco_language(extension: str) -> str: + """ + Maps a file extension to a source code language for Monaco. + + Ref: https://microsoft.github.io/monaco-editor/ + """ + + language_mappings = { + "bat": "batch", + "c": "c", + "cpp": "cpp", + "cs": "csharp", + "css": "css", + "cypher": "cypher", + "dockerfile": "dockerfile", + "fs": "fsharp", + "go": "go", + "gql": "graphql", + "graphql": "graphql", + "html": "html", + "ini": "ini", + "java": "java", + "js": "javascript", + "lua": "lua", + "md": "markdown", + "mysql": "mysql", + "php": "php", + "php3": "php", + "php4": "php", + "php5": "php", + "pl": "perl", + "ps1": "powershell", + "psd1": "powershell", + "psm1": "powershell", + "proto": "proto", + "py": "python", + "r": "r", + "rb": "ruby", + "rs": "rust", + "shell": "shell", + "sh": "shell", + "sql": "sql", + "swift": "swift", + "ts": "typescript", + "ts": "typescript", + "vb": "vb", + "wgsl": "wgsl", + "xml": "xml", + "yaml": "yaml", + "json": "json" + } + return language_mappings.get(extension.lower(), "plaintext") + + def is_valid_chromium_file_path(file_path: str) -> bool: """Returns true if the supplied path is a valid Chromium file path.""" diff --git a/cmd/enrichment/enrichment/containers.py b/cmd/enrichment/enrichment/containers.py index aed032f..eff678c 100644 --- a/cmd/enrichment/enrichment/containers.py +++ b/cmd/enrichment/enrichment/containers.py @@ -421,7 +421,7 @@ class Container(containers.DeclarativeContainer): # # Services # - alerter_service = providers.Factory(NemesisAlerter, outputq_alert, config.public_kibana_url) + alerter_service = providers.Factory(NemesisAlerter, outputq_alert, config.public_nemesis_url) elasticsearch_client = providers.Factory( AsyncElasticsearch, diff --git a/cmd/enrichment/enrichment/lib/helpers.py b/cmd/enrichment/enrichment/lib/helpers.py index 5f59bd4..56790e0 100644 --- a/cmd/enrichment/enrichment/lib/helpers.py +++ b/cmd/enrichment/enrichment/lib/helpers.py @@ -907,7 +907,7 @@ def is_pe_extension(file_path: str) -> bool: def is_source_code(file_path: str) -> bool: """Returns True if the supplied file_path matches a number of supported source code file extensions.""" - source_code_regex = "^.*\\.(aspx|c|cpp|cs|go|groovy|java|jsp|js|lua|php|php3" "|php4|php5|ps1|psd1|psm1|py|rb|rs|sql|sh|swift|vb|vbs)$" + source_code_regex = "^.*\\.(aspx|c|cpp|cs|go|groovy|java|jsp|js|lua|php|php3|php4|php5|ps1|psd1|psm1|py|rb|rs|sql|sh|swift|vb|vbs)$" return re.match(source_code_regex, file_path, re.IGNORECASE) is not None diff --git a/cmd/enrichment/enrichment/settings.py b/cmd/enrichment/enrichment/settings.py index e3b666a..6d5bbb7 100644 --- a/cmd/enrichment/enrichment/settings.py +++ b/cmd/enrichment/enrichment/settings.py @@ -53,6 +53,7 @@ class EnrichmentSettings(FileProcessingService): # type: ignore elasticsearch_password: str elasticsearch_url: HttpUrlWithSlash web_api_url: HttpUrlWithSlash + public_nemesis_url: HttpUrlWithSlash public_kibana_url: HttpUrlWithSlash slack_webhook_url: Optional[str] slack_username: str diff --git a/cmd/passwordcracker/passwordcracker/containers.py b/cmd/passwordcracker/passwordcracker/containers.py index 4d81412..68ba975 100644 --- a/cmd/passwordcracker/passwordcracker/containers.py +++ b/cmd/passwordcracker/passwordcracker/containers.py @@ -9,13 +9,12 @@ import nemesispb.nemesis_pb2 as pb import passwordcracker.settings as settings from dependency_injector import containers, providers from nemesiscommon.constants import NemesisQueue -from nemesiscommon.messaging_rabbitmq import ( - NemesisRabbitMQConsumer, - NemesisRabbitMQProducer, -) +from nemesiscommon.messaging_rabbitmq import (NemesisRabbitMQConsumer, + NemesisRabbitMQProducer) from nemesiscommon.services.alerter import NemesisAlerter from nemesiscommon.tasking import TaskDispatcher -from passwordcracker.services.john_the_ripper_cracker import JohnTheRipperCracker +from passwordcracker.services.john_the_ripper_cracker import \ + JohnTheRipperCracker from passwordcracker.settings import PasswordCrackerSettings from passwordcracker.tasks.password_cracker import PasswordCracker @@ -76,7 +75,7 @@ class Container(containers.DeclarativeContainer): alerting_service = providers.Factory( NemesisAlerter, outputq_alert, - config.public_kibana_url, + config.public_nemesis_url, ) cracker_service = providers.Factory( JohnTheRipperCracker, diff --git a/cmd/passwordcracker/passwordcracker/settings.py b/cmd/passwordcracker/passwordcracker/settings.py index f6f607f..67845cf 100644 --- a/cmd/passwordcracker/passwordcracker/settings.py +++ b/cmd/passwordcracker/passwordcracker/settings.py @@ -13,7 +13,7 @@ class CrackWordlistSize(IntEnum): class PasswordCrackerSettings(NemesisServiceSettings): # type: ignore rabbitmq_connection_uri: AnyUrl - public_kibana_url: HttpUrlWithSlash + public_nemesis_url: HttpUrlWithSlash data_download_dir: str crack_wordlist_top_words: CrackWordlistSize # either 10000 or 100000 for now diff --git a/kubernetes/enrichment/deployment.yaml b/kubernetes/enrichment/deployment.yaml index 8a9a74f..16fbd62 100644 --- a/kubernetes/enrichment/deployment.yaml +++ b/kubernetes/enrichment/deployment.yaml @@ -80,6 +80,8 @@ spec: configMapKeyRef: name: operation-config key: nemesis-http-server + - name: PUBLIC_NEMESIS_URL + value: "$(NEMESIS_HTTP_SERVER)/dashboard/" - name: PUBLIC_KIBANA_URL value: "$(NEMESIS_HTTP_SERVER)/kibana/" - name: WEB_API_URL diff --git a/kubernetes/passwordcracker/deployment.yaml b/kubernetes/passwordcracker/deployment.yaml index 97cbcd1..0f07e15 100644 --- a/kubernetes/passwordcracker/deployment.yaml +++ b/kubernetes/passwordcracker/deployment.yaml @@ -46,8 +46,8 @@ spec: value: "True" - name: PROMETHEUS_PORT value: "9090" - - name: PUBLIC_KIBANA_URL - value: "$(NEMESIS_HTTP_SERVER)/kibana/" + - name: PUBLIC_NEMESIS_URL + value: "$(NEMESIS_HTTP_SERVER)/dashboard/" - name: RABBITMQ_CONNECTION_URI valueFrom: secretKeyRef: diff --git a/packages/python/nemesiscommon/nemesiscommon/services/alerter.py b/packages/python/nemesiscommon/nemesiscommon/services/alerter.py index 9798d11..0696374 100644 --- a/packages/python/nemesiscommon/nemesiscommon/services/alerter.py +++ b/packages/python/nemesiscommon/nemesiscommon/services/alerter.py @@ -28,11 +28,11 @@ class AlerterInterface: class NemesisAlerter(AlerterInterface): alert_queue: NemesisRabbitMQProducer - kibana_url: str + nemesis_url: str - def __init__(self, alert_queue: NemesisRabbitMQProducer, kibana_url: str): + def __init__(self, alert_queue: NemesisRabbitMQProducer, nemesis_url: str): self.alert_queue = alert_queue - self.kibana_url = kibana_url + self.nemesis_url = nemesis_url async def alert(self, text: str) -> None: alert_msg = pb.Alert() @@ -48,15 +48,15 @@ class NemesisAlerter(AlerterInterface): header = f"*{title}*\n" if title else "" text = f"\n{text}" if text else "" - full_kibana_url = f"{self.kibana_url}app/discover#/?_a=(filters:!((query:(match_phrase:(objectId:'{file_data.object_id}')))),index:'26360ae8-a518-4dac-b499-ef682d3f6bac')&_g=(time:(from:now-1y%2Fd,to:now))" - kibana_footer = f"\n<{full_kibana_url}|*File in Kibana*>" + full_nemesis_url = f"{self.nemesis_url}File_Viewer?object_id={file_data.object_id}" + nemesis_footer = f"\n<{full_nemesis_url}|*File in Nemesis*>" try: metadata_dict = MessageToDict(metadata, preserving_proto_field_name=True) timestamp = metadata_dict["timestamp"] agent_type = metadata_dict["agent_type"] agent_id = metadata_dict["agent_id"] - message = f"{header}*File:* {file_name}\n*SHA1:* {sha1_hash}\n*Downloaded:* {timestamp}\n*Agent:* {agent_id} (type: {agent_type}){text}{kibana_footer}" + message = f"{header}*File:* {file_name}\n*SHA1:* {sha1_hash}\n*Downloaded:* {timestamp}\n*Agent:* {agent_id} (type: {agent_type}){text}{nemesis_footer}" await self.alert(text=message) From f1f0ea985f3c0fd64602f45056b34c423bcd82d0 Mon Sep 17 00:00:00 2001 From: HarmJ0y Date: Wed, 6 Sep 2023 11:08:01 -0700 Subject: [PATCH 2/3] Small wording tweak -Small wording tweak --- packages/python/nemesiscommon/nemesiscommon/services/alerter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/python/nemesiscommon/nemesiscommon/services/alerter.py b/packages/python/nemesiscommon/nemesiscommon/services/alerter.py index 0696374..3bf049c 100644 --- a/packages/python/nemesiscommon/nemesiscommon/services/alerter.py +++ b/packages/python/nemesiscommon/nemesiscommon/services/alerter.py @@ -49,7 +49,7 @@ class NemesisAlerter(AlerterInterface): text = f"\n{text}" if text else "" full_nemesis_url = f"{self.nemesis_url}File_Viewer?object_id={file_data.object_id}" - nemesis_footer = f"\n<{full_nemesis_url}|*File in Nemesis*>" + nemesis_footer = f"\n<{full_nemesis_url}|*View File in Nemesis*>" try: metadata_dict = MessageToDict(metadata, preserving_proto_field_name=True) From 264bc4acd9cb4dcf79db86f5daf63d72bb57cc5e Mon Sep 17 00:00:00 2001 From: HarmJ0y Date: Wed, 6 Sep 2023 13:34:21 -0700 Subject: [PATCH 3/3] Moved PDF/Monaco display to the Basic File View tab -Moved PDF/Monaco display to the Basic File View tab --- .../dashboard/pages/3_File_Viewer.py | 134 +++++++++++------- 1 file changed, 80 insertions(+), 54 deletions(-) diff --git a/cmd/dashboard/dashboard/pages/3_File_Viewer.py b/cmd/dashboard/dashboard/pages/3_File_Viewer.py index 250ef51..870bd00 100644 --- a/cmd/dashboard/dashboard/pages/3_File_Viewer.py +++ b/cmd/dashboard/dashboard/pages/3_File_Viewer.py @@ -114,7 +114,6 @@ if st.session_state["authentication_status"]: tabs = [ stx.TabBarItemData(id="basic_file_info", title="Basic File Info", description="Basic File Information"), - stx.TabBarItemData(id="elasticsearch_info", title="Elasticsearch Info", description="Elasticsearch Information Dump") ] es_results = utils.elastic_file_search(object_id) @@ -123,10 +122,7 @@ if st.session_state["authentication_status"]: if "noseyparker" in es_results["hits"]["hits"][0]["_source"]: tabs.append(stx.TabBarItemData(id="noseyparker_results", title="Noseyparker Results", description="Noseyparker Results")) - if pdf_download_url: - tabs.append(stx.TabBarItemData(id="pdf_viewer", title="PDF Viewer", description="PDF View of the File")) - if is_ascii: - tabs.append(stx.TabBarItemData(id="file_view", title="File View", description="View of File Data")) + tabs.append(stx.TabBarItemData(id="elasticsearch_info", title="Elasticsearch Info", description="Elasticsearch Information Dump")) chosen_tab = stx.tab_bar( data=tabs, @@ -137,6 +133,7 @@ if st.session_state["authentication_status"]: layout = [ # Grid layout parameters: element_identifier, x_pos, y_pos, width, height, [item properties...] dashboard.Item("1", 0, 0, 10, 2.5, isDraggable=False, isResizable=False, sx={"height": "100%"}), + dashboard.Item("2", 0, 0, 10, 5, isDraggable=False, isResizable=False, sx={"height": "100%"}), ] with elements("dashboard"): with dashboard.Grid(layout=layout): @@ -240,6 +237,84 @@ if st.session_state["authentication_status"]: InputProps={"endAdornment": end}, ) + if is_ascii: + # Monaco editor display for ascii files + with mui.Card( + key="2", + sx={ + "display": "flex", + "flexDirection": "column", + "borderRadius": 2, + "overflow": "auto", + "overflowY": "auto", + "m": "10", + "gap": "10px", + }, + padding=1, + elevation=1, + spacing=10, + ): + response = requests.get(download_url) + if response.status_code != 200: + st.error(f"Error retrieving text data from {download_url}, status code: {response.status_code}", icon="🚨") + else: + with mui.Card( + sx={ + "display": "flex", + "overflow": "auto", + "overflowY": "auto", + }, + padding=1, + elevation=1, + spacing=10, + ): + editor.Monaco( + height="64vh", + defaultValue=response.content.decode('utf-8'), + language=utils.map_extension_to_monaco_language(extension) + ) + elif pdf_download_url: + # Inline PDF file file display, if PDF is present + with mui.Card( + key="2", + sx={ + "display": "flex", + "flexDirection": "column", + "borderRadius": 2, + "overflow": "auto", + "overflowY": "auto", + "m": "10", + "gap": "10px", + }, + padding=1, + elevation=1, + spacing=10, + ): + try: + response = requests.get(pdf_download_url) + if response.status_code != 200: + st.error(f"Error retrieving PDF data from {pdf_download_url}, status code: {response.status_code}", icon="🚨") + else: + with mui.Card( + sx={ + "display": "flex", + "overflow": "auto", + "overflowY": "auto", + }, + padding=1, + elevation=1, + spacing=10, + ): + b64_data = base64.b64encode(response.content).decode('utf-8') + html.iframe( + src=f"data:application/pdf;base64,{b64_data}", + height=785, + width="100%", + type="application/pdf" + ) + except Exception as e: + st.error(f"Error retrieving PDF data from {pdf_download_url} : {e}", icon="🚨") + elif chosen_tab == "noseyparker_results": if es_results != {}: total_hits = es_results["hits"]["total"]["value"] @@ -282,55 +357,6 @@ if st.session_state["authentication_status"]: annotated_text(annotation(before, "context", color="#8ef"), annotation(matching, "match"), annotation(after, "context", color="#8ef")) st.divider() - elif chosen_tab == "file_view": - with elements("file_view"): - response = requests.get(download_url) - if response.status_code != 200: - st.error(f"Error retrieving text data from {download_url}, status code: {response.status_code}", icon="🚨") - else: - with mui.Card( - sx={ - "display": "flex", - "overflow": "auto", - "overflowY": "auto", - }, - padding=1, - elevation=1, - spacing=10, - ): - editor.Monaco( - height=600, - defaultValue=response.content.decode('utf-8'), - language=utils.map_extension_to_monaco_language(extension) - ) - - elif chosen_tab == "pdf_viewer": - with elements("pdf_viewer"): - try: - response = requests.get(pdf_download_url) - if response.status_code != 200: - st.error(f"Error retrieving PDF data from {pdf_download_url}, status code: {response.status_code}", icon="🚨") - else: - with mui.Card( - sx={ - "display": "flex", - "overflow": "auto", - "overflowY": "auto", - }, - padding=1, - elevation=1, - spacing=10, - ): - b64_data = base64.b64encode(response.content).decode('utf-8') - html.iframe( - src=f"data:application/pdf;base64,{b64_data}", - height="700", - width="100%", - type="application/pdf" - ) - except Exception as e: - st.error(f"Error retrieving PDF data from {pdf_download_url} : {e}", icon="🚨") - elif chosen_tab == "elasticsearch_info": if es_results != {}: total_hits = es_results["hits"]["total"]["value"]