From e00a549074245061112ae990f28861842aa3369a Mon Sep 17 00:00:00 2001 From: Lee Christensen Date: Wed, 27 Sep 2023 17:51:01 -0400 Subject: [PATCH] move File Upload to own page to fix double submit problem, set default search end day to today+1 --- cmd/dashboard/dashboard/pages/15_Hashes.py | 2 + cmd/dashboard/dashboard/pages/1_Files.py | 139 +----------------- .../dashboard/pages/3_File_Viewer.py | 51 ------- .../dashboard/pages/4_File_Upload.py | 133 +++++++++++++++++ 4 files changed, 140 insertions(+), 185 deletions(-) create mode 100644 cmd/dashboard/dashboard/pages/4_File_Upload.py diff --git a/cmd/dashboard/dashboard/pages/15_Hashes.py b/cmd/dashboard/dashboard/pages/15_Hashes.py index 112336e..8898c80 100644 --- a/cmd/dashboard/dashboard/pages/15_Hashes.py +++ b/cmd/dashboard/dashboard/pages/15_Hashes.py @@ -59,6 +59,8 @@ FROM extracted_hashes if object_id: query += " WHERE originating_object_id = :object_id" params["object_id"] = object_id + + st.write("Filters are currently applied.") if st.button("Clear Filters"): st.experimental_set_query_params() diff --git a/cmd/dashboard/dashboard/pages/1_Files.py b/cmd/dashboard/dashboard/pages/1_Files.py index 7d03251..22d9cc1 100644 --- a/cmd/dashboard/dashboard/pages/1_Files.py +++ b/cmd/dashboard/dashboard/pages/1_Files.py @@ -1,6 +1,5 @@ # Standard Libraries import datetime -import ntpath import os import re import urllib.parse @@ -12,7 +11,6 @@ import humanize import streamlit as st import templates import utils -from streamlit_cookies_manager import CookieManager from streamlit_elements import dashboard, elements, html, lazy, mui, sync from streamlit_searchbox import st_searchbox from streamlit_toggle import st_toggle_switch @@ -170,7 +168,7 @@ def build_and_get_search_filters(): with col2: subcol1, subcol2 = st.columns(2) with subcol1: - end_date = st.date_input("Filter end date (UTC)", datetime.datetime.now()) + end_date = st.date_input("Filter end date (UTC)", datetime.datetime.now() + datetime.timedelta(days=1)) with subcol2: end_time = st.time_input("Filter end time (UTC)", datetime.datetime.now()) @@ -213,7 +211,7 @@ def build_and_get_search_filters(): return search_filters -def build_file_triage_tab(): +def build_file_listing(): if not st.session_state.file_download_page: st.session_state.file_download_page = 1 @@ -434,121 +432,6 @@ def build_file_triage_tab(): st.write(pagination_html, unsafe_allow_html=True) -def build_upload_file_tab(): - cookies = CookieManager() - if not cookies.ready(): - st.stop() - - cols = st.columns(4) - with cols[0]: - default_project = cookies["nemesus_project"] if "nemesus_project" in cookies else "" - nemesus_project = st.text_input("The name of the current project", value=default_project, placeholder="Please enter a project name") - if nemesus_project: - cookies["nemesus_project"] = nemesus_project.upper() - with cols[1]: - default_operator = cookies["nemesis_operator"] if "nemesis_operator" in cookies else "" - nemesis_operator = st.text_input("The name of the current operator", value=default_operator, placeholder="Please enter an operator name") - if nemesis_operator: - cookies["nemesis_operator"] = nemesis_operator.upper() - with cols[2]: - default_source = cookies["nemesis_source"] if "nemesis_source" in cookies else "" - nemesis_source = st.text_input("The name of the source machine", value=default_source, placeholder="Please enter a source") - if nemesis_source: - cookies["nemesis_source"] = nemesis_source.upper() - with cols[3]: - default_expiration_days = cookies["nemesis_expiration_days"] if "nemesis_expiration_days" in cookies else "100" - nemesis_expiration_days = st.text_input("Days until the data should expire", value=default_expiration_days) - if nemesis_expiration_days: - if nemesis_expiration_days.isdigit(): - cookies["nemesis_expiration_days"] = nemesis_expiration_days - - else: - st.warning("Expiration days must be an integer!") - - original_file_path = st.text_input("Original file path (optional, used for masterkeys, Chromium data, etc.)") - if original_file_path: - original_file_path = original_file_path.replace("\\", "/") - - cookies.save() - - if ( - "nemesus_project" in cookies - and cookies["nemesus_project"] != "" - and "nemesis_operator" in cookies - and cookies["nemesis_operator"] != "" - and "nemesis_source" in cookies - and cookies["nemesis_source"] != "" - and "nemesis_expiration_days" in cookies - and cookies["nemesis_expiration_days"] != "" - ): - uploaded_files = st.file_uploader("Choose a file for Nemesis to process", accept_multiple_files=True) - - for uploaded_file in uploaded_files: - proceed = True - file_name = ntpath.basename(uploaded_file.name) - - if original_file_path: - if original_file_path.lower().endswith(file_name.lower()): - file_path = original_file_path - elif original_file_path.endswith("/"): - file_path = f"{original_file_path}{file_name}" - else: - file_path = f"{original_file_path}/{file_name}" - else: - file_path = uploaded_file.name - - # if the file is a Chromium based file, a proper path is needed - if re.match("^(Local State|History|Login Data|Cookies)$", file_name, re.IGNORECASE): - if not utils.is_valid_chromium_file_path(file_path): - proceed = False - st.warning("Chromium data (Local State, History, Login Data, Cookies) needs a valid originating path!") - - if proceed: - bytes_data = uploaded_file.read() - - nemesis_file_id = utils.nemesis_post_file(bytes_data) - - if nemesis_file_id: - submission_time = datetime.datetime.utcnow() - expiration = submission_time + datetime.timedelta(days=int(cookies["nemesis_expiration_days"])) - submission_time_s = submission_time.strftime("%Y-%m-%dT%H:%M:%S.000Z") - expiration_s = expiration.strftime("%Y-%m-%dT%H:%M:%S.000Z") - metadata = { - "agent_id": cookies["nemesis_operator"], - "agent_type": "dashboard", - "automated": False, - "data_type": "file_data", - "expiration": expiration_s, - "source": cookies["nemesis_source"], - "project": cookies["nemesus_project"], - "timestamp": submission_time_s, - } - - file_data = {"path": f"{file_path}", "size": len(bytes_data), "object_id": nemesis_file_id} - - submission_id = utils.nemesis_post_data({"metadata": metadata, "data": [file_data]}) - - if submission_id: - st.success("Successful Nemesis submission", icon="✅") - nemesis_upload = f""" - | Property | Value | - | ------------------- | ----------- | - | __Submission ID__ | {submission_id} | - | __Nemesis File ID__ | [{nemesis_file_id}]({NEMESIS_HTTP_SERVER}/dashboard/File_Viewer?object_id={nemesis_file_id}) | - | __Project__ | {cookies["nemesus_project"]} | - | __Source__ | {cookies["nemesis_source"]} | - | __Agent ID__ | {cookies["nemesis_operator"]} | - | __Agent Type__ | dashboard | - | __Automated__ | False | - | __Data Type__ | file_data | - | __Timestamp__ | {submission_time_s} | - | __Expiration__ | {expiration_s} | - | __Filename__ | {uploaded_file.name} | - | __Size__ | {len(bytes_data)} | - """ - st.markdown(nemesis_upload, unsafe_allow_html=True) - - def init_session_state(): if "file_download_page" not in st.session_state: st.session_state.file_download_page = None @@ -576,9 +459,9 @@ def build_about_expander(): with st.expander("About Files"): st.markdown( """ - This page shows downloaded files processed by Nemesis, searchable by a variety of filters. + This page shows files processed by Nemesis. Files are searchable using variety of filters. - The `Upload File` tab allows you to upload one or more files to Nemesis for processing. + Operators can triage files as "useful", "not useful", or "needs more investigation" by clicking the thumbs up, thumbs down, and question mark buttons in the top right of each file. """ ) @@ -604,19 +487,7 @@ def build_page(username: str): utils.local_css("./css/files_style.css") init_session_state() build_about_expander() - - chosen_tab = stx.tab_bar( - data=[ - stx.TabBarItemData(id=1, title="Downloaded Files", description="Downloaded Files Processed by Nemesis"), - stx.TabBarItemData(id=2, title="Upload File", description="Manually Upload a File to Nemesis for Processing"), - ], - default="1", - ) - - if chosen_tab == str(1): # downloaded_files - build_file_triage_tab() - elif chosen_tab == str(2): # upload_file - build_upload_file_tab() + build_file_listing() utils.render_nemesis_page(build_page) diff --git a/cmd/dashboard/dashboard/pages/3_File_Viewer.py b/cmd/dashboard/dashboard/pages/3_File_Viewer.py index deb1f44..61405a0 100644 --- a/cmd/dashboard/dashboard/pages/3_File_Viewer.py +++ b/cmd/dashboard/dashboard/pages/3_File_Viewer.py @@ -192,21 +192,10 @@ def build_page(username: str): extension = pathlib.Path(file["name"]).suffix.strip(".").lower() download_url_internal = f"http://enrichment-webapi:9910/download/{object_id}" - download_url_public = f"{NEMESIS_HTTP_SERVER}/api/download/{object_id}" - is_text = False - - if "magic_type" in file and file["magic_type"] and ("ASCII text" in file["magic_type"] or "UTF-8 Unicode" in file["magic_type"]): - is_text = True - if "converted_pdf_id" in file and file["converted_pdf_id"] != "00000000-0000-0000-0000-000000000000": - pdf_download_url_public = f"{NEMESIS_HTTP_SERVER}/api/download/{file['converted_pdf_id']}?action=view&name=test.pdf" - - if file["name"].endswith(".pdf"): - pdf_download_url_public = f"{download_url_public}?action=view&name=test.pdf" tabs = [stx.TabBarItemData(id=1, title="Basic File Info", description="Basic File Information"), stx.TabBarItemData(id=3, title="Elasticsearch Info", description="Elasticsearch Information Dump")] es_results = utils.elastic_file_search(object_id) - has_np_results = False if es_results and es_results["hits"]["total"]["value"] == 1: if "noseyparker" in es_results["hits"]["hits"][0]["_source"]: tabs.append(stx.TabBarItemData(id=2, title="Noseyparker Results", description="Noseyparker Results")) @@ -273,35 +262,6 @@ def build_page(username: str): except Exception as e: st.error(f"Error displaying file in Monaco editor: {e}", icon="🚨") - # elif pdf_download_url_public: - # # Inline PDF file file display, if PDF is present - # with mui.Card( - # key="2", - # sx={ - # "display": "flex", - # "flexDirection": "column", - # "borderRadius": 2, - # "overflow": "auto", - # "overflowY": "auto", - # "m": "10", - # "gap": "10px", - # }, - # padding=1, - # elevation=1, - # spacing=10, - # src=pdf_download_url_public - # ): - # # TODO: Iframe and embed tags and base64 blobs no longer work. This is due to the "sandbox" property on the streamlit MUI component's iframe - # # Would be nice to use something like PDF.js, but streamlit makes hosting static HTLM/JS files a pain - # # See https://github.com/whatwg/html/issues/3958 - # html.iframe( - # src=f"{pdf_download_url_public}", - # height=785, - # width="100%", - # type="application/pdf" - # ) - # pass - elif chosen_tab == str(2): # noseyparker_results if es_results != {}: total_hits = es_results["hits"]["total"]["value"] @@ -311,17 +271,6 @@ def build_page(username: str): object_id = es_results["hits"]["hits"][i]["_source"]["objectId"] file_name = es_results["hits"]["hits"][i]["_source"]["name"] download_url_internal = f"{NEMESIS_HTTP_SERVER}/api/download/{object_id}?name={file_name}" - kibana_link = f"{NEMESIS_HTTP_SERVER}/kibana/app/discover#/?_a=(filters:!((query:(match_phrase:(objectId:'{object_id}')))),index:'26360ae8-a518-4dac-b499-ef682d3f6bac')&_g=(time:(from:now-1y%2Fd,to:now))" - path = es_results["hits"]["hits"][i]["_source"]["path"] - sha1 = es_results["hits"]["hits"][i]["_source"]["hashes"]["sha1"] - source = "" - if "metadata" in es_results["hits"]["hits"][i]["_source"] and "source" in es_results["hits"]["hits"][i]["_source"]["metadata"]: - source = es_results["hits"]["hits"][i]["_source"]["metadata"]["source"] - - if source: - expander_text = f"{source} : **{path}** (SHA1: {sha1})" - else: - expander_text = f"**{path}** (SHA1: {sha1})" for ruleMatch in es_results["hits"]["hits"][i]["_source"]["noseyparker"]["ruleMatches"]: for match in ruleMatch["matches"]: diff --git a/cmd/dashboard/dashboard/pages/4_File_Upload.py b/cmd/dashboard/dashboard/pages/4_File_Upload.py new file mode 100644 index 0000000..4925615 --- /dev/null +++ b/cmd/dashboard/dashboard/pages/4_File_Upload.py @@ -0,0 +1,133 @@ +# Standard Libraries +import datetime +import ntpath +import os +import re + +# 3rd Party Libraries +import streamlit as st +import utils +from streamlit_cookies_manager import CookieManager + +NEMESIS_HTTP_SERVER = os.environ.get("NEMESIS_HTTP_SERVER") + + +def build_page(authenticate_user: str): + cookies = CookieManager() + if not cookies.ready(): + st.stop() + + cols = st.columns(4) + with cols[0]: + default_project = cookies["nemesus_project"] if "nemesus_project" in cookies else "" + nemesus_project = st.text_input("The name of the current project", value=default_project, placeholder="Please enter a project name") + if nemesus_project: + cookies["nemesus_project"] = nemesus_project.upper() + with cols[1]: + default_operator = cookies["nemesis_operator"] if "nemesis_operator" in cookies else "" + nemesis_operator = st.text_input("The name of the current operator", value=default_operator, placeholder="Please enter an operator name") + if nemesis_operator: + cookies["nemesis_operator"] = nemesis_operator.upper() + with cols[2]: + default_source = cookies["nemesis_source"] if "nemesis_source" in cookies else "" + nemesis_source = st.text_input("The name of the source machine", value=default_source, placeholder="Please enter a source") + if nemesis_source: + cookies["nemesis_source"] = nemesis_source.upper() + with cols[3]: + default_expiration_days = cookies["nemesis_expiration_days"] if "nemesis_expiration_days" in cookies else "100" + nemesis_expiration_days = st.text_input("Days until the data should expire", value=default_expiration_days) + if nemesis_expiration_days: + if nemesis_expiration_days.isdigit(): + cookies["nemesis_expiration_days"] = nemesis_expiration_days + + else: + st.warning("Expiration days must be an integer!") + + original_file_path = st.text_input("Original file path (optional, used for masterkeys, Chromium data, etc.)") + if original_file_path: + original_file_path = original_file_path.replace("\\", "/") + + cookies.save() + + if ( + "nemesus_project" in cookies + and cookies["nemesus_project"] != "" + and "nemesis_operator" in cookies + and cookies["nemesis_operator"] != "" + and "nemesis_source" in cookies + and cookies["nemesis_source"] != "" + and "nemesis_expiration_days" in cookies + and cookies["nemesis_expiration_days"] != "" + ): + uploaded_files = st.file_uploader("Choose a file for Nemesis to process", accept_multiple_files=True) + + if not uploaded_files: + return + + for uploaded_file in uploaded_files: + proceed = True + file_name = ntpath.basename(uploaded_file.name) + + if original_file_path: + if original_file_path.lower().endswith(file_name.lower()): + file_path = original_file_path + elif original_file_path.endswith("/"): + file_path = f"{original_file_path}{file_name}" + else: + file_path = f"{original_file_path}/{file_name}" + else: + file_path = uploaded_file.name + + # if the file is a Chromium based file, a proper path is needed + if re.match("^(Local State|History|Login Data|Cookies)$", file_name, re.IGNORECASE): + if not utils.is_valid_chromium_file_path(file_path): + proceed = False + st.warning("Chromium data (Local State, History, Login Data, Cookies) needs a valid originating path!") + + if proceed: + bytes_data = uploaded_file.read() + + nemesis_file_id = utils.nemesis_post_file(bytes_data) + + if nemesis_file_id: + submission_time = datetime.datetime.utcnow() + expiration = submission_time + datetime.timedelta(days=int(cookies["nemesis_expiration_days"])) + submission_time_s = submission_time.strftime("%Y-%m-%dT%H:%M:%S.000Z") + expiration_s = expiration.strftime("%Y-%m-%dT%H:%M:%S.000Z") + metadata = { + "agent_id": cookies["nemesis_operator"], + "agent_type": "dashboard", + "automated": False, + "data_type": "file_data", + "expiration": expiration_s, + "source": cookies["nemesis_source"], + "project": cookies["nemesus_project"], + "timestamp": submission_time_s, + } + + file_data = {"path": f"{file_path}", "size": len(bytes_data), "object_id": nemesis_file_id} + + submission_id = utils.nemesis_post_data({"metadata": metadata, "data": [file_data]}) + + if submission_id: + st.success("Successful Nemesis submission", icon="✅") + nemesis_upload = f""" + | Property | Value | + | ------------------- | ----------- | + | __Submission ID__ | {submission_id} | + | __Nemesis File ID__ | [{nemesis_file_id}]({NEMESIS_HTTP_SERVER}/dashboard/File_Viewer?object_id={nemesis_file_id}) | + | __Project__ | {cookies["nemesus_project"]} | + | __Source__ | {cookies["nemesis_source"]} | + | __Agent ID__ | {cookies["nemesis_operator"]} | + | __Agent Type__ | dashboard | + | __Automated__ | False | + | __Data Type__ | file_data | + | __Timestamp__ | {submission_time_s} | + | __Expiration__ | {expiration_s} | + | __Filename__ | {uploaded_file.name} | + | __Size__ | {len(bytes_data)} | + """ + st.markdown(nemesis_upload, unsafe_allow_html=True) + + +utils.render_nemesis_page(build_page)