admin/SpecterOps-Nemesis
1
0
Fork 0
mirror of https://github.com/SpecterOps/Nemesis synced 2026-06-08 12:36:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

move File Upload to own page to fix double submit problem, set default search end day to today+1

Browse Source
This commit is contained in:
Lee Christensen
2023-09-27 17:51:01 -04:00
parent 36846fcae9
commit e00a549074
4 changed files with 140 additions and 185 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/dashboard/dashboard/pages/15_Hashes.py
+2
View File
@@ -59,6 +59,8 @@ FROM extracted_hashes
if object_id:
query += " WHERE originating_object_id = :object_id"
params["object_id"] = object_id
st.write("Filters are currently applied.")
if st.button("Clear Filters"):
st.experimental_set_query_params()
cmd/dashboard/dashboard/pages/1_Files.py
+5 -134
View File
@@ -1,6 +1,5 @@
# Standard Libraries
import datetime
import ntpath
import os
import re
import urllib.parse
@@ -12,7 +11,6 @@ import humanize
import streamlit as st
import templates
import utils
from streamlit_cookies_manager import CookieManager
from streamlit_elements import dashboard, elements, html, lazy, mui, sync
from streamlit_searchbox import st_searchbox
from streamlit_toggle import st_toggle_switch
@@ -170,7 +168,7 @@ def build_and_get_search_filters():
with col2:
subcol1, subcol2 = st.columns(2)
with subcol1:
end_date = st.date_input("Filter end date (UTC)", datetime.datetime.now())
end_date = st.date_input("Filter end date (UTC)", datetime.datetime.now() + datetime.timedelta(days=1))
with subcol2:
end_time = st.time_input("Filter end time (UTC)", datetime.datetime.now())
@@ -213,7 +211,7 @@ def build_and_get_search_filters():
return search_filters
def build_file_triage_tab():
def build_file_listing():
if not st.session_state.file_download_page:
st.session_state.file_download_page = 1
@@ -434,121 +432,6 @@ def build_file_triage_tab():
st.write(pagination_html, unsafe_allow_html=True)
def build_upload_file_tab():
cookies = CookieManager()
if not cookies.ready():
st.stop()
cols = st.columns(4)
with cols[0]:
default_project = cookies["nemesus_project"] if "nemesus_project" in cookies else ""
nemesus_project = st.text_input("The name of the current project", value=default_project, placeholder="Please enter a project name")
if nemesus_project:
cookies["nemesus_project"] = nemesus_project.upper()
with cols[1]:
default_operator = cookies["nemesis_operator"] if "nemesis_operator" in cookies else ""
nemesis_operator = st.text_input("The name of the current operator", value=default_operator, placeholder="Please enter an operator name")
if nemesis_operator:
cookies["nemesis_operator"] = nemesis_operator.upper()
with cols[2]:
default_source = cookies["nemesis_source"] if "nemesis_source" in cookies else ""
nemesis_source = st.text_input("The name of the source machine", value=default_source, placeholder="Please enter a source")
if nemesis_source:
cookies["nemesis_source"] = nemesis_source.upper()
with cols[3]:
default_expiration_days = cookies["nemesis_expiration_days"] if "nemesis_expiration_days" in cookies else "100"
nemesis_expiration_days = st.text_input("Days until the data should expire", value=default_expiration_days)
if nemesis_expiration_days:
if nemesis_expiration_days.isdigit():
cookies["nemesis_expiration_days"] = nemesis_expiration_days
else:
st.warning("Expiration days must be an integer!")
original_file_path = st.text_input("Original file path (optional, used for masterkeys, Chromium data, etc.)")
if original_file_path:
original_file_path = original_file_path.replace("\\", "/")
cookies.save()
if (
"nemesus_project" in cookies
and cookies["nemesus_project"] != ""
and "nemesis_operator" in cookies
and cookies["nemesis_operator"] != ""
and "nemesis_source" in cookies
and cookies["nemesis_source"] != ""
and "nemesis_expiration_days" in cookies
and cookies["nemesis_expiration_days"] != ""
):
uploaded_files = st.file_uploader("Choose a file for Nemesis to process", accept_multiple_files=True)
for uploaded_file in uploaded_files:
proceed = True
file_name = ntpath.basename(uploaded_file.name)
if original_file_path:
if original_file_path.lower().endswith(file_name.lower()):
file_path = original_file_path
elif original_file_path.endswith("/"):
file_path = f"{original_file_path}{file_name}"
else:
file_path = f"{original_file_path}/{file_name}"
else:
file_path = uploaded_file.name
# if the file is a Chromium based file, a proper path is needed
if re.match("^(Local State|History|Login Data|Cookies)$", file_name, re.IGNORECASE):
if not utils.is_valid_chromium_file_path(file_path):
proceed = False
st.warning("Chromium data (Local State, History, Login Data, Cookies) needs a valid originating path!")
if proceed:
bytes_data = uploaded_file.read()
nemesis_file_id = utils.nemesis_post_file(bytes_data)
if nemesis_file_id:
submission_time = datetime.datetime.utcnow()
expiration = submission_time + datetime.timedelta(days=int(cookies["nemesis_expiration_days"]))
submission_time_s = submission_time.strftime("%Y-%m-%dT%H:%M:%S.000Z")
expiration_s = expiration.strftime("%Y-%m-%dT%H:%M:%S.000Z")
metadata = {
"agent_id": cookies["nemesis_operator"],
"agent_type": "dashboard",
"automated": False,
"data_type": "file_data",
"expiration": expiration_s,
"source": cookies["nemesis_source"],
"project": cookies["nemesus_project"],
"timestamp": submission_time_s,
}
file_data = {"path": f"{file_path}", "size": len(bytes_data), "object_id": nemesis_file_id}
submission_id = utils.nemesis_post_data({"metadata": metadata, "data": [file_data]})
if submission_id:
st.success("Successful Nemesis submission", icon="✅")
nemesis_upload = f"""
| Property | Value |
| ------------------- | ----------- |
| __Submission ID__ | {submission_id} |
| __Nemesis File ID__ | [{nemesis_file_id}]({NEMESIS_HTTP_SERVER}/dashboard/File_Viewer?object_id={nemesis_file_id}) |
| __Project__ | {cookies["nemesus_project"]} |
| __Source__ | {cookies["nemesis_source"]} |
| __Agent ID__ | {cookies["nemesis_operator"]} |
| __Agent Type__ | dashboard |
| __Automated__ | False |
| __Data Type__ | file_data |
| __Timestamp__ | {submission_time_s} |
| __Expiration__ | {expiration_s} |
| __Filename__ | {uploaded_file.name} |
| __Size__ | {len(bytes_data)} |
"""
st.markdown(nemesis_upload, unsafe_allow_html=True)
def init_session_state():
if "file_download_page" not in st.session_state:
st.session_state.file_download_page = None
@@ -576,9 +459,9 @@ def build_about_expander():
with st.expander("About Files"):
st.markdown(
"""
This page shows downloaded files processed by Nemesis, searchable by a variety of filters.
This page shows files processed by Nemesis. Files are searchable using variety of filters.
The `Upload File` tab allows you to upload one or more files to Nemesis for processing.
Operators can triage files as "useful", "not useful", or "needs more investigation" by clicking the thumbs up, thumbs down, and question mark buttons in the top right of each file.
"""
)
@@ -604,19 +487,7 @@ def build_page(username: str):
utils.local_css("./css/files_style.css")
init_session_state()
build_about_expander()
chosen_tab = stx.tab_bar(
data=[
stx.TabBarItemData(id=1, title="Downloaded Files", description="Downloaded Files Processed by Nemesis"),
stx.TabBarItemData(id=2, title="Upload File", description="Manually Upload a File to Nemesis for Processing"),
],
default="1",
)
if chosen_tab == str(1): # downloaded_files
build_file_triage_tab()
elif chosen_tab == str(2): # upload_file
build_upload_file_tab()
build_file_listing()
utils.render_nemesis_page(build_page)
cmd/dashboard/dashboard/pages/3_File_Viewer.py
-51
View File
@@ -192,21 +192,10 @@ def build_page(username: str):
extension = pathlib.Path(file["name"]).suffix.strip(".").lower()
download_url_internal = f"http://enrichment-webapi:9910/download/{object_id}"
download_url_public = f"{NEMESIS_HTTP_SERVER}/api/download/{object_id}"
is_text = False
if "magic_type" in file and file["magic_type"] and ("ASCII text" in file["magic_type"] or "UTF-8 Unicode" in file["magic_type"]):
is_text = True
if "converted_pdf_id" in file and file["converted_pdf_id"] != "00000000-0000-0000-0000-000000000000":
pdf_download_url_public = f"{NEMESIS_HTTP_SERVER}/api/download/{file['converted_pdf_id']}?action=view&name=test.pdf"
if file["name"].endswith(".pdf"):
pdf_download_url_public = f"{download_url_public}?action=view&name=test.pdf"
tabs = [stx.TabBarItemData(id=1, title="Basic File Info", description="Basic File Information"), stx.TabBarItemData(id=3, title="Elasticsearch Info", description="Elasticsearch Information Dump")]
es_results = utils.elastic_file_search(object_id)
has_np_results = False
if es_results and es_results["hits"]["total"]["value"] == 1:
if "noseyparker" in es_results["hits"]["hits"][0]["_source"]:
tabs.append(stx.TabBarItemData(id=2, title="Noseyparker Results", description="Noseyparker Results"))
@@ -273,35 +262,6 @@ def build_page(username: str):
except Exception as e:
st.error(f"Error displaying file in Monaco editor: {e}", icon="🚨")
# elif pdf_download_url_public:
# # Inline PDF file file display, if PDF is present
# with mui.Card(
# key="2",
# sx={
# "display": "flex",
# "flexDirection": "column",
# "borderRadius": 2,
# "overflow": "auto",
# "overflowY": "auto",
# "m": "10",
# "gap": "10px",
# },
# padding=1,
# elevation=1,
# spacing=10,
# src=pdf_download_url_public
# ):
# # TODO: Iframe and embed tags and base64 blobs no longer work. This is due to the "sandbox" property on the streamlit MUI component's iframe
# # Would be nice to use something like PDF.js, but streamlit makes hosting static HTLM/JS files a pain
# # See https://github.com/whatwg/html/issues/3958
# html.iframe(
# src=f"{pdf_download_url_public}",
# height=785,
# width="100%",
# type="application/pdf"
# )
# pass
elif chosen_tab == str(2): # noseyparker_results
if es_results != {}:
total_hits = es_results["hits"]["total"]["value"]
@@ -311,17 +271,6 @@ def build_page(username: str):
object_id = es_results["hits"]["hits"][i]["_source"]["objectId"]
file_name = es_results["hits"]["hits"][i]["_source"]["name"]
download_url_internal = f"{NEMESIS_HTTP_SERVER}/api/download/{object_id}?name={file_name}"
kibana_link = f"{NEMESIS_HTTP_SERVER}/kibana/app/discover#/?_a=(filters:!((query:(match_phrase:(objectId:'{object_id}')))),index:'26360ae8-a518-4dac-b499-ef682d3f6bac')&_g=(time:(from:now-1y%2Fd,to:now))"
path = es_results["hits"]["hits"][i]["_source"]["path"]
sha1 = es_results["hits"]["hits"][i]["_source"]["hashes"]["sha1"]
source = ""
if "metadata" in es_results["hits"]["hits"][i]["_source"] and "source" in es_results["hits"]["hits"][i]["_source"]["metadata"]:
source = es_results["hits"]["hits"][i]["_source"]["metadata"]["source"]
if source:
expander_text = f"{source} : **{path}** (SHA1: {sha1})"
else:
expander_text = f"**{path}** (SHA1: {sha1})"
for ruleMatch in es_results["hits"]["hits"][i]["_source"]["noseyparker"]["ruleMatches"]:
for match in ruleMatch["matches"]:
cmd/dashboard/dashboard/pages/4_File_Upload.py
+133
View File
@@ -0,0 +1,133 @@
# Standard Libraries
import datetime
import ntpath
import os
import re
# 3rd Party Libraries
import streamlit as st
import utils
from streamlit_cookies_manager import CookieManager
NEMESIS_HTTP_SERVER = os.environ.get("NEMESIS_HTTP_SERVER")
def build_page(authenticate_user: str):
cookies = CookieManager()
if not cookies.ready():
st.stop()
cols = st.columns(4)
with cols[0]:
default_project = cookies["nemesus_project"] if "nemesus_project" in cookies else ""
nemesus_project = st.text_input("The name of the current project", value=default_project, placeholder="Please enter a project name")
if nemesus_project:
cookies["nemesus_project"] = nemesus_project.upper()
with cols[1]:
default_operator = cookies["nemesis_operator"] if "nemesis_operator" in cookies else ""
nemesis_operator = st.text_input("The name of the current operator", value=default_operator, placeholder="Please enter an operator name")
if nemesis_operator:
cookies["nemesis_operator"] = nemesis_operator.upper()
with cols[2]:
default_source = cookies["nemesis_source"] if "nemesis_source" in cookies else ""
nemesis_source = st.text_input("The name of the source machine", value=default_source, placeholder="Please enter a source")
if nemesis_source:
cookies["nemesis_source"] = nemesis_source.upper()
with cols[3]:
default_expiration_days = cookies["nemesis_expiration_days"] if "nemesis_expiration_days" in cookies else "100"
nemesis_expiration_days = st.text_input("Days until the data should expire", value=default_expiration_days)
if nemesis_expiration_days:
if nemesis_expiration_days.isdigit():
cookies["nemesis_expiration_days"] = nemesis_expiration_days
else:
st.warning("Expiration days must be an integer!")
original_file_path = st.text_input("Original file path (optional, used for masterkeys, Chromium data, etc.)")
if original_file_path:
original_file_path = original_file_path.replace("\\", "/")
cookies.save()
if (
"nemesus_project" in cookies
and cookies["nemesus_project"] != ""
and "nemesis_operator" in cookies
and cookies["nemesis_operator"] != ""
and "nemesis_source" in cookies
and cookies["nemesis_source"] != ""
and "nemesis_expiration_days" in cookies
and cookies["nemesis_expiration_days"] != ""
):
uploaded_files = st.file_uploader("Choose a file for Nemesis to process", accept_multiple_files=True)
if not uploaded_files:
return
for uploaded_file in uploaded_files:
proceed = True
file_name = ntpath.basename(uploaded_file.name)
if original_file_path:
if original_file_path.lower().endswith(file_name.lower()):
file_path = original_file_path
elif original_file_path.endswith("/"):
file_path = f"{original_file_path}{file_name}"
else:
file_path = f"{original_file_path}/{file_name}"
else:
file_path = uploaded_file.name
# if the file is a Chromium based file, a proper path is needed
if re.match("^(Local State|History|Login Data|Cookies)$", file_name, re.IGNORECASE):
if not utils.is_valid_chromium_file_path(file_path):
proceed = False
st.warning("Chromium data (Local State, History, Login Data, Cookies) needs a valid originating path!")
if proceed:
bytes_data = uploaded_file.read()
nemesis_file_id = utils.nemesis_post_file(bytes_data)
if nemesis_file_id:
submission_time = datetime.datetime.utcnow()
expiration = submission_time + datetime.timedelta(days=int(cookies["nemesis_expiration_days"]))
submission_time_s = submission_time.strftime("%Y-%m-%dT%H:%M:%S.000Z")
expiration_s = expiration.strftime("%Y-%m-%dT%H:%M:%S.000Z")
metadata = {
"agent_id": cookies["nemesis_operator"],
"agent_type": "dashboard",
"automated": False,
"data_type": "file_data",
"expiration": expiration_s,
"source": cookies["nemesis_source"],
"project": cookies["nemesus_project"],
"timestamp": submission_time_s,
}
file_data = {"path": f"{file_path}", "size": len(bytes_data), "object_id": nemesis_file_id}
submission_id = utils.nemesis_post_data({"metadata": metadata, "data": [file_data]})
if submission_id:
st.success("Successful Nemesis submission", icon="✅")
nemesis_upload = f"""
| Property | Value |
| ------------------- | ----------- |
| __Submission ID__ | {submission_id} |
| __Nemesis File ID__ | [{nemesis_file_id}]({NEMESIS_HTTP_SERVER}/dashboard/File_Viewer?object_id={nemesis_file_id}) |
| __Project__ | {cookies["nemesus_project"]} |
| __Source__ | {cookies["nemesis_source"]} |
| __Agent ID__ | {cookies["nemesis_operator"]} |
| __Agent Type__ | dashboard |
| __Automated__ | False |
| __Data Type__ | file_data |
| __Timestamp__ | {submission_time_s} |
| __Expiration__ | {expiration_s} |
| __Filename__ | {uploaded_file.name} |
| __Size__ | {len(bytes_data)} |
"""
st.markdown(nemesis_upload, unsafe_allow_html=True)
utils.render_nemesis_page(build_page)