From 6dd4e1a16c2a5084544b2bf71402fa9be5b126d7 Mon Sep 17 00:00:00 2001
From: harmj0y <will@harmj0y.net>
Date: Tue, 12 Sep 2023 11:48:50 -0700
Subject: [PATCH] Added the ability to set starting Redis IDs for
 mythic-connector

-Added the ability to set starting Redis IDs for mythic-connector
---
 cmd/connectors/mythic-connector/README.md | 40 +++++++++++++++++++++++
 cmd/connectors/mythic-connector/sync.py   | 15 +++++++++
 2 files changed, 55 insertions(+)

diff --git a/cmd/connectors/mythic-connector/README.md b/cmd/connectors/mythic-connector/README.md
index 9432026..80b4c8a 100644
--- a/cmd/connectors/mythic-connector/README.md
+++ b/cmd/connectors/mythic-connector/README.md
@@ -45,6 +45,46 @@ Ensure the host where `mythic_nemesis_sync` is running has network access to the
 
 The container uses Redis to keep a persistent store of Mythic data that's been submitted to Nemesis. If you want to reprocess data, set `CLEAR_REDIS=True` in settings.env to clear the Redis database. There will be a 30 second pause on startup with a warning message indicating aborting the standup will avoid clearing the database.
 
+## Fixing Processing Starting Points
+
+If the Redis database is wiped and you don't want to reprocess all existing data again, you can set the Mythic ID starting points (0 being starting from the beginning, i.e. reprocessing) for syncing of files, file listings, and processes by adding the following to settings.env:
+
+```
+REDIS_LAST_FILE_ID=123
+REDIS_LAST_FILEBROWSER_ID=456
+REDIS_LAST_PROCESS_ID=789
+```
+
+You can find the last processed file ID through Hasura (http://MYTHIC/console/, using the HASURA_SECRET from Mythic's .env file) with:
+
+```
+query MyQuery {
+  filemeta(order_by: {id: desc}, limit: 1, where: {is_download_from_agent: {_eq: true}, complete: {_eq: true}, is_screenshot: {_eq: false}}) {
+    id
+  }
+}
+```
+
+You can find the last processed file listing ID through Hasura (http://MYTHIC/console/, using the HASURA_SECRET from Mythic's .env file) with:
+
+```
+query MyQuery {
+  mythictree(order_by: {id: desc}, limit: 1, where: {tree_type: {_eq: "file"}}) {
+    id
+  }
+}
+```
+
+You can find the last processed processing listing ID through Hasura (http://MYTHIC/console/, using the HASURA_SECRET from Mythic's .env file) with:
+
+```
+query MyQuery {
+  mythictree(order_by: {id: desc}, limit: 1, where: {tree_type: {_eq: "process"}}) {
+    id
+  }
+}
+```
+
 ## References
 
 - [Mythic](https://github.com/its-a-feature/Mythic) - Multi-platform C2 Framework
diff --git a/cmd/connectors/mythic-connector/sync.py b/cmd/connectors/mythic-connector/sync.py
index 7970f9b..961cb8d 100644
--- a/cmd/connectors/mythic-connector/sync.py
+++ b/cmd/connectors/mythic-connector/sync.py
@@ -66,6 +66,14 @@ try:
 except:
     pass
 
+# if we want to manually set at which specific ID (including 0) each file type processing starts
+# processed files
+REDIS_LAST_FILE_ID = os.environ.get("REDIS_LAST_FILE_ID")
+# listed files
+REDIS_LAST_FILEBROWSER_ID = os.environ.get("REDIS_LAST_FILEBROWSER_ID")
+# process listings
+REDIS_LAST_PROCESS_ID = os.environ.get("REDIS_LAST_PROCESS_ID")
+
 # Redis connector
 rconn = None
 
@@ -862,6 +870,13 @@ async def wait_for_redis() -> None:
                 await asyncio.sleep(30)
                 for key in rconn.keys('*'):
                     rconn.delete(key)
+            # set starting points if they're specified
+            if REDIS_LAST_FILE_ID:
+                rconn.mset({"last_file_id": int(REDIS_LAST_FILE_ID)})
+            if REDIS_LAST_FILEBROWSER_ID:
+                rconn.mset({"last_filebrowser_id": int(REDIS_LAST_FILEBROWSER_ID)})
+            if REDIS_LAST_PROCESS_ID:
+                rconn.mset({"last_process_id": int(REDIS_LAST_PROCESS_ID)})
             return True
         except Exception:
             mythic_sync_log.exception(