From 605d1ff9ce5b5188078f574c1f0d92026a6079c4 Mon Sep 17 00:00:00 2001 From: Will Date: Fri, 19 Jan 2024 15:26:15 -0800 Subject: [PATCH 1/3] Ingest extracted hash into database immediately -Ingest extracted hash into database immediately before cracking attempted -Roll back JTR image to previous format due to speed issues --- cmd/enrichment/enrichment/lib/nemesis_db.py | 14 +++ .../services/john_the_ripper_cracker.py | 4 +- .../passwordcracker/tasks/password_cracker.py | 18 +-- dockerfiles/passwordcracker.Dockerfile | 116 ++++++++++++++---- 4 files changed, 114 insertions(+), 38 deletions(-) diff --git a/cmd/enrichment/enrichment/lib/nemesis_db.py b/cmd/enrichment/enrichment/lib/nemesis_db.py index 08fa252..88e7611 100644 --- a/cmd/enrichment/enrichment/lib/nemesis_db.py +++ b/cmd/enrichment/enrichment/lib/nemesis_db.py @@ -824,6 +824,20 @@ class NemesisDb(NemesisDbInterface): auth_data.plaintext_value, ) + if auth_data.is_cracked: + # update any values in the table where the hash matches this cracked value + async with self.pool.acquire() as conn: + await conn.execute( + """ + UPDATE nemesis.extracted_hashes + SET is_cracked = True, plaintext_value = $1 + WHERE is_cracked= False AND hash_value = $2 + """, + auth_data.plaintext_value, + auth_data.hash_value + ) + + async def add_dpapi_blob(self, dpapi_blob: DpapiBlob) -> None: """Adds a new `nemesis.dpapi_blobs` entry from a DpapiBlob class object.""" diff --git a/cmd/passwordcracker/passwordcracker/services/john_the_ripper_cracker.py b/cmd/passwordcracker/passwordcracker/services/john_the_ripper_cracker.py index fae4b64..122bfef 100644 --- a/cmd/passwordcracker/passwordcracker/services/john_the_ripper_cracker.py +++ b/cmd/passwordcracker/passwordcracker/services/john_the_ripper_cracker.py @@ -40,7 +40,7 @@ class JohnTheRipperCracker(PasswordCrackerInterface): # TODO: convert this to using asyncio's subprocess functions result = subprocess.run( [ - "/john/run/john", + "/opt/john/run/john", f"--format={format}", f"--wordlist={wordlist_file_path}", "--no-log", @@ -53,7 +53,7 @@ class JohnTheRipperCracker(PasswordCrackerInterface): else: result = subprocess.run( [ - "/john/run/john", + "/opt/john/run/john", f"--wordlist={wordlist_file_path}", "--no-log", f"--pot={pot_file.name}", diff --git a/cmd/passwordcracker/passwordcracker/tasks/password_cracker.py b/cmd/passwordcracker/passwordcracker/tasks/password_cracker.py index f777c5b..bbcf792 100644 --- a/cmd/passwordcracker/passwordcracker/tasks/password_cracker.py +++ b/cmd/passwordcracker/passwordcracker/tasks/password_cracker.py @@ -5,13 +5,12 @@ import os # 3rd Party Libraries import nemesispb.nemesis_pb2 as pb import structlog -from nemesiscommon.messaging import ( - MessageQueueConsumerInterface, - MessageQueueProducerInterface, -) +from nemesiscommon.messaging import (MessageQueueConsumerInterface, + MessageQueueProducerInterface) from nemesiscommon.services.alerter import AlerterInterface from nemesiscommon.tasking import TaskInterface -from passwordcracker.services.john_the_ripper_cracker import PasswordCrackerInterface +from passwordcracker.services.john_the_ripper_cracker import \ + PasswordCrackerInterface from passwordcracker.settings import PasswordCrackerSettings from prometheus_async import aio from prometheus_client import Summary @@ -54,7 +53,7 @@ class PasswordCracker(TaskInterface): self.semaphore = asyncio.Semaphore() async def run(self) -> None: - await logger.ainfo("Starting the Auth Data service") + await logger.ainfo("Starting the password cracking service") await asyncio.gather( self.auth_data_q_in.Read(self.handle_auth_data), # type: ignore @@ -84,6 +83,9 @@ class PasswordCracker(TaskInterface): # TODO: formatting for Hashcat/JTR formats extracted_hash.jtr_formatted_value = data.data + # send the message _uncracked_ so it can be displayed ASAP + await self.extracted_hash_q_out.Send(extracted_hash_msg.SerializeToString()) + async with self.semaphore: match extracted_hash.hash_type: # handle specific hash types that need the type specified @@ -97,12 +99,10 @@ class PasswordCracker(TaskInterface): if jtr_pot_line: extracted_hash.jtr_pot_line = jtr_pot_line extracted_hash.is_cracked = True - - # regarding spltting on the : - # yes this is stupid, but don't see another way to do this plaintext = jtr_pot_line extracted_hash.plaintext_value = plaintext + await logger.ainfo(f"Hash cracked: {extracted_hash}") await self.send_hash_cracked_alert(extracted_hash, extracted_hash_msg.metadata.message_id) # publish the extracted hash out to the extracted_hash_q_out queue diff --git a/dockerfiles/passwordcracker.Dockerfile b/dockerfiles/passwordcracker.Dockerfile index 6afbce3..2ed3689 100644 --- a/dockerfiles/passwordcracker.Dockerfile +++ b/dockerfiles/passwordcracker.Dockerfile @@ -1,39 +1,22 @@ #################################### # Common python dependencies layer #################################### -FROM ghcr.io/openwall/john:latest_1.9.20240102 as debcommon +FROM python:3.11.2-bullseye AS debcommon +WORKDIR /app/cmd/passwordcracker ENV PYTHONUNBUFFERED=true #################################### -# Install Python +# OS dependencies #################################### -FROM debcommon AS dependencies-os-python +FROM debcommon AS dependencies-os -USER root -WORKDIR /tmp/python/ +# install our necessary dependencies +RUN apt-get update -y && apt-get install yara -y && apt-get install git -y && apt-get install wamerican -y && apt-get install libcompress-raw-lzma-perl -y -# install Python -ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update -y && apt install wget build-essential libncursesw5-dev libssl-dev libsqlite3-dev tk-dev libgdbm-dev libc6-dev libbz2-dev libffi-dev zlib1g-dev -y -RUN wget https://www.python.org/ftp/python/3.11.3/Python-3.11.3.tgz -RUN tar xzf Python-3.11.3.tgz -RUN cd Python-3.11.3 && ./configure --enable-optimizations && make altinstall - - -#################################### -# Other OS dependencies -#################################### -FROM dependencies-os-python AS dependencies-os - -WORKDIR /app/cmd/passwordcracker - -# install the rest of our dependencies -RUN apt install python3-pip libssl-dev yara git wamerican libcompress-raw-lzma-perl -y - -# rename the John binary -RUN cp /john/run/john-avx /john/run/john +# build JTR so we build get various X-2john binaries for file hash extraction +RUN cd /opt/ && git clone https://github.com/openwall/john && cd john/src && ./configure && make #################################### @@ -73,6 +56,85 @@ COPY cmd/passwordcracker/passwordcracker/ ./passwordcracker/ FROM build AS runtime ENV PATH="/app/cmd/passwordcracker/.venv/bin:$PATH" -RUN python3 --version +CMD ["python3", "-m", "passwordcracker"] -ENTRYPOINT ["python3", "-m", "passwordcracker"] + +## Running WAY slower for some reason... +# #################################### +# # Common python dependencies layer +# #################################### +# FROM ghcr.io/openwall/john:latest_1.9.20240102 as debcommon + +# ENV PYTHONUNBUFFERED=true + + +# #################################### +# # Install Python +# #################################### +# FROM debcommon AS dependencies-os-python + +# USER root +# WORKDIR /tmp/python/ + +# # install Python +# ENV DEBIAN_FRONTEND=noninteractive +# RUN apt-get update -y && apt install wget build-essential libncursesw5-dev libssl-dev libsqlite3-dev tk-dev libgdbm-dev libc6-dev libbz2-dev libffi-dev zlib1g-dev -y +# RUN wget https://www.python.org/ftp/python/3.11.3/Python-3.11.3.tgz +# RUN tar xzf Python-3.11.3.tgz +# RUN cd Python-3.11.3 && ./configure --enable-optimizations && make altinstall + + +# #################################### +# # Other OS dependencies +# #################################### +# FROM dependencies-os-python AS dependencies-os + +# WORKDIR /app/cmd/passwordcracker + +# # install the rest of our dependencies +# RUN apt install python3-pip libssl-dev yara git wamerican libcompress-raw-lzma-perl -y + +# # rename the John binary +# RUN cp /john/run/john-avx /john/run/john + + +# #################################### +# # Python dependencies +# #################################### +# FROM dependencies-os AS dependencies-python + +# ARG ENVIRONMENT=dev +# ENV POETRY_HOME=/opt/poetry +# ENV POETRY_VIRTUALENVS_IN_PROJECT=true +# ENV PATH="$POETRY_HOME/bin:$PATH" + +# # install Poetry +# RUN python3 -c 'from urllib.request import urlopen; print(urlopen("https://install.python-poetry.org").read().decode())' | python3 - + + +# #################################### +# # Container specific dependencies +# #################################### +# FROM dependencies-python AS build + +# COPY cmd/passwordcracker/poetry.lock cmd/passwordcracker/pyproject.toml ./ + +# # copy local libraries +# COPY packages/python/nemesispb/ /app/packages/python/nemesispb/ +# COPY packages/python/nemesiscommon/ /app/packages/python/nemesiscommon/ + +# # use Poetry to install the local packages +# RUN poetry install $(if [ "${ENVIRONMENT}" = 'production' ]; then echo "--without dev"; fi;) --no-root --no-interaction --no-ansi -vvv + +# COPY cmd/passwordcracker/passwordcracker/ ./passwordcracker/ + + +# #################################### +# # Runtime +# #################################### +# FROM build AS runtime +# ENV PATH="/app/cmd/passwordcracker/.venv/bin:$PATH" + +# RUN python3 --version + +# ENTRYPOINT ["python3", "-m", "passwordcracker"] From 81cb8a3267564c5487a975c321b54a301bc11fba Mon Sep 17 00:00:00 2001 From: Will Date: Fri, 19 Jan 2024 18:35:40 -0800 Subject: [PATCH 2/3] Uniquness for extracted hashes in DB -Uniquness for extracted hashes in DB --- kubernetes/postgres/configmap.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/kubernetes/postgres/configmap.yaml b/kubernetes/postgres/configmap.yaml index a813df0..d69045c 100644 --- a/kubernetes/postgres/configmap.yaml +++ b/kubernetes/postgres/configmap.yaml @@ -381,7 +381,9 @@ data: is_submitted_to_cracker BOOLEAN, -- True if the hash has submitted to a longer-run cracking job cracker_submission_time TIMESTAMP WITH TIME ZONE, -- Time the hash was submitted to a longer-run cracking job cracker_cracked_time TIMESTAMP WITH TIME ZONE, -- Time the hash was cracked by a longer-run cracking job - plaintext_value TEXT -- The data value if the hash has been cracked + plaintext_value TEXT, -- The data value if the hash has been cracked + hash_value_md5_hash UUID GENERATED ALWAYS AS (MD5(hash_value)::uuid) STORED, -- used in case the hash value is vvv longboi + UNIQUE (timestamp, originating_object_id, hash_value_md5_hash) ) INHERITS (project_data); From 8fd06aa7caf9e77c400b30a8aac9d098aedc70ed Mon Sep 17 00:00:00 2001 From: Will Date: Mon, 22 Jan 2024 10:14:00 -0800 Subject: [PATCH 3/3] Don't re-crack already cracked hashes -Check if a hash is already present and cracked in the database and use the cracked value instead of re-cracking the hash --- cmd/enrichment/enrichment/lib/nemesis_db.py | 19 +++++- .../passwordcracker/bootstrap.py | 6 +- .../passwordcracker/containers.py | 10 ++++ .../passwordcracker/settings.py | 3 +- .../passwordcracker/tasks/password_cracker.py | 60 +++++++++++++------ kubernetes/passwordcracker/deployment.yaml | 20 +++++++ 6 files changed, 92 insertions(+), 26 deletions(-) diff --git a/cmd/enrichment/enrichment/lib/nemesis_db.py b/cmd/enrichment/enrichment/lib/nemesis_db.py index 88e7611..344a734 100644 --- a/cmd/enrichment/enrichment/lib/nemesis_db.py +++ b/cmd/enrichment/enrichment/lib/nemesis_db.py @@ -1270,7 +1270,7 @@ class NemesisDb(NemesisDbInterface): async with self.pool.acquire() as conn: results = await conn.fetch( - "SELECT data from nemesis.authentication_data WHERE type = 'password' AND username ILIKE $1", + "SELECT data FROM nemesis.authentication_data WHERE type = 'password' AND username ILIKE $1", username, ) return [result[0] for result in results] @@ -1283,7 +1283,7 @@ class NemesisDb(NemesisDbInterface): async with self.pool.acquire() as conn: results = await conn.fetch( - "SELECT data from nemesis.authentication_data WHERE type = 'ntlm_hash' AND username ILIKE $1", + "SELECT data FROM nemesis.authentication_data WHERE type = 'ntlm_hash' AND username ILIKE $1", username, ) return [result[0] for result in results] @@ -1293,11 +1293,24 @@ class NemesisDb(NemesisDbInterface): async with self.pool.acquire() as conn: results = await conn.fetch( - "SELECT plaintext_value from nemesis.authentication_data is_cracked = True AND username ILIKE $1", + "SELECT plaintext_value FROM nemesis.extracted_hashes WHERE is_cracked = True AND username ILIKE $1", username, ) return [result[0] for result in results] + async def get_cracked_hash_value(self, hash_value: str): + """Returns the plaintext value for a hash if it's already cracked.""" + + async with self.pool.acquire() as conn: + results = await conn.fetch( + "SELECT plaintext_value FROM nemesis.extracted_hashes WHERE is_cracked = True AND hash_value = $1", + hash_value, + ) + if results: + return results[0][0] + else: + return None + async def get_encrypted_dpapi_masterkeys(self, username: str = "%", machine: bool = False): """Gets encrypted DPAPI masterkeys linked to a specific domain backupkey guid. diff --git a/cmd/passwordcracker/passwordcracker/bootstrap.py b/cmd/passwordcracker/passwordcracker/bootstrap.py index 7d6fc4e..061c420 100644 --- a/cmd/passwordcracker/passwordcracker/bootstrap.py +++ b/cmd/passwordcracker/passwordcracker/bootstrap.py @@ -88,13 +88,13 @@ def handle_exception(loop: asyncio.AbstractEventLoop, context: dict[str, Any]): async def wait_for_services(config: PasswordCrackerSettings) -> None: rabbitUri = urlparse(config.rabbitmq_connection_uri) + postgresUri = urlparse(config.postgres_connection_uri) - if rabbitUri.hostname is None: + if rabbitUri.hostname is None or postgresUri.hostname is None: raise Exception("Invalid connection URI") if rabbitUri.port is None: raise Exception("Invalid connection URI") - # TODO: Check that JohnTheRipper is installed - SocketWaiter(rabbitUri.hostname, rabbitUri.port).wait() + SocketWaiter(postgresUri.hostname, postgresUri.port).wait() await logger.ainfo("All services are online!") diff --git a/cmd/passwordcracker/passwordcracker/containers.py b/cmd/passwordcracker/passwordcracker/containers.py index 68ba975..2d38314 100644 --- a/cmd/passwordcracker/passwordcracker/containers.py +++ b/cmd/passwordcracker/passwordcracker/containers.py @@ -1,6 +1,7 @@ # Standard Libraries from typing import AsyncGenerator +import asyncpg # 3rd Party Libraries import google.protobuf.message import httpx @@ -40,6 +41,12 @@ async def create_producer(rabbitmq_connection_uri: str, queue: NemesisQueue): yield outputQ +async def create_nemesis_db_pool(postgres_connection_uri: str): + pool = await asyncpg.create_pool(dsn=postgres_connection_uri) + yield pool + await pool.Close() + + async def create_http_retry_client() -> AsyncGenerator[httpx.AsyncClient, None]: transport = httpx.AsyncHTTPTransport(retries=5) async with httpx.AsyncClient(transport=transport) as client: @@ -82,6 +89,8 @@ class Container(containers.DeclarativeContainer): config.data_download_dir, ) + database_pool = providers.Resource(create_nemesis_db_pool, config.postgres_connection_uri) + # # passwordcracker Service Tasks # @@ -89,6 +98,7 @@ class Container(containers.DeclarativeContainer): PasswordCracker, config2, alerting_service, + database_pool, cracker_service, inputq_passwordcracker_passwordcrackertask, outputq_extractedhash, diff --git a/cmd/passwordcracker/passwordcracker/settings.py b/cmd/passwordcracker/passwordcracker/settings.py index 4fac43e..6598800 100644 --- a/cmd/passwordcracker/passwordcracker/settings.py +++ b/cmd/passwordcracker/passwordcracker/settings.py @@ -3,7 +3,7 @@ from enum import IntEnum # 3rd Party Libraries from nemesiscommon.settings import HttpUrlWithSlash, NemesisServiceSettings -from pydantic.networks import AnyUrl +from pydantic.networks import AnyUrl, PostgresDsn class CrackWordlistSize(IntEnum): @@ -13,6 +13,7 @@ class CrackWordlistSize(IntEnum): class PasswordCrackerSettings(NemesisServiceSettings): # type: ignore rabbitmq_connection_uri: AnyUrl + postgres_connection_uri: PostgresDsn public_nemesis_url: HttpUrlWithSlash public_kibana_url: HttpUrlWithSlash data_download_dir: str diff --git a/cmd/passwordcracker/passwordcracker/tasks/password_cracker.py b/cmd/passwordcracker/passwordcracker/tasks/password_cracker.py index bbcf792..6f5cfd6 100644 --- a/cmd/passwordcracker/passwordcracker/tasks/password_cracker.py +++ b/cmd/passwordcracker/passwordcracker/tasks/password_cracker.py @@ -3,6 +3,7 @@ import asyncio import os # 3rd Party Libraries +import asyncpg import nemesispb.nemesis_pb2 as pb import structlog from nemesiscommon.messaging import (MessageQueueConsumerInterface, @@ -21,6 +22,7 @@ logger = structlog.get_logger(module=__name__) class PasswordCracker(TaskInterface): cfg: PasswordCrackerSettings alerter: AlerterInterface + db_pool: asyncpg.pool.Pool cracker: PasswordCrackerInterface semaphore: asyncio.Semaphore @@ -32,12 +34,14 @@ class PasswordCracker(TaskInterface): self, cfg: PasswordCrackerSettings, alerter: AlerterInterface, + db_pool: asyncpg.pool.Pool, cracker: PasswordCrackerInterface, auth_data_q_in: MessageQueueConsumerInterface, extracted_hash_q_out: MessageQueueProducerInterface, ): self.cfg = cfg self.alerter = alerter + self.db_pool = db_pool self.cracker = cracker self.auth_data_q_in = auth_data_q_in self.extracted_hash_q_out = extracted_hash_q_out @@ -63,6 +67,19 @@ class PasswordCracker(TaskInterface): async def handle_auth_data(self, q_msg: pb.AuthenticationDataIngestionMessage) -> None: await self.process_auth_data(q_msg) + async def get_cracked_hash_value(self, hash_value: str): + """Returns the plaintext value for a hash if it's already cracked.""" + + async with self.db_pool.acquire() as conn: + results = await conn.fetch( + "SELECT plaintext_value FROM nemesis.extracted_hashes WHERE is_cracked = True AND hash_value = $1", + hash_value, + ) + if results: + return results[0][0] + else: + return None + @aio.time(Summary("process_auth_data", "Time spent processing an Auth Data event")) # type: ignore async def process_auth_data(self, event: pb.AuthenticationDataIngestionMessage): """Main function to process authentication data events.""" @@ -83,27 +100,32 @@ class PasswordCracker(TaskInterface): # TODO: formatting for Hashcat/JTR formats extracted_hash.jtr_formatted_value = data.data - # send the message _uncracked_ so it can be displayed ASAP - await self.extracted_hash_q_out.Send(extracted_hash_msg.SerializeToString()) - - async with self.semaphore: - match extracted_hash.hash_type: - # handle specific hash types that need the type specified - case "hash_crypt": - jtr_pot_line = await self.cracker.crack(data.data, self.wordlist_path, "crypt") - case _: - jtr_pot_line = await self.cracker.crack(data.data, self.wordlist_path) - - extracted_hash.checked_against_top_passwords = True - - if jtr_pot_line: - extracted_hash.jtr_pot_line = jtr_pot_line + cracked_hash_value = await self.get_cracked_hash_value(data.data) + if cracked_hash_value: + # this means the hash is already cracked, so don't just JTR extracted_hash.is_cracked = True - plaintext = jtr_pot_line - extracted_hash.plaintext_value = plaintext + extracted_hash.plaintext_value = cracked_hash_value + await logger.ainfo("Hash is already cracked using existing value.") + else: + # send the message before using JTR so it can be displayed ASAP + await self.extracted_hash_q_out.Send(extracted_hash_msg.SerializeToString()) - await logger.ainfo(f"Hash cracked: {extracted_hash}") - await self.send_hash_cracked_alert(extracted_hash, extracted_hash_msg.metadata.message_id) + async with self.semaphore: + match extracted_hash.hash_type: + # handle specific hash types that need the type specified + case "hash_crypt": + jtr_pot_line = await self.cracker.crack(data.data, self.wordlist_path, "crypt") + case _: + jtr_pot_line = await self.cracker.crack(data.data, self.wordlist_path) + + extracted_hash.checked_against_top_passwords = True + + if jtr_pot_line: + extracted_hash.jtr_pot_line = jtr_pot_line + extracted_hash.is_cracked = True + plaintext = jtr_pot_line + extracted_hash.plaintext_value = plaintext + await self.send_hash_cracked_alert(extracted_hash, extracted_hash_msg.metadata.message_id) # publish the extracted hash out to the extracted_hash_q_out queue await self.extracted_hash_q_out.Send(extracted_hash_msg.SerializeToString()) diff --git a/kubernetes/passwordcracker/deployment.yaml b/kubernetes/passwordcracker/deployment.yaml index 5f55e02..548d10b 100644 --- a/kubernetes/passwordcracker/deployment.yaml +++ b/kubernetes/passwordcracker/deployment.yaml @@ -58,6 +58,26 @@ spec: secretKeyRef: name: rabbitmq-creds key: rabbitmq-connectionuri + + - name: POSTGRES_SERVER + value: postgres + - name: POSTGRES_PORT + value: "5432" + - name: POSTGRES_DATABASE + value: nemesis + - name: POSTGRES_USER + valueFrom: + secretKeyRef: + name: postgres-creds + key: postgres-user + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: postgres-creds + key: postgres-password + - name: POSTGRES_CONNECTION_URI + value: "postgresql://$(POSTGRES_USER):$(POSTGRES_PASSWORD)@$(POSTGRES_SERVER):$(POSTGRES_PORT)/$(POSTGRES_DATABASE)" + - name: SLACK_CHANNEL valueFrom: configMapKeyRef: