admin/SpecterOps-Nemesis
1
0
Fork 0
mirror of https://github.com/SpecterOps/Nemesis synced 2026-06-08 12:36:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #34 from SpecterOps/passwordcracker_mods

Browse Source 
Passwordcracker mods
This commit is contained in:
Will
2024-01-22 10:15:22 -08:00
committed by GitHub
parent c60d094843 8fd06aa7ca
commit 3afc422ce2
9 changed files with 204 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/enrichment/enrichment/lib/nemesis_db.py
+30 -3
View File
@@ -824,6 +824,20 @@ class NemesisDb(NemesisDbInterface):
auth_data.plaintext_value,
)
if auth_data.is_cracked:
# update any values in the table where the hash matches this cracked value
async with self.pool.acquire() as conn:
await conn.execute(
"""
UPDATE nemesis.extracted_hashes
SET is_cracked = True, plaintext_value = $1
WHERE is_cracked= False AND hash_value = $2
""",
auth_data.plaintext_value,
auth_data.hash_value
)
async def add_dpapi_blob(self, dpapi_blob: DpapiBlob) -> None:
"""Adds a new `nemesis.dpapi_blobs` entry from a DpapiBlob class object."""
@@ -1256,7 +1270,7 @@ class NemesisDb(NemesisDbInterface):
async with self.pool.acquire() as conn:
results = await conn.fetch(
"SELECT data from nemesis.authentication_data WHERE type = 'password' AND username ILIKE $1",
"SELECT data FROM nemesis.authentication_data WHERE type = 'password' AND username ILIKE $1",
username,
)
return [result[0] for result in results]
@@ -1269,7 +1283,7 @@ class NemesisDb(NemesisDbInterface):
async with self.pool.acquire() as conn:
results = await conn.fetch(
"SELECT data from nemesis.authentication_data WHERE type = 'ntlm_hash' AND username ILIKE $1",
"SELECT data FROM nemesis.authentication_data WHERE type = 'ntlm_hash' AND username ILIKE $1",
username,
)
return [result[0] for result in results]
@@ -1279,11 +1293,24 @@ class NemesisDb(NemesisDbInterface):
async with self.pool.acquire() as conn:
results = await conn.fetch(
"SELECT plaintext_value from nemesis.authentication_data is_cracked = True AND username ILIKE $1",
"SELECT plaintext_value FROM nemesis.extracted_hashes WHERE is_cracked = True AND username ILIKE $1",
username,
)
return [result[0] for result in results]
async def get_cracked_hash_value(self, hash_value: str):
"""Returns the plaintext value for a hash if it's already cracked."""
async with self.pool.acquire() as conn:
results = await conn.fetch(
"SELECT plaintext_value FROM nemesis.extracted_hashes WHERE is_cracked = True AND hash_value = $1",
hash_value,
)
if results:
return results[0][0]
else:
return None
async def get_encrypted_dpapi_masterkeys(self, username: str = "%", machine: bool = False):
"""Gets encrypted DPAPI masterkeys linked to a specific domain backupkey guid.
cmd/passwordcracker/passwordcracker/bootstrap.py
+3 -3
View File
@@ -88,13 +88,13 @@ def handle_exception(loop: asyncio.AbstractEventLoop, context: dict[str, Any]):
async def wait_for_services(config: PasswordCrackerSettings) -> None:
rabbitUri = urlparse(config.rabbitmq_connection_uri)
postgresUri = urlparse(config.postgres_connection_uri)
if rabbitUri.hostname is None:
if rabbitUri.hostname is None or postgresUri.hostname is None:
raise Exception("Invalid connection URI")
if rabbitUri.port is None:
raise Exception("Invalid connection URI")
# TODO: Check that JohnTheRipper is installed
SocketWaiter(rabbitUri.hostname, rabbitUri.port).wait()
SocketWaiter(postgresUri.hostname, postgresUri.port).wait()
await logger.ainfo("All services are online!")
cmd/passwordcracker/passwordcracker/containers.py
+10
View File
@@ -1,6 +1,7 @@
# Standard Libraries
from typing import AsyncGenerator
import asyncpg
# 3rd Party Libraries
import google.protobuf.message
import httpx
@@ -40,6 +41,12 @@ async def create_producer(rabbitmq_connection_uri: str, queue: NemesisQueue):
yield outputQ
async def create_nemesis_db_pool(postgres_connection_uri: str):
pool = await asyncpg.create_pool(dsn=postgres_connection_uri)
yield pool
await pool.Close()
async def create_http_retry_client() -> AsyncGenerator[httpx.AsyncClient, None]:
transport = httpx.AsyncHTTPTransport(retries=5)
async with httpx.AsyncClient(transport=transport) as client:
@@ -82,6 +89,8 @@ class Container(containers.DeclarativeContainer):
config.data_download_dir,
)
database_pool = providers.Resource(create_nemesis_db_pool, config.postgres_connection_uri)
#
# passwordcracker Service Tasks
#
@@ -89,6 +98,7 @@ class Container(containers.DeclarativeContainer):
PasswordCracker,
config2,
alerting_service,
database_pool,
cracker_service,
inputq_passwordcracker_passwordcrackertask,
outputq_extractedhash,
cmd/passwordcracker/passwordcracker/services/john_the_ripper_cracker.py
+2 -2
View File
@@ -40,7 +40,7 @@ class JohnTheRipperCracker(PasswordCrackerInterface):
# TODO: convert this to using asyncio's subprocess functions
result = subprocess.run(
[
"/john/run/john",
"/opt/john/run/john",
f"--format={format}",
f"--wordlist={wordlist_file_path}",
"--no-log",
@@ -53,7 +53,7 @@ class JohnTheRipperCracker(PasswordCrackerInterface):
else:
result = subprocess.run(
[
"/john/run/john",
"/opt/john/run/john",
f"--wordlist={wordlist_file_path}",
"--no-log",
f"--pot={pot_file.name}",
cmd/passwordcracker/passwordcracker/settings.py
+2 -1
View File
@@ -3,7 +3,7 @@ from enum import IntEnum
# 3rd Party Libraries
from nemesiscommon.settings import HttpUrlWithSlash, NemesisServiceSettings
from pydantic.networks import AnyUrl
from pydantic.networks import AnyUrl, PostgresDsn
class CrackWordlistSize(IntEnum):
@@ -13,6 +13,7 @@ class CrackWordlistSize(IntEnum):
class PasswordCrackerSettings(NemesisServiceSettings): # type: ignore
rabbitmq_connection_uri: AnyUrl
postgres_connection_uri: PostgresDsn
public_nemesis_url: HttpUrlWithSlash
public_kibana_url: HttpUrlWithSlash
data_download_dir: str
cmd/passwordcracker/passwordcracker/tasks/password_cracker.py
+45 -23
View File
@@ -3,15 +3,15 @@ import asyncio
import os
# 3rd Party Libraries
import asyncpg
import nemesispb.nemesis_pb2 as pb
import structlog
from nemesiscommon.messaging import (
MessageQueueConsumerInterface,
MessageQueueProducerInterface,
)
from nemesiscommon.messaging import (MessageQueueConsumerInterface,
MessageQueueProducerInterface)
from nemesiscommon.services.alerter import AlerterInterface
from nemesiscommon.tasking import TaskInterface
from passwordcracker.services.john_the_ripper_cracker import PasswordCrackerInterface
from passwordcracker.services.john_the_ripper_cracker import \
PasswordCrackerInterface
from passwordcracker.settings import PasswordCrackerSettings
from prometheus_async import aio
from prometheus_client import Summary
@@ -22,6 +22,7 @@ logger = structlog.get_logger(module=__name__)
class PasswordCracker(TaskInterface):
cfg: PasswordCrackerSettings
alerter: AlerterInterface
db_pool: asyncpg.pool.Pool
cracker: PasswordCrackerInterface
semaphore: asyncio.Semaphore
@@ -33,12 +34,14 @@ class PasswordCracker(TaskInterface):
self,
cfg: PasswordCrackerSettings,
alerter: AlerterInterface,
db_pool: asyncpg.pool.Pool,
cracker: PasswordCrackerInterface,
auth_data_q_in: MessageQueueConsumerInterface,
extracted_hash_q_out: MessageQueueProducerInterface,
):
self.cfg = cfg
self.alerter = alerter
self.db_pool = db_pool
self.cracker = cracker
self.auth_data_q_in = auth_data_q_in
self.extracted_hash_q_out = extracted_hash_q_out
@@ -54,7 +57,7 @@ class PasswordCracker(TaskInterface):
self.semaphore = asyncio.Semaphore()
async def run(self) -> None:
await logger.ainfo("Starting the Auth Data service")
await logger.ainfo("Starting the password cracking service")
await asyncio.gather(
self.auth_data_q_in.Read(self.handle_auth_data), # type: ignore
@@ -64,6 +67,19 @@ class PasswordCracker(TaskInterface):
async def handle_auth_data(self, q_msg: pb.AuthenticationDataIngestionMessage) -> None:
await self.process_auth_data(q_msg)
async def get_cracked_hash_value(self, hash_value: str):
"""Returns the plaintext value for a hash if it's already cracked."""
async with self.db_pool.acquire() as conn:
results = await conn.fetch(
"SELECT plaintext_value FROM nemesis.extracted_hashes WHERE is_cracked = True AND hash_value = $1",
hash_value,
)
if results:
return results[0][0]
else:
return None
@aio.time(Summary("process_auth_data", "Time spent processing an Auth Data event")) # type: ignore
async def process_auth_data(self, event: pb.AuthenticationDataIngestionMessage):
"""Main function to process authentication data events."""
@@ -84,26 +100,32 @@ class PasswordCracker(TaskInterface):
# TODO: formatting for Hashcat/JTR formats
extracted_hash.jtr_formatted_value = data.data
async with self.semaphore:
match extracted_hash.hash_type:
# handle specific hash types that need the type specified
case "hash_crypt":
jtr_pot_line = await self.cracker.crack(data.data, self.wordlist_path, "crypt")
case _:
jtr_pot_line = await self.cracker.crack(data.data, self.wordlist_path)
extracted_hash.checked_against_top_passwords = True
if jtr_pot_line:
extracted_hash.jtr_pot_line = jtr_pot_line
cracked_hash_value = await self.get_cracked_hash_value(data.data)
if cracked_hash_value:
# this means the hash is already cracked, so don't just JTR
extracted_hash.is_cracked = True
extracted_hash.plaintext_value = cracked_hash_value
await logger.ainfo("Hash is already cracked using existing value.")
else:
# send the message before using JTR so it can be displayed ASAP
await self.extracted_hash_q_out.Send(extracted_hash_msg.SerializeToString())
# regarding spltting on the :
# yes this is stupid, but don't see another way to do this
plaintext = jtr_pot_line
extracted_hash.plaintext_value = plaintext
async with self.semaphore:
match extracted_hash.hash_type:
# handle specific hash types that need the type specified
case "hash_crypt":
jtr_pot_line = await self.cracker.crack(data.data, self.wordlist_path, "crypt")
case _:
jtr_pot_line = await self.cracker.crack(data.data, self.wordlist_path)
await self.send_hash_cracked_alert(extracted_hash, extracted_hash_msg.metadata.message_id)
extracted_hash.checked_against_top_passwords = True
if jtr_pot_line:
extracted_hash.jtr_pot_line = jtr_pot_line
extracted_hash.is_cracked = True
plaintext = jtr_pot_line
extracted_hash.plaintext_value = plaintext
await self.send_hash_cracked_alert(extracted_hash, extracted_hash_msg.metadata.message_id)
# publish the extracted hash out to the extracted_hash_q_out queue
await self.extracted_hash_q_out.Send(extracted_hash_msg.SerializeToString())
dockerfiles/passwordcracker.Dockerfile
+89 -27
View File
@@ -1,39 +1,22 @@
####################################
# Common python dependencies layer
####################################
FROM ghcr.io/openwall/john:latest_1.9.20240102 as debcommon
FROM python:3.11.2-bullseye AS debcommon
WORKDIR /app/cmd/passwordcracker
ENV PYTHONUNBUFFERED=true
####################################
# Install Python
# OS dependencies
####################################
FROM debcommon AS dependencies-os-python
FROM debcommon AS dependencies-os
USER root
WORKDIR /tmp/python/
# install our necessary dependencies
RUN apt-get update -y && apt-get install yara -y && apt-get install git -y && apt-get install wamerican -y && apt-get install libcompress-raw-lzma-perl -y
# install Python
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update -y && apt install wget build-essential libncursesw5-dev libssl-dev libsqlite3-dev tk-dev libgdbm-dev libc6-dev libbz2-dev libffi-dev zlib1g-dev -y
RUN wget https://www.python.org/ftp/python/3.11.3/Python-3.11.3.tgz
RUN tar xzf Python-3.11.3.tgz
RUN cd Python-3.11.3 && ./configure --enable-optimizations && make altinstall
####################################
# Other OS dependencies
####################################
FROM dependencies-os-python AS dependencies-os
WORKDIR /app/cmd/passwordcracker
# install the rest of our dependencies
RUN apt install python3-pip libssl-dev yara git wamerican libcompress-raw-lzma-perl -y
# rename the John binary
RUN cp /john/run/john-avx /john/run/john
# build JTR so we build get various X-2john binaries for file hash extraction
RUN cd /opt/ && git clone https://github.com/openwall/john && cd john/src && ./configure && make
####################################
@@ -73,6 +56,85 @@ COPY cmd/passwordcracker/passwordcracker/ ./passwordcracker/
FROM build AS runtime
ENV PATH="/app/cmd/passwordcracker/.venv/bin:$PATH"
RUN python3 --version
CMD ["python3", "-m", "passwordcracker"]
ENTRYPOINT ["python3", "-m", "passwordcracker"]
## Running WAY slower for some reason...
# ####################################
# # Common python dependencies layer
# ####################################
# FROM ghcr.io/openwall/john:latest_1.9.20240102 as debcommon
# ENV PYTHONUNBUFFERED=true
# ####################################
# # Install Python
# ####################################
# FROM debcommon AS dependencies-os-python
# USER root
# WORKDIR /tmp/python/
# # install Python
# ENV DEBIAN_FRONTEND=noninteractive
# RUN apt-get update -y && apt install wget build-essential libncursesw5-dev libssl-dev libsqlite3-dev tk-dev libgdbm-dev libc6-dev libbz2-dev libffi-dev zlib1g-dev -y
# RUN wget https://www.python.org/ftp/python/3.11.3/Python-3.11.3.tgz
# RUN tar xzf Python-3.11.3.tgz
# RUN cd Python-3.11.3 && ./configure --enable-optimizations && make altinstall
# ####################################
# # Other OS dependencies
# ####################################
# FROM dependencies-os-python AS dependencies-os
# WORKDIR /app/cmd/passwordcracker
# # install the rest of our dependencies
# RUN apt install python3-pip libssl-dev yara git wamerican libcompress-raw-lzma-perl -y
# # rename the John binary
# RUN cp /john/run/john-avx /john/run/john
# ####################################
# # Python dependencies
# ####################################
# FROM dependencies-os AS dependencies-python
# ARG ENVIRONMENT=dev
# ENV POETRY_HOME=/opt/poetry
# ENV POETRY_VIRTUALENVS_IN_PROJECT=true
# ENV PATH="$POETRY_HOME/bin:$PATH"
# # install Poetry
# RUN python3 -c 'from urllib.request import urlopen; print(urlopen("https://install.python-poetry.org").read().decode())' | python3 -
# ####################################
# # Container specific dependencies
# ####################################
# FROM dependencies-python AS build
# COPY cmd/passwordcracker/poetry.lock cmd/passwordcracker/pyproject.toml ./
# # copy local libraries
# COPY packages/python/nemesispb/ /app/packages/python/nemesispb/
# COPY packages/python/nemesiscommon/ /app/packages/python/nemesiscommon/
# # use Poetry to install the local packages
# RUN poetry install $(if [ "${ENVIRONMENT}" = 'production' ]; then echo "--without dev"; fi;) --no-root --no-interaction --no-ansi -vvv
# COPY cmd/passwordcracker/passwordcracker/ ./passwordcracker/
# ####################################
# # Runtime
# ####################################
# FROM build AS runtime
# ENV PATH="/app/cmd/passwordcracker/.venv/bin:$PATH"
# RUN python3 --version
# ENTRYPOINT ["python3", "-m", "passwordcracker"]
kubernetes/passwordcracker/deployment.yaml
+20
View File
@@ -58,6 +58,26 @@ spec:
secretKeyRef:
name: rabbitmq-creds
key: rabbitmq-connectionuri
- name: POSTGRES_SERVER
value: postgres
- name: POSTGRES_PORT
value: "5432"
- name: POSTGRES_DATABASE
value: nemesis
- name: POSTGRES_USER
valueFrom:
secretKeyRef:
name: postgres-creds
key: postgres-user
- name: POSTGRES_PASSWORD
valueFrom:
secretKeyRef:
name: postgres-creds
key: postgres-password
- name: POSTGRES_CONNECTION_URI
value: "postgresql://$(POSTGRES_USER):$(POSTGRES_PASSWORD)@$(POSTGRES_SERVER):$(POSTGRES_PORT)/$(POSTGRES_DATABASE)"
- name: SLACK_CHANNEL
valueFrom:
configMapKeyRef:
kubernetes/postgres/configmap.yaml
+3 -1
View File
@@ -381,7 +381,9 @@ data:
is_submitted_to_cracker BOOLEAN, -- True if the hash has submitted to a longer-run cracking job
cracker_submission_time TIMESTAMP WITH TIME ZONE, -- Time the hash was submitted to a longer-run cracking job
cracker_cracked_time TIMESTAMP WITH TIME ZONE, -- Time the hash was cracked by a longer-run cracking job
plaintext_value TEXT -- The data value if the hash has been cracked
plaintext_value TEXT, -- The data value if the hash has been cracked
hash_value_md5_hash UUID GENERATED ALWAYS AS (MD5(hash_value)::uuid) STORED, -- used in case the hash value is vvv longboi
UNIQUE (timestamp, originating_object_id, hash_value_md5_hash)
) INHERITS (project_data);