Merge pull request #82 from SpecterOps/nemesis-2.1.3

Nemesis 2.1.3

.github/scripts/generate_api_docs.py

@@ -0,0 +1,238 @@
+#!/usr/bin/env python3
+"""
+Generate simplified API documentation from FastAPI OpenAPI spec.
+This script extracts the OpenAPI specification from the web-api service
+and generates clean markdown documentation in docs/api/
+"""
+
+import json
+import os
+import sys
+from pathlib import Path
+from typing import Any
+
+# Add the web_api module to the path
+sys.path.insert(0, str(Path(__file__).parent.parent / "projects" / "web_api"))
+
+
+def extract_openapi_spec() -> dict[str, Any]:
+    """Extract OpenAPI specification from the FastAPI app."""
+    try:
+        # Set minimal environment variables to avoid database connection issues
+        os.environ.setdefault("POSTGRES_CONNECTION_URI", "postgresql://test:test@localhost/test")
+        os.environ.setdefault("MINIO_ROOT_USER", "test")
+        os.environ.setdefault("MINIO_ROOT_PASSWORD", "test")
+        os.environ.setdefault("MINIO_SERVER", "localhost:9000")
+        os.environ.setdefault("BUCKET_NAME", "test")
+
+        # Mock Dapr dependencies to avoid connection issues
+        import sys
+        from unittest.mock import MagicMock
+
+        # Mock dapr modules before importing web_api.main
+        mock_dapr_clients = MagicMock()
+        mock_dapr_ext_fastapi = MagicMock()
+        sys.modules["dapr.clients"] = mock_dapr_clients
+        sys.modules["dapr.ext.fastapi"] = mock_dapr_ext_fastapi
+
+        # Mock the specific classes that are imported
+        mock_dapr_clients.DaprClient = MagicMock()
+        mock_dapr_ext_fastapi.DaprApp = MagicMock()
+
+        from web_api.main import app
+
+        return app.openapi()
+    except ImportError as e:
+        print(f"Error importing web_api.main: {e}")
+        print("Make sure to run: cd projects/web_api && poetry install")
+        sys.exit(1)
+    except Exception as e:
+        print(f"Error generating OpenAPI spec: {e}")
+        sys.exit(1)
+
+
+def format_parameter(param: dict[str, Any]) -> str:
+    """Format a parameter for documentation."""
+    param_type = param.get("schema", {}).get("type", "unknown")
+    required = "**required**" if param.get("required", False) else "optional"
+    description = param.get("description", "")
+
+    return f"- `{param['name']}` ({param_type}, {required}): {description}"
+
+
+def format_request_body(request_body: dict[str, Any]) -> str:
+    """Format request body information."""
+    if not request_body:
+        return ""
+
+    content = request_body.get("content", {})
+    if "application/json" in content:
+        schema = content["application/json"].get("schema", {})
+        if "$ref" in schema:
+            # Extract model name from reference
+            model_name = schema["$ref"].split("/")[-1]
+            return f"**Request Body:** `{model_name}` (JSON)"
+        elif schema.get("type") == "object":
+            return "**Request Body:** JSON object"
+
+    return "**Request Body:** See OpenAPI spec for details"
+
+
+def generate_endpoint_docs(paths: dict[str, Any]) -> str:
+    """Generate documentation for all endpoints."""
+    docs = []
+
+    # Group endpoints by tags
+    tagged_endpoints = {}
+    untagged_endpoints = []
+
+    for path, methods in paths.items():
+        for method, details in methods.items():
+            if method.upper() not in ["GET", "POST", "PUT", "DELETE", "PATCH"]:
+                continue
+
+            tags = details.get("tags", [])
+            endpoint_info = {"path": path, "method": method.upper(), "details": details}
+
+            if tags:
+                tag = tags[0]  # Use first tag
+                if tag not in tagged_endpoints:
+                    tagged_endpoints[tag] = []
+                tagged_endpoints[tag].append(endpoint_info)
+            else:
+                untagged_endpoints.append(endpoint_info)
+
+    # Generate docs for each tag group
+    for tag, endpoints in sorted(tagged_endpoints.items()):
+        docs.append(f"## {tag.title()}")
+        docs.append("")
+
+        for endpoint in sorted(endpoints, key=lambda x: (x["path"], x["method"])):
+            docs.extend(format_endpoint(endpoint))
+
+        docs.append("")
+
+    # Add untagged endpoints if any
+    if untagged_endpoints:
+        docs.append("## Other Endpoints")
+        docs.append("")
+
+        for endpoint in sorted(untagged_endpoints, key=lambda x: (x["path"], x["method"])):
+            docs.extend(format_endpoint(endpoint))
+
+        docs.append("")
+
+    return "\n".join(docs)
+
+
+def format_endpoint(endpoint: dict[str, Any]) -> list[str]:
+    """Format a single endpoint for documentation."""
+    path = endpoint["path"]
+    method = endpoint["method"]
+    details = endpoint["details"]
+
+    docs = []
+
+    # Endpoint header
+    summary = details.get("summary", "")
+    docs.append(f"### `{method} {path}`")
+    docs.append("")
+
+    if summary:
+        docs.append(summary)
+        docs.append("")
+
+    # Description
+    description = details.get("description", "")
+    if description and description != summary:
+        docs.append(description)
+        docs.append("")
+
+    # Parameters
+    parameters = details.get("parameters", [])
+    if parameters:
+        docs.append("**Parameters:**")
+        docs.append("")
+        for param in parameters:
+            docs.append(format_parameter(param))
+        docs.append("")
+
+    # Request body
+    request_body = details.get("requestBody", {})
+    if request_body:
+        body_docs = format_request_body(request_body)
+        if body_docs:
+            docs.append(body_docs)
+            docs.append("")
+
+    # Response summary (simplified)
+    responses = details.get("responses", {})
+    if responses:
+        success_responses = [code for code in responses.keys() if code.startswith("2")]
+        if success_responses:
+            docs.append(f"**Returns:** {', '.join(success_responses)} on success")
+            docs.append("")
+
+    docs.append("---")
+    docs.append("")
+
+    return docs
+
+
+def generate_markdown_docs(spec: dict[str, Any]) -> str:
+    """Generate complete markdown documentation."""
+    info = spec.get("info", {})
+    title = info.get("title", "API Documentation")
+    version = info.get("version", "Unknown")
+    description = info.get("description", "")
+
+    docs = [
+        f"# {title}",
+        "",
+        f"**Version:** {version}",
+        "",
+    ]
+
+    if description:
+        docs.extend([description, ""])
+
+    docs.extend(["This documentation is automatically generated from the OpenAPI specification.", "", "---", ""])
+
+    # Add endpoints
+    paths = spec.get("paths", {})
+    if paths:
+        docs.append(generate_endpoint_docs(paths))
+
+    return "\n".join(docs)
+
+
+def main():
+    """Main function to generate API documentation."""
+    print("Extracting OpenAPI specification...")
+    spec = extract_openapi_spec()
+
+    print("Generating markdown documentation...")
+    markdown_content = generate_markdown_docs(spec)
+
+    # Ensure docs directory exists (repo root)
+    docs_dir = Path(__file__).parent.parent.parent / "docs"
+    docs_dir.mkdir(parents=True, exist_ok=True)
+
+    # Write main API documentation
+    api_docs_file = docs_dir / "api.md"
+    with open(api_docs_file, "w", encoding="utf-8") as f:
+        f.write(markdown_content)
+
+    print(f"API documentation generated: {api_docs_file}")
+
+    # Also save the raw OpenAPI spec for reference
+    openapi_file = docs_dir / "openapi.json"
+    with open(openapi_file, "w", encoding="utf-8") as f:
+        json.dump(spec, f, indent=2)
+
+    print(f"OpenAPI specification saved: {openapi_file}")
+    print("Documentation generation complete!")
+
+
+if __name__ == "__main__":
+    main()

.github/workflows/docker-build-base.yml

@@ -5,7 +5,6 @@ on:
     branches: [ "main" ]
     paths:
       - 'infra/docker/python_base/**'
-      - 'projects/InspectAssembly/**'
   workflow_dispatch:
 
 env:
@@ -91,29 +90,6 @@ jobs:
           cache-from: type=gha,scope=python-base-prod-${{ matrix.arch }}
           cache-to: type=gha,mode=max,scope=python-base-prod-${{ matrix.arch }}
 
-      # InspectAssembly Base Image
-      - name: Extract metadata for InspectAssembly image
-        id: meta-inspect-assembly
-        uses: docker/metadata-action@v5
-        with:
-          images: ${{ env.REGISTRY }}/${{ env.IMAGE_PREFIX }}/inspect-assembly
-          tags: |
-            type=sha,format=short,suffix=-${{ matrix.arch }}
-            type=ref,event=branch,suffix=-${{ matrix.arch }}
-            type=raw,value=latest-${{ matrix.arch }},enable={{is_default_branch}}
-
-      - name: Build and push InspectAssembly image
-        uses: docker/build-push-action@v5
-        with:
-          context: ./projects/InspectAssembly
-          file: ./projects/InspectAssembly/Dockerfile
-          push: true
-          platforms: ${{ matrix.platform }}
-          tags: ${{ steps.meta-inspect-assembly.outputs.tags }}
-          labels: ${{ steps.meta-inspect-assembly.outputs.labels }}
-          cache-from: type=gha,scope=inspect-assembly-${{ matrix.arch }}
-          cache-to: type=gha,mode=max,scope=inspect-assembly-${{ matrix.arch }}
-
   # Create multi-arch manifests
   create-manifests:
     needs: build-base-images
@@ -175,26 +151,3 @@ jobs:
                 "${tag}-arm64"
             fi
           done
-
-      # InspectAssembly Manifest
-      - name: Extract metadata for InspectAssembly manifest
-        id: meta-inspect-assembly
-        uses: docker/metadata-action@v5
-        with:
-          images: ${{ env.REGISTRY }}/${{ env.IMAGE_PREFIX }}/inspect-assembly
-          tags: |
-            type=sha,format=short
-            type=ref,event=branch
-            type=raw,value=latest,enable={{is_default_branch}}
-
-      - name: Create and push InspectAssembly multi-arch manifest
-        run: |
-          echo '${{ steps.meta-inspect-assembly.outputs.tags }}' | while IFS= read -r tag; do
-            if [ -n "$tag" ]; then
-              echo "Creating manifest for: $tag"
-              docker buildx imagetools create \
-                --tag "$tag" \
-                "${tag}-amd64" \
-                "${tag}-arm64"
-            fi
-          done

.github/workflows/docker-build.yml

@@ -24,36 +24,36 @@ jobs:
       matrix:
         runner: [ubuntu-22.04, ubuntu-22.04-arm]
         service:
-          - name: web-api
+          - name: agents
             context: .
-            dockerfile: ./projects/web_api/Dockerfile
-          - name: dotnet-api
+            dockerfile: ./projects/agents/Dockerfile
+          - name: alerting
             context: .
-            dockerfile: ./projects/dotnet_api/Dockerfile
+            dockerfile: ./projects/alerting/Dockerfile
+          - name: cli
+            context: .
+            dockerfile: ./projects/cli/Dockerfile
+          - name: document-conversion
+            context: .
+            dockerfile: ./projects/document_conversion/Dockerfile
+          - name: dotnet-service
+            context: .
+            dockerfile: ./projects/dotnet_service/Dockerfile
           - name: file-enrichment
             context: .
             dockerfile: ./projects/file_enrichment/Dockerfile
           - name: frontend
             context: ./projects/frontend
             dockerfile: ./projects/frontend/Dockerfile
-          - name: jupyter
-            context: ./projects/jupyter
-            dockerfile: ./projects/jupyter/Dockerfile
-          - name: alerting
-            context: .
-            dockerfile: ./projects/alerting/Dockerfile
-          - name: triage
-            context: .
-            dockerfile: ./projects/triage/Dockerfile
-          - name: cli
-            context: .
-            dockerfile: ./projects/cli/Dockerfile
           - name: housekeeping
             context: .
             dockerfile: ./projects/housekeeping/Dockerfile
-          - name: document-conversion
+          - name: jupyter
+            context: ./projects/jupyter
+            dockerfile: ./projects/jupyter/Dockerfile
+          - name: web-api
             context: .
-            dockerfile: ./projects/document_conversion/Dockerfile
+            dockerfile: ./projects/web_api/Dockerfile
 
     steps:
       - name: Checkout repository

.github/workflows/generate-api-docs.yml

@@ -0,0 +1,46 @@
+name: Generate API Documentation
+
+on:
+  push:
+    branches: [main]
+    paths: 
+      - 'projects/web_api/**'
+      - '.github/workflows/generate-api-docs.yml'
+      - '.github/scripts/generate_api_docs.py'
+  workflow_dispatch: # For manual triggering
+
+jobs:
+  generate-docs:
+    runs-on: ubuntu-22.04
+    
+    permissions:
+      contents: write
+    
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setup Python virtualenv
+        run: |
+          pip install --upgrade pip
+          python -m venv env
+          source env/bin/activate
+          cd projects/web_api
+          pip install poetry
+          poetry install --only main
+
+      - name: Generate API documentation
+        run: |
+          source env/bin/activate
+          cd projects/web_api
+          poetry run python ../../.github/scripts/generate_api_docs.py
+
+      - name: Commit documentation changes
+        uses: stefanzweifel/git-auto-commit-action@v5
+        with:
+          commit_message: 'docs: update API documentation [skip ci]'
+          file_pattern: 'docs/api/*'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -1,4 +1,5 @@
 .DS_STORE
+version.json
 
 # Byte-compiled / optimized / DLL files
 __pycache__/
@@ -175,3 +176,9 @@ cython_debug/
 old/
 
 
+projects/dotnet_service/obj/
+projects/dotnet_service/bin/
+
+projects/jupyter/notebooks/*.csv
+
+projects/frontend/public/env.js

.vscode/nemesis.code-workspace

@@ -12,6 +12,10 @@
             "path": "../infra",
             "name": "infra"
         },
+        {
+            "path": "../libs/chromium",
+            "name": "libs → chromium"
+        },
         {
             "path": "../libs/common",
             "name": "libs → common"
@@ -20,6 +24,18 @@
             "path": "../libs/file_enrichment_modules",
             "name": "libs → file_enrichment_modules",
         },
+        {
+            "path": "../libs/file_linking",
+            "name": "libs → file_linking"
+        },
+        {
+            "path": "../libs/nemesis_dpapi",
+            "name": "libs → nemesis_dpapi"
+        },
+        {
+            "path": "../projects/agents",
+            "name": "projects → agents"
+        },
         {
             "path": "../projects/alerting",
             "name": "projects → alerting"
@@ -56,10 +72,6 @@
             "path": "../projects/noseyparker_scanner",
             "name": "projects → noseyparker_scanner"
         },
-        {
-            "path": "../projects/triage",
-            "name": "projects → triage"
-        },
         {
             "path": "../projects/web_api",
             "name": "projects → web_api"
@@ -72,6 +84,8 @@
     "settings": {
         "files.exclude": {
             "**/__pycache__": true,
+            "**/.benchmarks": true,
+            "**/.ruff_cache": true,
             "**/.DS_Store": true,
             "**/.git": true,
             "**/.hg": true,
@@ -81,16 +95,9 @@
             "**/.venv": true,
             "**/CVS": true,
             "**/Thumbs.db": true,
+            "projects/frontend/node_modules": true,
+            "projects/frontend/dist": true
         },
-        "[python]": {
-            "diffEditor.ignoreTrimWhitespace": false,
-            "editor.wordBasedSuggestions": "off",
-            "editor.formatOnSave": true,
-            "editor.defaultFormatter": "charliermarsh.ruff"
-        },
-        "python.defaultInterpreterPath": "python",
-        // "python.pixiToolPath": "",
-        // "python.pipenvPath": "",
     },
     "extensions": {
         "recommendations": [

.vscode/settings.json

@@ -6,12 +6,12 @@
         "editor.formatOnSave": false
     },
     "[python]": {
-        "editor.formatOnSave": true,
+        // "editor.formatOnSave": true,
         "editor.codeActionsOnSave": {
-            "source.fixAll": "explicit",
-            "source.organizeImports": "explicit"
+            // "source.fixAll": "explicit",
+            // "source.organizeImports": "explicit"
         },
-        "editor.defaultFormatter": "charliermarsh.ruff"
+        // "editor.defaultFormatter": "charliermarsh.ruff"
     },
     "autoDocstring.docstringFormat": "google",
     "editor.formatOnSave": false,

CHANGELOG.md

@@ -4,6 +4,207 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+
+## [2.1.4]
+
+### Added
+
+- Auto-decryption of Chromium and DPAPI related data:
+  - Cookie and Login Data(saved passwords) DPAPI value decryption
+  - Added Chromium UI page to display Cookie/Login Data
+  - CNG/Chromekey file enrichment module (parser + decryptor)
+  - Chromium ABE v3 decryption (via decrypted CNG keys)
+- nemesis_dpapi support library:
+  - Added Postgres support for DPAPI backend
+  - Dapr pubsub integration for DPAPI-related event broadcasting
+  - Added uniqueness and write constraints to prevent duplicate master/backup keys
+  - Can now differentiate between user/system masterkeys
+  - Added docs, examples, tests, and decryption benchmarks
+  - Added support for v3 masterkeys decrypted with backup key
+- New file enrichment modules:
+  - `dpapi_masterkey` - Extracts encrypted masterkeys from user/system DPAPI masterkey files and decrypts them, if possible.
+  - `exif_metadata` file enrichment module for supported image files
+  - Added `cng_file`
+- Added support for async code in Dapr activities used in enrichment modules
+- Findings Page: Modified the Severities filter button to use checkboxes
+- Multi-language Tika OCR support (`TIKA_OCR_LANGUAGES` ENV var, see `compose.yaml`)
+- Text translation agent
+- Retroactive DPAPI decryption:
+  - Chromium Local State files when plaintext masterkeys are submitted/decrypted
+  - Google Chromekeys (CNG file based), including decrypting applicable Local State files
+  - Chromium Cookies/Login Data files
+- File linking:
+  - Enchanced file linking with placeholders in the path that resolve once a matching file is collected.
+  - File Viewer: Added ability to delete file linkings from the FileViewer
+  - File Browser: Added the collection reason and "Linked to by" fields on the "Files that need collection" option
+- CLI: add a `--folder` option to the submit command that allows you to specify the path to the root folder of uploaded files.
+- Reporting functionality
+  - SYSTEM wide and per SOURCE
+  - API endpoints for statistic and PDF generation (via Gotenberg conversion)
+  - Reporting summarization agent
+
+### Changed
+
+- Now use DPAPIck3 for blob decryption
+- Bumped Dapr version to 1.16.1
+- DPAPI_SYSTEM key pulled from registry parsing is now registered with the backend
+- Updated category filters for frontend
+- Collapsed inbound/outbound labels for linked files in dashboard
+- Update Prometheus endpoints
+- Chromium Local State, Cookies, and Login Data files now don't require hard paths
+- Reg hive file linkings now done programmatically instead of via rules
+- SYSTEM masterkey file linkings now done programmatically instead of via rules
+- Added more details to errors that cause a workflow to die
+- Optimized DPAPI masterkey decryption based on the type of masterkey
+- Converted several DB calls to async code
+- Optimized the housekeeping code to run in parallel and use transactions (where possible)
+
+### Fixed
+
+- Properly use entropy for DPAPI blob decryption
+- Lots of async issues
+- Fixed tag search issue
+- Fix to keep strings.txt from sqlite dbs from processing
+- Path normalization: Fixed many bugs, removed lots of duplicate normalization, standardized normalization upon initial ingestion.
+- Countless linting and other fixes
+
+
+## [2.1.3]
+
+### Added
+
+- "Chromium" page in the web frontend to display history, downloads, cookies, logins, and state keys
+  - Includes filtering + CSV downloads for displayed data
+- "File Browser" in the web frontend
+- "DPAPI" viewer/submission pages in the web frontend
+- Linked file tracking (via the ./libs/file_linking/ library)
+  - Linked files exposed in the "File Browser" frontend and file viewer pages
+- Adaptation of @Dreadnode's .NET reversing agent to `agents`
+- "chromium" standard library in ./libs/ for parsing Chromium based files
+- "nemesis_dpapi" library in ./libs/ for handling DPAPI related data/decryption
+  - Includes in-memory storage of keys as well as postgres
+  - Allows subscriptions to react to DPAPI-related events (e.g. new backup key, new plaintext masterkey, etc.)
+- `registry_hive` parsing module that extracts bootkeys + local accounts + lsa secrets from linked hives
+- Refactored file enrichment web API code to be more modular
+- API route to submit DPAPI credential material
+- Auto-building API documents for ./docs/api.md from the FastAPI routes in `web_api` container
+- Documentation for "Containers" and LLM functionality
+- Changed default FileList view to All Files, added unviewed indicator dot, and change search default to always include wildcards.
+
+### Changed
+
+- Standardize logging + fix suppressed logs
+- DPAPI keys carved from LSASS dumps now saved via the nemesis_dpapi library
+
+### Fixed
+
+- Frontend live file reload
+- `certificate` and `keytab` file enrichment modules
+
+
+## [2.1.2] - 2025-08-22
+
+### Added
+
+- Ability to drag/drop folders onto the file upload page
+- basic `triage` container greatly expanded to `agents`
+  - JWT + finding validator agents implemented
+  - Generalized/expandable agent infrastructure built
+  - Confidence score, explanation, and risk detail returned by finding triage
+  - "triage consensus" added for multiple triage values for the same file
+- Tracing for `agents` added with Arize Phoenix (/phoenix, if --monitoring is enabled)
+  - Token costs pulled from LiteLLM instance and manually synced to Phoenix for cost tracking
+- Triage details added to finding table entries and findings modal
+- Settings frontend page now has commit/build date/etc. info and Slack alert channel info (if configured)
+- Repeat option added to submit script
+- Conditionally shown "Agents" page in the frontend that shows current agents and token spend stats
+  - Also allows for editing Agent prompts in the UI
+
+### Changed
+
+- "explanation" field added to the findings_triage_history table in the schema
+- Help page in frontend only shows routes for services that are enabled
+- Logs suppressed during Dapr replay (only show on first run)
+- Removed loud FastAPI tracer
+- Enrichment modules rolled into one activity (no longer each their own) for optimization
+  - Also means a reduction in file download actions for enrichment modules - modules modified to support this
+- LLM credential analysis and text summarization enrichment modules ported to `agents`
+
+### Fixed
+
+- Markdown escape for displayed extracted hashes
+- Fixed PE parsing not throwing an exception
+- Fixed runtime deprecation warning
+- Maintain references to various asyncio tasks
+
+
+## [2.1.1] - 2025-08-01
+
+### Added
+
+- LiteLLM server (with "llm" profile in Docker) to serve future LLM integrations
+  - Includes cost limits
+- Display/linking to originating container for files derived from containers
+- Support for include/exclude filters for the `/containers` API
+  - Added filter support into `cli` container + submit.sh
+- Processing for a number of disk image formats
+- File monitoring for (large) containers copied to MOUNTED_CONTAINER_PATH
+  - Containers have files extracted + processed
+  - Used for workflows with very large containers/disk images
+- Velociraptor connector (server event .yaml option)
+- `noseyparker_scanner` now can scan zips and .git repos
+  - Includes relevant match info in results (can be set by ENV vars)
+
+### Changed
+
+- Expired containers now cleaned up
+- Made "timestamp" and "expiration" submission fields optional (filled with defaults)
+- Bumped Dapr version to 1.15.8
+- Filtering by URL for containers
+- Bulk enrichment system now uses pub/sub
+- `triage` connecter now uses LiteLLM for models via Rigging
+- Pagination in FileList view for large number of files
+
+### Fixed
+
+- "source" field propagation for containers
+- Container filtering in dashboard
+- Arguments with value ordering error in submit.sh
+
+
+## [2.1.0] - 2025-07-20
+
+### Added
+
+- Retries for submit.py
+- `3_workflow_performance.ipynb` Jupyter notebook to assess pipeline performance
+- "source" field (to represent hostname, source site, etc.) integrated into schema + frontend
+- Start of bulk-enrichment re-rerunning, including re-running Yara rules from the dashboard
+- New system for large "container" triaging
+  - New `/api/containers` route
+  - Container process tracking system using pub/sub from `file_enrichment` -> `web_api`
+  - Live updating container tracking in the dashboard (new "Containers" tab)
+  - submit.sh/monitor.sh scripts now can submit "containers"
+- Internal queues now cleaned on up system delete/reset
+- PostgreSQL NOTIFY/LISTEN system for `file_enrichment` workers
+
+### Changed
+
+- Timeouts/improved submit logic for the web_api
+- Bumped Dapr version to 1.15.6
+- Combined `dotnet_api` and `InspectAssembly` into single, streamlined pure .NET `dotnet_service` container
+- Eliminated the internal file-enrichment queue
+  - Now relies on the Dapr pub/sub queue (RabbitMQ) to provide backpressure
+- Stale workflows periodically cleaned up
+
+### Fixed
+
+- Limits/concurrency fixes for NoseyParker scanner to prevent OOM errors
+- Implemented queue/workflow persistence
+  - RabbitMQ queues now restored properly even if containers are completely removed
+  - In-flight workflows re-submitted for processing
+
+
 ## [2.0.1]
 
 ### Changed

compose.base.yaml

@@ -13,13 +13,3 @@ services:
     build:
       context: ./infra/docker/python_base
       dockerfile: prod.Dockerfile
-
-  # Long term, this shouldn't be with the base image.
-  # We should publish the build artifact and use that
-  # in the final image (instead of copying from this one).
-  # Until then. we'll keep it here so we don't have to
-  # rebuild it each time we rebuild just the services.
-  inspect-assembly:
-    build:
-      context: ./projects/InspectAssembly/
-      dockerfile: Dockerfile

compose.override.yaml

@@ -14,6 +14,8 @@ services:
       - ./libs/:/src/libs/
       - ./projects/web_api/web_api:/src/projects/web_api/web_api
       - /src/projects/web_api/.venv
+    environment:
+      - LOG_LEVEL=${LOG_LEVEL:-DEBUG}
 
   noseyparker-scanner:
     image: !reset null
@@ -24,16 +26,14 @@ services:
     environment:
       - RUST_LOG=debug
 
-  dotnet-api:
+  dotnet-service:
     image: !reset null
     build:
       context: .
-      dockerfile: ./projects/dotnet_api/Dockerfile
+      dockerfile: ./projects/dotnet_service/Dockerfile
       target: dev
-    volumes:
-      - ./libs/:/src/libs/
-      - ./projects/dotnet_api/dotnet_api:/src/projects/dotnet_api/dotnet_api
-      - /src/projects/dotnet_api/.venv
+    environment:
+      - ASPNETCORE_ENVIRONMENT=Development
 
   file-enrichment:
     image: !reset null
@@ -42,7 +42,7 @@ services:
       dockerfile: ./projects/file_enrichment/Dockerfile
       target: dev
     environment:
-      - LOG_LEVEL=DEBUG
+      - LOG_LEVEL=${LOG_LEVEL:-DEBUG}
     volumes:
       - ./libs/:/src/libs/
       - ./projects/file_enrichment/file_enrichment:/src/projects/file_enrichment/file_enrichment
@@ -57,19 +57,19 @@ services:
       target: dev
     command: >
       sh -c "
-        mkdir -p /app-runtime &&
-        cp -r /app/* /app-runtime/ &&
+        if [ -f /version.json ]; then cp /version.json /app/public/version.json; fi &&
         SECRET=\"$$HASURA_ADMIN_SECRET\" &&
-        sed -i \"s/\\$$HASURA_ADMIN_SECRET/$$SECRET/g\" /app-runtime/index.html &&
-        cd /app-runtime &&
+        echo \"window.ENV = { HASURA_ADMIN_SECRET: '$$SECRET' };\" > /app/public/env.js &&
+        cd /app &&
         npm run dev
       "
     volumes:
       - ./projects/frontend/index.html:/app/index.html:ro
       - ./projects/frontend/package.json:/app/package.json:ro
-      - ./projects/frontend/public:/app/public:ro
+      - ./projects/frontend/public:/app/public
       - ./projects/frontend/src:/app/src:ro
       - ./projects/frontend/vite.config.js:/app/vite.config.js:ro
+      - ./version.json:/version.json:ro
     labels:
       - "traefik.http.services.frontend.loadbalancer.server.port=3000"
 
@@ -88,13 +88,21 @@ services:
       context: .
       dockerfile: ./projects/alerting/Dockerfile
       target: dev
+    environment:
+      - LOG_LEVEL=${LOG_LEVEL:-DEBUG}
 
-  triage:
+  agents:
     image: !reset null
     build:
       context: .
-      dockerfile: ./projects/triage/Dockerfile
+      dockerfile: ./projects/agents/Dockerfile
       target: dev
+    environment:
+      - LOG_LEVEL=${LOG_LEVEL:-DEBUG}
+    volumes:
+      - ./libs/:/src/libs/
+      - ./projects/agents/agents:/src/projects/agents/agents
+      - /src/projects/agents/.venv
 
   housekeeping:
     image: !reset null
@@ -108,6 +116,7 @@ services:
       - /src/projects/housekeeping/.venv
     environment:
       - CLEANUP_SCHEDULE=*/3 * * * * # Test every 3 minutes
+      - LOG_LEVEL=${LOG_LEVEL:-DEBUG}
 
   document-conversion:
     image: !reset null
@@ -115,7 +124,16 @@ services:
       context: .
       dockerfile: ./projects/document_conversion/Dockerfile
       target: dev
+      args:
+        - TIKA_OCR_LANGUAGES=${TIKA_OCR_LANGUAGES:-eng}
     volumes:
       - ./libs/:/src/libs/
       - ./projects/document_conversion/document_conversion:/src/projects/document_conversion/document_conversion
-      - /src/projects/document_conversion/.venv
+      - /src/projects/document_conversion/.venv
+    environment:
+      - LOG_LEVEL=${LOG_LEVEL:-DEBUG}
+
+  postgres:
+    # expose the port locally
+    ports:
+      - "5432:5432"

compose.prod.build.yaml

@@ -17,11 +17,11 @@ services:
       dockerfile: ./projects/noseyparker_scanner/Dockerfile
       target: prod
 
-  dotnet-api:
+  dotnet-service:
     image: !reset null
     build:
       context: .
-      dockerfile: ./projects/dotnet_api/Dockerfile
+      dockerfile: ./projects/dotnet_service/Dockerfile
       target: prod
 
   file-enrichment:
@@ -59,11 +59,11 @@ services:
       dockerfile: ./projects/alerting/Dockerfile
       target: prod
 
-  triage:
+  agents:
     image: !reset null
     build:
       context: .
-      dockerfile: ./projects/triage/Dockerfile
+      dockerfile: ./projects/agents/Dockerfile
       target: prod
 
   housekeeping:

compose.yaml

@@ -9,10 +9,12 @@ volumes:
   jaeger_data:
   loki_data:
   minio_data:
+  phoenix_data:
   postgres_data:
   prometheus_data:
   rabbitmq_data:
   empty:
+  empty-mounted-containers:
 
 services:
   ########################################
@@ -30,6 +32,9 @@ services:
       - APP_ID=web-api
       - DAPR_GRPC_PORT=50001
       - DAPR_HTTP_PORT=3500
+      - DEFAULT_EXPIRATION_DAYS=${DEFAULT_EXPIRATION_DAYS:-100}
+    volumes:
+      - ${MOUNTED_CONTAINER_PATH:-empty-mounted-containers}:/mounted-containers
     logging: &logging-config
       driver: "json-file"
       options: { max-size: "10m", max-file: "3" }
@@ -45,14 +50,14 @@ services:
       - "traefik.http.routers.web-api.tls=true"
       - "traefik.http.routers.web-api.middlewares=auth"
   web-api-dapr:
-    image: "daprio/daprd:1.15.5"
+    image: "daprio/daprd:1.16.1"
     command:
       [
         "./daprd",
         "--app-id",
         "web-api",
         "--max-body-size",
-        "300Mi",
+        "1Gi",
         "--app-port",
         "8000",
         "--dapr-http-port",
@@ -67,6 +72,9 @@ services:
         "/dapr/components",
         "--config",
         "/dapr/configuration/config.yaml",
+        "--enable-metrics",
+        "--dapr-graceful-shutdown-seconds",
+        "5",
       ]
     volumes:
       - ./infra/dapr/components/pubsub.yaml:/dapr/components/pubsub.yaml:ro
@@ -76,6 +84,14 @@ services:
       - MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD:?}
       - MINIO_ROOT_USER=${MINIO_ROOT_USER:?}
       - RABBITMQ_CONNECTION_STRING=amqp://${RABBITMQ_USER}:${RABBITMQ_PASSWORD}@rabbitmq:5672
+      - POSTGRES_USER=${POSTGRES_USER:?}
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:?}
+      - POSTGRES_HOST=${POSTGRES_HOST:-postgres}
+      - POSTGRES_PORT=${POSTGRES_PORT:-5432}
+      - POSTGRES_DB=${POSTGRES_DB:-enrichment}
+      - POSTGRES_PARAMETERS=${POSTGRES_PARAMETERS:-sslmode=disable}
+      - RABBITMQ_PASSWORD=${RABBITMQ_PASSWORD:?}
+      - RABBITMQ_USER=${RABBITMQ_USER:?}
     depends_on:
       web-api: { condition: service_started }
       placement: { condition: service_started }
@@ -96,16 +112,20 @@ services:
       - MINIO_ENDPOINT=http://minio:9000
       - MINIO_SECRET_KEY=${MINIO_ROOT_PASSWORD:?}
       - RUST_LOG=info
-      - SNIPPET_LENGTH=512
+      - SNIPPET_LENGTH=512        # context length around any Nosey Parker matches
+      - MAX_CONCURRENT_FILES=2    # maximum number of concurrent files to scan
+      - MAX_FILE_SIZE_MB=200      # maximum file size to scan
+      - DECOMPRESS_ZIPS=true      # whether to decompress+scan zips
+      - MAX_EXTRACT_SIZE_MB=1000  # maximum number of bytes to extract if decompressing
     volumes:
       - ./projects/noseyparker_scanner/custom_rules/:/opt/noseyparker:ro
   noseyparker-scanner-dapr:
-    image: "daprio/daprd:1.15.5"
+    image: "daprio/daprd:1.16.1"
     command:
       [
         "./daprd",
         "--max-body-size",
-        "300Mi",
+        "1Gi",
         "--app-id",
         "noseyparker-scanner",
         "--app-port",
@@ -124,6 +144,9 @@ services:
         "/dapr/configuration/config.yaml",
         "--app-max-concurrency",
         "1",
+        "--enable-metrics",
+        "--dapr-graceful-shutdown-seconds",
+        "5",
       ]
     environment:
       - RABBITMQ_CONNECTION_STRING=amqp://${RABBITMQ_USER}:${RABBITMQ_PASSWORD}@rabbitmq:5672
@@ -134,60 +157,64 @@ services:
     depends_on: [noseyparker-scanner, placement]
     network_mode: "service:noseyparker-scanner"
 
-  dotnet-api:
-    image: ghcr.io/specterops/nemesis/dotnet-api:latest
+  dotnet-service:
+    image: ghcr.io/specterops/nemesis/dotnet-service:latest
     healthcheck:
-      test: ["CMD", "wget", "--spider", "-q", "http://localhost:1337/healthz"]
-      interval: 10s
-      timeout: 5s
-      retries: 5
-    environment:
-      - APP_ID=dotnet-api
-      - DAPR_GRPC_PORT=50010
-      - DAPR_HTTP_PORT=3507
-    logging: *logging-config
+      test: ["CMD", "curl", "-f", "http://localhost:5000/health"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 30s
     depends_on:
-      postgres: { condition: service_started }
-      placement: { condition: service_started }
+      minio: { condition: service_healthy }
       rabbitmq: { condition: service_healthy }
-  dotnet-api-dapr:
-    image: "daprio/daprd:1.15.5"
+    environment:
+      - DAPR_GRPC_PORT=50014
+      - DAPR_PORT=3514
+      - MINIO_ACCESS_KEY=${MINIO_ROOT_USER:?}
+      - MINIO_BUCKET=files
+      - MINIO_ENDPOINT=http://minio:9000
+      - MINIO_SECRET_KEY=${MINIO_ROOT_PASSWORD:?}
+      - ASPNETCORE_URLS=http://0.0.0.0:5000
+    expose:
+      - "5000"
+  dotnet-service-dapr:
+    image: "daprio/daprd:1.16.1"
     command:
       [
         "./daprd",
         "--max-body-size",
-        "300Mi",
+        "1Gi",
         "--app-id",
-        "dotnet-api",
+        "dotnet-service",
         "--app-port",
-        "1337",
+        "5000",
+        "--app-protocol",
+        "http",
         "--dapr-http-port",
-        "3507",
+        "3514",
         "--dapr-grpc-port",
-        "50010",
+        "50014",
         "--placement-host-address",
         "placement:50006",
-        "--scheduler-host-address",
-        "scheduler:50007",
         "--resources-path",
         "/dapr/components",
         "--config",
         "/dapr/configuration/config.yaml",
+        "--app-max-concurrency",
+        "1",
+        "--enable-metrics",
+        "--dapr-graceful-shutdown-seconds",
+        "5",
       ]
+    environment:
+      - RABBITMQ_CONNECTION_STRING=amqp://${RABBITMQ_USER}:${RABBITMQ_PASSWORD}@rabbitmq:5672
     volumes:
+      - ./infra/dapr/components/pubsub.yaml:/dapr/components/pubsub.yaml:ro
       - ./infra/dapr/components/secretstore.yaml:/dapr/components/secretstore.yaml:ro
       - ./infra/dapr/configuration/config_monitoring_${NEMESIS_MONITORING:-disabled}.yaml:/dapr/configuration/config.yaml:ro
-    environment:
-      - MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD:?}
-      - MINIO_ROOT_USER=${MINIO_ROOT_USER:?}
-      - POSTGRES_CONNECTION_STRING=host=postgres user=${POSTGRES_USER} password=${POSTGRES_PASSWORD} dbname=enrichment port=5432 sslmode=disable
-      - RABBITMQ_CONNECTION_STRING=amqp://${RABBITMQ_USER}:${RABBITMQ_PASSWORD}@rabbitmq:5672
-    depends_on:
-      dotnet-api: { condition: service_started }
-      placement: { condition: service_started }
-      scheduler: { condition: service_started }
-      rabbitmq: { condition: service_healthy }
-    network_mode: "service:dotnet-api"
+    depends_on: [dotnet-service, placement]
+    network_mode: "service:dotnet-service"
 
   file-enrichment:
     image: ghcr.io/specterops/nemesis/file-enrichment:latest
@@ -203,30 +230,31 @@ services:
       - APP_ID=file-enrichment
       - DAPR_GRPC_PORT=50003
       - DAPR_HTTP_PORT=3503
-      - LOG_LEVEL=INFO
-      - MAX_PARALLEL_ENRICHMENT_MODULES=${MAX_PARALLEL_ENRICHMENT_MODULES:-5}
-      - MAX_PARALLEL_WORKFLOWS=${MAX_PARALLEL_WORKFLOWS:-5}
+      - LOG_LEVEL=${LOG_LEVEL:-INFO}
       - MAX_WORKFLOW_EXECUTION_TIME=${MAX_WORKFLOW_EXECUTION_TIME:-300}
       - NEMESIS_MONITORING=${NEMESIS_MONITORING:-disabled}
       - NEMESIS_URL=${NEMESIS_URL:?}
       - OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://otel-collector:4317
       - OTEL_EXPORTER_OTLP_TRACES_ENDPOINT_INSECURE=true
+      - PYTHONASYNCIOTHREADPOOLSIZE=100  # Increase thread pool for asyncio.to_thread()
       - RIGGING_GENERATOR_CREDENTIALS=${RIGGING_GENERATOR_CREDENTIALS:-}
       - RIGGING_GENERATOR_SUMMARY=${RIGGING_GENERATOR_SUMMARY:-}
       - RIGGING_GENERATOR_TRIAGE=${RIGGING_GENERATOR_TRIAGE:-}
-      - UVICORN_WORKERS=${UVICORN_WORKERS:-2}
+      - UVICORN_WORKERS=1  # Reduce to 1 worker to avoid cross-process contention
+      - WORKFLOW_RUNTIME_LOG_LEVEL=${WORKFLOW_RUNTIME_LOG_LEVEL:-WARNING}
+      - WORKFLOW_CLIENT_LOG_LEVEL=${WORKFLOW_CLIENT_LOG_LEVEL:-WARNING}
     logging: *logging-config
     depends_on:
       postgres: { condition: service_started }
       placement: { condition: service_started }
       rabbitmq: { condition: service_healthy }
   file-enrichment-dapr:
-    image: "daprio/daprd:1.15.5"
+    image: "daprio/daprd:1.16.1"
     command:
       [
         "./daprd",
         "--max-body-size",
-        "300Mi",
+        "1Gi",
         "--app-id",
         "file-enrichment",
         "--app-port",
@@ -243,6 +271,9 @@ services:
         "/dapr/components",
         "--config",
         "/dapr/configuration/config.yaml",
+        "--enable-metrics",
+        "--dapr-graceful-shutdown-seconds",
+        "5",
       ]
     volumes:
       - ./infra/dapr/components/pubsub.yaml:/dapr/components/pubsub.yaml:ro
@@ -250,7 +281,12 @@ services:
       - ./infra/dapr/components/workflowstate.yaml:/dapr/components/workflowstate.yaml:ro
       - ./infra/dapr/configuration/config_monitoring_${NEMESIS_MONITORING:-disabled}.yaml:/dapr/configuration/config.yaml:ro
     environment:
-      - POSTGRES_CONNECTION_STRING=host=postgres user=${POSTGRES_USER} password=${POSTGRES_PASSWORD} dbname=enrichment port=5432 sslmode=disable
+      - POSTGRES_USER=${POSTGRES_USER:?}
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:?}
+      - POSTGRES_HOST=${POSTGRES_HOST:-postgres}
+      - POSTGRES_PORT=${POSTGRES_PORT:-5432}
+      - POSTGRES_DB=${POSTGRES_DB:-enrichment}
+      - POSTGRES_PARAMETERS=${POSTGRES_PARAMETERS:-sslmode=disable}
       - RABBITMQ_CONNECTION_STRING=amqp://${RABBITMQ_USER}:${RABBITMQ_PASSWORD}@rabbitmq:5672
       - MINIO_ROOT_USER=${MINIO_ROOT_USER:?}
       - MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD:?}
@@ -296,7 +332,7 @@ services:
     volumes:
     - "./projects/jupyter/notebooks:/home/jovyan/work"
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:8888/jupyter/"]
+      test: ["CMD", "curl", "-f", "http://localhost:8888/jupyter/api"]
       interval: 30s
       timeout: 10s
       retries: 3
@@ -328,12 +364,12 @@ services:
       retries: 5
       start_period: 20s
   alerting-dapr:
-    image: "daprio/daprd:1.15.5"
+    image: "daprio/daprd:1.16.1"
     command:
       [
         "./daprd",
         "--max-body-size",
-        "300Mi",
+        "1Gi",
         "--app-id",
         "alerting",
         "--app-port",
@@ -350,6 +386,9 @@ services:
         "/dapr/components",
         "--config",
         "/dapr/configuration/config.yaml",
+        "--enable-metrics",
+        "--dapr-graceful-shutdown-seconds",
+        "5",
       ]
     volumes:
       - ./infra/dapr/components/pubsub.yaml:/dapr/components/pubsub.yaml:ro
@@ -365,13 +404,28 @@ services:
       rabbitmq: { condition: service_healthy }
     network_mode: "service:alerting"
 
-  triage:
-    image: ghcr.io/specterops/nemesis/triage:latest
+  agents:
+    image: ghcr.io/specterops/nemesis/agents:latest
     environment:
       - DAPR_HTTP_PORT=3509
       - DAPR_GRPC_PORT=50009
       - NEMESIS_URL=${NEMESIS_URL:?}
-      - RIGGING_GENERATOR_TRIAGE=${RIGGING_GENERATOR_TRIAGE:-}
+      - WORKFLOW_RUNTIME_LOG_LEVEL=${WORKFLOW_RUNTIME_LOG_LEVEL:-WARNING}
+      - WORKFLOW_CLIENT_LOG_LEVEL=${WORKFLOW_CLIENT_LOG_LEVEL:-WARNING}
+      - LITELLM_ADMIN_KEY=sk-${LITELLM_ADMIN_PASSWORD:-admin123}
+      - MAX_BUDGET=${LLM_MAX_BUDGET:-100}
+      - BUDGET_DURATION=${LLM_BUDGET_DURATION:-30d}
+      - DOTNET_ANALYSIS_RUN_REQUEST_LIMIT=${DOTNET_ANALYSIS_RUN_REQUEST_LIMIT:-25}    # max LLM calls to use per .NET program analysis run
+      - DOTNET_ANALYSIS_RUN_TOKENS_LIMIT=${DOTNET_ANALYSIS_RUN_TOKENS_LIMIT:-1000000} # max tokens to use per .NET program analysis run
+      # if we hit this number of the same triage values for the same file, all future findings get that value
+      - TRIAGE_CONSENSUS_THRESHOLD=${TRIAGE_CONSENSUS_THRESHOLD:-3}
+      # Phoenix LLM tracing configuration
+      - PHOENIX_ENABLED=${PHOENIX_ENABLED:-false}
+      - PHOENIX_ENDPOINT=${PHOENIX_ENDPOINT:-http://phoenix:6006/v1/traces}
+      - PHOENIX_SQL_DATABASE_URL=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST:-postgres}:${POSTGRES_PORT:-5432}/phoenix
+      - NEMESIS_MONITORING=${NEMESIS_MONITORING:-disabled}
+      - OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://otel-collector:4317
+      - OTEL_EXPORTER_OTLP_TRACES_ENDPOINT_INSECURE=true
     logging: *logging-config
     depends_on:
       postgres: { condition: service_healthy }
@@ -379,15 +433,15 @@ services:
       rabbitmq: { condition: service_healthy }
       hasura: { condition: service_healthy }
     healthcheck: *healthcheck-python-svc
-  triage-dapr:
-    image: "daprio/daprd:1.15.5"
+  agents-dapr:
+    image: "daprio/daprd:1.16.1"
     command:
       [
         "./daprd",
         "--max-body-size",
-        "300Mi",
+        "1Gi",
         "--app-id",
-        "triage",
+        "agents",
         "--app-port",
         "8000",
         "--dapr-http-port",
@@ -402,19 +456,31 @@ services:
         "/dapr/components",
         "--config",
         "/dapr/configuration/config.yaml",
+        "--enable-metrics",
+        "--dapr-graceful-shutdown-seconds",
+        "5",
       ]
     volumes:
       - ./infra/dapr/components/secretstore.yaml:/dapr/components/secretstore.yaml:ro
+      - ./infra/dapr/components/workflowstate.yaml:/dapr/components/workflowstate.yaml:ro
       - ./infra/dapr/configuration/config_monitoring_${NEMESIS_MONITORING:-disabled}.yaml:/dapr/configuration/config.yaml:ro
     environment:
+      - POSTGRES_USER=${POSTGRES_USER:?}
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:?}
+      - POSTGRES_HOST=${POSTGRES_HOST:-postgres}
+      - POSTGRES_PORT=${POSTGRES_PORT:-5432}
+      - POSTGRES_DB=${POSTGRES_DB:-enrichment}
+      - POSTGRES_PARAMETERS=${POSTGRES_PARAMETERS:-sslmode=disable}
       - HASURA_ADMIN_SECRET=${HASURA_ADMIN_SECRET:-pass456}
       - RABBITMQ_CONNECTION_STRING=amqp://${RABBITMQ_USER}:${RABBITMQ_PASSWORD}@rabbitmq:5672
+      - MINIO_ROOT_USER=${MINIO_ROOT_USER:?}
+      - MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD:?}
     depends_on:
-      triage: { condition: service_started }
+      agents: { condition: service_started }
       placement: { condition: service_started }
       scheduler: { condition: service_started }
       rabbitmq: { condition: service_healthy }
-    network_mode: "service:triage"
+    network_mode: "service:agents"
 
   housekeeping:
     image: ghcr.io/specterops/nemesis/housekeeping:latest
@@ -430,12 +496,12 @@ services:
       rabbitmq: { condition: service_healthy }
     healthcheck: *healthcheck-python-svc
   housekeeping-dapr:
-    image: "daprio/daprd:1.15.5"
+    image: "daprio/daprd:1.16.1"
     command:
       [
         "./daprd",
         "--max-body-size",
-        "300Mi",
+        "1Gi",
         "--app-id",
         "housekeeping",
         "--app-port",
@@ -452,6 +518,9 @@ services:
         "/dapr/components",
         "--config",
         "/dapr/configuration/config.yaml",
+        "--enable-metrics",
+        "--dapr-graceful-shutdown-seconds",
+        "5",
       ]
     volumes:
       - ./infra/dapr/components/secretstore.yaml:/dapr/components/secretstore.yaml:ro
@@ -459,7 +528,12 @@ services:
     environment:
       - MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD:?}
       - MINIO_ROOT_USER=${MINIO_ROOT_USER:?}
-      - POSTGRES_CONNECTION_STRING=host=postgres user=${POSTGRES_USER} password=${POSTGRES_PASSWORD} dbname=enrichment port=5432 sslmode=disable
+      - POSTGRES_USER=${POSTGRES_USER:?}
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:?}
+      - POSTGRES_HOST=${POSTGRES_HOST:-postgres}
+      - POSTGRES_PORT=${POSTGRES_PORT:-5432}
+      - POSTGRES_DB=${POSTGRES_DB:-enrichment}
+      - POSTGRES_PARAMETERS=${POSTGRES_PARAMETERS:-sslmode=disable}
     depends_on:
       housekeeping: { condition: service_started }
       placement: { condition: service_started }
@@ -476,6 +550,14 @@ services:
       - DAPR_GRPC_PORT=50002
       - DAPR_HTTP_PORT=3501
       - TIKA_CONFIG=/tika-config.xml
+      # If you want to have additional language packs supported (see https://github.com/tesseract-ocr/tessdata for a full list):
+      #   $ export TIKA_OCR_LANGUAGES="eng chi_sim chi_tra jpn rus deu spa"
+      # To change the default:
+      #   - TIKA_OCR_LANGUAGES=${TIKA_OCR_LANGUAGES:-eng chi_sim chi_tra jpn rus deu spa}
+      # Note: each package installed will increase the image size!
+      - TIKA_OCR_LANGUAGES=${TIKA_OCR_LANGUAGES:-eng}
+      - MAX_PARALLEL_WORKFLOWS=${MAX_PARALLEL_WORKFLOWS:-5}
+      - MAX_WORKFLOW_EXECUTION_TIME=${MAX_WORKFLOW_EXECUTION_TIME:-300}
     healthcheck:
       test: ["CMD", "wget", "--spider", "-q", "http://localhost:8000/healthz"]
       interval: 10s
@@ -487,12 +569,12 @@ services:
       placement: { condition: service_started }
       rabbitmq: { condition: service_healthy }
   document-conversion-dapr:
-    image: "daprio/daprd:1.15.5"
+    image: "daprio/daprd:1.16.1"
     command:
       [
         "./daprd",
         "--max-body-size",
-        "300Mi",
+        "1Gi",
         "--app-id",
         "document-conversion",
         "--app-port",
@@ -509,6 +591,9 @@ services:
         "/dapr/components",
         "--config",
         "/dapr/configuration/config.yaml",
+        "--enable-metrics",
+        "--dapr-graceful-shutdown-seconds",
+        "5",
       ]
     volumes:
       - ./infra/dapr/components/pubsub.yaml:/dapr/components/pubsub.yaml:ro
@@ -518,7 +603,12 @@ services:
     environment:
       - MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD:?}
       - MINIO_ROOT_USER=${MINIO_ROOT_USER:?}
-      - POSTGRES_CONNECTION_STRING=host=postgres user=${POSTGRES_USER} password=${POSTGRES_PASSWORD} dbname=enrichment port=5432 sslmode=disable
+      - POSTGRES_USER=${POSTGRES_USER:?}
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:?}
+      - POSTGRES_HOST=${POSTGRES_HOST:-postgres}
+      - POSTGRES_PORT=${POSTGRES_PORT:-5432}
+      - POSTGRES_DB=${POSTGRES_DB:-enrichment}
+      - POSTGRES_PARAMETERS=${POSTGRES_PARAMETERS:-sslmode=disable}
       - RABBITMQ_CONNECTION_STRING=amqp://${RABBITMQ_USER}:${RABBITMQ_PASSWORD}@rabbitmq:5672
     depends_on:
       document-conversion: { condition: service_started }
@@ -535,6 +625,7 @@ services:
         "--api-timeout=180s",
         "--libreoffice-restart-after=5",
         "--libreoffice-auto-start=true",
+        "--prometheus-collect-interval=10s",
       ]
     environment: { DISABLE_GOOGLE_CHROME: "1" }
     healthcheck:
@@ -543,12 +634,12 @@ services:
       timeout: 10s
       retries: 3
   gotenberg-dapr:
-    image: "daprio/daprd:1.15.5"
+    image: "daprio/daprd:1.16.1"
     command:
       [
         "./daprd",
         "--max-body-size",
-        "300Mi",
+        "1Gi",
         "--app-id",
         "gotenberg",
         "--app-port",
@@ -565,6 +656,9 @@ services:
         "/dapr/components",
         "--config",
         "/dapr/configuration/config.yaml",
+        "--enable-metrics",
+        "--dapr-graceful-shutdown-seconds",
+        "5",
       ]
     volumes:
       - ./infra/dapr/configuration/config_monitoring_${NEMESIS_MONITORING:-disabled}.yaml:/dapr/configuration/config.yaml:ro
@@ -605,18 +699,21 @@ services:
       [
         "/bin/sh",
         "-c",
-        "sleep 1; until mc alias set minio http://minio:9000 $${MINIO_ROOT_USER} $${MINIO_ROOT_PASSWORD}; do echo 'Minio is not online. Waiting for it to start...'; sleep 2; done && mc mb minio/loki-data --ignore-existing",
+        "sleep 1; until mc alias set minio http://minio:9000 $${MINIO_ROOT_USER} $${MINIO_ROOT_PASSWORD}; do echo 'Minio is not online. Waiting for it to start...'; sleep 2; done && mc mb minio/loki-data --ignore-existing && mc mb minio/files --ignore-existing",
       ]
     restart: "no"
 
   rabbitmq:
-    image: rabbitmq:4.1.1-management
+    image: rabbitmq:4.1.2-management
+    hostname: rabbitmq-node # have to do this for persistence reasons
     environment:
       - RABBITMQ_DEFAULT_PASS=${RABBITMQ_PASSWORD:?}
       - RABBITMQ_DEFAULT_USER=${RABBITMQ_USER:?}
       - RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS=-rabbitmq_management path_prefix "/rabbitmq"
+      - RABBITMQ_NODENAME=rabbit@rabbitmq-node # have to do this for persistence reasons
     volumes:
-    - rabbitmq_data:/var/lib/rabbitmq
+      - rabbitmq_data:/var/lib/rabbitmq
+      - ./infra/rabbitmq/enabled_plugins:/etc/rabbitmq/enabled_plugins:ro
     healthcheck:
       test: ["CMD", "rabbitmq-diagnostics", "check_port_connectivity"]
       interval: 10s
@@ -630,7 +727,12 @@ services:
       - "traefik.http.routers.rabbitmq-ui.rule=PathPrefix(`/rabbitmq`)"
 
   postgres:
-    image: postgres:17.5-alpine
+    image: postgres:17.6-alpine
+    command: [
+      "postgres",
+      "-c", "max_connections=200",
+      "-c", "shared_buffers=256MB"
+    ]
     environment:
       POSTGRES_DB: ${POSTGRES_DB:-enrichment}
       POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?}
@@ -644,20 +746,30 @@ services:
       timeout: 5s
       retries: 5
 
+  postgres-exporter:
+    profiles: ["monitoring"]
+    image: prometheuscommunity/postgres-exporter:latest
+    environment:
+      DATA_SOURCE_NAME: "postgresql://${POSTGRES_USER:?}:${POSTGRES_PASSWORD:?}@${POSTGRES_HOST:-postgres}:${POSTGRES_PORT:-5432}/${POSTGRES_DB:-enrichment}?${POSTGRES_PARAMETERS:-sslmode=disable}"
+    volumes:
+      - ./infra/postgres-exporter/postgres_exporter.yml:/postgres_exporter.yml:ro
+    depends_on:
+      postgres: { condition: service_healthy }
+
   hasura:
-    image: hasura/graphql-engine:v2.45.1.cli-migrations-v2
+    image: hasura/graphql-engine:v2.48.6.cli-migrations-v2
     depends_on:
       postgres: { condition: service_healthy }
     environment:
       HASURA_GRAPHQL_ADMIN_SECRET: "${HASURA_ADMIN_SECRET:-pass456}"
       HASURA_GRAPHQL_BASE_PATH: "/hasura"
-      HASURA_GRAPHQL_DATABASE_URL: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/enrichment
+      HASURA_GRAPHQL_DATABASE_URL: postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST:-postgres}:${POSTGRES_PORT:-5432}/${POSTGRES_DB:-enrichment}
       HASURA_GRAPHQL_ENABLE_CONSOLE: "true"
       HASURA_GRAPHQL_ENABLE_METADATA_SYNC: "true"
       HASURA_GRAPHQL_ENABLE_TELEMETRY: "false"
       HASURA_GRAPHQL_ENABLED_LOG_TYPES: startup, http-log, webhook-log, websocket-log, query-log
       HASURA_GRAPHQL_LOG_LEVEL: "warn"
-      HASURA_GRAPHQL_METADATA_DATABASE_URL: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/enrichment
+      HASURA_GRAPHQL_METADATA_DATABASE_URL: postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST:-postgres}:${POSTGRES_PORT:-5432}/${POSTGRES_DB:-enrichment}
       HASURA_GRAPHQL_METADATA_DIR: /hasura-metadata
       HASURA_GRAPHQL_UNAUTHORIZED_ROLE: anonymous
     volumes:
@@ -672,7 +784,7 @@ services:
       - "traefik.http.routers.hasura.tls=true"
 
   traefik:
-    image: traefik:v3.3.2
+    image: traefik:v3.4.4
     command:
       - "--api.insecure=true"
       # - "--log.level=DEBUG"
@@ -700,18 +812,20 @@ services:
       - ./auth:/auth:ro
       - ./infra/traefik/certs:/certs:ro
       - ./infra/traefik/config:/config:ro
+
   placement:
-    image: "daprio/dapr:1.15.5"
-    command: ["./placement", "-port", "50006"]
+    image: "daprio/dapr:1.16.1"
+    command: ["./placement", "-port", "50006", "--enable-metrics"]
 
   scheduler:
-    image: "daprio/dapr:1.15.5"
+    image: "daprio/dapr:1.16.1"
     command:
       [
         "./scheduler",
         "--port",
         "50007",
         "--etcd-data-dir=/var/lock/dapr/scheduler",
+        "--enable-metrics",
       ]
 
   ############################
@@ -750,6 +864,35 @@ services:
       retries: 3
       start_period: 30s
 
+  phoenix:
+    profiles: ["monitoring"]
+    image: arizephoenix/phoenix:11.24.1
+    environment:
+      - PHOENIX_WORKING_DIR=/data
+      - PHOENIX_HOST_ROOT_PATH=/phoenix
+      - PHOENIX_SQL_DATABASE_URL=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST:-postgres}:${POSTGRES_PORT:-5432}/phoenix
+      - PHOENIX_DEFAULT_RETENTION_POLICY_DAYS=100
+    volumes:
+      - phoenix_data:/data
+    # ports:
+    #   - "6006:6006"  # Phoenix UI port, for debugging
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.services.phoenix.loadbalancer.server.port=6006"
+      - "traefik.http.routers.phoenix.rule=PathPrefix(`/phoenix`)"
+      - "traefik.http.routers.phoenix.entrypoints=websecure"
+      - "traefik.http.routers.phoenix.tls=true"
+      - "traefik.http.routers.phoenix.middlewares=auth,phoenix-stripprefix"
+      - "traefik.http.middlewares.phoenix-stripprefix.stripprefix.prefixes=/phoenix"
+    depends_on:
+      postgres: { condition: service_healthy }
+    healthcheck:
+      test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:6006/healthz').read()"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 30s
+
   loki:
     profiles: ["monitoring"]
     image: grafana/loki:3.3.2
@@ -793,6 +936,7 @@ services:
       - GF_SERVER_DOMAIN=${EXTERNAL_HOST:-https://localhost}
       - GF_SERVER_ROOT_URL=${EXTERNAL_HOST:-https://localhost}/grafana
       - GF_SERVER_SERVE_FROM_SUB_PATH=true
+      - POSTGRES_CONNECTION_STRING=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST:-postgres}:${POSTGRES_PORT:-5432}/${POSTGRES_DB:-enrichment}?${POSTGRES_PARAMETERS:-sslmode=disable}
     volumes:
       - ./infra/grafana/provisioning:/etc/grafana/provisioning:ro
       - grafana_data:/var/lib/grafana
@@ -867,3 +1011,42 @@ services:
     - /var/lib/docker/:/var/lib/docker:ro
     - /var/run:/var/run:ro
     devices: ["/dev/kmsg:/dev/kmsg"]
+
+
+  ############################
+  # LLM Services (optional profile)
+  ############################
+
+  litellm:
+    profiles: ["llm"]
+    image: ghcr.io/berriai/litellm:v1.74.0-stable
+    environment:
+      - DATABASE_URL=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST:-postgres}:${POSTGRES_PORT:-5432}/litellm
+      - STORE_MODEL_IN_DB=True
+      - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-}
+      - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-}
+      - AWS_REGION_NAME=${AWS_REGION_NAME:-us-east-1}
+      - LITELLM_MASTER_KEY=sk-${LITELLM_ADMIN_PASSWORD:-admin123}
+      - LITELLM_SALT_KEY=sk-${LITELLM_ADMIN_PASSWORD:-admin123}
+      - SERVER_ROOT_PATH=/llm
+      - PROXY_BASE_URL=${NEMESIS_URL:?}
+      - UI_BASE_PATH=/llm/ui
+    volumes:
+      - ./infra/litellm/config.yml:/app/config.yml:ro
+    command: ["--config", "/app/config.yml"]
+    depends_on:
+      postgres: { condition: service_healthy }
+    healthcheck:
+      test: ["CMD", "wget", "--quiet", "--tries=1", "-O", "/dev/null", "--header=Authorization: Bearer sk-${LITELLM_ADMIN_PASSWORD:-admin123}", "http://localhost:4000/health"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 30s
+      disable: true
+    logging: *logging-config
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.litellm.rule=PathPrefix(`/llm`)"
+      - "traefik.http.services.litellm.loadbalancer.server.port=4000"
+      - "traefik.http.routers.litellm.entrypoints=websecure"
+      - "traefik.http.routers.litellm.tls=true"

docs/agents.md

@@ -0,0 +1,95 @@
+# Nemesis Agents
+
+Nemesis integrates a number of LLM agents as well as some rule-based agents for different actions. These agents are run from the `agents` container and utilize Dapr workflows as well as [Pydantic AI](https://ai.pydantic.dev/) for the core agent execution. LLM powered agents are not enabled by default (see Setup and Configuration below for details).
+
+## Overview
+
+Nemesis currently has the following agents:
+
+| Agent Name            | Type       | Runs Automatically | Purpose                                                                                            |
+| --------------------- | ---------- | ------------------ | -------------------------------------------------------------------------------------------------- |
+| `validate`            | LLM-based  | true               | Validates security findings by triaging them as true positives, false positives, or needing review |
+| `credential_analyzer` | LLM-based  | false              | Extracts credentials and passwords from text content using LLM analysis                            |
+| `dotnet_analyzer`     | LLM-based  | false              | Adapted .NET vulnerability analyzer from @Dreadnode.                                               |
+| `summarizer`          | LLM-based  | false              | Creates concise summaries of text content using LLM analysis                                       |
+| `jwt`                 | Rule-based | true               | Rule-based JWT analysis that checks expiry status and identifies sample data                       |
+
+## Setup
+
+In order to enable any LLM-powered agents, you first need to configure one or more models in `./infra/litellm/config.yaml` . You can use any model provider that LiteLLM supports (more details [here](https://docs.litellm.ai/docs/proxy/configs)) - **ensure that at a minimum you have a model called "default"**. In the `agents` section of `./compose.yaml` you can adjust the MAX_BUDGET (default $100) and BUDGET_DURATION (default 30d) enrivonment variables as wanted. Then launch the "llm" profile for Nemesis by supplying `--llm` to the `./tools/nemesis-ctl.sh` script.
+
+***Note:*** service keys/tokens/etc. should be configured in your .env file and can be used in `./infra/litellm/config.yaml` as demonstrated in the current config file.
+
+## Agent Details
+
+### Finding Validator
+
+The `validate` agent uses the configured LLM to triage findings as true_positive, false_positive, or needs_review. This is based off of the finding details and the file path/name. It also will provide a 1-sentence reason for the decision along with a confidence score. Automated triage values are displayed in the main file-listing interface with a small robot icon for agent-powered triage. Mousing over the robot icon will give the explanation and confidence score:
+
+![Nemesis Agents Finding Triage](images/agents_finding_triage.png)
+
+If the finding is true_positive, a 1-sentence risk statement is also generated. Clicking the finding will give more details along with the risk statement if appliable
+
+![True Positive Finding Details](images/agents_true_positive_details.png)
+
+This agent will run/triage all findings that come in except for the following categories: "extracted_hash", "yara_match", "extracted_data".
+
+### Credential Analyzer
+
+The `credential_analyzer` agent is an LLM-powered agent that will examine a text file for any credentials that might be present. It does **not** run automatically, but is triggered manually from a file details interface:
+
+![Agent Credential Extraction](images/agent_credential_extraction.png)
+
+Once processing is complete, a markdown file will appear with any results in the transforms tab:
+
+![Agent Credential Extraction Results](images/agent_credential_extraction_results.png)
+
+### .NET Analyzer
+
+The `dotnet_analyzer` agent is an LLM-powered agent directly adapted from [@Dreadnode](https://x.com/dreadnode)'s [example-agents](https://github.com/dreadnode/example-agents) repo. It will analyze a .NET binary using a number of callable tools, searching for security issues.
+
+It does **not** run automatically, but is triggered manually from a file details interface. A confirmation dialog will confirm running the agent, as it may take a bit of time and tokens.
+
+![.NET Analysis](images/agent_dotnet_analysis.png)
+
+Once processing is complete, a markdown file will appear with any results in the transforms tab under ".NET Assembly Analysis":
+
+![.NET Analysis Results](images/agent_dotnet_analysis_results.png)
+
+### Text Summarizer
+
+The text `summarizer` agent is an LLM-powered agent that will summarize. Like the credential analizer, it does **not** run automatically, but is triggered manually from a file details interface. After processing, it will display the markdown-formatted text summary:
+
+![Agent Text Summarizer](images/agent_text_summarizer.png)
+
+### JWT Validator
+
+The JWT validator is a rule-based agent that will extract JWT tokens from Nosey Parker findings and mark the finding as "false_positive" if the token has expired.
+
+## The Nemesis Web Interface
+
+### Agents Web Interface
+
+If the llm profile is used, the Nemesis frontend will detect that LiteLLM was deployed and enable a new "Agents" tab on the left. Clicking on this tab will show you the currently enabled agents, as well as the current token and cost spend for the system:
+
+![Agents Web Interface](images/nemesis_dashboard_agents.png)
+
+For LLM-powered agents, you can modify the main system prompt used by clicking "Edit", making your changes to the prompt, and clicking "Save":
+
+![Agents Web Interface Prompts](images/nemesis_dashboard_agents_prompt.png)
+
+### Monitoring
+
+If the `--monitoring` flag is also passed to the `./tools/nemesis-ctl.sh` script, the [Arize Phoenix](https://github.com/Arize-ai/phoenix) will be deployed to allow tracking of the inputs/outputs sent to the LLM (at the /phoenix route):
+
+![Arize Phoenix Tracing](images/arize_phoenix_tracing.png)
+
+If the monitoring profile is used and Arize Phoenix is deployed, the Nemesis frontend will dynamically display links to the Phoenix interface in the Help menu as well.
+
+![Nemesis Dynamic Help Menu](images/nemesis_dynamic_help_menu.png)
+
+### LiteLLM Interface
+
+If the lm profile is used and LiteLLM is deployed, the Nemesis frontend will dynamically display links to the LiteLLM in the Help menu as well. Clicking on the link you can log in with `admin` and the value of the LITELLM_MASTER_KEY (sk-admin123 by default). This interface can give you additional breakdowns for tokens and costs:
+
+![LiteLLM Main Interface](images/litellm_main_interface.png)

docs/api.md

@@ -0,0 +1,329 @@
+# Enrichment API
+
+**Version:** 0.1.0
+
+API for file enrichment services
+
+This documentation is automatically generated from the OpenAPI specification.
+
+---
+
+## Enrichments
+
+### `GET /enrichments`
+
+List enrichment modules
+
+Get a list of all available enrichment modules
+
+**Returns:** 200 on success
+
+---
+
+### `POST /enrichments/{enrichment_name}`
+
+Run enrichment module
+
+Run a specific enrichment module on a file
+
+**Parameters:**
+
+- `enrichment_name` (string, **required**): Name of the enrichment module to run
+
+**Request Body:** `EnrichmentRequest` (JSON)
+
+**Returns:** 200 on success
+
+---
+
+### `POST /enrichments/{enrichment_name}/bulk`
+
+Start bulk enrichment
+
+Start bulk enrichment for a specific module against all files in the system using distributed processing
+
+**Parameters:**
+
+- `enrichment_name` (string, **required**): Name of the enrichment module to run
+
+**Returns:** 200 on success
+
+---
+
+### `GET /enrichments/{enrichment_name}/bulk/status`
+
+Get bulk enrichment status
+
+Bulk enrichment status tracking has been simplified
+
+**Parameters:**
+
+- `enrichment_name` (string, **required**): Name of the enrichment module to check status for
+
+**Returns:** 200 on success
+
+---
+
+### `POST /enrichments/{enrichment_name}/bulk/stop`
+
+Stop bulk enrichment
+
+Bulk enrichment cannot be stopped once tasks are published
+
+**Parameters:**
+
+- `enrichment_name` (string, **required**): Name of the enrichment module to stop
+
+**Returns:** 200 on success
+
+---
+
+
+## Files
+
+### `POST /containers`
+
+Submit large container file for processing with optional filtering
+
+...
+
+**Request Body:** See OpenAPI spec for details
+
+**Returns:** 200 on success
+
+---
+
+### `GET /containers/{container_id}/status`
+
+Get container processing status
+
+Get the current processing status and progress of a submitted container
+
+**Parameters:**
+
+- `container_id` (string, **required**): Unique identifier of the container
+
+**Returns:** 200 on success
+
+---
+
+### `POST /files`
+
+Upload file with metadata
+
+Upload a file using multipart/form-data with metadata.
+    Returns an object_id for the uploaded file and submission_id for the metadata submission.
+
+    Example:
+    ```
+    curl -k -u n:n -F "file=@example.txt"          -F 'metadata={"agent_id":"agent123","project":"proj1","timestamp":"2024-01-29T12:00:00Z","expiration":"2024-02-29T12:00:00Z","path":"/tmp/example.txt"}'          https://nemesis:7443/api/files
+    ```
+
+    Example:
+    ```
+    curl -k -u n:n -F "file=@example.txt"          -F 'metadata={"agent_id":"agent123","project":"proj1","path":"/tmp/example.txt"}'          https://nemesis:7443/api/files
+    ```
+
+**Request Body:** See OpenAPI spec for details
+
+**Returns:** 200 on success
+
+---
+
+### `GET /files/{object_id}`
+
+Download a file
+
+Download a file by its object ID with optional raw format and custom filename
+
+**Parameters:**
+
+- `object_id` (string, **required**): Unique identifier of the file to download
+- `raw` (boolean, optional): Whether to return the file in raw format
+- `name` (string, optional): Custom filename for the downloaded file
+
+**Returns:** 200 on success
+
+---
+
+
+## Queues
+
+### `GET /queues`
+
+Get queue statistics
+
+Get comprehensive queue metrics for all workflow topics
+
+**Returns:** 200 on success
+
+---
+
+### `GET /queues/{queue_name}`
+
+Get single queue statistics
+
+Get metrics for a specific queue topic
+
+**Parameters:**
+
+- `queue_name` (string, **required**): Name of the queue to get metrics for
+
+**Returns:** 200 on success
+
+---
+
+
+## System
+
+### `GET /agents`
+
+Get available agents
+
+Get a list of available AI agents with their metadata and capabilities
+
+**Returns:** 200 on success
+
+---
+
+### `POST /agents/dotnet_analysis`
+
+Run .NET assembly analysis
+
+Forward .NET assembly analysis request to agents service
+
+**Request Body:** JSON object
+
+**Returns:** 200 on success
+
+---
+
+### `POST /agents/llm_credential_analysis`
+
+Run credential analysis
+
+Forward credential analysis request to agents service
+
+**Request Body:** JSON object
+
+**Returns:** 200 on success
+
+---
+
+### `GET /agents/spend-data`
+
+Get LLM spend and usage data
+
+Get total spend and token usage statistics from LiteLLM logs
+
+**Returns:** 200 on success
+
+---
+
+### `POST /agents/text_summarizer`
+
+Run text summarization
+
+Forward text summarization request to agents service
+
+**Request Body:** JSON object
+
+**Returns:** 200 on success
+
+---
+
+### `GET /system/apprise-info`
+
+Get Apprise alert information
+
+Get information about configured alert channels (currently Slack only)
+
+**Returns:** 200 on success
+
+---
+
+### `GET /system/available-services`
+
+Get available services
+
+Query Traefik to determine which optional services are currently available
+
+**Returns:** 200 on success
+
+---
+
+### `POST /system/cleanup`
+
+Trigger database and datalake cleanup
+
+Trigger the housekeeping service to clean up expired files and database entries, and reset the workflow manager state. Optionally specify an expiration date or 'all' to remove all files.
+
+**Request Body:** `CleanupRequest` (JSON)
+
+**Returns:** 200 on success
+
+---
+
+### `GET /system/container-monitor/status`
+
+Container monitor status
+
+Get the status of the container file monitor
+
+**Returns:** 200 on success
+
+---
+
+### `GET /system/health`
+
+Health check
+
+Health check endpoint for Docker healthcheck
+
+**Returns:** 200 on success
+
+---
+
+### `GET /system/info`
+
+API information
+
+Root endpoint that shows API information
+
+**Returns:** 200 on success
+
+---
+
+### `POST /system/yara/reload`
+
+Reload Yara rules
+
+Trigger a reload of all Yara rules in the backend across all workers/replicas
+
+**Returns:** 200 on success
+
+---
+
+
+## Workflows
+
+### `GET /workflows/failed`
+
+Get failed workflows
+
+Get the set of failed enrichment workflows
+
+**Returns:** 200 on success
+
+---
+
+### `GET /workflows/status`
+
+Get workflow enrichment workflow status
+
+Get the current status of the enrichment workflow system
+
+**Returns:** 200 on success
+
+---
+

docs/chromium.md

@@ -0,0 +1,119 @@
+# Nemesis Chromium Analysis
+
+Nemesis includes comprehensive support for analyzing Chromium-based browser data including Chrome, Edge, Brave, and other Chromium-based browsers. The system automatically detects and processes various Chromium database files to extract browsing history, saved credentials, cookies, downloads, and encryption state information.
+
+## Overview
+
+Nemesis currently analyzes the following Chromium data sources:
+
+| Data Type  | File Source   | Purpose                                                                        |
+| ---------- | ------------- | ------------------------------------------------------------------------------ |
+| History    | `History`     | Extracts browsing history including URLs, titles, visit counts, and timestamps |
+| Downloads  | `History`     | Extracts download history with file paths, URLs, and download metadata         |
+| Logins     | `Login Data`  | Extracts saved login credentials including usernames and passwords             |
+| Cookies    | `Cookies`     | Extracts browser cookies with domain, name, value, and expiration data         |
+| State Keys | `Local State` | Extracts OS encryption keys used to decrypt passwords and sensitive data       |
+
+
+## Chrome Web Tab
+
+### Chromium Data Viewer
+
+The Nemesis frontend provides a dedicated Chromium interface accessible through the main navigation. This interface organizes all extracted Chromium data into five main categories:
+
+### History Tab
+
+The History tab displays extracted browsing history with searchable and filterable tables. Each entry includes:
+
+- **Object ID**: The object_id from the originating file
+- **Source**: the "source" label for the originating file (host, url, etc.)
+- **Username**: Host-based username, extracted from file path context (i.e., `C:\Users\USER\*`)
+- **Browser**: Browser name, extracted from file path context
+- **Title**: Page title as recorded by the browser
+- **Visits**: Number of times the URL was visited
+- **Last Visit**: Timestamp of most recent visit
+- **URL**: The visited website URL
+
+![Chromium History Tab](images/chromium-history-tab.png)
+
+### Downloads Tab
+
+The Downloads tab shows extracted download history with detailed information about each downloaded file:
+
+- **Object ID**: The object_id from the originating file
+- **Source**: the "source" label for the originating file (host, url, etc.)
+- **Username**: Host-based username, extracted from file path context (i.e., `C:\Users\USER\*`)
+- **Browser**: Browser name, extracted from file path context
+- **URL**: Original source URL of the downloaded file
+- **End Time**: Time the download ended
+- **Download Path**: Path the file was downloaded to on the host
+
+![Chromium Downloads Tab](images/chromium-downloads-tab.png)
+
+### Logins Tab
+
+The Logins tab displays extracted login credentials:
+
+- **Object ID**: The object_id from the originating file
+- **Decrypted**: Yes/No indication if the login entry has been decrypted (by a state key) or not
+- **Source**: the "source" label for the originating file (host, url, etc.)
+- **Username**: Host-based username, extracted from file path context (i.e., `C:\Users\USER\*`)
+- **Password**: Plaintext value of the password (if decrypted)
+- **Browser**: Browser name, extracted from file path context
+- **Login Name**: Extracted login name for the login data entry (NOT the host-based path username of the Login Data file)
+- **Times Used**: Number of times the login entry has been used
+- **Signon Realm**: Extracted signon realm for the login entry
+- **Origin URL**: Website where credentials were saved
+
+![Chromium Logins Tab](images/chromium-logins-tab.png)
+
+### Cookies Tab
+
+The Cookies tab provides access to extracted browser cookies:
+
+- **Object ID**: The object_id from the originating file
+- **Decrypted**: Yes/No indication if the coookie has been decrypted (by a state key) or not
+- **Source**: the "source" label for the originating file (host, url, etc.)
+- **Username**: Host-based username, extracted from file path context (i.e., `C:\Users\USER\*`)
+- **Password**: Plaintext value of the password (if decrypted)
+- **Browser**: Browser name, extracted from file path context
+- **Host Key**: Domain or host the cookie belongs to
+- **Expires UTC**: Time (in UTC) the cookie value expires
+- **Last Access UTC**: Time (in UTC) the cookie value was last uased
+- **Name**: Name identifier of the cookie
+
+**Note**: If the cookie value has been decrypted, click "Download CSV" to download the currently filtered/viewable
+cookies on the page *including* decrypted values.
+
+![Chromium Cookies Tab](images/chromium-cookies-tab.png)
+
+### State Keys Tab
+
+The State Keys tab displays OS encryption keys used by Chromium to protect sensitive data:
+
+- **Object ID**: The object_id from the originating file
+- **Source**: the "source" label for the originating file (host, url, etc.)
+- **Username**: Host-based username, extracted from file path context (i.e., `C:\Users\USER\*`)
+- **Browser**: Browser name, extracted from file path context
+- **Key Decrypted**: If the pre-v127 Chromium local state encryption key has been decrypted
+- **App Bound Key Decrypted**: If the post-v127 Chromium App-bound encryption key has been decrypted
+
+![Chromium State Keys Tab](images/chromium-state-keys-tab.png)
+
+## Data Export and Analysis
+
+### CSV Export Functionality
+
+All Chromium data tables support CSV export for external analysis:
+
+1. Use the table interface to filter and search desired records
+2. Click the "Download CSV" button in the top-right of each tab
+3. All currently visible/filtered records will be exported
+
+### Copy Operations
+
+Individual records or entire result sets can be copied to clipboard:
+
+- **Single Row**: Double-click any table row to copy all fields
+- **Multiple Rows**: Select rows and use Ctrl+C (or Cmd+C on Mac)
+- **Filtered Results**: Copy button will copy all currently visible records

docs/cli.md

@@ -72,6 +72,7 @@ The `./tools/submit.sh` script wraps the docker syntax automatically.
   --username your-username \
   --password your-password \
   --project my-project \
+  --source host://HOST1 \
   --agent-id my-agent \
   --workers 5 \
   --recursive \
@@ -103,6 +104,7 @@ Options:
   -u, --username TEXT    Basic auth username  [default: n]
   -p, --password TEXT    Basic auth password  [default: n]
   --project TEXT         Project name for metadata  [default: assess-test]
+  --source TEXT          Source name for metadata (e.g., 'host://HOST1')
   --agent-id TEXT        Agent ID for metadata  [default:
                          submitunknown_user@docker-desktop]
   -f, --file FILE        Path to single file to submit (alternative to PATHS
@@ -110,16 +112,17 @@ Options:
   --help                 Show this message and exit.
 ```
 
-| Option        | Default               | Description               |
-| ------------- | --------------------- | ------------------------- |
-| `--host`      | `0.0.0.0:7443`        | Nemesis host and port     |
-| `--recursive` | `false`               | Process subdirectories    |
-| `--workers`   | `10`                  | Number of upload threads  |
-| `--username`  | `n`                   | Basic auth username       |
-| `--password`  | `n`                   | Basic auth password       |
-| `--project`   | `assess-test`         | Project name for metadata |
-| `--agent-id`  | `submit<user>@<host>` | Agent ID for metadata     |
-| `--debug`     | `false`               | Enable debug logging      |
+| Option        | Default               | Description                                                             |
+| ------------- | --------------------- | ----------------------------------------------------------------------- |
+| `--host`      | `0.0.0.0:7443`        | Nemesis host and port                                                   |
+| `--recursive` | `false`               | Process subdirectories                                                  |
+| `--workers`   | `10`                  | Number of upload threads                                                |
+| `--username`  | `n`                   | Basic auth username                                                     |
+| `--password`  | `n`                   | Basic auth password                                                     |
+| `--project`   | `assess-test`         | Project name for metadata                                               |
+| `--source`    |                       | Source name (e.g., 'host://HOST1' or 'https://domain.com') for metadata |
+| `--agent-id`  | `submit<user>@<host>` | Agent ID for metadata                                                   |
+| `--debug`     | `false`               | Enable debug logging                                                    |
 
 ## Folder Monitoring
 
@@ -173,6 +176,7 @@ docker run \
   --username your-username \
   --password your-password \
   --project my-project \
+  --source host://HOST1 \
   --agent-id my-agent \
   --workers 5  \
   --debug
@@ -188,16 +192,17 @@ poetry run python -m cli monitor /path/to/directory
 
 ### Options Reference
 
-| Option           | Default                | Description                                     |
-| ---------------- | ---------------------- | ----------------------------------------------- |
-| `--host`         | `0.0.0.0:7443`         | Nemesis host and port                           |
-| `--username`     | `n`                    | Basic auth username                             |
-| `--password`     | `n`                    | Basic auth password                             |
-| `--project`      | `assess-test`          | Project name for metadata                       |
-| `--agent-id`     | `monitor<user>@<host>` | Agent ID for metadata                           |
-| `--workers`      | `10`                   | Number of upload threads for initial submission |
-| `--only-monitor` | `false`                | Skip existing files, only monitor for new ones  |
-| `--debug`        | `false`                | Enable debug logging                            |
+| Option           | Default                | Description                                                             |
+| ---------------- | ---------------------- | ----------------------------------------------------------------------- |
+| `--host`         | `0.0.0.0:7443`         | Nemesis host and port                                                   |
+| `--username`     | `n`                    | Basic auth username                                                     |
+| `--password`     | `n`                    | Basic auth password                                                     |
+| `--project`      | `assess-test`          | Project name for metadata                                               |
+| `--source`       |                        | Source name (e.g., 'host://HOST1' or 'https://domain.com') for metadata |
+| `--agent-id`     | `monitor<user>@<host>` | Agent ID for metadata                                                   |
+| `--workers`      | `10`                   | Number of upload threads for initial submission                         |
+| `--only-monitor` | `false`                | Skip existing files, only monitor for new ones                          |
+| `--debug`        | `false`                | Enable debug logging                                                    |
 
 
 ## Mythic Connector

docs/containers.md

@@ -0,0 +1,88 @@
+# Containers
+
+Nemesis has the ability to process the files extracted/carved from various container formats _without_ saving the container itself to the internal datalake. This is useful the the following situations:
+
+- When you want to process a large number of files without doubling the storage (storing .zip + extracted files)
+- When you want to process large "containers" like forensic disk images
+
+## Submitting Containers
+
+### "Regular" Containers
+
+To process a regular container, you have two options. First, you can submit the .zip/etc. as normal, and then click "Extract/Process Container Contents" on the file viewer page:
+
+![Extract/Process Container Contents](images/extract_process_container_contents.png)
+
+Alternatively, you can configure and drop containers into the mounted folder as described in the **"Large" Containers** section below.
+
+You can also submit the container with the `nemesis-cli` (and `./tools/submit.sh` script) with something like (note the --container flag):
+
+```bash
+% ./tools/submit.sh --project PROJECT-123 --source DEV --container zip_test.zip
+Uploading (✓:1 ✗:0 | 1.59 KB): 100%|██████████| 1/1 [00:00<00:00]
+INFO
+Upload Summary:
+INFO     ────────────────────────────────────────
+INFO     Total Files:     1
+INFO     Successful:      1
+INFO     Failed:          0
+INFO     Success Rate:    100.0%
+INFO     Total Uploaded:  1.59 KB
+```
+
+### "Large" Containers
+
+For large things like disk images, a straight REST API doesn't cut it - lots of things mess up. Getting a multi-gigabyte file into Nemesis can be a challenge, but the current process uses a mountained container and large container monitoring abilities in the `web-api` service.
+
+In order to process really large containers, first create a folder on your host and set the MOUNTED_CONTAINER_PATH ENV variable to that path. This folder is mounted into the `web-api` and will process containers that appear there (after they're done copying in). Then just start Nemesis and copy containers into that folder, it's that easy!
+
+Wait, but what about metadata?
+
+#### Large Container Configs/Metadata
+
+Since we're changing the normal way we submit files, we need a new way to pass metadata into Nemesis for a disk image/large container. In order to accomplish this, we support a YAML-based metadata file format that can be placed in the parent folder, or any sub folders. This file can be named config.[yaml|yml] or settings.[yaml|yml] and takes the form:
+
+```yaml
+metadata:
+  project: project123
+  agent: collector
+  source: BLAH
+file_filters:
+  pattern_type: regex
+  include:
+    - "^(?:[A-Za-z]://|/)?[Ww]indows/[Ss]ystem32/config/"
+  exclude:
+    - "^(?:[A-Za-z]://|/)?[Ww]indows/"
+```
+
+The metadata will be passed through for every file carved from the container. The optional `file_filters` allow you specify which extracted/carved files to include, or not include, for processing.
+
+The logic for inclusion varies based on which patterns are provided:
+
+1. No filters: Include everything
+2. Only include patterns: Only include files matching include patterns (allowlist mode)
+3. Only exclude patterns: Include everything except files matching exclude patterns (blocklist mode)
+4. Both include and exclude: Include everything, apply excludes, then re-include matches from include patterns (exception mode)
+
+This creates a natural hierarchy where include patterns act as exceptions to exclusions when both are present.
+
+You can also create sub-directories in the submission folder, for example:
+
+```
+MOUNTED_CONTAINER_PATH  /   settings.yaml
+                        /   disk1.dd
+                        /   windows /
+                                    / settings.yaml
+                                    / windows_disk.dd
+```
+
+In this case, the `settings.yaml` closest to the hierarchy of the file takes precedence - the "windows_disk.dd" will use the settings.yaml in its current folder, but would use the MOUNTED_CONTAINER_PATH/settings.yaml file if one wasn't present lower down. This lets you create a nested structure with multiple config options depending on where you drop your disk image.
+
+
+## Tracking Containers
+
+Whether a container is submitted via the cli or the mounted folder option, it will appear in the "Containers" page accessible from the left navigation page:
+
+![Containers Dashboard](images/containers_dashboard.png)
+
+This page will show the status of the container file extraction, and lets you filter by various fields.

docs/dapr.md

@@ -20,13 +20,13 @@ The [document_conversion](https://github.com/SpecterOps/Nemesis/tree/main/projec
 
 ## Secrets
 
-Nemesis uses the [Dapr Secrets management](https://docs.dapr.io/developing-applications/building-blocks/secrets/secrets-overview/) building block to protect secrets internally (like Postgres connection strings). Currently the [Local environment variables](https://docs.dapr.io/reference/components-reference/supported-secret-stores/envvar-secret-store/) component is used. These secrets are also refereced within some Dapr files such as [pubsub.yaml](https://github.com/SpecterOps/Nemesis/tree/main/infra/dapr/components/pubsub.yaml).
+Nemesis uses the [Dapr Secrets management](https://docs.dapr.io/developing-applications/building-blocks/secrets/secrets-overview/) building block to protect secrets internally (like PostgreSQL connection parameters). Currently the [Local environment variables](https://docs.dapr.io/reference/components-reference/supported-secret-stores/envvar-secret-store/) component is used. These secrets are also referenced within some Dapr files such as [pubsub.yaml](https://github.com/SpecterOps/Nemesis/tree/main/infra/dapr/components/pubsub.yaml).
 
 This reason for using this abstraction is so alternative secret management systems like [Vault or Kubernetes secrets](https://docs.dapr.io/reference/components-reference/supported-secret-stores/) can be used in the future:
 
 ![Dapr Secrets](images/dapr-secrets-overview-cloud-stores.png)
 
-An example of retrieving a secret is at the top of the the [housekeeping code](https://github.com/SpecterOps/Nemesis/blob/main/projects/housekeeping/housekeeping/main.py) to retrieve the `POSTGRES_CONNECTION_STRING` string.
+An example of retrieving secrets is in [libs/common/common/db.py](https://github.com/SpecterOps/Nemesis/blob/main/libs/common/common/db.py) which retrieves individual PostgreSQL connection parameters (`POSTGRES_USER`, `POSTGRES_PASSWORD`, `POSTGRES_HOST`, `POSTGRES_PORT`, `POSTGRES_DB`, `POSTGRES_PARAMETERS`) and constructs the connection string.
 
 ## Service Invocation
 

docs/docker_compose.md

@@ -73,7 +73,7 @@ Development images are not published and must be built locally. If you make any
 
 The easiest method to build + run dev images is to just use the `dev` target instead of `prod` with `./tools/nemesis-ctl.sh` :
 ```bash
-./tools/nemesis-ctl.sh start dev [--monitoring] [--jupyter]
+./tools/nemesis-ctl.sh start dev [--monitoring] [--jupyter] [--llm]
 ```
 
 ### Step 1 - Configure environment variables

docs/dpapi.md

@@ -0,0 +1,116 @@
+# Nemesis DPAPI Analysis
+
+Nemesis includes comprehensive support for analyzing Microsoft's Data Protection API (DPAPI) artifacts, which are used to encrypt sensitive data on Windows systems.
+
+The system provides functionality to view extracted DPAPI master keys, domain backup keys, and submit credential material to decrypt protected data.
+
+Additionally, "retroactive" decryption will occur when plaintext DPAPI masterkeys are submitted (or scraped from LSASS dumps), or DPAPI masterkeys are successfully decrypted (via domain backup keys, NTLM hashes, passwords, etc.). Specifically, new plaintext masterkeys are used to decrypt Google Chromekey1's and Chromium Local State files.
+
+## Overview
+
+Nemesis currently analyzes and manages the following DPAPI data sources:
+
+| Data Type                      | Purpose                                                                            |
+| ------------------------------ | ---------------------------------------------------------------------------------- |
+| Master Keys                    | User-specific DPAPI master keys used to encrypt/decrypt user data                  |
+| Domain Backup Keys             | Domain controller backup keys for enterprise DPAPI key recovery                    |
+| Credential Material Submission | Interface to submit various credential types to decrypt existing DPAPI master keys |
+
+## DPAPI Web Tab
+
+### DPAPI Data Viewer
+
+The Nemesis frontend provides a dedicated DPAPI interface accessible through the main navigation. This interface organizes all DPAPI-related functionality into three main categories:
+
+### Master Keys Tab
+
+The Master Keys tab displays extracted DPAPI master keys with searchable and filterable tables. Each entry includes:
+
+- **Object ID**: The object_id from the originating file
+- **Source**: The "source" label for the originating file (host, url, etc.)
+- **Username**: Host-based username, extracted from file path context (i.e., `C:\Users\USER\*`)
+- **Key GUID**: Unique identifier for the master key
+- **Key Status**: Decryption status of the master key
+- **Created Date**: When the master key was created
+- **Master Key Data**: The actual key material (if decrypted)
+
+![DPAPI masterkeys](images/dpapi_masterkeys.png)
+
+### Domain Backup Keys Tab
+
+The Domain Backup Keys tab shows extracted domain backup keys used for enterprise DPAPI recovery:
+
+- **Object ID**: The object_id from the originating file
+- **Source**: The "source" label for the originating file (host, url, etc.)
+- **Domain**: Domain name associated with the backup key
+- **Key GUID**: Unique identifier for the backup key
+- **Key Status**: Whether the backup key has been successfully extracted
+- **Domain Controller**: Source domain controller (if available)
+- **Backup Key Data**: The actual backup key material
+
+![DPAPI domain backup key](images/dpapi_domain_backupkey.png)
+
+### Submit Credential Material Tab
+
+The Submit Credential Material tab provides an interface for submitting various types of credential material to decrypt existing DPAPI master keys stored in the system. This tab includes:
+
+#### Credential Types Supported
+
+The interface supports multiple credential types for DPAPI decryption:
+
+- **Password**: Plain text passwords for user accounts
+- **NTLM Hash**: NT hash values (16 bytes) for password-equivalent authentication
+- **SHA1**: SHA1 credential keys (20 bytes) derived from NTLM hashes
+- **Secure Credential Key (PBKDF2)**: 16-byte keys derived using PBKDF2
+- **Domain Backup Key**: Domain controller DPAPI backup keys in base64 PVK format
+- **Master Keys {GUID}:SHA1 Pairs**: Plaintext DPAPI master keys as {GUID}:SHA1 pairs
+- **DPAPI_SYSTEM Secret**: System-wide DPAPI credentials for machine encryption
+
+![DPAPI credential submission types](images/dpapi_submit_credential_types.png)
+
+#### Form Fields
+
+Based on the selected credential type, the form dynamically displays relevant fields:
+
+- **User SID**: Required for user-specific credential types (password, NTLM hash, SHA1, PBKDF2)
+- **Backup Key GUID**: Required for domain backup keys
+- **Domain Controller**: Optional field for domain backup key submissions
+- **Credential Value**: Main input field for the credential material
+- **Master Key Data**: Structured input for {GUID}:SHA1 pairs (multi-line format)
+
+Additionally, each credential type includes detailed descriptions explaining:
+
+- What the credential type is used for
+- How to obtain or generate the credential material
+- Common sources where these credentials can be collected
+- Technical details about the key format and length requirements
+
+![DPAPI domain backup key submission](images/dpapi_domain_backupkey_submission.png)
+
+## Data Export and Analysis
+
+### CSV Export Functionality
+
+DPAPI data tables support CSV export for external analysis:
+
+1. Use the table interface to filter and search desired records
+2. Click the "Download CSV" button in the top-right of each tab
+3. All currently visible/filtered records will be exported
+
+### Copy Operations
+
+Individual records or entire result sets can be copied to clipboard:
+
+- **Single Row**: Double-click any table row to copy all fields
+- **Multiple Rows**: Select rows and use Ctrl+C (or Cmd+C on Mac)
+- **Filtered Results**: Copy button will copy all currently visible records
+
+## Integration with Other Nemesis Components
+
+### Automatic Processing
+
+DPAPI artifacts are automatically processed when:
+
+- Windows registry hives are uploaded (SYSTEM, SECURITY, SAM)
+- LSASS dumps are analized
+- User/system profile directories containing DPAPI folders/masterkeys are processed

docs/file_enrichment_modules.md

@@ -33,8 +33,13 @@ The `should_process()` function determines if the module should run on a file. Y
 
 ```python
 ...
-def should_process(self, state_key: str) -> bool:
-    """Determine if this module should run based on file type."""
+def should_process(self, object_id: str, file_path: str | None = None) -> bool:
+    """Determine if this module should run based on file type
+
+    Args:
+        object_id: The object ID of the file
+        file_path: Optional path to already downloaded file
+    """
     file_enriched = get_file_enriched(state_key)
     # Check if file appears to be a VNC config file
     should_run = (
@@ -65,18 +70,31 @@ rule has_dpapi_blob
 }
     """)
 
-def should_process(self, object_id: str) -> bool:
+def should_process(self, object_id: str, file_path: str | None = None) -> bool:
+    """Check if this file should be processed by scanning for DPAPI blobs.
+
+    Args:
+        object_id: The object ID of the file
+        file_path: Optional path to already downloaded file
+    """
     file_enriched = get_file_enriched(object_id)
+    logger.debug(f"File {object_id} should be processed by DPAPI blob analyzer")
     if file_enriched.size > self.size_limit:
-        logger.warning(
+        logger.debug(
             f"[dpapi_analyzer] file {file_enriched.path} ({file_enriched.object_id} / {file_enriched.size} bytes) exceeds the size limit of {self.size_limit} bytes, only analyzing the first {self.size_limit} bytes"
         )
 
-    num_bytes = file_enriched.size if file_enriched.size < self.size_limit else self.size_limit
-    file_bytes = self.storage.download_bytes(file_enriched.object_id, length=num_bytes)
+    if file_path:
+        # Use provided file path - read only the needed bytes
+        with open(file_path, "rb") as f:
+            num_bytes = min(file_enriched.size, self.size_limit)
+            file_bytes = f.read(num_bytes)
+    else:
+        # Fallback to downloading the file itself
+        num_bytes = file_enriched.size if file_enriched.size < self.size_limit else self.size_limit
+        file_bytes = self.storage.download_bytes(file_enriched.object_id, length=num_bytes)
 
     should_run = len(self.yara_rule.scan(file_bytes).matching_rules) > 0
-    logger.debug(f"[dpapi_analyzer] should_run: {should_run}")
     return should_run
 ...
 ```

docs/index.md

@@ -28,23 +28,27 @@ Nemesis is an offensive file enrichment pipeline.
 
 Nemesis 2.0 is built on [Docker](https://www.docker.com/) with heavy [Dapr integration](https://dapr.io/), our goal with Nemesis was to create a centralized file processing platform that functions as an "offensive VirusTotal".
 
-## Additional Information
-Blog Posts:
+***Note that Nemesis v1.0 is incompatible with Nemesis 2.0!***
 
-| Title                                                                                                                                                            | Date         |
-|------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------|
-| [*Nemesis 1.0.0*](https://posts.specterops.io/nemesis-1-0-0-8c6b745dc7c5)                                                                                        | Apr 25, 2024 |
-| [*Summoning RAGnarok With Your Nemesis*](https://posts.specterops.io/summoning-ragnarok-with-your-nemesis-7c4f0577c93b)                                          | Mar 13, 2024 |
-| [*Shadow Wizard Registry Gang: Structured Registry Querying*](https://posts.specterops.io/shadow-wizard-registry-gang-structured-registry-querying-9a2fab62a26f) | Sep 5, 2023  |
-| [*Hacking With Your Nemesis*](https://posts.specterops.io/hacking-with-your-nemesis-7861f75fcab4)                                                                | Aug 9, 2023  |
-| [*Challenges In Post-Exploitation Workflows*](https://posts.specterops.io/challenges-in-post-exploitation-workflows-2b3469810fe9)                                | Aug 2, 2023  |
-| [*On (Structured) Data*](https://posts.specterops.io/on-structured-data-707b7d9876c6)                                                                            | Jul 26, 2023 |
+## Additional Information
+
+Blog Posts
+
+| Title                                                                                                                                                            | Nemesis Version | Date         |
+| ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------- | ------------ |
+| [*Nemesis 2.0*](https://specterops.io/blog/2025/08/05/nemesis-2-0/)                                                                                              | v2.0            | Aug 5, 2025  |
+| [*Nemesis 1.0.0*](https://posts.specterops.io/nemesis-1-0-0-8c6b745dc7c5)                                                                                        | v1.0            | Apr 25, 2024 |
+| [*Summoning RAGnarok With Your Nemesis*](https://posts.specterops.io/summoning-ragnarok-with-your-nemesis-7c4f0577c93b)                                          | v1.0            | Mar 13, 2024 |
+| [*Shadow Wizard Registry Gang: Structured Registry Querying*](https://posts.specterops.io/shadow-wizard-registry-gang-structured-registry-querying-9a2fab62a26f) | v1.0            | Sep 5, 2023  |
+| [*Hacking With Your Nemesis*](https://posts.specterops.io/hacking-with-your-nemesis-7861f75fcab4)                                                                | v1.0            | Aug 9, 2023  |
+| [*Challenges In Post-Exploitation Workflows*](https://posts.specterops.io/challenges-in-post-exploitation-workflows-2b3469810fe9)                                | v1.0            | Aug 2, 2023  |
+| [*On (Structured) Data*](https://posts.specterops.io/on-structured-data-707b7d9876c6)                                                                            | v1.0            | Jul 26, 2023 |
 
 
 Presentations:
 
 | Title                                                                      | Date         |
-|----------------------------------------------------------------------------|--------------|
+| -------------------------------------------------------------------------- | ------------ |
 | [*SAINTCON 2023*](https://www.youtube.com/watch?v=0q9u2hDcpIo)             | Oct 24, 2023 |
 | [*BSidesAugusta 2023*](https://www.youtube.com/watch?v=Ug9r7lCF_FA)        | Oct 7, 2023  |
 | [*44CON 2023*](https://www.youtube.com/watch?v=tjPTLBGI7K8)                | Sep 15, 2023 |

docs/odr.md

@@ -22,6 +22,7 @@ Schema definition for the public `file` message POSTed to the API frontend.
 | Field                   | Type     | Description                                                                                                      |
 | ----------------------- | -------- | ---------------------------------------------------------------------------------------------------------------- |
 | `path`                  | string   | File system path to the relevant resource. Can use either forward (/) or backward (\\) slashes                   |
+| `source`                | string   | String representing a `host://IP`, `host://HOSTNAME`, or traditional URI (i.e., `https://domain.com)             |
 | `originating_object_id` | string   | UUID v4 format identifier referencing a parent or source object                                                  |
 | `nesting_level`         | number   | The level of nesting for the file within an originating container. Used to prevent indefinite container nesting. |
 | `creation_time`         | datetime | ISO 8601 formatted timestamp for when the file was created                                                       |

docs/performance.md

@@ -10,11 +10,8 @@ If workflows begin to fail, or you are experiencing major performance issues (as
 For production (non-dev) deployments, multiple UVICORN_WORKERS are used for the `file-enrichment` service. The default value is 2 and is defined in the `file-enrichment` section in [compose.yaml](https://github.com/SpecterOps/Nemesis/blob/71406afc12f855140ea68aae337076f9b8dc292f/compose.yaml#L217). This value can be set to 1 for troubleshooting, or increased to 4+ for potential performance gains. You can modify this value by defining the `export UVICORN_WORKERS=4` environment variable before launching Nemesis.
 
 
-### MAX_PARALLEL_WORKFLOWS
-
-The `file-enrichment` container runs a number of file-enrichment workflows in parallel, defaulting to 5. You can modify this value by defining the `export MAX_PARALLEL_WORKFLOWS=3` environment variable before launching Nemesis.
-
-
-### MAX_PARALLEL_ENRICHMENT_MODULES
-
-For each file enrichment workflow, the `file-enrichment` container runs multiple file enrichment modules in parallel, defaulting to 5. You can modify this value by defining the `export MAX_PARALLEL_ENRICHMENT_MODULES=3` environment variable before launching Nemesis.
+# Useful Prometheus Metrics
+Minio
+```
+minio_cluster_usage_objects_count{}
+```

docs/quickstart.md

@@ -17,6 +17,12 @@ Ensure your machine meets the following requirements:
   - Docker version 28.0.0 or higher is recommended. See [Docker's installation instructions](https://docs.docker.com/engine/install/) for instructions on installing Docker. Running the Docker Engine on Linux or on OS X via Docker Desktop is recommended. If using Docker Desktop, ensure that the VM is configured with sufficient RAM/Disk/swap.
 
 
+**NOTE:** for multi-language support for OCR/document processing, set the `TIKA_OCR_LANGUAGES` ENV var before launching with the [Tesseract language code](https://github.com/tesseract-ocr/tessdata):
+```bash
+$ export TIKA_OCR_LANGUAGES="eng chi_sim chi_tra jpn rus deu spa"
+```
+
+
 ### Step 1: Clone the Nemesis Repository
 ```bash
 git clone https://github.com/SpecterOps/Nemesis
@@ -46,10 +52,10 @@ To start Nemesis's core services, run the `./tools/nemesis-ctl.sh` script:
 ./tools/nemesis-ctl.sh start prod
 ```
 
-If you'd like to install the monitoring services and/or jupyter notebooks, use the associated optional command line arguments:
+If you'd like to install the monitoring services, jupyter notebooks, and/or LLM agents use the associated optional command line arguments:
 
 ```bash
-./tools/nemesis-ctl.sh start prod --monitoring --jupyter
+./tools/nemesis-ctl.sh start prod --monitoring --jupyter [--llm]
 ```
 `nemesis-ctl.sh` effectively is a wrapper for `docker compose` commands and is in charge of pulling and starting the appropriate published Nemesis docker images. In general, we recommend people use `nemesis-ctl.sh` instead of manually invoking `docker compose`. For more complex deployment scenarios, see Nemesis's [Docker Compose documentation](docker_compose.md) to understand what `nemesis-ctl.sh` does underneath.
 
@@ -90,17 +96,19 @@ Click on the "Help" button on the bottom left to view the additionally exposed N
 
 **NOTE:** The /jupyter/ route will only be available if you started with it enabled (`--jupyter`).
 
+**NOTE:** The /jupyter/ route will only be available if you started with it enabled (`--jupyter`).
+
 ![Nemesis services](images/nemesis-dashboard-services.png)
 
 ### Step 7: Shutting Nemesis Down
 
-To shutdown Nemesis, use the `nemesis-ctl.sh` script's `stop` or `clean` commands ***with the same arguments you used to start it***. For example, if you started it with monitoring and jupyter enabled, then run the following:
+To shutdown Nemesis, use the `nemesis-ctl.sh` script's `stop` or `clean` commands ***with the same arguments you used to start it***. For example, if you started it with monitoring, jupyter, or LLM support enabled, then run the following:
 - To stop Nemesis containers:
 ```bash
-./tools/nemesis-ctl.sh stop prod --monitoring --jupyter
+./tools/nemesis-ctl.sh stop prod --monitoring --jupyter --llm
 ```
 
 - To stop Nemesis containers and delete their associated volumes:
 ```bash
-./tools/nemesis-ctl.sh clean prod --monitoring --jupyter
+./tools/nemesis-ctl.sh clean prod --monitoring --jupyter --llm
 ```

env.example

@@ -1,20 +1,27 @@
-MINIO_ROOT_PASSWORD=Qwerty12345
-MINIO_ROOT_USER=nemesis
-POSTGRES_PASSWORD=Qwerty12345
-POSTGRES_USER=nemesis
-RABBITMQ_PASSWORD=Qwerty12345
-RABBITMQ_USER=nemesis
+MINIO_ROOT_PASSWORD="Qwerty12345"
+MINIO_ROOT_USER="nemesis"
+
+RABBITMQ_PASSWORD="Qwerty12345"
+RABBITMQ_USER="nemesis"
+
+POSTGRES_PASSWORD="Qwerty12345"
+POSTGRES_USER="nemesis"
+POSTGRES_HOST="postgres"
+POSTGRES_PORT=5432
+POSTGRES_DB="enrichment"
+POSTGRES_PARAMETERS="sslmode=disable"
 
 
-# You can change the password used by HASURA. If not defined a default password will be used.
+
+# You can change the password used by HASURA. If not defined a default password will be used (pass456)
 # Example:
-#   HASURA_ADMIN_SECRET=Qwerty12345
+#   HASURA_ADMIN_SECRET="Qwerty12345"
 
 
 # NEMESIS_URL is used when building hyperlinks for findings and Apprise alerts.
 # If you change the port Nemesis listens on using NEMESIS_PORT (below), ensure this URL's port matches.
 # If using a host/domain name, a FQDN (with a top level domain) is recommended.
-NEMESIS_URL=https://localhost:7443/
+NEMESIS_URL="https://localhost:7443/"
 
 
 ####################
@@ -50,4 +57,11 @@ NEMESIS_URL=https://localhost:7443/
 # (Optional) Set Jupyter credentials using JUPYTER_PASSWORD.
 # If not defined, a random password will be generated and printed in the jupyter container's logs.
 # Example:
-#   JUPYTER_PASSWORD=Qwerty12345
+#   JUPYTER_PASSWORD="Qwerty12345"
+
+
+# (Optional) Enable Phoenix LLM tracing for Pydantic AI agents.
+# Requires starting Nemesis with monitoring profile: ./tools/nemesis-ctl.sh start dev --monitoring
+# Phoenix UI will be available at http://localhost:6006
+# Example:
+#   PHOENIX_ENABLED=true

infra/dapr/components/pubsub.yaml

@@ -16,15 +16,54 @@ spec:
       value: "false"
     - name: autoAck
       value: "false"
+    - name: requeueInFailure
+      value: "true"
     - name: reconnectWait
-      value: "5"
+      value: "3"
     - name: concurrencyMode
       value: parallel
-    # - name: backOffMaxRetries # doesn't appear to be a used value
-    #   value: "5"
-    # - name: backOffMaxInterval # doesn't appear to be a used value
-    #   value: "5s"
-    - name: prefetchCount
-      value: "10"              # Set to 1 for processing one message at a time
+    - name: deliveryMode  # persistence setting
+      value: "2"
+    - name: prefetchCount # should be equal or less than the max concurrent workflows
+      value: "5"
+    - name: maxPriority   # set so we can prioritize some messages if wanted
+      value: "3"
 auth:
   secretStore: nemesis-secret-store
+---
+apiVersion: dapr.io/v1alpha1
+kind: Component
+metadata:
+  name: broadcast
+spec:
+  type: pubsub.rabbitmq
+  version: v1
+  metadata:
+  - name: connectionString
+    secretKeyRef:
+      name: RABBITMQ_CONNECTION_STRING
+      key: RABBITMQ_CONNECTION_STRING
+  - name: consumerID
+    value: mytest-{appID}-{uuid}    # Unique consumerID causes the message to be sent to each subscriber. Each subscriber has a RabbitMQ queue named accoridingly.
+  - name: durable
+    value: true
+  - name: deleteWhenUnused
+    value: true
+  - name: autoAck
+    value: true
+  - name: deliveryMode
+    value: 2                        # 1=transient messages(messages stay in-memory), 2=Persistent messages(use in conjunction with durable queues)
+  - name: publisherConfirm
+    value: true
+  - name: requeueInFailure
+    value: "true"
+  - name: reconnectWait
+    value: "3"
+  - name: concurrencyMode
+    value: parallel
+  - name: prefetchCount # should be equal or less than the max concurrent workflows
+    value: "5"
+  - name: maxPriority   # set so we can prioritize some messages if wanted
+    value: "3"
+auth:
+  secretStore: nemesis-secret-store

infra/dapr/components/workflowstate.yaml

@@ -7,9 +7,27 @@ spec:
   version: v1
   metadata:
   - name: connectionString
+    value: "postgresql://username:password@hostname:5432/databasename?sslmode=disable"
+  - name: host
     secretKeyRef:
-      name: POSTGRES_CONNECTION_STRING
-      key: POSTGRES_CONNECTION_STRING
+      name: POSTGRES_HOST
+      key: POSTGRES_HOST
+  - name: port
+    secretKeyRef:
+      name: POSTGRES_PORT
+      key: POSTGRES_PORT
+  - name: database
+    secretKeyRef:
+      name: POSTGRES_DB
+      key: POSTGRES_DB
+  - name: user
+    secretKeyRef:
+      name: POSTGRES_USER
+      key: POSTGRES_USER
+  - name: password
+    secretKeyRef:
+      name: POSTGRES_PASSWORD
+      key: POSTGRES_PASSWORD  
   - name: actorStateStore
     value: "true"
   - name: table

infra/dapr/configuration/config_monitoring_disabled.yaml

@@ -3,6 +3,9 @@ kind: Configuration
 metadata:
   name: schedulerconfig
 spec:
+  # workflow:
+  #   maxConcurrentWorkflowInvocations: 1
+  #   maxConcurrentActivityInvocations: 1
   httpPipeline:
     handlers:
     - name: maximum-request-size

infra/dapr/configuration/config_monitoring_enabled.yaml

@@ -3,6 +3,9 @@ kind: Configuration
 metadata:
   name: schedulerconfig
 spec:
+  # workflow:
+  #   maxConcurrentWorkflowInvocations: 1
+  #   maxConcurrentActivityInvocations: 1
   tracing:
     expandParams: true
     samplingRate: "1"

infra/grafana/provisioning/datasources/postgres.yaml

@@ -7,13 +7,8 @@ deleteDatasources:
 datasources:
   - name: Postgres
     type: postgres
-    url: postgres:5432
-    user: ${POSTGRES_USER}
-    secureJsonData:
-      password: ${POSTGRES_PASSWORD}
+    url: ${POSTGRES_CONNECTION_STRING}
     jsonData:
-      database: enrichment
-      sslmode: disable
       maxOpenConns: 100
       maxIdleConns: 100
       maxIdleConnsAuto: true

infra/hasura/metadata/tables.yaml

@@ -2,6 +2,9 @@
 - table:
     name: document_search_results
     schema: public
+- table:
+    name: container_processing
+    schema: public
 - table:
     name: workflows
     schema: public
@@ -139,3 +142,59 @@
 - table:
     name: yara_rules
     schema: public
+- table:
+    name: agent_prompts
+    schema: public
+- table:
+    name: file_linkings
+    schema: public
+- table:
+    name: file_listings
+    schema: public
+
+# `chromium` schema
+- table:
+    name: history
+    schema: chromium
+  object_relationships:
+    - name: files_enriched
+      using:
+        foreign_key_constraint_on: originating_object_id
+- table:
+    name: downloads
+    schema: chromium
+  object_relationships:
+    - name: files_enriched
+      using:
+        foreign_key_constraint_on: originating_object_id
+- table:
+    name: logins
+    schema: chromium
+  object_relationships:
+    - name: files_enriched
+      using:
+        foreign_key_constraint_on: originating_object_id
+- table:
+    name: cookies
+    schema: chromium
+  object_relationships:
+    - name: files_enriched
+      using:
+        foreign_key_constraint_on: originating_object_id
+- table:
+    name: state_keys
+    schema: chromium
+- table:
+    name: chrome_keys
+    schema: chromium
+
+# `dpapi` schema
+- table:
+    name: masterkeys
+    schema: dpapi
+- table:
+    name: system_credentials
+    schema: dpapi
+- table:
+    name: domain_backup_keys
+    schema: dpapi

infra/litellm/config.yml

@@ -0,0 +1,19 @@
+model_list:
+  # used if other task models are not defined
+  - model_name: default
+    litellm_params:
+      model: bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0
+      aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
+      aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
+      aws_region_name: "us-east-1"
+  # # used to triage findings and determine their validity
+  # - model_name: triage
+  #   litellm_params:
+  #     model: bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0
+  #     aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
+  #     aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
+  #     aws_region_name: "us-east-1"
+
+general_settings:
+  database_connection_pool_limit: 30
+  database_connection_timeout: 60

infra/postgres-exporter/postgres_exporter.yml

@@ -0,0 +1,2 @@
+# Postgres Exporter Configuration
+auth_modules: {}

infra/postgres/01-schema.sql

@@ -6,11 +6,13 @@ CREATE EXTENSION IF NOT EXISTS pg_trgm;
 CREATE TABLE IF NOT EXISTS files (
     object_id UUID PRIMARY KEY,
     agent_id VARCHAR(255),
+    source VARCHAR(1000),
     project VARCHAR(255),
     timestamp TIMESTAMP WITH TIME ZONE,
     expiration TIMESTAMP WITH TIME ZONE,
     path TEXT,
     originating_object_id UUID,
+    originating_container_id UUID, -- for large container processing
     nesting_level INTEGER,
     file_creation_time TIMESTAMP WITH TIME ZONE,
     file_access_time TIMESTAMP WITH TIME ZONE,
@@ -27,6 +29,7 @@ CREATE TABLE IF NOT EXISTS files (
 CREATE TABLE IF NOT EXISTS files_enriched (
     object_id UUID PRIMARY KEY,
     agent_id VARCHAR(255),
+    source VARCHAR(1000),
     project VARCHAR(255),
     timestamp TIMESTAMP WITH TIME ZONE,
     expiration TIMESTAMP WITH TIME ZONE,
@@ -39,6 +42,7 @@ CREATE TABLE IF NOT EXISTS files_enriched (
     is_plaintext BOOLEAN,
     is_container BOOLEAN,
     originating_object_id UUID,
+    originating_container_id UUID, -- for large container processing
     nesting_level INTEGER,
     file_creation_time TIMESTAMP WITH TIME ZONE,
     file_access_time TIMESTAMP WITH TIME ZONE,
@@ -105,6 +109,7 @@ CREATE TABLE IF NOT EXISTS files_feedback (
 CREATE TABLE IF NOT EXISTS files_enriched_dataset (
     object_id UUID PRIMARY KEY,
     agent_id VARCHAR(255),
+    source VARCHAR(1000),
     project VARCHAR(255),
     timestamp TIMESTAMP WITH TIME ZONE,
     expiration TIMESTAMP WITH TIME ZONE,
@@ -169,6 +174,9 @@ CREATE TABLE IF NOT EXISTS findings_triage_history (
     username VARCHAR(255) NOT NULL,
     automated BOOLEAN,
     value VARCHAR(255) NOT NULL,
+    explanation VARCHAR(5000),
+    confidence REAL,
+    true_positive_context VARCHAR(5000),
     timestamp TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP
 );
 
@@ -256,6 +264,7 @@ SELECT
     ef.extension,
     ef.project,
     ef.agent_id,
+    ef.source,
     ef."timestamp"::timestamp with time zone
 FROM plaintext_content pc
 JOIN files_enriched ef ON pc.object_id = ef.object_id;
@@ -268,7 +277,8 @@ CREATE OR REPLACE FUNCTION public.search_documents(
     project_name text DEFAULT NULL,
     start_date timestamp with time zone DEFAULT NULL,
     end_date timestamp with time zone DEFAULT NULL,
-    max_results integer DEFAULT 100
+    max_results integer DEFAULT 100,
+    source_pattern text DEFAULT NULL
 ) RETURNS SETOF document_search_results
 STABLE
 LANGUAGE sql
@@ -281,15 +291,13 @@ AS $$
             ) as rn
         FROM document_search_results
         WHERE
-            (
-                content_vector @@ plainto_tsquery('simple', search_query)
-                OR content ILIKE '%' || search_query || '%'
-                OR file_name ILIKE '%' || search_query || '%'
-            )            AND (path_pattern IS NULL OR path LIKE path_pattern)
+            content_vector @@ plainto_tsquery('simple', search_query)
+            AND (path_pattern IS NULL OR path LIKE path_pattern)
             AND (agent_pattern IS NULL OR agent_id LIKE agent_pattern)
             AND (project_name IS NULL OR project = project_name)
             AND (start_date IS NULL OR "timestamp" >= start_date)
             AND (end_date IS NULL OR "timestamp" <= end_date)
+            AND (source_pattern IS NULL OR source IS NULL OR source LIKE source_pattern)
     )
     SELECT
         object_id,
@@ -301,6 +309,7 @@ AS $$
         extension,
         project,
         agent_id,
+        source,
         "timestamp"
     FROM ranked_chunks
     WHERE rn = 1
@@ -331,6 +340,160 @@ CREATE TABLE IF NOT EXISTS yara_rules (
 );
 
 
+-----------------------
+-- Agent Prompts
+-----------------------
+CREATE TABLE IF NOT EXISTS agent_prompts (
+    name VARCHAR(255) NOT NULL PRIMARY KEY,
+    description TEXT,
+    prompt TEXT NOT NULL,
+    enabled BOOLEAN NOT NULL DEFAULT true,
+    created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP
+);
+
+
+-----------------------
+-- FILE LINKINGS
+-----------------------
+CREATE TABLE IF NOT EXISTS file_linkings (
+    linking_id BIGSERIAL PRIMARY KEY,
+    source VARCHAR(1000) NOT NULL,
+    file_path_1 TEXT NOT NULL,
+    file_path_2 TEXT NOT NULL,
+    link_type VARCHAR(255),  -- Optional: to specify the type of relationship
+    created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+    UNIQUE(source, file_path_1, file_path_2)
+);
+
+-- Create indexes for efficient lookups in both directions
+CREATE INDEX IF NOT EXISTS idx_file_linkings_file_1 ON file_linkings(file_path_1);
+CREATE INDEX IF NOT EXISTS idx_file_linkings_file_2 ON file_linkings(file_path_2);
+CREATE INDEX IF NOT EXISTS idx_file_linkings_source ON file_linkings(source);
+
+
+-----------------------
+-- FILE LISTINGS
+-----------------------
+CREATE TABLE IF NOT EXISTS file_listings (
+    listing_id BIGSERIAL PRIMARY KEY,
+    source VARCHAR(1000) NOT NULL,
+    path TEXT NOT NULL,
+    object_id UUID,
+    status VARCHAR(50) NOT NULL CHECK (status IN ('needs_to_be_collected', 'not_exists', 'collected', 'not_wanted')),
+    created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+    path_lower TEXT GENERATED ALWAYS AS (LOWER(path)) STORED,
+    UNIQUE(source, path_lower),
+    FOREIGN KEY (object_id) REFERENCES files_enriched(object_id) ON DELETE CASCADE
+);
+
+-- Create indexes for efficient queries
+CREATE INDEX IF NOT EXISTS idx_file_listings_source ON file_listings(source);
+CREATE INDEX IF NOT EXISTS idx_file_listings_status ON file_listings(status);
+CREATE INDEX IF NOT EXISTS idx_file_listings_object_id ON file_listings(object_id);
+-- Create composite index for efficient path prefix queries
+CREATE INDEX IF NOT EXISTS idx_file_listings_source_path ON file_listings(source, path);
+-- Create trigram index for path pattern matching
+CREATE INDEX IF NOT EXISTS idx_file_listings_path_trgm ON file_listings USING gist (path gist_trgm_ops);
+
+-- Helper functions for file browser hierarchical navigation
+CREATE OR REPLACE FUNCTION get_path_depth(file_path text)
+RETURNS integer AS $$
+BEGIN
+    -- Count forward slashes to determine depth
+    -- Root files (no slash) are depth 0, /folder/file is depth 1, etc.
+    IF file_path = '' OR file_path IS NULL THEN
+        RETURN 0;
+    END IF;
+    RETURN array_length(string_to_array(file_path, '/'), 1) - 1;
+END;
+$$ LANGUAGE plpgsql IMMUTABLE;
+
+CREATE OR REPLACE FUNCTION get_path_parent(file_path text)
+RETURNS text AS $$
+BEGIN
+    -- Return parent path, handling edge cases
+    IF file_path IS NULL OR file_path = '' OR position('/' in file_path) = 0 THEN
+        RETURN '';
+    END IF;
+    RETURN regexp_replace(file_path, '/[^/]*$', '');
+END;
+$$ LANGUAGE plpgsql IMMUTABLE;
+
+CREATE OR REPLACE FUNCTION get_path_filename(file_path text)
+RETURNS text AS $$
+BEGIN
+    -- Extract just the filename/folder name from full path
+    IF file_path IS NULL OR file_path = '' THEN
+        RETURN '';
+    END IF;
+    RETURN regexp_replace(file_path, '^.*/', '');
+END;
+$$ LANGUAGE plpgsql IMMUTABLE;
+
+CREATE OR REPLACE FUNCTION is_file_path(file_path text)
+RETURNS boolean AS $$
+BEGIN
+    -- Simple heuristic: if path has an extension, it's likely a file
+    -- This isn't perfect but works for most cases
+    IF file_path IS NULL OR file_path = '' THEN
+        RETURN false;
+    END IF;
+    -- Check if the last part after the final slash contains a dot
+    RETURN get_path_filename(file_path) LIKE '%.%';
+END;
+$$ LANGUAGE plpgsql IMMUTABLE;
+
+-- View for hierarchical file browser queries
+-- This creates virtual folder entries for efficient navigation
+CREATE OR REPLACE VIEW file_listings_hierarchy AS
+WITH RECURSIVE folder_paths AS (
+    -- Get all unique folder paths from file paths
+    SELECT DISTINCT
+        source,
+        get_path_parent(path) as folder_path,
+        get_path_depth(get_path_parent(path)) as depth
+    FROM file_listings
+    WHERE get_path_parent(path) != ''
+
+    UNION
+
+    -- Add parent folders recursively
+    SELECT
+        source,
+        get_path_parent(folder_path) as folder_path,
+        get_path_depth(get_path_parent(folder_path)) as depth
+    FROM folder_paths
+    WHERE get_path_parent(folder_path) != '' AND get_path_parent(folder_path) != folder_path
+)
+SELECT
+    source,
+    folder_path as path,
+    'folder' as item_type,
+    null::uuid as object_id,
+    'folder' as status,
+    depth,
+    get_path_parent(folder_path) as parent_path,
+    get_path_filename(folder_path) as name
+FROM folder_paths
+WHERE folder_path != ''
+
+UNION ALL
+
+SELECT
+    source,
+    path,
+    'file' as item_type,
+    object_id,
+    status,
+    get_path_depth(path) as depth,
+    get_path_parent(path) as parent_path,
+    get_path_filename(path) as name
+FROM file_listings;
+
+
 -----------------------
 -- CREATE UPDATE FIELD TRIGGER
 -----------------------
@@ -362,6 +525,11 @@ CREATE OR REPLACE TRIGGER update_yara_rules_updated_at
     FOR EACH ROW
     EXECUTE FUNCTION update_updated_at_column();
 
+CREATE OR REPLACE TRIGGER update_agent_prompts_updated_at
+    BEFORE UPDATE ON agent_prompts
+    FOR EACH ROW
+    EXECUTE FUNCTION update_updated_at_column();
+
 CREATE OR REPLACE TRIGGER update_enrichments_updated_at
     BEFORE UPDATE ON enrichments
     FOR EACH ROW
@@ -382,6 +550,15 @@ CREATE OR REPLACE TRIGGER update_files_enriched_dataset_updated_at
     FOR EACH ROW
     EXECUTE FUNCTION update_updated_at_column();
 
+CREATE OR REPLACE TRIGGER update_file_linkings_updated_at
+    BEFORE UPDATE ON file_linkings
+    FOR EACH ROW
+    EXECUTE FUNCTION update_updated_at_column();
+
+CREATE OR REPLACE TRIGGER update_file_listings_updated_at
+    BEFORE UPDATE ON file_listings
+    FOR EACH ROW
+    EXECUTE FUNCTION update_updated_at_column();
 
 
 -----------------------
@@ -397,3 +574,244 @@ CREATE TABLE IF NOT EXISTS workflows (
     runtime_seconds REAL,
     start_time TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP
 );
+
+
+-----------------------
+-- Container Processing Tracking
+-----------------------
+CREATE TABLE IF NOT EXISTS container_processing (
+    container_id UUID NOT NULL PRIMARY KEY DEFAULT gen_random_uuid(),
+    container_type VARCHAR(50) NOT NULL,
+    original_filename VARCHAR(255),
+    original_size BIGINT,
+    agent_id VARCHAR(255),
+    source VARCHAR(1000),
+    project VARCHAR(255),
+    status VARCHAR(50) NOT NULL DEFAULT 'submitted',
+    total_files_extracted INTEGER DEFAULT 0,
+    total_bytes_extracted  INTEGER DEFAULT 0,
+    submitted_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+    expiration TIMESTAMP WITH TIME ZONE,
+    processing_started_at TIMESTAMP WITH TIME ZONE,
+    processing_completed_at TIMESTAMP WITH TIME ZONE,
+    error_message TEXT,
+    workflows_completed INTEGER DEFAULT 0,
+    workflows_failed INTEGER DEFAULT 0,
+    workflows_total INTEGER DEFAULT 0,
+    total_bytes_processed  INTEGER DEFAULT 0
+);
+
+
+-- Create phoenix database
+CREATE DATABASE phoenix;
+
+
+-----------------------
+-- Chromium schema/tables
+-----------------------
+
+CREATE SCHEMA chromium;
+
+-- "urls" table in "History" file
+CREATE TABLE IF NOT EXISTS chromium.history (
+    id SERIAL PRIMARY KEY,
+    originating_object_id UUID,
+    agent_id VARCHAR(255),
+    source VARCHAR(1000),
+    project VARCHAR(255),
+    username TEXT,                              -- username extracted from user data directory, if applicable
+    browser TEXT,                               -- browser name extracted from user data directory, if applicable
+
+    url TEXT,                                   -- extracted from the Chromium DB
+    title TEXT,                                 -- extracted from the Chromium DB
+    visit_count INTEGER,                        -- extracted from the Chromium DB
+    last_visit_time TIMESTAMP WITH TIME ZONE,   -- extracted from the Chromium DB
+
+    FOREIGN KEY (originating_object_id) REFERENCES files_enriched(object_id) ON DELETE CASCADE,
+    UNIQUE (source, username, browser, url, title, last_visit_time)
+);
+
+-- "downloads" table in "History" file
+CREATE TABLE IF NOT EXISTS chromium.downloads (
+    id SERIAL PRIMARY KEY,
+    originating_object_id UUID,
+    agent_id VARCHAR(255),
+    source VARCHAR(1000),
+    project VARCHAR(255),
+    username TEXT,                              -- username extracted from user data directory, if applicable
+    browser TEXT,                               -- browser name extracted from user data directory, if applicable
+
+    url TEXT,                                   -- extracted from the Chromium DB
+    download_path TEXT,                         -- extracted from the Chromium DB `target_path` field
+    start_time TIMESTAMP WITH TIME ZONE,        -- extracted from the Chromium DB
+    end_time TIMESTAMP WITH TIME ZONE,          -- extracted from the Chromium DB
+    total_bytes INTEGER,                        -- extracted from the Chromium DB
+
+    FOREIGN KEY (originating_object_id) REFERENCES files_enriched(object_id) ON DELETE CASCADE,
+    UNIQUE (source, username, browser, url, download_path, start_time)
+);
+
+-- extracted from a Chromium "Local State" file
+CREATE TABLE chromium.state_keys (
+    id SERIAL PRIMARY KEY,
+    originating_object_id UUID,
+    agent_id VARCHAR(255),
+    source VARCHAR(1000),
+    project VARCHAR(255),
+    username TEXT,                                      -- username extracted from user data directory, if applicable
+    browser TEXT,                                       -- browser name extracted from user data directory, if applicable
+
+    key_masterkey_guid UUID,                            -- associated masterkey GUID for key_bytes_enc
+    key_bytes_enc BYTEA,                                -- os_crypt.encrypted_key in Chromium `Local State` file (pre v127)
+    key_bytes_dec BYTEA,
+    key_is_decrypted BOOLEAN,
+
+    app_bound_key_enc BYTEA,                            -- os_crypt.app_bound_encrypted_key in Chromium `Local State` file (post v127)
+    app_bound_key_system_masterkey_guid UUID,           -- associated _system_ masterkey GUID for key_bytes_enc
+    app_bound_key_system_dec BYTEA,                     -- intermediate dec value after the SYSTEM key has been used
+    app_bound_key_user_masterkey_guid UUID,             -- associated _user_ masterkey GUID for app_bound_key_system_dec
+    app_bound_key_user_dec BYTEA,                       -- intermediate dec value after the USER key has been used (before chromekey for v3)
+    app_bound_key_dec BYTEA,                            -- completely dec value
+    app_bound_key_is_decrypted BOOLEAN,
+
+    FOREIGN KEY (originating_object_id) REFERENCES files_enriched(object_id) ON DELETE CASCADE,
+    UNIQUE (source, username, browser)
+);
+
+-- AES keys extracted from a CNG "Google Chromekey1" CNG file from C:\ProgramData\Microsoft\Crypto\SystemKeys\
+--      Used in v3 of the Chromium ABE decryption
+CREATE TABLE chromium.chrome_keys (
+    id SERIAL PRIMARY KEY,
+    originating_object_id UUID,
+    agent_id VARCHAR(255),
+    source VARCHAR(1000),                               -- should only be one key per host/source
+    project VARCHAR(255),
+
+    key_masterkey_guid UUID,                            -- associated _system_ masterkey GUID for key_bytes_enc
+    key_bytes_enc BYTEA,                                -- the raw DPAPI blob bytes from the CNG file
+    key_bytes_dec BYTEA,                                -- completely dec AES key value
+    key_is_decrypted BOOLEAN,
+
+    FOREIGN KEY (originating_object_id) REFERENCES files_enriched(object_id) ON DELETE CASCADE,
+    UNIQUE (key_masterkey_guid)
+);
+
+-- Create indexes for masterkey GUID lookups on state_keys
+CREATE INDEX IF NOT EXISTS idx_state_keys_key_masterkey_guid ON chromium.state_keys(key_masterkey_guid) WHERE key_is_decrypted = FALSE;
+CREATE INDEX IF NOT EXISTS idx_state_keys_app_bound_system_mk_guid ON chromium.state_keys(app_bound_key_system_masterkey_guid) WHERE length(app_bound_key_system_dec) = 0;
+CREATE INDEX IF NOT EXISTS idx_state_keys_app_bound_user_mk_guid ON chromium.state_keys(app_bound_key_user_masterkey_guid) WHERE length(app_bound_key_user_dec) = 0;
+
+-- "logins" table in "Login Data" file
+CREATE TABLE IF NOT EXISTS chromium.logins (
+    id SERIAL PRIMARY KEY,
+    originating_object_id UUID,
+    agent_id VARCHAR(255),
+    source VARCHAR(1000),
+    project VARCHAR(255),
+    username TEXT,                                      -- username extracted from user data directory, if applicable
+    browser TEXT,                                       -- browser name extracted from user data directory, if applicable
+
+    origin_url TEXT,                                    -- extracted from the Chromium DB
+    username_value TEXT,                                -- extracted from the Chromium DB
+    signon_realm TEXT,                                  -- extracted from the Chromium DB
+    date_created TIMESTAMP WITH TIME ZONE,              -- extracted from the Chromium DB
+    date_last_used TIMESTAMP WITH TIME ZONE,            -- extracted from the Chromium DB
+    date_password_modified TIMESTAMP WITH TIME ZONE,    -- extracted from the Chromium DB
+    times_used INTEGER,                                 -- extracted from the Chromium DB
+
+    encryption_type TEXT,                               -- carved from the `password_value_enc` bytes - dpapi, key, or abe (app-bound-encryption)
+    masterkey_guid UUID,                                -- if encryption_type == dpapi, associated masterkey GUID
+    state_key_id INTEGER,                               -- if encryption_type != dpapi, linked to "id" in `chromium.state_keys`
+    is_decrypted BOOLEAN,
+    password_value_enc BYTEA,                           -- extracted from the Chromium DB `password_value` field
+    password_value_dec TEXT,
+
+    FOREIGN KEY (originating_object_id) REFERENCES files_enriched(object_id) ON DELETE CASCADE,
+    FOREIGN KEY (state_key_id) REFERENCES chromium.state_keys(id) ON DELETE SET NULL,
+    UNIQUE (source, username, browser, origin_url, username_value)
+);
+
+-- "cookies" table in "Cookies" file
+CREATE TABLE IF NOT EXISTS chromium.cookies (
+    id SERIAL PRIMARY KEY,
+    originating_object_id UUID,
+    agent_id VARCHAR(255),
+    source VARCHAR(1000),
+    project VARCHAR(255),
+    username TEXT,                                      -- username extracted from user data directory, if applicable
+    browser TEXT,                                       -- browser name extracted from user data directory, if applicable
+
+    host_key TEXT,                                      -- extracted from the Chromium DB
+    name TEXT,                                          -- extracted from the Chromium DB
+    path TEXT,                                          -- extracted from the Chromium DB
+    creation_utc TIMESTAMP WITH TIME ZONE,              -- extracted from the Chromium DB
+    expires_utc TIMESTAMP WITH TIME ZONE,               -- extracted from the Chromium DB
+    last_access_utc TIMESTAMP WITH TIME ZONE,           -- extracted from the Chromium DB
+    last_update_utc TIMESTAMP WITH TIME ZONE,           -- extracted from the Chromium DB
+    is_secure BOOLEAN,                                  -- extracted from the Chromium DB
+    is_httponly BOOLEAN,                                -- extracted from the Chromium DB
+    is_persistent BOOLEAN,                              -- extracted from the Chromium DB
+    samesite TEXT,                                      -- extracted from the Chromium DB, translated from int
+    source_port INTEGER,                                -- extracted from the Chromium DB
+
+    encryption_type TEXT,                               -- carved from the `encrypted_value` field - dpapi, key, or abe (app-bound-encryption)
+    masterkey_guid UUID,                                -- if encryption_type == dpapi, associated masterkey GUID
+    state_key_id INTEGER,                               -- if encryption_type != dpapi, linked to "id" in `chromium.state_keys`
+    is_decrypted BOOLEAN,
+    value_enc BYTEA,                                    -- extracted from the Chromium DB `encrypted_value` field
+    value_dec TEXT,
+
+    FOREIGN KEY (originating_object_id) REFERENCES files_enriched(object_id) ON DELETE CASCADE,
+    FOREIGN KEY (state_key_id) REFERENCES chromium.state_keys(id) ON DELETE SET NULL,
+    UNIQUE (source, username, browser, host_key, name, path)
+);
+
+-- DPAPI tables
+CREATE SCHEMA dpapi;
+
+CREATE TABLE IF NOT EXISTS dpapi.masterkeys (
+    id SERIAL PRIMARY KEY,
+    guid TEXT UNIQUE NOT NULL,
+    encrypted_key_usercred BYTEA,
+    encrypted_key_backup BYTEA,
+    plaintext_key BYTEA,
+    plaintext_key_sha1 BYTEA,
+    backup_key_guid TEXT,
+    masterkey_type TEXT DEFAULT 'unknown',
+    created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP
+);
+
+CREATE TABLE IF NOT EXISTS dpapi.domain_backup_keys (
+    id SERIAL PRIMARY KEY,
+    guid TEXT UNIQUE NOT NULL,
+    key_data BYTEA NOT NULL,
+    domain_controller TEXT,
+    created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP
+);
+
+CREATE TABLE IF NOT EXISTS dpapi.system_credentials (
+    id SERIAL PRIMARY KEY,
+    user_key BYTEA NOT NULL,
+    machine_key BYTEA NOT NULL,
+    created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+    UNIQUE (user_key, machine_key)
+);
+
+-- Create triggers for DPAPI tables
+CREATE OR REPLACE TRIGGER update_dpapi_masterkeys_updated_at
+    BEFORE UPDATE ON dpapi.masterkeys
+    FOR EACH ROW
+    EXECUTE FUNCTION update_updated_at_column();
+
+CREATE OR REPLACE TRIGGER update_dpapi_domain_backup_keys_updated_at
+    BEFORE UPDATE ON dpapi.domain_backup_keys
+    FOR EACH ROW
+    EXECUTE FUNCTION update_updated_at_column();
+
+CREATE OR REPLACE TRIGGER update_dpapi_system_credentials_updated_at
+    BEFORE UPDATE ON dpapi.system_credentials
+    FOR EACH ROW
+    EXECUTE FUNCTION update_updated_at_column();

infra/prometheus/prometheus.yml

@@ -9,20 +9,128 @@ scrape_configs:
       - targets: ['traefik:8080']
 
   - job_name: 'minio'
-    metrics_path: /minio/v2/metrics/cluster
+    metrics_path: /minio/metrics/v3/cluster/health
     scheme: http
     static_configs:
       - targets: ['minio:9000']
 
+  - job_name: 'minio-api'
+    metrics_path: /minio/metrics/v3/api/requests
+    scheme: http
+    static_configs:
+      - targets: ['minio:9000']
+
+  - job_name: 'minio-bucket-api'
+    metrics_path: /minio/metrics/v3/bucket/api
+    scheme: http
+    static_configs:
+      - targets: ['minio:9000']
+
+  - job_name: 'minio-cluster-usage-buckets'
+    metrics_path: /minio/metrics/v3/cluster/usage/buckets
+    scheme: http
+    static_configs:
+      - targets: ['minio:9000']
+
+  - job_name: 'minio-cluster-usage-objects'
+    metrics_path: /minio/metrics/v3/cluster/usage/objects
+    scheme: http
+    static_configs:
+      - targets: ['minio:9000']
+
+  - job_name: 'minio-drives'
+    metrics_path: /minio/metrics/v3/system/drive
+    scheme: http
+    static_configs:
+      - targets: ['minio:9000']
+
+  - job_name: 'loki'
+    metrics_path: '/metrics'
+    static_configs:
+      - targets: ['loki:3100']
+
   - job_name: 'prometheus'
     metrics_path: '/prometheus/metrics'
     static_configs:
       - targets: ['localhost:9090']
 
   - job_name: 'node-exporter'
+    metrics_path: '/metrics'
     static_configs:
       - targets: ['node-exporter:9100']
 
   - job_name: 'cadvisor'
+    metrics_path: '/metrics'
     static_configs:
-      - targets: ['cadvisor:8080']
+      - targets: ['cadvisor:8080']
+
+  - job_name: 'postgres'
+    metrics_path: '/metrics'
+    static_configs:
+      - targets: ['postgres-exporter:9187']
+
+  - job_name: 'rabbitmq'
+    metrics_path: '/metrics'
+    static_configs:
+      - targets: ['rabbitmq:15692']
+
+  # Reserved for enterprise edition only?
+  # - job_name: 'hasura'
+  #   metrics_path: '/v1/metrics'
+  #   static_configs:
+  #     - targets: ['hasura:8080']
+
+  - job_name: 'nemesis-web-api'
+    metrics_path: '/metrics'
+    static_configs:
+      - targets: ['web-api:9090']
+
+  - job_name: 'nemesis-noseyparker-scanner'
+    metrics_path: '/metrics'
+    static_configs:
+      - targets: ['noseyparker-scanner:9090']
+
+  - job_name: 'nemesis-dotnet-service'
+    metrics_path: '/metrics'
+    static_configs:
+      - targets: ['dotnet-service:9090']
+
+  - job_name: 'nemesis-file-enrichment'
+    metrics_path: '/metrics'
+    static_configs:
+      - targets: ['file-enrichment:9090']
+
+  - job_name: 'nemesis-alerting'
+    metrics_path: '/metrics'
+    static_configs:
+      - targets: ['alerting:9090']
+
+  - job_name: 'nemesis-agents'
+    metrics_path: '/metrics'
+    static_configs:
+      - targets: ['agents:9090']
+
+  - job_name: 'nemesis-housekeeping'
+    metrics_path: '/metrics'
+    static_configs:
+      - targets: ['housekeeping:9090']
+
+  - job_name: 'nemesis-document-conversion'
+    metrics_path: '/metrics'
+    static_configs:
+      - targets: ['document-conversion:9090']
+
+  - job_name: 'nemesis-gotenberg'
+    metrics_path: '/prometheus/metrics'
+    static_configs:
+      - targets: ['gotenberg:3000']
+
+  - job_name: 'nemesis-placement'
+    metrics_path: '/metrics'
+    static_configs:
+      - targets: ['placement:9090']
+
+  - job_name: 'nemesis-scheduler'
+    metrics_path: '/metrics'
+    static_configs:
+      - targets: ['scheduler:9090']

infra/rabbitmq/enabled_plugins

@@ -0,0 +1 @@
+[rabbitmq_management,rabbitmq_prometheus].

infra/tika/tika-config.xml

@@ -6,6 +6,64 @@
       <maxFiles>10</maxFiles>
       <numConsumers>5</numConsumers>
       <timeoutThresholdMillis>300000</timeoutThresholdMillis>
+      <!-- Increase max file size for large documents -->
+      <maxFileSize>1073741824</maxFileSize>
     </params>
   </server>
+
+  <parsers>
+    <!-- Needed to ensure existing parsers fire -->
+    <parser class="org.apache.tika.parser.DefaultParser"/>
+
+    <parser class="org.apache.tika.parser.ocr.TesseractOCRParser">
+      <params>
+        <!-- Multi-language support: add languages as needed (e.g., eng+spa+fra+deu) -->
+        <param name="language" type="string">eng</param>
+
+        <!-- Timeout in seconds - adjust based on document complexity -->
+        <param name="timeout" type="int">300</param>
+
+        <!-- Image preprocessing improves accuracy -->
+        <param name="enableImageProcessing" type="bool">true</param>
+
+        <!-- Page segmentation mode: 3 is fully automatic (best for most documents) -->
+        <param name="pageSegMode" type="string">3</param>
+
+        <!-- DPI for rendering PDFs to images - 300 is optimal for OCR -->
+        <param name="density" type="int">300</param>
+
+        <!-- Image depth - 4 bits is good balance between speed and quality -->
+        <param name="depth" type="int">4</param>
+
+        <!-- Color space - gray is faster, sufficient for most text -->
+        <param name="colorspace" type="string">gray</param>
+
+        <!-- Filter for better preprocessing -->
+        <param name="filter" type="string">triangle</param>
+
+        <!-- Resize images if too large (max dimension in pixels) -->
+        <param name="resize" type="int">900</param>
+
+        <!-- Apply image processing enhancements -->
+        <param name="applyRotation" type="bool">true</param>
+      </params>
+    </parser>
+
+    <!-- PDF Parser with OCR support -->
+    <parser class="org.apache.tika.parser.pdf.PDFParser">
+      <params>
+        <!-- Auto: use OCR only when needed -->
+        <param name="ocrStrategy" type="string">auto</param>
+        <param name="extractInlineImages" type="bool">true</param>
+        <param name="extractFontNames" type="bool">true</param>
+        <!-- Prevent OOM on large PDFs -->
+        <param name="maxMainMemoryBytes" type="long">536870912</param>
+      </params>
+    </parser>
+
+  </parsers>
+
+  <!-- Enable language detection -->
+  <detector class="org.apache.tika.langdetect.optimaize.OptimaizeLangDetector"/>
+
 </properties>

libs/chromium/.vscode/settings.json

@@ -6,15 +6,13 @@
         "editor.formatOnSave": false
     },
     "[python]": {
-        "editor.formatOnSave": true,
         "editor.codeActionsOnSave": {
-            "source.fixAll": "explicit",
-            "source.organizeImports": "explicit"
+            "source.fixAll.ruff": "explicit",
+            "source.organizeImports.ruff": "explicit"
         },
         "editor.defaultFormatter": "charliermarsh.ruff"
     },
     "autoDocstring.docstringFormat": "google",
-    "editor.formatOnSave": true,
     "files.exclude": {
         "**/.DS_Store": true,
         "**/.git": true,
@@ -39,27 +37,18 @@
         "**/*.egg-info/**": true,
         "**/build/**": true,
         "**/dist/**": true,
-        "**/node_modules/*/**": true,
+        "**/node_modules/*/**": true
     },
     "python.analysis.diagnosticSeverityOverrides": {
         "reportMissingImports": "none",
-        "reportMissingModuleSource": "none",
+        "reportMissingModuleSource": "none"
     },
     "python.analysis.useLibraryCodeForTypes": true, // Pyright
-    // "python.formatting.provider": "none",
     "python.languageServer": "Pylance",
-    "python.linting.enabled": true,
-    "python.linting.flake8Args": [
-        "--max-line-length=240",
-        "--ignore=E203,E722,W503",
-    ],
-    "python.linting.flake8Enabled": true,
-    "python.linting.lintOnSave": true,
-    "python.linting.pylintEnabled": false,
     "python.testing.pytestArgs": [
         "tests"
     ],
     "python.testing.unittestEnabled": false,
     "python.testing.pytestEnabled": true,
-    "python.analysis.typeCheckingMode": "basic",
+    "python.analysis.typeCheckingMode": "basic"
 }

libs/chromium/README.md

@@ -0,0 +1,2 @@
+# Chromium
+A Chromium helper library.

libs/chromium/chromium/__init__.py

@@ -0,0 +1,25 @@
+"""Chromium utility library for Chromium focused operations."""
+
+from .chromekey import retry_decrypt_chrome_keys_for_masterkey
+from .cookies import process_chromium_cookies
+from .helpers import convert_chromium_timestamp
+from .history import process_chromium_history
+from .local_state import (
+    process_chromium_local_state,
+    retry_decrypt_state_keys_for_chromekey,
+    retry_decrypt_state_keys_for_masterkey,
+)
+from .logins import process_chromium_logins
+from .retry import retry_decrypt_chromium_data
+
+__all__ = [
+    "convert_chromium_timestamp",
+    "process_chromium_history",
+    "process_chromium_cookies",
+    "process_chromium_logins",
+    "process_chromium_local_state",
+    "retry_decrypt_chrome_keys_for_masterkey",
+    "retry_decrypt_chromium_data",
+    "retry_decrypt_state_keys_for_chromekey",
+    "retry_decrypt_state_keys_for_masterkey",
+]

libs/chromium/chromium/chromekey.py

@@ -0,0 +1,219 @@
+"""Chrome Key and database operations."""
+
+from uuid import UUID
+
+import psycopg
+from common.logger import get_logger
+from file_enrichment_modules.cng_file.cng_parser import check_bcrypt_key_blob, extract_final_key_material
+from nemesis_dpapi import Blob, DpapiManager, MasterKeyNotDecryptedError, MasterKeyNotFoundError
+
+from .helpers import get_postgres_connection_str
+from .local_state import retry_decrypt_state_keys_for_chromekey
+
+logger = get_logger(__name__)
+
+
+async def _retry_state_keys_for_chromekey_after_decrypt(source: str, chromekey: bytes):
+    """Helper to retry state key decryption after a chromekey is decrypted.
+
+    Import is done here to avoid circular dependency issues.
+    """
+    return await retry_decrypt_state_keys_for_chromekey(source, chromekey)
+
+
+async def retry_decrypt_chrome_key(chrome_key_id: int, dpapi_manager: DpapiManager, pg_conn) -> dict:
+    """Attempt to decrypt a single chrome_key record using currently available masterkeys.
+
+    Args:
+        chrome_key_id: The ID of the chrome_key record to decrypt
+        dpapi_manager: DpapiManager instance for decryption
+        pg_conn: PostgreSQL connection
+
+    Returns:
+        Dict with decryption results: {
+            "decrypted": bool,
+            "state_keys_result": dict (optional, if decryption succeeded)
+        }
+    """
+    result = {"decrypted": False}
+    cng_key_blob_entropy = b'xT5rZW5qVVbrvpuA\x00'
+
+    # Fetch the chrome_key record
+    with pg_conn.cursor() as cur:
+        cur.execute(
+            """
+            SELECT id, source, key_masterkey_guid, key_bytes_enc,
+                   key_bytes_dec, key_is_decrypted
+            FROM chromium.chrome_keys
+            WHERE id = %s
+            """,
+            (chrome_key_id,),
+        )
+        row = cur.fetchone()
+
+        if not row:
+            logger.warning("Chrome key not found", chrome_key_id=chrome_key_id)
+            return result
+
+        (record_id, source, key_masterkey_guid, key_bytes_enc, key_bytes_dec, key_is_decrypted) = row
+
+    # Skip if already decrypted
+    if key_is_decrypted:
+        return result
+
+    # Try to decrypt the chrome key
+    if key_bytes_enc and len(key_bytes_enc) > 0:
+        try:
+            dpapi_blob = Blob.from_bytes(key_bytes_enc)
+            try:
+                decrypted_blob = await dpapi_manager.decrypt_blob(dpapi_blob, entropy=cng_key_blob_entropy)
+                if decrypted_blob:
+                    # Check for BCRYPT_KEY_DATA_BLOB and log details
+                    if check_bcrypt_key_blob(decrypted_blob):
+
+                        # Extract the final 32-byte key material
+                        key_bytes_dec = extract_final_key_material(decrypted_blob)
+
+                        if key_bytes_dec:
+                            key_is_decrypted = True
+                            result["decrypted"] = True
+                            logger.info(
+                                "Successfully decrypted and extracted chrome_key material",
+                                chrome_key_id=chrome_key_id,
+                                masterkey_guid=dpapi_blob.masterkey_guid,
+                            )
+
+                            # Update database
+                            with pg_conn.cursor() as cur:
+                                cur.execute(
+                                    """
+                                    UPDATE chromium.chrome_keys
+                                    SET key_bytes_dec = %s, key_is_decrypted = %s,
+                                        key_masterkey_guid = %s
+                                    WHERE id = %s
+                                    """,
+                                    (key_bytes_dec, key_is_decrypted, dpapi_blob.masterkey_guid, chrome_key_id),
+                                )
+
+                            # Commit the chrome_key update before trying state_keys
+                            pg_conn.commit()
+
+                            # Now try to decrypt any state_keys waiting for this chromekey
+                            try:
+                                state_keys_result = await _retry_state_keys_for_chromekey_after_decrypt(source, key_bytes_dec)
+                                result["state_keys_result"] = state_keys_result
+                                logger.info(
+                                    "Completed retroactive state_key decryption for newly decrypted chromekey",
+                                    chrome_key_id=chrome_key_id,
+                                    source=source,
+                                    state_keys_result=state_keys_result,
+                                )
+                            except Exception as e:
+                                logger.warning(
+                                    "Error retrying state_keys after chromekey decryption",
+                                    chrome_key_id=chrome_key_id,
+                                    source=source,
+                                    error=str(e),
+                                )
+                        else:
+                            logger.warning(
+                                "Failed to extract final key material from decrypted chrome_key",
+                                chrome_key_id=chrome_key_id,
+                            )
+
+            except (MasterKeyNotFoundError, MasterKeyNotDecryptedError):
+                # Masterkey still not available, skip silently
+                pass
+            except Exception as e:
+                logger.warning("Error decrypting chrome_key", chrome_key_id=chrome_key_id, error=str(e))
+
+        except Exception as e:
+            logger.warning("Error processing chrome_key", chrome_key_id=chrome_key_id, error=str(e))
+    else:
+        # Commit even if no decryption happened (for consistency)
+        pg_conn.commit()
+
+    return result
+
+
+async def retry_decrypt_chrome_keys_for_masterkey(
+    masterkey_guid: UUID, dpapi_manager: DpapiManager, masterkey_type: str | None = None
+) -> dict:
+    """Find all chrome_keys waiting for this masterkey and try to decrypt them.
+
+    Args:
+        masterkey_guid: The GUID of the newly available masterkey
+        dpapi_manager: DpapiManager instance for decryption
+        masterkey_type: Optional masterkey type ('system', 'user', 'unknown') for optimization
+
+    Returns:
+        Dict with statistics: {
+            "chrome_keys_attempted": int,
+            "chrome_keys_decrypted": int,
+            "errors": list
+        }
+    """
+    result = {"chrome_keys_attempted": 0, "chrome_keys_decrypted": 0, "errors": []}
+
+    # Chrome keys only use SYSTEM masterkeys, so skip if this is a USER key
+    if masterkey_type == "user":
+        logger.debug(
+            "Skipping chrome_key decryption for USER masterkey",
+            masterkey_guid=masterkey_guid,
+            masterkey_type=masterkey_type,
+        )
+        return result
+
+    conn_str = get_postgres_connection_str()
+
+    try:
+        with psycopg.connect(conn_str) as pg_conn:
+            # Find all chrome_keys that might need this masterkey
+            with pg_conn.cursor() as cur:
+                # SYSTEM keys are used for chrome_keys, but if type is unknown, still try
+                query = """
+                    SELECT DISTINCT id FROM chromium.chrome_keys
+                    WHERE key_masterkey_guid = %s AND key_is_decrypted = FALSE
+                """
+                cur.execute(query, (masterkey_guid,))
+
+                chrome_key_ids = [row[0] for row in cur.fetchall()]
+
+            logger.debug(
+                "Found chrome_keys potentially waiting for masterkey",
+                masterkey_guid=masterkey_guid,
+                masterkey_type=masterkey_type,
+                count=len(chrome_key_ids),
+            )
+
+            # Try to decrypt each chrome_key
+            for chrome_key_id in chrome_key_ids:
+                result["chrome_keys_attempted"] += 1
+                try:
+                    decrypt_result = await retry_decrypt_chrome_key(chrome_key_id, dpapi_manager, pg_conn)
+
+                    # Check if decryption succeeded
+                    if decrypt_result["decrypted"]:
+                        result["chrome_keys_decrypted"] += 1
+
+                except Exception as e:
+                    error_msg = f"Error processing chrome_key {chrome_key_id}: {str(e)}"
+                    logger.warning("Failed to retry decrypt chrome_key", chrome_key_id=chrome_key_id, error=str(e))
+                    result["errors"].append(error_msg)
+
+            logger.info(
+                "Completed retroactive chrome_key decryption for masterkey",
+                masterkey_guid=masterkey_guid,
+                attempted=result["chrome_keys_attempted"],
+                decrypted=result["chrome_keys_decrypted"],
+                errors=len(result["errors"]),
+            )
+
+    except Exception as e:
+        error_msg = f"Database error during retroactive chrome_key decryption: {str(e)}"
+        logger.exception(
+            "Error in retry_decrypt_chrome_keys_for_masterkey", masterkey_guid=masterkey_guid, error=str(e)
+        )
+        result["errors"].append(error_msg)
+
+    return result

libs/chromium/chromium/cookies.py

@@ -0,0 +1,282 @@
+"""Chromium Cookies file parsing and database operations."""
+
+import asyncio
+import sqlite3
+
+import psycopg
+from common.logger import get_logger
+from common.state_helpers import get_file_enriched
+from common.storage import StorageMinio
+from nemesis_dpapi import Blob, DpapiManager
+
+from .helpers import (
+    convert_chromium_timestamp,
+    decrypt_chrome_string,
+    detect_encryption_type,
+    get_postgres_connection_str,
+    get_state_key_bytes,
+    get_state_key_id,
+    is_sqlite3,
+    parse_chromium_file_path,
+    try_decrypt_with_all_keys,
+)
+
+logger = get_logger(__name__)
+
+
+def process_chromium_cookies(
+    object_id: str, file_path: str | None = None, dpapi_manager: DpapiManager | None = None
+) -> None:
+    """Process Chromium Cookies file and insert cookies into database.
+
+    Args:
+        object_id: The object ID of the Cookies file
+        file_path: Optional path to already downloaded file
+    """
+    logger.info("Processing Chromium Cookies file", object_id=object_id)
+
+    file_enriched = get_file_enriched(object_id)
+
+    # Extract username and browser from file path
+    username, browser = parse_chromium_file_path(file_enriched.path or "")
+    logger.debug("[process_chromium_cookies]", username=username, browser=browser)
+
+    # Get database file
+    if file_path:
+        db_path = file_path
+    else:
+        storage = StorageMinio()
+        with storage.download(file_enriched.object_id) as temp_file:
+            db_path = temp_file.name
+
+    if is_sqlite3(db_path) is False:
+        logger.warning("File is not a valid SQLite3 database", object_id=object_id, file_path=file_enriched.path)
+        return
+
+    conn_str = get_postgres_connection_str()
+    with psycopg.connect(conn_str) as pg_conn:
+        _insert_cookies(file_enriched, username, browser, db_path, pg_conn, dpapi_manager)
+
+    logger.debug("Completed processing Chromium Cookies", object_id=object_id)
+
+
+def _translate_samesite(samesite_int: int) -> str:
+    """Translate samesite integer value to string.
+
+    Args:
+        samesite_int: Integer value from database
+
+    Returns:
+        String representation of samesite value
+    """
+    samesite_map = {-1: "Unspecified", 0: "None", 1: "Lax", 2: "Strict"}
+    return samesite_map.get(samesite_int, "Unknown")
+
+
+def _insert_cookies(
+    file_enriched, username: str | None, browser: str, db_path: str, pg_conn, dpapi_manager: DpapiManager | None = None
+) -> None:
+    """Extract cookies from Cookies database and insert into chromium.cookies table."""
+    try:
+        # Read from SQLite
+        with sqlite3.connect(db_path) as conn:
+            conn.text_factory = bytes  # Get raw bytes, we'll handle text decoding manually
+            cursor = conn.cursor()
+
+            # Query cookies table
+            cursor.execute("""
+                SELECT host_key, name, path, creation_utc, expires_utc, last_access_utc,
+                       last_update_utc, is_secure, is_httponly, is_persistent, samesite,
+                       source_port, encrypted_value
+                FROM cookies
+            """)
+            rows = cursor.fetchall()
+
+        if not rows:
+            return
+
+        # Prepare data for PostgreSQL
+        cookies_data = []
+        for row in rows:
+            (
+                host_key,
+                name,
+                path,
+                creation_utc,
+                expires_utc,
+                last_access_utc,
+                last_update_utc,
+                is_secure,
+                is_httponly,
+                is_persistent,
+                samesite,
+                source_port,
+                encrypted_value,
+            ) = row
+
+            # Decode text fields from bytes (since we set text_factory = bytes)
+            host_key = host_key.decode("utf-8", errors="replace") if host_key else None
+            name = name.decode("utf-8", errors="replace") if name else None
+            path = path.decode("utf-8", errors="replace") if path else None
+
+            # encrypted_value is already binary (what we want)
+
+            if encrypted_value is None:
+                encrypted_value = b""
+
+            # Detect encryption type and get masterkey GUID (if applicable)
+            encryption_type, masterkey_guid = detect_encryption_type(encrypted_value)
+            is_decrypted = False
+            value_dec = None
+            if masterkey_guid and dpapi_manager:
+                try:
+                    value_dec_bytes = asyncio.run(dpapi_manager.decrypt_blob(Blob.from_bytes(encrypted_value)))
+                    if value_dec_bytes:
+                        value_dec = value_dec_bytes.decode("utf-8", errors="replace")
+                        is_decrypted = True
+                except:
+                    pass
+
+            # Get state key ID for key/abe encryption
+            state_key_id = None
+            if encryption_type in ["key", "abe"]:
+                # Try primary approach: get state key based on username/browser
+                if username:  # Only try primary approach if username was extracted
+                    state_key_id = get_state_key_id(file_enriched.source, username, browser, pg_conn)
+                    if state_key_id:
+                        # Retrieve the pre-processed state key for decryption
+                        state_key_bytes = get_state_key_bytes(state_key_id, encryption_type, pg_conn)
+                        if state_key_bytes:
+                            try:
+                                value_dec_bytes = decrypt_chrome_string(
+                                    encrypted_value, state_key_bytes, encryption_type
+                                )
+                                if value_dec_bytes:
+                                    # For cookies, may need to strip offset bytes depending on version
+                                    if encryption_type == "abe" and len(value_dec_bytes) > 32:
+                                        # v20 cookies typically have 32-byte offset
+                                        value_dec_bytes = value_dec_bytes[32:]
+                                    elif encryption_type == "key" and len(value_dec_bytes) > 48:
+                                        # v10/v11 cookies may have 32-byte prefix + 16-byte suffix
+                                        value_dec_bytes = value_dec_bytes[32:-16]
+                                    elif encryption_type == "key" and len(value_dec_bytes) > 16:
+                                        # Or just 16-byte suffix
+                                        value_dec_bytes = value_dec_bytes[:-16]
+
+                                    value_dec = value_dec_bytes.decode("utf-8", errors="replace")
+                                    is_decrypted = True
+                            except Exception as e:
+                                logger.warning(
+                                    "Failed to decrypt cookie with state key",
+                                    state_key_id=state_key_id,
+                                    encryption_type=encryption_type,
+                                    error=str(e),
+                                )
+
+                # Backup approach: try all state keys from the same source if primary failed
+                if not is_decrypted:
+                    logger.debug(
+                        "Primary decryption failed, trying backup approach with all keys from source",
+                        source=file_enriched.source,
+                        encryption_type=encryption_type,
+                    )
+                    backup_decrypted_bytes, backup_state_key_id = try_decrypt_with_all_keys(
+                        encrypted_value, file_enriched.source, encryption_type, pg_conn
+                    )
+                    if backup_decrypted_bytes and backup_state_key_id:
+                        # Apply the same offset handling as above
+                        if encryption_type == "abe" and len(backup_decrypted_bytes) > 32:
+                            # v20 cookies typically have 32-byte offset
+                            backup_decrypted_bytes = backup_decrypted_bytes[32:]
+                        elif encryption_type == "key" and len(backup_decrypted_bytes) > 48:
+                            # v10/v11 cookies may have 32-byte prefix + 16-byte suffix
+                            backup_decrypted_bytes = backup_decrypted_bytes[32:-16]
+                        elif encryption_type == "key" and len(backup_decrypted_bytes) > 16:
+                            # Or just 16-byte suffix
+                            backup_decrypted_bytes = backup_decrypted_bytes[:-16]
+
+                        value_dec = backup_decrypted_bytes.decode("utf-8", errors="replace")
+                        is_decrypted = True
+                        state_key_id = backup_state_key_id
+                        logger.debug(
+                            "Successfully decrypted cookie using backup approach", state_key_id=backup_state_key_id
+                        )
+
+            cookie_data = {
+                "originating_object_id": file_enriched.object_id,
+                "agent_id": file_enriched.agent_id,
+                "source": file_enriched.source,
+                "project": file_enriched.project,
+                "username": username,
+                "browser": browser,
+                "host_key": host_key,
+                "name": name,
+                "path": path,
+                "creation_utc": convert_chromium_timestamp(creation_utc),
+                "expires_utc": convert_chromium_timestamp(expires_utc),
+                "last_access_utc": convert_chromium_timestamp(last_access_utc),
+                "last_update_utc": convert_chromium_timestamp(last_update_utc),
+                "is_secure": bool(is_secure),
+                "is_httponly": bool(is_httponly),
+                "is_persistent": bool(is_persistent),
+                "samesite": _translate_samesite(samesite) if samesite is not None else "Unknown",
+                "source_port": source_port,
+                "encryption_type": encryption_type,
+                "masterkey_guid": masterkey_guid,
+                "state_key_id": state_key_id,
+                "is_decrypted": is_decrypted,
+                "value_enc": encrypted_value,
+                "value_dec": value_dec,
+            }
+            cookies_data.append(cookie_data)
+
+        # Insert into PostgreSQL
+        with pg_conn.cursor() as cur:
+            insert_sql = """
+                INSERT INTO chromium.cookies
+                (originating_object_id, agent_id, source, project, username, browser,
+                    host_key, name, path, creation_utc, expires_utc, last_access_utc,
+                    last_update_utc, is_secure, is_httponly, is_persistent, samesite,
+                    source_port, encryption_type, masterkey_guid, state_key_id,
+                    is_decrypted, value_enc, value_dec)
+                VALUES (%(originating_object_id)s, %(agent_id)s, %(source)s, %(project)s,
+                        %(username)s, %(browser)s, %(host_key)s, %(name)s, %(path)s,
+                        %(creation_utc)s, %(expires_utc)s, %(last_access_utc)s,
+                        %(last_update_utc)s, %(is_secure)s, %(is_httponly)s, %(is_persistent)s,
+                        %(samesite)s, %(source_port)s, %(encryption_type)s, %(masterkey_guid)s,
+                        %(state_key_id)s, %(is_decrypted)s, %(value_enc)s, %(value_dec)s)
+                ON CONFLICT (source, username, browser, host_key, name, path)
+                DO UPDATE SET
+                    host_key = EXCLUDED.host_key,
+                    name = EXCLUDED.name,
+                    path = EXCLUDED.path,
+                    creation_utc = EXCLUDED.creation_utc,
+                    expires_utc = EXCLUDED.expires_utc,
+                    last_access_utc = EXCLUDED.last_access_utc,
+                    last_update_utc = EXCLUDED.last_update_utc,
+                    is_secure = EXCLUDED.is_secure,
+                    is_httponly = EXCLUDED.is_httponly,
+                    is_persistent = EXCLUDED.is_persistent,
+                    samesite = EXCLUDED.samesite,
+                    source_port = EXCLUDED.source_port,
+                    encryption_type = EXCLUDED.encryption_type,
+                    masterkey_guid = EXCLUDED.masterkey_guid,
+                    state_key_id = EXCLUDED.state_key_id,
+                    is_decrypted = EXCLUDED.is_decrypted,
+                    value_enc = EXCLUDED.value_enc,
+                    value_dec = EXCLUDED.value_dec
+            """
+
+            cur.executemany(insert_sql, cookies_data)
+            pg_conn.commit()
+
+        logger.info("Inserted cookies into database", count=len(cookies_data))
+
+    except Exception as e:
+        logger.exception(
+            e,
+            "Error processing Cookies",
+            object_id=file_enriched.object_id,
+            file_path=file_enriched.path,
+        )
+        raise

libs/chromium/chromium/helpers.py

@@ -0,0 +1,482 @@
+"""Helper functions for Chromium data processing."""
+
+import re
+import struct
+from datetime import UTC, datetime, timedelta
+
+import psycopg
+from common.db import get_postgres_connection_str
+from common.logger import get_logger
+from Crypto.Cipher import AES, ChaCha20_Poly1305
+from nemesis_dpapi import Blob
+
+logger = get_logger(__name__)
+
+def is_sqlite3(filename):
+    try:
+        with open(filename, "rb") as f:
+            header = f.read(16)
+        return header.startswith(b"SQLite format 3\0")
+    except OSError:
+        return False
+
+
+
+def convert_chromium_timestamp(timestamp: int, str_format: bool = False) -> datetime | str | None:
+    """Convert Chromium timestamp to datetime.
+
+    Args:
+        timestamp: Chromium timestamp (microseconds since 1601-01-01)
+        str_format: Whether to return the timestamp in a string iso format (default is a datetime)
+
+    Returns:
+        datetime object or None if invalid
+    """
+    if not timestamp or timestamp == 0:
+        return None
+
+    try:
+        epoch = datetime(1601, 1, 1, tzinfo=UTC)
+        dt = epoch + timedelta(microseconds=timestamp)
+        if str_format:
+            return dt.isoformat()
+        else:
+            return dt
+    except (ValueError, OverflowError):
+        return None
+
+
+def parse_chromium_file_path(file_path: str) -> tuple[str | None, str]:
+    """Extract username and browser from Chromium file path.
+
+    Args:
+        file_path: Path to the Chromium file
+
+    Returns:
+        Tuple of (username, browser_name)
+    """
+
+    # Chrome/Edge/Brave pattern
+    match = re.search(
+        r".*/(?P<username>[^/]+)/AppData/Local/(?:Google|Microsoft|BraveSoftware)/(?P<browser>Chrome|Edge|Brave-Browser)/",
+        file_path,
+        re.IGNORECASE,
+    )
+
+    if match:
+        username = match.group("username").lower()
+        browser_str = match.group("browser").lower()
+
+        if "chrome" in browser_str:
+            return username, "chrome"
+        elif "edge" in browser_str:
+            return username, "edge"
+        elif "brave" in browser_str:
+            return username, "brave"
+
+    # Opera pattern
+    match = re.search(r".*/(?P<username>[^/]+)/AppData/Roaming/Opera Software/Opera Stable/", file_path, re.IGNORECASE)
+
+    if match:
+        username = match.group("username").lower()
+        return username, "opera"
+
+    return None, "unknown"
+
+
+def detect_encryption_type(encrypted_value: bytes) -> tuple[str, str | None]:
+    """Detect encryption type and extract masterkey GUID if applicable.
+
+    Args:
+        encrypted_value: Raw encrypted value bytes
+
+    Returns:
+        Tuple of (encryption_type, masterkey_guid)
+    """
+    if not encrypted_value or len(encrypted_value) < 4:
+        return "unknown", None
+
+    # Check for DPAPI (first 4 bytes are \x01\x00\x00\x00)
+    if encrypted_value[:4] == b"\x01\x00\x00\x00":
+        try:
+            blob = Blob.from_bytes(encrypted_value)
+            return "dpapi", str(blob.masterkey_guid)
+        except Exception as e:
+            raise Exception(f"Found DPAPI app bound key, but couldn't parse blob: {str(e)}") from e
+
+    # Check for key-based encryption (v10, v11)
+    if len(encrypted_value) >= 3:
+        prefix = encrypted_value[:3]
+        try:
+            prefix_str = prefix.decode("ascii")
+            if prefix_str in ["v10", "v11"]:
+                return "key", None
+            elif prefix_str == "v20":
+                return "abe", None
+        except UnicodeDecodeError:
+            pass
+
+    return "unknown", None
+
+
+def get_state_key_id(source: str, username: str | None, browser: str, pg_conn=None) -> int | None:
+    """Get state key ID for key/abe encryption types.
+
+    Args:
+        source: Source value
+        username: Username value
+        browser: Browser value
+        pg_conn: existing Postgres connection
+
+    Returns:
+        State key ID if found, None otherwise
+    """
+    if pg_conn:
+        try:
+            with pg_conn.cursor() as cur:
+                cur.execute(
+                    "SELECT id FROM chromium.state_keys WHERE source = %s AND username = %s AND browser = %s",
+                    (source, username, browser),
+                )
+                result = cur.fetchone()
+                return result[0] if result else None
+        except Exception as e:
+            logger.warning("Failed to lookup state key ID", error=str(e))
+            return None
+    else:
+        try:
+            conn_str = get_postgres_connection_str()
+            with psycopg.connect(conn_str) as pg_conn:
+                with pg_conn.cursor() as cur:
+                    cur.execute(
+                        "SELECT id FROM chromium.state_keys WHERE source = %s AND username = %s AND browser = %s",
+                        (source, username, browser),
+                    )
+                    result = cur.fetchone()
+                    return result[0] if result else None
+        except Exception as e:
+            logger.warning("Failed to lookup state key ID", error=str(e))
+            return None
+
+
+def get_state_key_bytes(state_key_id: int, encryption_type: str, pg_conn=None) -> bytes | None:
+    """Get decrypted state key bytes for key/abe encryption types.
+
+    Args:
+        state_key_id: State key ID
+        encryption_type: Either 'key' or 'abe'
+        pg_conn: existing Postgres connection
+
+    Returns:
+        Decrypted key bytes if found and decrypted, None otherwise
+    """
+    if encryption_type not in ["key", "abe"]:
+        return None
+
+    column_name = "key_bytes_dec" if encryption_type == "key" else "app_bound_key_dec"
+    is_decrypted_col = "key_is_decrypted" if encryption_type == "key" else "app_bound_key_is_decrypted"
+
+    if pg_conn:
+        try:
+            with pg_conn.cursor() as cur:
+                cur.execute(
+                    f"SELECT {column_name}, {is_decrypted_col} FROM chromium.state_keys WHERE id = %s",
+                    (state_key_id,),
+                )
+                result = cur.fetchone()
+                if result and result[1]:  # Check if is_decrypted is True
+                    return result[0]
+                return None
+        except Exception as e:
+            logger.warning("Failed to lookup state key bytes", error=str(e))
+            return None
+    else:
+        try:
+            conn_str = get_postgres_connection_str()
+            with psycopg.connect(conn_str) as pg_conn:
+                with pg_conn.cursor() as cur:
+                    cur.execute(
+                        f"SELECT {column_name}, {is_decrypted_col} FROM chromium.state_keys WHERE id = %s",
+                        (state_key_id,),
+                    )
+                    result = cur.fetchone()
+                    if result and result[1]:  # Check if is_decrypted is True
+                        return result[0]
+                    return None
+        except Exception as e:
+            logger.warning("Failed to lookup state key bytes", error=str(e))
+            return None
+
+
+def get_all_state_keys_from_source(source: str, pg_conn=None) -> list[dict]:
+    """Get all decrypted state keys from the same source.
+
+    Args:
+        source: Source value
+        pg_conn: existing Postgres connection
+
+    Returns:
+        List of dictionaries containing state key information
+    """
+    state_keys = []
+
+    if pg_conn:
+        try:
+            with pg_conn.cursor() as cur:
+                cur.execute(
+                    """SELECT id, username, browser, key_bytes_dec, key_is_decrypted,
+                              app_bound_key_dec, app_bound_key_is_decrypted
+                       FROM chromium.state_keys WHERE source = %s""",
+                    (source,),
+                )
+                results = cur.fetchall()
+                for result in results:
+                    state_key_info = {
+                        "id": result[0],
+                        "username": result[1],
+                        "browser": result[2],
+                        "key_bytes_dec": result[3] if result[4] else None,  # Only if decrypted
+                        "app_bound_key_dec": result[5] if result[6] else None,  # Only if decrypted
+                    }
+                    state_keys.append(state_key_info)
+                return state_keys
+        except Exception as e:
+            logger.warning("Failed to lookup all state keys from source", error=str(e))
+            return []
+    else:
+        try:
+            conn_str = get_postgres_connection_str()
+            with psycopg.connect(conn_str) as pg_conn:
+                with pg_conn.cursor() as cur:
+                    cur.execute(
+                        """SELECT id, username, browser, key_bytes_dec, key_is_decrypted,
+                                  app_bound_key_dec, app_bound_key_is_decrypted
+                           FROM chromium.state_keys WHERE source = %s""",
+                        (source,),
+                    )
+                    results = cur.fetchall()
+                    for result in results:
+                        state_key_info = {
+                            "id": result[0],
+                            "username": result[1],
+                            "browser": result[2],
+                            "key_bytes_dec": result[3] if result[4] else None,  # Only if decrypted
+                            "app_bound_key_dec": result[5] if result[6] else None,  # Only if decrypted
+                        }
+                        state_keys.append(state_key_info)
+                    return state_keys
+        except Exception as e:
+            logger.warning("Failed to lookup all state keys from source", error=str(e))
+            return []
+
+
+def is_valid_text(data: bytes) -> bool:
+    """Check if decrypted bytes represent valid ASCII or UTF-8 text.
+
+    Args:
+        data: Bytes to validate
+
+    Returns:
+        True if data is valid text, False otherwise
+    """
+    if not data:
+        return False
+
+    try:
+        # Try to decode as UTF-8
+        text = data.decode("utf-8")
+        # Check if it contains mostly printable characters
+        printable_chars = sum(1 for c in text if c.isprintable() or c.isspace())
+        return printable_chars / len(text) > 0.8  # At least 80% printable
+    except UnicodeDecodeError:
+        try:
+            # Try ASCII as fallback
+            text = data.decode("ascii")
+            printable_chars = sum(1 for c in text if c in "\x20-\x7e\t\n\r")
+            return printable_chars / len(text) > 0.8
+        except UnicodeDecodeError:
+            return False
+
+
+def try_decrypt_with_all_keys(
+    encrypted_value: bytes, source: str, encryption_type: str, pg_conn=None
+) -> tuple[bytes | None, int | None]:
+    """Try to decrypt with all available state keys from the same source.
+
+    Args:
+        encrypted_value: Raw encrypted value bytes
+        source: Source value to look up keys from
+        encryption_type: Either 'key' or 'abe'
+        pg_conn: existing Postgres connection
+
+    Returns:
+        Tuple of (decrypted_bytes, state_key_id) if successful, (None, None) otherwise
+    """
+    if encryption_type not in ["key", "abe"]:
+        return None, None
+
+    state_keys = get_all_state_keys_from_source(source, pg_conn)
+
+    for state_key in state_keys:
+        key_bytes = state_key.get("key_bytes_dec") if encryption_type == "key" else state_key.get("app_bound_key_dec")
+
+        if not key_bytes:
+            continue
+
+        try:
+            decrypted_bytes = decrypt_chrome_string(encrypted_value, key_bytes, encryption_type)
+            if decrypted_bytes:
+                # Apply offset handling for cookies
+                if encryption_type == "abe" and len(decrypted_bytes) > 32:
+                    # v20 cookies typically have 32-byte offset
+                    test_bytes = decrypted_bytes[32:]
+                elif encryption_type == "key" and len(decrypted_bytes) > 48:
+                    # v10/v11 cookies may have 32-byte prefix + 16-byte suffix
+                    test_bytes = decrypted_bytes[32:-16]
+                elif encryption_type == "key" and len(decrypted_bytes) > 16:
+                    # Or just 16-byte suffix
+                    test_bytes = decrypted_bytes[:-16]
+                else:
+                    test_bytes = decrypted_bytes
+
+                # Check if the result is valid text
+                if is_valid_text(test_bytes):
+                    logger.debug(
+                        "Successfully decrypted with backup key",
+                        state_key_id=state_key["id"],
+                        username=state_key["username"],
+                        browser=state_key["browser"],
+                    )
+                    return decrypted_bytes, state_key["id"]
+        except Exception:
+            # Continue trying other keys
+            continue
+
+    return None, None
+
+
+def byte_xor(ba1, ba2):
+    return bytes([_a ^ _b for _a, _b in zip(ba1, ba2)])
+
+def parse_abe_blob(abe_data: bytes, chromekey: bytes | None = None) -> dict | None:
+    """Parse ABE (App-Bound Encryption) blob data.
+
+    Args:
+        abe_data: Raw ABE blob bytes
+
+    Returns:
+        Dictionary containing parsed ABE data, None if parsing fails
+    """
+    try:
+        abe_parsed = {}
+        header_len = struct.unpack("<I", abe_data[:4])[0]
+        abe_parsed["header"] = abe_data[4 : 4 + header_len].strip(b"\x02").decode(errors="ignore")
+        content_len = struct.unpack("<I", abe_data[4 + header_len : 4 + header_len + 4])[0]
+        content = abe_data[8 + header_len : 8 + header_len + content_len]
+
+        abe_parsed["version"] = int(content[0])
+        content = content[1:]
+        if abe_parsed["version"] <= 2:  # Versions 1 and 2
+            # Version|IV|ciphertext|tag, 1|12|32|16 bytes
+            abe_parsed["iv"] = content[:12]
+            abe_parsed["cipherdata"] = content[12 : 12 + 32]
+            abe_parsed["tag"] = content[12 + 32 : 12 + 32 + 16]
+        else:  # Version 3
+            # Version|encAES|IV|ciphertext|tag, 1|32|12|32|16 bytes
+            # adapted from:
+            #   https://github.com/runassu/chrome_v20_decryption/blob/e8f244543e98266d50884aba2778e0ccedefa45d/decrypt_chrome_v20_cookie.py#L42-L69
+            #   https://github.com/tijldeneut/diana/blob/b9473b5004ecf1d7bdd5852232b5cd06a5378e5e/diana-browserdec.py
+            abe_parsed["encrAES"] = content[:32]
+            abe_parsed["iv"] = content[32 : 32 + 12]
+            abe_parsed["cipherdata"] = content[32 + 12 : 32 + 12 + 32]
+            abe_parsed["tag"] = content[32 + 12 + 32 : 32 + 12 + 32 + 16]
+
+            if chromekey:
+                # gotta make sure to specify the \x00*16 IV here
+                cipher = AES.new(chromekey, AES.MODE_CBC, b'\x00' * 16)
+                result = cipher.decrypt(abe_parsed["encrAES"])
+                xor_key = bytes.fromhex("CCF8A1CEC56605B8517552BA1A2D061C03A29E90274FB2FCF59BA4B75C392390")
+                abe_parsed["xored_aes_key"] = byte_xor(result, xor_key)
+        return abe_parsed
+    except Exception as e:
+        logger.warning("Failed to parse ABE blob", error=str(e))
+        return None
+
+
+# Ref: https://github.com/runassu/chrome_v20_decryption/blob/e8f244543e98266d50884aba2778e0ccedefa45d/decrypt_chrome_v20_cookie.py#L121-L135
+def derive_abe_key(abe_data: dict) -> bytes | None:
+    """Derive ABE key from parsed ABE data.
+
+    Args:
+        abe_data: Parsed ABE data dictionary
+
+    Returns:
+        Derived ABE key bytes, None if derivation fails
+    """
+    try:
+        if abe_data["version"] == 1:
+            cipher = AES.new(
+                bytes.fromhex("B31C6E241AC846728DA9C1FAC4936651CFFB944D143AB816276BCC6DA0284787"),
+                AES.MODE_GCM,
+                nonce=abe_data["iv"],
+            )
+        elif abe_data["version"] == 2:
+            cipher = ChaCha20_Poly1305.new(
+                key=bytes.fromhex("E98F37D7F4E1FA433D19304DC2258042090E2D1D7EEA7670D41F738D08729660"),
+                nonce=abe_data["iv"],
+            )
+        elif abe_data["version"] == 3:
+            if abe_data["xored_aes_key"]:
+                cipher = AES.new(abe_data["xored_aes_key"], AES.MODE_GCM, nonce=abe_data["iv"])
+            else:
+                # Version 3 requires CNG decryption of encrypted AES key
+                logger.warning("xored_aes_key not present, ABE version 3 requires CNG decrypted 'Google Chromekey1'")
+                return None
+        else:
+            logger.warning("Unknown ABE version", version=abe_data["version"])
+            return None
+
+        return cipher.decrypt_and_verify(abe_data["cipherdata"], abe_data["tag"])
+    except Exception as e:
+        logger.warning("Failed to derive ABE key", error=str(e))
+        return None
+
+
+def decrypt_chrome_string(encrypted_data: bytes, key_bytes: bytes, encryption_type: str) -> bytes | None:
+    """Decrypt Chrome encrypted string using key or ABE encryption.
+
+    Args:
+        encrypted_data: Raw encrypted data bytes
+        key_bytes: Decrypted key bytes (BME key for 'key', ABE key for 'abe')
+        encryption_type: Either 'key' or 'abe'
+
+    Returns:
+        Decrypted bytes, None if decryption fails
+    """
+    if not encrypted_data or len(encrypted_data) < 3:
+        return None
+
+    try:
+        if encryption_type == "key" and encrypted_data[:3] in [b"v10", b"v11"]:
+            # Version|IV|ciphertext, 4|12|<var>
+            iv = encrypted_data[3 : 3 + 12]
+            ciphertext = encrypted_data[15:]
+            cipher = AES.new(key_bytes, AES.MODE_GCM, iv)
+            return cipher.decrypt(ciphertext)
+
+        elif encryption_type == "abe" and encrypted_data[:3] == b"v20":
+            # Version|IV|ciphertext|tag, 3|12|<var>|16 bytes
+            iv = encrypted_data[3 : 3 + 12]
+            ciphertext = encrypted_data[15:-16]
+            tag = encrypted_data[-16:]
+            cipher = AES.new(key_bytes, AES.MODE_GCM, iv)
+            decrypted = cipher.decrypt_and_verify(ciphertext, tag)
+            # v20 cookies have 32-byte offset, but this varies by data type
+            return decrypted
+        else:
+            logger.warning("Unsupported encryption format", prefix=encrypted_data[:3], encryption_type=encryption_type)
+            return None
+
+    except Exception as e:
+        logger.warning("Failed to decrypt Chrome string", error=str(e))
+        return None

libs/chromium/chromium/history.py

@@ -0,0 +1,165 @@
+"""Chromium History file parsing and database operations."""
+
+import sqlite3
+
+import psycopg
+from common.logger import get_logger
+from common.state_helpers import get_file_enriched
+from common.storage import StorageMinio
+
+from .helpers import convert_chromium_timestamp, get_postgres_connection_str, parse_chromium_file_path
+
+logger = get_logger(__name__)
+
+
+def process_chromium_history(object_id: str, file_path: str | None = None) -> None:
+    """Process Chromium History file and insert URLs and downloads into database.
+
+    Args:
+        object_id: The object ID of the History file
+        file_path: Optional path to already downloaded file
+    """
+    logger.info("Processing Chromium History file", object_id=object_id)
+
+    file_enriched = get_file_enriched(object_id)
+
+    # Extract username and browser from file path
+    username, browser = parse_chromium_file_path(file_enriched.path or "")
+    logger.debug("[process_chromium_history]", username=username, browser=browser)
+
+    # Get database file
+    if file_path:
+        db_path = file_path
+    else:
+        storage = StorageMinio()
+        with storage.download(file_enriched.object_id) as temp_file:
+            db_path = temp_file.name
+
+    # Process both tables
+    _insert_history_urls(object_id, file_enriched, username, browser, db_path)
+    _insert_history_downloads(object_id, file_enriched, username, browser, db_path)
+
+    logger.debug("Completed processing Chromium History", object_id=object_id)
+
+
+def _insert_history_urls(object_id: str, file_enriched, username: str | None, browser: str, db_path: str) -> None:
+    """Extract URLs from History and insert into chromium.history table."""
+    try:
+        # Read from SQLite
+        with sqlite3.connect(db_path) as conn:
+            conn.text_factory = lambda x: x.decode("utf-8", errors="replace")
+            cursor = conn.cursor()
+
+            cursor.execute("SELECT url, title, visit_count, last_visit_time FROM urls")
+            rows = cursor.fetchall()
+
+        if not rows:
+            return
+
+        # Prepare data for PostgreSQL
+        urls_data = []
+        for url, title, visit_count, last_visit_time in rows:
+            urls_data.append(
+                {
+                    "originating_object_id": file_enriched.object_id,
+                    "agent_id": file_enriched.agent_id,
+                    "source": file_enriched.source,
+                    "project": file_enriched.project,
+                    "username": username,
+                    "browser": browser,
+                    "url": url,
+                    "title": title,
+                    "visit_count": visit_count,
+                    "last_visit_time": convert_chromium_timestamp(last_visit_time),
+                }
+            )
+
+        # Insert into PostgreSQL
+        conn_str = get_postgres_connection_str()
+        with psycopg.connect(conn_str) as pg_conn:
+            with pg_conn.cursor() as cur:
+                insert_sql = """
+                    INSERT INTO chromium.history
+                    (originating_object_id, agent_id, source, project, username, browser,
+                     url, title, visit_count, last_visit_time)
+                    VALUES (%(originating_object_id)s, %(agent_id)s, %(source)s, %(project)s,
+                            %(username)s, %(browser)s, %(url)s, %(title)s, %(visit_count)s, %(last_visit_time)s)
+                    ON CONFLICT (source, username, browser, url, title, last_visit_time)
+                    DO UPDATE SET
+                        url = EXCLUDED.url,
+                        title = EXCLUDED.title,
+                        visit_count = EXCLUDED.visit_count,
+                        last_visit_time = EXCLUDED.last_visit_time
+                """
+
+                cur.executemany(insert_sql, urls_data)
+                pg_conn.commit()
+
+        logger.info("Inserted URLs into database", count=len(urls_data))
+
+    except Exception as e:
+        logger.exception("Error processing History URLs", error=str(e))
+        raise
+
+
+def _insert_history_downloads(object_id: str, file_enriched, username: str | None, browser: str, db_path: str) -> None:
+    """Extract downloads from History and insert into chromium_downloads table."""
+    try:
+        # Read from SQLite
+        with sqlite3.connect(db_path) as conn:
+            conn.text_factory = lambda x: x.decode("utf-8", errors="replace")
+            cursor = conn.cursor()
+
+            cursor.execute("SELECT tab_url, target_path, start_time, end_time, total_bytes FROM downloads")
+            rows = cursor.fetchall()
+
+        if not rows:
+            return
+
+        # Prepare data for PostgreSQL
+        downloads_data = []
+        for tab_url, target_path, start_time, end_time, total_bytes in rows:
+            downloads_data.append(
+                {
+                    "originating_object_id": file_enriched.object_id,
+                    "agent_id": file_enriched.agent_id,
+                    "source": file_enriched.source,
+                    "project": file_enriched.project,
+                    "username": username,
+                    "browser": browser,
+                    "url": tab_url,
+                    "download_path": target_path,
+                    "start_time": convert_chromium_timestamp(start_time),
+                    "end_time": convert_chromium_timestamp(end_time),
+                    "total_bytes": total_bytes,
+                }
+            )
+
+        # Insert into PostgreSQL
+        conn_str = get_postgres_connection_str()
+        with psycopg.connect(conn_str) as pg_conn:
+            with pg_conn.cursor() as cur:
+                insert_sql = """
+                    INSERT INTO chromium.downloads
+                    (originating_object_id, agent_id, source, project, username, browser,
+                     url, download_path, start_time, end_time, total_bytes)
+                    VALUES (%(originating_object_id)s, %(agent_id)s, %(source)s, %(project)s,
+                            %(username)s, %(browser)s, %(url)s, %(download_path)s,
+                            %(start_time)s, %(end_time)s, %(total_bytes)s)
+                    ON CONFLICT (source, username, browser, url, download_path, start_time)
+                    DO UPDATE SET
+                        url = EXCLUDED.url,
+                        download_path = EXCLUDED.download_path,
+                        start_time = EXCLUDED.start_time,
+                        end_time = EXCLUDED.end_time,
+                        total_bytes = EXCLUDED.total_bytes
+                """
+
+                cur.executemany(insert_sql, downloads_data)
+                pg_conn.commit()
+
+        logger.info("Inserted downloads into database", count=len(downloads_data))
+
+    except Exception as e:
+        logger.exception("Error processing History downloads", error=str(e))
+        raise

libs/chromium/chromium/local_state.py

@@ -0,0 +1,968 @@
+"""Chromium Local State file parsing and database operations."""
+
+import base64
+import json
+import posixpath
+from uuid import UUID
+
+import psycopg
+from common.helpers import get_drive_from_path
+from common.logger import get_logger
+from common.state_helpers import get_file_enriched
+from common.storage import StorageMinio
+from file_linking import add_file_linking
+from impacket.dpapi import DPAPI_BLOB
+from impacket.uuid import bin_to_string
+from nemesis_dpapi import Blob, DpapiManager, MasterKeyNotDecryptedError, MasterKeyNotFoundError
+
+from .helpers import (
+    derive_abe_key,
+    detect_encryption_type,
+    get_postgres_connection_str,
+    parse_abe_blob,
+    parse_chromium_file_path,
+)
+
+logger = get_logger(__name__)
+
+
+async def process_chromium_local_state(
+    dpapi_manager: DpapiManager,
+    object_id: str,
+    file_path: str | None = None,
+) -> dict | None:
+    """Process Chromium Local State file and insert state keys into database.
+
+    Args:
+        object_id: The object ID of the Local State file
+        file_path: Optional path to already downloaded file
+    """
+    logger.info("Processing Chromium Local State file", object_id=object_id)
+
+    file_enriched = get_file_enriched(object_id)
+
+    # Extract username and browser from file path
+    username, browser = parse_chromium_file_path(file_enriched.path or "")
+    logger.debug("[process_chromium_local_state()]", username=username, browser=browser)
+
+    # Get file content
+    if file_path:
+        with open(file_path, encoding="utf-8") as f:
+            content = f.read()
+    else:
+        storage = StorageMinio()
+        with storage.download(file_enriched.object_id) as temp_file:
+            with open(temp_file.name, encoding="utf-8") as f:
+                content = f.read()
+
+    conn_str = get_postgres_connection_str()
+    with psycopg.connect(conn_str) as pg_conn:
+        state_key_data = await _insert_state_keys(file_enriched, username, browser, content, pg_conn, dpapi_manager)
+
+    logger.debug("Completed processing Chromium Local State", object_id=object_id)
+    return state_key_data
+
+
+def _parse_app_bound_key(app_bound_key_b64: str) -> tuple[bytes, str | None]:
+    """Parse app-bound encrypted key and extract system masterkey GUID.
+
+    Args:
+        app_bound_key_b64: Base64 encoded app-bound key
+
+    Returns:
+        Tuple of (decoded_bytes, system_masterkey_guid)
+
+    Raises:
+        ValueError: If the key doesn't have APPB header
+    """
+    try:
+        app_bound_key_bytes = base64.b64decode(app_bound_key_b64, validate=True)
+
+        # Check for APPB header (first 4 bytes)
+        if len(app_bound_key_bytes) < 4 or app_bound_key_bytes[:4] != b"APPB":
+            raise ValueError("App-bound key does not have APPB header")
+
+        # Remove APPB header and parse remaining as DPAPI blob
+        dpapi_bytes = app_bound_key_bytes[4:]
+
+        blob = DPAPI_BLOB(dpapi_bytes)
+        if blob.rawData is not None:
+            blob.rawData = blob.rawData[: len(blob.getData())]
+            system_masterkey_guid = bin_to_string(blob["GuidMasterKey"]).lower()
+            return app_bound_key_bytes, system_masterkey_guid
+
+        return app_bound_key_bytes, None
+
+    except Exception as e:
+        logger.warning("Failed to parse app-bound key", error=str(e))
+        return base64.b64decode(app_bound_key_b64), None
+
+
+
+async def _add_user_masterkey_link(file_enriched, username: str | None, masterkey_guid: UUID) -> None:
+    """Add file linking entry for user masterkey."""
+
+    # Skip trying to figure out the username/drive if we can
+    if r"AppData/Local/Google/Chrome/User Data" in file_enriched.path:
+        masterkey_path = posixpath.normpath(
+            posixpath.join(
+                file_enriched.path,
+                "../../../../../Roaming/Microsoft/Protect/<WINDOWS_SECURITY_IDENTIFIER>",
+                str(masterkey_guid),
+            )
+        )
+    else:
+        # Not an AppData path, so build path based on the drive letter and username (if available)
+        drive = get_drive_from_path(file_enriched.path) or ""
+
+        if not username:
+            username = "<WINDOWS_USERNAME>"
+
+        masterkey_path = (
+            f"{drive}/Users/{username}/AppData/Roaming/Microsoft/Protect/<WINDOWS_SECURITY_IDENTIFIER>/{masterkey_guid}"
+        )
+
+    await add_file_linking(file_enriched.source, file_enriched.path, masterkey_path, "windows:user_masterkey")
+
+
+def _get_chromekey_from_source(source: str, pg_conn) -> bytes | None:
+    """Get decrypted Chrome key from chrome_keys table by source.
+
+    Args:
+        source: Source value to match
+        pg_conn: PostgreSQL connection
+
+    Returns:
+        Decrypted key bytes if found and decrypted, None otherwise
+    """
+    try:
+        with pg_conn.cursor() as cur:
+            cur.execute(
+                "SELECT key_bytes_dec FROM chromium.chrome_keys WHERE source = %s AND key_is_decrypted = TRUE",
+                (source,),
+            )
+            result = cur.fetchone()
+            return result[0] if result else None
+    except Exception as e:
+        logger.warning("Failed to lookup chrome key from source", error=str(e))
+        return None
+
+
+async def _insert_state_keys(
+    file_enriched,
+    username: str | None,
+    browser: str,
+    content: str,
+    pg_conn,
+    dpapi_manager: DpapiManager,
+) -> dict | None:
+    """Parse Local State JSON and insert state keys into chromium.state_keys table."""
+    try:
+        # Parse JSON content
+        data = json.loads(content)
+
+        # Extract os_crypt section
+        os_crypt = data.get("os_crypt", {})
+
+        # Get Chrome key from chrome_keys table if available
+        chromekey = _get_chromekey_from_source(file_enriched.source, pg_conn)
+
+        key_bytes_dec = b""
+        key_is_decrypted = False
+        app_bound_key_system_dec = b""
+        app_bound_key_user_dec = b""
+        app_bound_key_dec = b""
+        app_bound_key_is_decrypted = False
+
+        # Get encrypted_key (pre v127)
+        encrypted_key_b64 = os_crypt.get("encrypted_key")
+        key_bytes_enc = b""
+        key_masterkey_guid = None
+
+        if encrypted_key_b64:
+            logger.debug("Found app bound key encrypted_key in Local State")
+            key_bytes_enc = base64.b64decode(encrypted_key_b64)
+
+            if len(key_bytes_enc) < 5 or key_bytes_enc[:5] != b"DPAPI":
+                raise ValueError("Encrypted key does not have DPAPI header")
+
+            # Remove DPAPI header and parse remaining as DPAPI blob
+            dpapi_blob_bytes = key_bytes_enc[5:]
+
+            encryption_type, _ = detect_encryption_type(dpapi_blob_bytes)
+            if encryption_type != "dpapi":
+                raise Exception(f"Unsupported encryption type for v1 state key: {encryption_type}")
+
+            dpapi_blob = Blob.from_bytes(dpapi_blob_bytes)
+            key_masterkey_guid = str(dpapi_blob.masterkey_guid)
+            try:
+                key_bytes_dec = await dpapi_manager.decrypt_blob(dpapi_blob)
+                if key_bytes_dec:
+                    key_is_decrypted = True
+                    logger.debug(
+                        "Successfully decrypted encrypted_key state key",
+                        masterkey_guid=dpapi_blob.masterkey_guid,
+                    )
+                else:
+                    logger.debug("Failed to decrypt encrypted_key state key with DPAPI")
+            except (MasterKeyNotFoundError, MasterKeyNotDecryptedError) as e:
+                logger.debug(
+                    "Masterkey not found or not decrypted for encrypted_key state key",
+                    masterkey_guid=dpapi_blob.masterkey_guid,
+                    reason=type(e).__name__,
+                )
+
+                await _add_user_masterkey_link(file_enriched, username, dpapi_blob.masterkey_guid)
+
+            except Exception as e:
+                logger.warning(f"Unable to decrypt state key DPAPI blob: {dpapi_blob.masterkey_guid}", error=str(e))
+
+        # Get app_bound_encrypted_key (post v127)
+        app_bound_key_b64 = os_crypt.get("app_bound_encrypted_key")
+        app_bound_key_enc = b""
+        app_bound_key_system_masterkey_guid = None
+        app_bound_key_user_masterkey_guid = None
+
+        if app_bound_key_b64:
+            logger.debug("Found v2 app bound key encrypted_key in Local State")
+            app_bound_key_enc, app_bound_key_system_masterkey_guid = _parse_app_bound_key(app_bound_key_b64)
+            logger.debug(f"app_bound_key_system_masterkey_guid: {app_bound_key_system_masterkey_guid}")
+
+            drive = get_drive_from_path(file_enriched.path) or ""
+            masterkey_path = (
+                f"{drive}/Windows/System32/Microsoft/Protect/S-1-5-18/User/{app_bound_key_system_masterkey_guid}"
+            )
+
+            # Filename format: <hash>_<machineGuid>
+            # Hash comes from Chromium calling NCryptOpenKey with the key name of "Google Chromekey1"
+            # Hashing algorithm is described here: https://gist.github.com/leechristensen/40acb67ff5b788d6b78d81443b66b444
+            cng_system_private_key_path = (
+                f"{drive}/ProgramData/Microsoft/Crypto/SystemKeys/7096db7aeb75c0d3497ecd56d355a695_<WINDOWS_MACHINE_GUID>"
+            )
+
+            # add the masterkey file path (now that we know the key GUID) as a link/listing
+            await add_file_linking(file_enriched.source, file_enriched.path, masterkey_path, "windows:system_masterkey")
+            await add_file_linking(file_enriched.source, file_enriched.path, cng_system_private_key_path, "windows:cng_system_private_key - Contains Chrome key used to encrypt the Local State")
+
+            try:
+                # Parse only the DPAPI portion (after APPB header)
+                if len(app_bound_key_enc) >= 4 and app_bound_key_enc[:4] == b"APPB":
+                    dpapi_portion = app_bound_key_enc[4:]
+                    # Step 1 - decrypt with a SYSTEM masterkey
+                    app_bound_key_system_dec = await dpapi_manager.decrypt_blob(Blob.from_bytes(dpapi_portion))
+                else:
+                    logger.warning("App-bound key missing APPB header, cannot decrypt")
+                    app_bound_key_system_dec = b""
+
+                if app_bound_key_system_dec:
+                    user_blob = Blob.from_bytes(app_bound_key_system_dec)
+                    app_bound_key_user_masterkey_guid = str(user_blob.masterkey_guid)
+
+                    try:
+                        # Step 2 - decrypt with a _user_ masterkey
+                        abe_blob_bytes = await dpapi_manager.decrypt_blob(user_blob)
+                        if abe_blob_bytes:
+                            # Store the intermediate value after USER key decryption
+                            app_bound_key_user_dec = abe_blob_bytes
+
+                            # Step 3 - parse and derive the final ABE key (using the Chromekey for v3)
+                            abe_parsed = parse_abe_blob(abe_blob_bytes, chromekey)
+
+                            if abe_parsed:
+                                app_bound_key_dec = derive_abe_key(abe_parsed)
+
+                                if app_bound_key_dec:
+                                    app_bound_key_is_decrypted = True
+                                    logger.debug(
+                                        "Successfully derived ABE key",
+                                        version=abe_parsed.get("version"),
+                                        system_masterkey_guid=app_bound_key_system_masterkey_guid,
+                                    )
+                                else:
+                                    raise Exception("Failed to derive ABE key")
+                            else:
+                                raise Exception("Failed to parse ABE blob")
+                        else:
+                            raise Exception("Failed to decrypt ABE blob with user masterkey")
+                    except (MasterKeyNotFoundError, MasterKeyNotDecryptedError) as e:
+                        logger.debug(
+                            "ABE key not decrypted. Masterkey not found or not decrypted",
+                            masterkey_guid=user_blob.masterkey_guid,
+                            reason=type(e).__name__,
+                        )
+
+                        await _add_user_masterkey_link(file_enriched, username, user_blob.masterkey_guid)
+
+                    except Exception as e:
+                        logger.warning(f"Unable to decrypt/process final app bound key blob: {e}")
+
+            except Exception as e:
+                logger.warning(f"Unable to decrypt intermediate/outer app bound key blob: {e}")
+
+        # Skip if no keys are present
+        if not key_bytes_enc and not app_bound_key_enc:
+            logger.warning("No encryption keys found in Local State file")
+            return None
+
+        # Prepare data for PostgreSQL
+        state_key_data = {
+            "originating_object_id": file_enriched.object_id,
+            "agent_id": file_enriched.agent_id,
+            "source": file_enriched.source,
+            "project": file_enriched.project,
+            "username": username,
+            "browser": browser,
+            "key_masterkey_guid": key_masterkey_guid,
+            "key_bytes_enc": key_bytes_enc,
+            "key_bytes_dec": key_bytes_dec,
+            "key_is_decrypted": key_is_decrypted,
+            "app_bound_key_enc": app_bound_key_enc,
+            "app_bound_key_system_masterkey_guid": app_bound_key_system_masterkey_guid,
+            "app_bound_key_user_masterkey_guid": app_bound_key_user_masterkey_guid,
+            "app_bound_key_system_dec": app_bound_key_system_dec,
+            "app_bound_key_user_dec": app_bound_key_user_dec,
+            "app_bound_key_dec": app_bound_key_dec,
+            "app_bound_key_is_decrypted": app_bound_key_is_decrypted,
+        }
+
+        # Create a serializable copy for return (hex encode binary data)
+        serializable_data = state_key_data.copy()
+        for key, value in serializable_data.items():
+            if isinstance(value, bytes):
+                serializable_data[key] = value.hex()
+
+        # Insert into PostgreSQL
+        with pg_conn.cursor() as cur:
+            insert_sql = """
+                INSERT INTO chromium.state_keys
+                (originating_object_id, agent_id, source, project, username, browser,
+                 key_masterkey_guid, key_bytes_enc, key_bytes_dec, key_is_decrypted,
+                 app_bound_key_enc, app_bound_key_system_masterkey_guid,
+                 app_bound_key_user_masterkey_guid, app_bound_key_system_dec, app_bound_key_user_dec,
+                 app_bound_key_dec, app_bound_key_is_decrypted)
+                VALUES (%(originating_object_id)s, %(agent_id)s, %(source)s, %(project)s,
+                        %(username)s, %(browser)s, %(key_masterkey_guid)s, %(key_bytes_enc)s,
+                        %(key_bytes_dec)s, %(key_is_decrypted)s, %(app_bound_key_enc)s,
+                        %(app_bound_key_system_masterkey_guid)s, %(app_bound_key_user_masterkey_guid)s,
+                        %(app_bound_key_system_dec)s, %(app_bound_key_user_dec)s,
+                        %(app_bound_key_dec)s, %(app_bound_key_is_decrypted)s)
+                ON CONFLICT (source, username, browser)
+                DO UPDATE SET
+                    key_masterkey_guid = EXCLUDED.key_masterkey_guid,
+                    key_bytes_enc = EXCLUDED.key_bytes_enc,
+                    key_bytes_dec = EXCLUDED.key_bytes_dec,
+                    key_is_decrypted = EXCLUDED.key_is_decrypted,
+                    app_bound_key_enc = EXCLUDED.app_bound_key_enc,
+                    app_bound_key_system_masterkey_guid = EXCLUDED.app_bound_key_system_masterkey_guid,
+                    app_bound_key_user_masterkey_guid = EXCLUDED.app_bound_key_user_masterkey_guid,
+                    app_bound_key_system_dec = EXCLUDED.app_bound_key_system_dec,
+                    app_bound_key_user_dec = EXCLUDED.app_bound_key_user_dec,
+                    app_bound_key_dec = EXCLUDED.app_bound_key_dec,
+                    app_bound_key_is_decrypted = EXCLUDED.app_bound_key_is_decrypted
+            """
+
+            cur.execute(insert_sql, state_key_data)
+
+            # Get the inserted state key ID
+            cur.execute(
+                "SELECT id FROM chromium.state_keys WHERE source = %s AND username = %s AND browser = %s",
+                (file_enriched.source, username, browser),
+            )
+            result = cur.fetchone()
+            if result:
+                state_key_id = result[0]
+
+                cur.execute(
+                    """
+                    UPDATE chromium.logins
+                    SET state_key_id = %s
+                    WHERE source = %s AND username = %s AND browser = %s
+                    AND state_key_id IS NULL
+                    AND encryption_type IN ('key', 'abe')
+                """,
+                    (state_key_id, file_enriched.source, username, browser),
+                )
+                logins_updated = cur.rowcount
+
+                cur.execute(
+                    """
+                    UPDATE chromium.cookies
+                    SET state_key_id = %s
+                    WHERE source = %s AND username = %s AND browser = %s
+                    AND state_key_id IS NULL
+                    AND encryption_type IN ('key', 'abe')
+                """,
+                    (state_key_id, file_enriched.source, username, browser),
+                )
+                cookies_updated = cur.rowcount
+
+                logger.info(
+                    "Updated existing entries with state key ID",
+                    state_key_id=state_key_id,
+                    logins_updated=logins_updated,
+                    cookies_updated=cookies_updated,
+                )
+
+            pg_conn.commit()
+
+        logger.info(
+            "Inserted state keys into database",
+            has_encrypted_key=bool(key_bytes_enc),
+            has_app_bound_key=bool(app_bound_key_enc),
+        )
+
+        return serializable_data
+
+    except json.JSONDecodeError as e:
+        logger.exception("Failed to parse Local State JSON", error=str(e))
+        raise
+    except Exception as e:
+        logger.exception("Error processing Local State", error=str(e))
+        raise
+
+
+async def retry_decrypt_state_key(state_key_id: int, dpapi_manager: DpapiManager, pg_conn) -> dict:
+    """Attempt to decrypt a single state_key record using currently available masterkeys.
+
+    Args:
+        state_key_id: The ID of the state_key record to decrypt
+        dpapi_manager: DpapiManager instance for decryption
+        pg_conn: PostgreSQL connection
+
+    Returns:
+        Dict with decryption results: {
+            "decrypted_v1": bool,
+            "decrypted_abe_stage1": bool,
+            "decrypted_abe_stage2": bool
+        }
+    """
+    result = {
+        "decrypted_v1": False,
+        "decrypted_abe_stage1": False,
+        "decrypted_abe_stage2": False,
+    }
+
+    # Fetch the state_key record
+    with pg_conn.cursor() as cur:
+        cur.execute(
+            """
+            SELECT id, source, username, browser,
+                   key_masterkey_guid, key_bytes_enc, key_bytes_dec, key_is_decrypted,
+                   app_bound_key_enc, app_bound_key_system_masterkey_guid,
+                   app_bound_key_user_masterkey_guid, app_bound_key_system_dec,
+                   app_bound_key_user_dec, app_bound_key_dec, app_bound_key_is_decrypted
+            FROM chromium.state_keys
+            WHERE id = %s
+            """,
+            (state_key_id,),
+        )
+        row = cur.fetchone()
+
+        if not row:
+            logger.warning("State key not found", state_key_id=state_key_id)
+            return result
+
+        (
+            record_id,
+            source,
+            username,
+            browser,
+            key_masterkey_guid,
+            key_bytes_enc,
+            key_bytes_dec,
+            key_is_decrypted,
+            app_bound_key_enc,
+            app_bound_key_system_masterkey_guid,
+            app_bound_key_user_masterkey_guid,
+            app_bound_key_system_dec,
+            app_bound_key_user_dec,
+            app_bound_key_dec,
+            app_bound_key_is_decrypted,
+        ) = row
+
+    # Try to decrypt pre-v127 encrypted_key
+    if key_bytes_enc and len(key_bytes_enc) > 0 and not key_is_decrypted:
+        try:
+            # Remove DPAPI header (first 5 bytes)
+            if len(key_bytes_enc) >= 5 and key_bytes_enc[:5] == b"DPAPI":
+                dpapi_blob_bytes = key_bytes_enc[5:]
+
+                dpapi_blob = Blob.from_bytes(dpapi_blob_bytes)
+                try:
+                    key_bytes_dec = await dpapi_manager.decrypt_blob(dpapi_blob)
+                    if key_bytes_dec:
+                        key_is_decrypted = True
+                        result["decrypted_v1"] = True
+                        logger.debug(
+                            "Successfully decrypted v1 state key",
+                            state_key_id=state_key_id,
+                            masterkey_guid=dpapi_blob.masterkey_guid,
+                        )
+
+                        # Update database
+                        with pg_conn.cursor() as cur:
+                            cur.execute(
+                                """
+                                UPDATE chromium.state_keys
+                                SET key_bytes_dec = %s, key_is_decrypted = %s,
+                                    key_masterkey_guid = %s
+                                WHERE id = %s
+                                """,
+                                (key_bytes_dec, key_is_decrypted, str(dpapi_blob.masterkey_guid), state_key_id),
+                            )
+
+                except (MasterKeyNotFoundError, MasterKeyNotDecryptedError):
+                    # Masterkey still not available, skip silently
+                    pass
+                except Exception as e:
+                    logger.warning("Error decrypting v1 state key", state_key_id=state_key_id, error=str(e))
+
+        except Exception as e:
+            logger.warning("Error processing v1 state key", state_key_id=state_key_id, error=str(e))
+
+    # Try to decrypt post-v127 app_bound_encrypted_key
+    if app_bound_key_enc and len(app_bound_key_enc) > 0:
+        # Stage 1: Decrypt outer layer with SYSTEM masterkey
+        if len(app_bound_key_system_dec) == 0:
+            try:
+                if len(app_bound_key_enc) >= 4 and app_bound_key_enc[:4] == b"APPB":
+                    dpapi_portion = app_bound_key_enc[4:]
+                    system_blob = Blob.from_bytes(dpapi_portion)
+                    try:
+                        app_bound_key_system_dec = await dpapi_manager.decrypt_blob(system_blob)
+                        if app_bound_key_system_dec:
+                            result["decrypted_abe_stage1"] = True
+                            logger.debug(
+                                "Successfully decrypted ABE stage 1 (SYSTEM key)",
+                                state_key_id=state_key_id,
+                                system_masterkey_guid=system_blob.masterkey_guid,
+                            )
+
+                            # Parse the intermediate blob to get user masterkey GUID
+                            try:
+                                user_blob = Blob.from_bytes(app_bound_key_system_dec)
+                                app_bound_key_user_masterkey_guid = str(user_blob.masterkey_guid)
+                            except Exception:
+                                app_bound_key_user_masterkey_guid = None
+
+                            # Update database with stage 1 results
+                            with pg_conn.cursor() as cur:
+                                cur.execute(
+                                    """
+                                    UPDATE chromium.state_keys
+                                    SET app_bound_key_system_dec = %s,
+                                        app_bound_key_system_masterkey_guid = %s,
+                                        app_bound_key_user_masterkey_guid = %s
+                                    WHERE id = %s
+                                    """,
+                                    (
+                                        app_bound_key_system_dec,
+                                        str(system_blob.masterkey_guid),
+                                        app_bound_key_user_masterkey_guid,
+                                        state_key_id,
+                                    ),
+                                )
+
+                    except (MasterKeyNotFoundError, MasterKeyNotDecryptedError):
+                        # SYSTEM masterkey still not available
+                        pass
+                    except Exception as e:
+                        logger.warning("Error decrypting ABE stage 1", state_key_id=state_key_id, error=str(e))
+
+            except Exception as e:
+                logger.warning("Error processing ABE stage 1", state_key_id=state_key_id, error=str(e))
+
+        # Stage 2: Decrypt inner layer with USER masterkey and derive final key
+        if len(app_bound_key_system_dec) > 0 and not app_bound_key_is_decrypted:
+            try:
+                user_blob = Blob.from_bytes(app_bound_key_system_dec)
+                try:
+                    abe_blob_bytes = await dpapi_manager.decrypt_blob(user_blob)
+                    if abe_blob_bytes:
+                        # Store the intermediate value after USER key decryption
+                        app_bound_key_user_dec = abe_blob_bytes
+
+                        # Get chrome_key from database
+                        chromekey = _get_chromekey_from_source(source, pg_conn)
+
+                        # Always attempt to parse the ABE blob (works for v2 without chromekey)
+                        abe_parsed = parse_abe_blob(abe_blob_bytes, chromekey)
+
+                        if abe_parsed:
+                            app_bound_key_dec = derive_abe_key(abe_parsed)
+                            if app_bound_key_dec:
+                                app_bound_key_is_decrypted = True
+                                result["decrypted_abe_stage2"] = True
+                                logger.debug(
+                                    "Successfully decrypted ABE stage 2 (USER key + ABE derivation)",
+                                    state_key_id=state_key_id,
+                                    user_masterkey_guid=user_blob.masterkey_guid,
+                                    abe_version=abe_parsed.get("version"),
+                                )
+
+                                # Update database with final key and intermediate user_dec
+                                with pg_conn.cursor() as cur:
+                                    cur.execute(
+                                        """
+                                        UPDATE chromium.state_keys
+                                        SET app_bound_key_user_dec = %s,
+                                            app_bound_key_dec = %s,
+                                            app_bound_key_is_decrypted = %s,
+                                            app_bound_key_user_masterkey_guid = %s
+                                        WHERE id = %s
+                                        """,
+                                        (
+                                            app_bound_key_user_dec,
+                                            app_bound_key_dec,
+                                            app_bound_key_is_decrypted,
+                                            str(user_blob.masterkey_guid),
+                                            state_key_id,
+                                        ),
+                                    )
+                            else:
+                                logger.warning("Failed to derive ABE key", state_key_id=state_key_id)
+                                # Save the intermediate user_dec value for later retry
+                                with pg_conn.cursor() as cur:
+                                    cur.execute(
+                                        """
+                                        UPDATE chromium.state_keys
+                                        SET app_bound_key_user_dec = %s,
+                                            app_bound_key_user_masterkey_guid = %s
+                                        WHERE id = %s
+                                        """,
+                                        (
+                                            app_bound_key_user_dec,
+                                            str(user_blob.masterkey_guid),
+                                            state_key_id,
+                                        ),
+                                    )
+                        else:
+                            # Parsing failed - likely v3 waiting for chromekey
+                            if chromekey is None:
+                                logger.warning(
+                                    "ABE parsing failed, likely v3 waiting for Chrome key",
+                                    state_key_id=state_key_id,
+                                    source=source,
+                                )
+                            else:
+                                logger.warning("Failed to parse ABE blob", state_key_id=state_key_id)
+
+                            # Save the intermediate user_dec value for later retry
+                            with pg_conn.cursor() as cur:
+                                cur.execute(
+                                    """
+                                    UPDATE chromium.state_keys
+                                    SET app_bound_key_user_dec = %s,
+                                        app_bound_key_user_masterkey_guid = %s
+                                    WHERE id = %s
+                                    """,
+                                    (
+                                        app_bound_key_user_dec,
+                                        str(user_blob.masterkey_guid),
+                                        state_key_id,
+                                    ),
+                                )
+
+                except (MasterKeyNotFoundError, MasterKeyNotDecryptedError):
+                    # USER masterkey still not available
+                    pass
+                except Exception as e:
+                    logger.warning("Error decrypting ABE stage 2", state_key_id=state_key_id, error=str(e))
+
+            except Exception as e:
+                logger.warning("Error processing ABE stage 2", state_key_id=state_key_id, error=str(e))
+
+    # If we made any progress, update linked logins and cookies
+    if result["decrypted_v1"] or result["decrypted_abe_stage2"]:
+        try:
+            with pg_conn.cursor() as cur:
+                cur.execute(
+                    """
+                    UPDATE chromium.logins
+                    SET state_key_id = %s
+                    WHERE source = %s AND username = %s AND browser = %s
+                    AND state_key_id IS NULL
+                    AND encryption_type IN ('key', 'abe')
+                    """,
+                    (state_key_id, source, username, browser),
+                )
+                logins_updated = cur.rowcount
+
+                cur.execute(
+                    """
+                    UPDATE chromium.cookies
+                    SET state_key_id = %s
+                    WHERE source = %s AND username = %s AND browser = %s
+                    AND state_key_id IS NULL
+                    AND encryption_type IN ('key', 'abe')
+                    """,
+                    (state_key_id, source, username, browser),
+                )
+                cookies_updated = cur.rowcount
+
+                if logins_updated > 0 or cookies_updated > 0:
+                    logger.info(
+                        "Linked state key to existing logins/cookies",
+                        state_key_id=state_key_id,
+                        logins_updated=logins_updated,
+                        cookies_updated=cookies_updated,
+                    )
+
+        except Exception as e:
+            logger.warning("Error linking logins/cookies to state key", state_key_id=state_key_id, error=str(e))
+
+    # Commit all changes
+    pg_conn.commit()
+
+    return result
+
+
+async def retry_decrypt_state_keys_for_masterkey(
+    masterkey_guid: UUID, dpapi_manager: DpapiManager, masterkey_type: str | None = None
+) -> dict:
+    """Find all state_keys waiting for this masterkey and try to decrypt them.
+
+    Args:
+        masterkey_guid: The GUID of the newly available masterkey
+        dpapi_manager: DpapiManager instance for decryption
+        masterkey_type: Optional masterkey type ('system', 'user', 'unknown') for optimization
+
+    Returns:
+        Dict with statistics: {
+            "state_keys_attempted": int,
+            "state_keys_progressed": int,
+            "errors": list
+        }
+    """
+    result = {"state_keys_attempted": 0, "state_keys_progressed": 0, "errors": []}
+
+    conn_str = get_postgres_connection_str()
+
+    try:
+        with psycopg.connect(conn_str) as pg_conn:
+            # Find all state_keys that might need this masterkey
+            with pg_conn.cursor() as cur:
+                # Build query based on masterkey type for optimization
+                if masterkey_type == "system":
+                    # SYSTEM keys only used for v2 ABE stage 1
+                    query = """
+                        SELECT DISTINCT id FROM chromium.state_keys
+                        WHERE app_bound_key_system_masterkey_guid = %s
+                        AND length(app_bound_key_system_dec) = 0
+                    """
+                    cur.execute(query, (str(masterkey_guid),))
+                elif masterkey_type == "user":
+                    # USER keys used for both v1 and v2 ABE stage 2
+                    query = """
+                        SELECT DISTINCT id FROM chromium.state_keys
+                        WHERE (key_masterkey_guid = %s AND key_is_decrypted = FALSE)
+                        OR (app_bound_key_user_masterkey_guid = %s AND app_bound_key_is_decrypted = FALSE)
+                    """
+                    cur.execute(query, (str(masterkey_guid), str(masterkey_guid)))
+                else:
+                    # Unknown type - check all possible uses
+                    query = """
+                        SELECT DISTINCT id FROM chromium.state_keys
+                        WHERE (key_masterkey_guid = %s AND key_is_decrypted = FALSE)
+                        OR (app_bound_key_system_masterkey_guid = %s AND length(app_bound_key_system_dec) = 0)
+                        OR (app_bound_key_user_masterkey_guid = %s AND app_bound_key_is_decrypted = FALSE)
+                    """
+                    cur.execute(query, (str(masterkey_guid), str(masterkey_guid), str(masterkey_guid)))
+
+                state_key_ids = [row[0] for row in cur.fetchall()]
+
+            logger.debug(
+                "Found state keys potentially waiting for masterkey",
+                masterkey_guid=masterkey_guid,
+                masterkey_type=masterkey_type,
+                count=len(state_key_ids),
+            )
+
+            # Try to decrypt each state_key
+            for state_key_id in state_key_ids:
+                result["state_keys_attempted"] += 1
+                try:
+                    decrypt_result = await retry_decrypt_state_key(state_key_id, dpapi_manager, pg_conn)
+
+                    # Check if any progress was made
+                    if (
+                        decrypt_result["decrypted_v1"]
+                        or decrypt_result["decrypted_abe_stage1"]
+                        or decrypt_result["decrypted_abe_stage2"]
+                    ):
+                        result["state_keys_progressed"] += 1
+
+                except Exception as e:
+                    error_msg = f"Error processing state_key {state_key_id}: {str(e)}"
+                    logger.warning("Failed to retry decrypt state key", state_key_id=state_key_id, error=str(e))
+                    result["errors"].append(error_msg)
+
+            logger.info(
+                "Completed retroactive state_key decryption for masterkey",
+                masterkey_guid=masterkey_guid,
+                attempted=result["state_keys_attempted"],
+                progressed=result["state_keys_progressed"],
+                errors=len(result["errors"]),
+            )
+
+    except Exception as e:
+        error_msg = f"Database error during retroactive decryption: {str(e)}"
+        logger.exception("Error in retry_decrypt_state_keys_for_masterkey", masterkey_guid=masterkey_guid, error=str(e))
+        result["errors"].append(error_msg)
+
+    return result
+
+
+async def retry_decrypt_state_keys_for_chromekey(source: str, chromekey: bytes) -> dict:
+    """Find all state_keys from a source waiting for chromekey and try to decrypt them.
+
+    This function handles v3 ABE decryption where the USER masterkey has already been
+    applied (app_bound_key_user_dec is populated) but the chromekey is needed to
+    complete the final derivation.
+
+    Args:
+        source: The source identifier (hostname) for the chromekey
+        chromekey: The decrypted Chrome key bytes
+
+    Returns:
+        Dict with statistics: {
+            "state_keys_attempted": int,
+            "state_keys_decrypted": int,
+            "errors": list
+        }
+    """
+    result = {"state_keys_attempted": 0, "state_keys_decrypted": 0, "errors": []}
+
+    conn_str = get_postgres_connection_str()
+
+    try:
+        with psycopg.connect(conn_str) as pg_conn:
+            # Find all state_keys from this source that have user_dec but aren't fully decrypted
+            with pg_conn.cursor() as cur:
+                query = """
+                    SELECT id, source, username, browser, app_bound_key_user_dec,
+                           app_bound_key_user_masterkey_guid
+                    FROM chromium.state_keys
+                    WHERE source = %s
+                    AND app_bound_key_is_decrypted = FALSE
+                    AND app_bound_key_user_dec IS NOT NULL
+                    AND length(app_bound_key_user_dec) > 0
+                """
+                cur.execute(query, (source,))
+                state_keys = cur.fetchall()
+
+            logger.debug(
+                "Found state keys waiting for chromekey",
+                source=source,
+                count=len(state_keys),
+            )
+
+            # Try to decrypt each state_key with the chromekey
+            for row in state_keys:
+                state_key_id, source, username, browser, app_bound_key_user_dec, user_masterkey_guid = row
+                result["state_keys_attempted"] += 1
+
+                try:
+                    # Parse and derive the final ABE key using the chromekey
+                    abe_parsed = parse_abe_blob(app_bound_key_user_dec, chromekey)
+                    logger.debug(
+                        "[retry_decrypt_state_keys_for_chromekey] Parsed ABE blob",
+                        state_key_id=state_key_id,
+                        abe_parsed=abe_parsed,
+                    )
+
+                    if abe_parsed:
+                        app_bound_key_dec = derive_abe_key(abe_parsed)
+                        logger.debug(
+                            "[retry_decrypt_state_keys_for_chromekey] Derived ABE key",
+                            state_key_id=state_key_id,
+                            success=bool(app_bound_key_dec),
+                        )
+
+                        if app_bound_key_dec:
+                            # Update database with final decrypted key
+                            with pg_conn.cursor() as cur:
+                                cur.execute(
+                                    """
+                                    UPDATE chromium.state_keys
+                                    SET app_bound_key_dec = %s,
+                                        app_bound_key_is_decrypted = TRUE
+                                    WHERE id = %s
+                                    """,
+                                    (app_bound_key_dec, state_key_id),
+                                )
+
+                                # Link to existing logins/cookies
+                                cur.execute(
+                                    """
+                                    UPDATE chromium.logins
+                                    SET state_key_id = %s
+                                    WHERE source = %s AND username = %s AND browser = %s
+                                    AND state_key_id IS NULL
+                                    AND encryption_type IN ('key', 'abe')
+                                    """,
+                                    (state_key_id, source, username, browser),
+                                )
+                                logins_updated = cur.rowcount
+
+                                cur.execute(
+                                    """
+                                    UPDATE chromium.cookies
+                                    SET state_key_id = %s
+                                    WHERE source = %s AND username = %s AND browser = %s
+                                    AND state_key_id IS NULL
+                                    AND encryption_type IN ('key', 'abe')
+                                    """,
+                                    (state_key_id, source, username, browser),
+                                )
+                                cookies_updated = cur.rowcount
+
+                            pg_conn.commit()
+                            result["state_keys_decrypted"] += 1
+
+                            logger.debug(
+                                "Successfully decrypted ABE v3 with chromekey",
+                                state_key_id=state_key_id,
+                                source=source,
+                                username=username,
+                                browser=browser,
+                                abe_version=abe_parsed.get("version"),
+                                logins_updated=logins_updated,
+                                cookies_updated=cookies_updated,
+                            )
+                        else:
+                            logger.warning(
+                                "Failed to derive ABE key with chromekey",
+                                state_key_id=state_key_id,
+                                source=source,
+                            )
+                    else:
+                        logger.warning(
+                            "Failed to parse ABE blob with chromekey",
+                            state_key_id=state_key_id,
+                            source=source,
+                        )
+
+                except Exception as e:
+                    error_msg = f"Error processing state_key {state_key_id}: {str(e)}"
+                    logger.warning(
+                        "Failed to decrypt state key with chromekey",
+                        state_key_id=state_key_id,
+                        error=str(e),
+                    )
+                    result["errors"].append(error_msg)
+
+            logger.info(
+                "Completed retroactive state_key decryption for chromekey",
+                source=source,
+                attempted=result["state_keys_attempted"],
+                decrypted=result["state_keys_decrypted"],
+                errors=len(result["errors"]),
+            )
+
+    except Exception as e:
+        error_msg = f"Database error during retroactive chromekey decryption: {str(e)}"
+        logger.exception("Error in retry_decrypt_state_keys_for_chromekey", source=source, error=str(e))
+        result["errors"].append(error_msg)
+
+    return result

libs/chromium/chromium/logins.py

@@ -0,0 +1,254 @@
+"""Chromium Login Data file parsing and database operations."""
+
+import asyncio
+import sqlite3
+
+import psycopg
+from common.logger import get_logger
+from common.state_helpers import get_file_enriched
+from common.storage import StorageMinio
+from nemesis_dpapi import Blob, DpapiManager
+
+from .helpers import (
+    convert_chromium_timestamp,
+    decrypt_chrome_string,
+    detect_encryption_type,
+    get_postgres_connection_str,
+    get_state_key_bytes,
+    get_state_key_id,
+    is_sqlite3,
+    parse_chromium_file_path,
+    try_decrypt_with_all_keys,
+)
+
+logger = get_logger(__name__)
+
+
+def process_chromium_logins(
+    object_id: str, file_path: str | None = None, dpapi_manager: DpapiManager | None = None
+) -> None:
+    """Process Chromium Login Data file and insert logins into database.
+
+    Args:
+        object_id: The object ID of the Login Data file
+        file_path: Optional path to already downloaded file
+        dpapi_manager: DPAPI manager for decryption
+    """
+    logger.info("Processing Chromium Login Data file", object_id=object_id)
+
+    file_enriched = get_file_enriched(object_id)
+
+    # Extract username and browser from file path
+    username, browser = parse_chromium_file_path(file_enriched.path or "")
+    logger.debug("[process_chromium_logins]", username=username, browser=browser)
+
+    # Get database file
+    if file_path:
+        db_path = file_path
+    else:
+        storage = StorageMinio()
+        with storage.download(file_enriched.object_id) as temp_file:
+            db_path = temp_file.name
+
+    if is_sqlite3(db_path) is False:
+        logger.warning(
+            "Login Data file is not a valid SQLite3 database", object_id=object_id, file_path=file_enriched.path
+        )
+        return
+
+    conn_str = get_postgres_connection_str()
+    with psycopg.connect(conn_str) as pg_conn:
+        _insert_logins(file_enriched, username, browser, db_path, pg_conn, dpapi_manager)
+
+    logger.debug("Completed processing Chromium Login Data", object_id=object_id)
+
+
+def _insert_logins(
+    file_enriched, username: str | None, browser: str, db_path: str, pg_conn, dpapi_manager: DpapiManager | None = None
+) -> None:
+    """Extract logins from Login Data database and insert into chromium.logins table."""
+    try:
+        # Read from SQLite
+        with sqlite3.connect(db_path) as conn:
+            conn.text_factory = bytes  # Get raw bytes, we'll handle text decoding manually
+            cursor = conn.cursor()
+
+            # Query logins table
+            cursor.execute("""
+                SELECT origin_url, username_value, signon_realm, date_created,
+                       date_last_used, date_password_modified, times_used, password_value
+                FROM logins
+            """)
+            rows = cursor.fetchall()
+
+        if not rows:
+            return
+
+        # Prepare data for PostgreSQL
+        logins_data = []
+        for row in rows:
+            (
+                origin_url,
+                username_value,
+                signon_realm,
+                date_created,
+                date_last_used,
+                date_password_modified,
+                times_used,
+                password_value,
+            ) = row
+
+            # Decode text fields from bytes (since we set text_factory = bytes)
+            origin_url = origin_url.decode("utf-8", errors="replace") if origin_url else None
+            username_value = username_value.decode("utf-8", errors="replace") if username_value else None
+            signon_realm = signon_realm.decode("utf-8", errors="replace") if signon_realm else None
+
+            # password_value is already binary (what we want)
+            if password_value is None:
+                password_value = b""
+
+            # Detect encryption type and get masterkey GUID
+            encryption_type, masterkey_guid = detect_encryption_type(password_value)
+            is_decrypted = False
+            password_value_dec = None
+
+            # Try DPAPI decryption first
+            if masterkey_guid and dpapi_manager:
+                try:
+                    password_dec_bytes = asyncio.run(dpapi_manager.decrypt_blob(Blob.from_bytes(password_value)))
+                    if password_dec_bytes:
+                        password_value_dec = password_dec_bytes.decode("utf-8", errors="replace")
+                        is_decrypted = True
+                except:
+                    pass
+
+            # Get state key ID for key/abe encryption
+            state_key_id = None
+            if encryption_type in ["key", "abe"]:
+                # Try primary approach: get state key based on username/browser
+                if username:  # Only try primary approach if username was extracted
+                    state_key_id = get_state_key_id(file_enriched.source, username, browser, pg_conn)
+                    if state_key_id:
+                        # Retrieve the pre-processed state key for decryption
+                        state_key_bytes = get_state_key_bytes(state_key_id, encryption_type, pg_conn)
+                        if state_key_bytes:
+                            try:
+                                password_dec_bytes = decrypt_chrome_string(
+                                    password_value, state_key_bytes, encryption_type
+                                )
+                                if password_dec_bytes:
+                                    # For passwords, may need to strip offset bytes depending on version
+                                    if encryption_type == "key" and len(password_dec_bytes) > 32:
+                                        # v10/v11 passwords may have 32-byte prefix + 16-byte suffix
+                                        password_dec_bytes = password_dec_bytes[32:-16]
+                                    elif encryption_type == "key" and len(password_dec_bytes) > 16:
+                                        # Or just 16-byte suffix
+                                        password_dec_bytes = password_dec_bytes[:-16]
+                                    # v20 passwords typically don't have offset like cookies
+
+                                    password_value_dec = password_dec_bytes.decode("utf-8", errors="replace")
+                                    is_decrypted = True
+                            except Exception as e:
+                                logger.debug(
+                                    "Failed to decrypt password with state key",
+                                    state_key_id=state_key_id,
+                                    encryption_type=encryption_type,
+                                    error=str(e),
+                                )
+
+                # Backup approach: try all state keys from the same source if primary failed
+                if not is_decrypted:
+                    logger.debug(
+                        "Primary decryption failed, trying backup approach with all keys from source",
+                        source=file_enriched.source,
+                        encryption_type=encryption_type,
+                    )
+                    backup_decrypted_bytes, backup_state_key_id = try_decrypt_with_all_keys(
+                        password_value, file_enriched.source, encryption_type, pg_conn
+                    )
+                    if backup_decrypted_bytes and backup_state_key_id:
+                        # Apply the same offset handling as above
+                        if encryption_type == "abe" and len(backup_decrypted_bytes) > 32:
+                            # v20 cookies typically have 32-byte offset
+                            backup_decrypted_bytes = backup_decrypted_bytes[32:]
+                        elif encryption_type == "key" and len(backup_decrypted_bytes) > 48:
+                            # v10/v11 cookies may have 32-byte prefix + 16-byte suffix
+                            backup_decrypted_bytes = backup_decrypted_bytes[32:-16]
+                        elif encryption_type == "key" and len(backup_decrypted_bytes) > 16:
+                            # Or just 16-byte suffix
+                            backup_decrypted_bytes = backup_decrypted_bytes[:-16]
+
+                        password_value_dec = backup_decrypted_bytes.decode("utf-8", errors="replace")
+                        is_decrypted = True
+                        state_key_id = backup_state_key_id
+                        logger.debug(
+                            "Successfully decrypted cookie using backup approach", state_key_id=backup_state_key_id
+                        )
+
+            login_data = {
+                "originating_object_id": file_enriched.object_id,
+                "agent_id": file_enriched.agent_id,
+                "source": file_enriched.source,
+                "project": file_enriched.project,
+                "username": username,
+                "browser": browser,
+                "origin_url": origin_url,
+                "username_value": username_value,
+                "signon_realm": signon_realm,
+                "date_created": convert_chromium_timestamp(date_created),
+                "date_last_used": convert_chromium_timestamp(date_last_used),
+                "date_password_modified": convert_chromium_timestamp(date_password_modified),
+                "times_used": times_used,
+                "encryption_type": encryption_type,
+                "masterkey_guid": masterkey_guid,
+                "state_key_id": state_key_id,
+                "is_decrypted": is_decrypted,
+                "password_value_enc": password_value,
+                "password_value_dec": password_value_dec,
+            }
+            logins_data.append(login_data)
+
+        # Insert into PostgreSQL
+        with pg_conn.cursor() as cur:
+            insert_sql = """
+                INSERT INTO chromium.logins
+                (originating_object_id, agent_id, source, project, username, browser,
+                 origin_url, username_value, signon_realm, date_created, date_last_used,
+                 date_password_modified, times_used, encryption_type, masterkey_guid,
+                 state_key_id, is_decrypted, password_value_enc, password_value_dec)
+                VALUES (%(originating_object_id)s, %(agent_id)s, %(source)s, %(project)s,
+                        %(username)s, %(browser)s, %(origin_url)s, %(username_value)s,
+                        %(signon_realm)s, %(date_created)s, %(date_last_used)s,
+                        %(date_password_modified)s, %(times_used)s, %(encryption_type)s,
+                        %(masterkey_guid)s, %(state_key_id)s, %(is_decrypted)s,
+                        %(password_value_enc)s, %(password_value_dec)s)
+                ON CONFLICT (source, username, browser, origin_url, username_value)
+                DO UPDATE SET
+                    origin_url = EXCLUDED.origin_url,
+                    username_value = EXCLUDED.username_value,
+                    signon_realm = EXCLUDED.signon_realm,
+                    date_created = EXCLUDED.date_created,
+                    date_last_used = EXCLUDED.date_last_used,
+                    date_password_modified = EXCLUDED.date_password_modified,
+                    times_used = EXCLUDED.times_used,
+                    encryption_type = EXCLUDED.encryption_type,
+                    masterkey_guid = EXCLUDED.masterkey_guid,
+                    state_key_id = EXCLUDED.state_key_id,
+                    is_decrypted = EXCLUDED.is_decrypted,
+                    password_value_enc = EXCLUDED.password_value_enc,
+                    password_value_dec = EXCLUDED.password_value_dec
+            """
+
+            cur.executemany(insert_sql, logins_data)
+            pg_conn.commit()
+
+        logger.info("Inserted logins into database", count=len(logins_data))
+
+    except Exception as e:
+        logger.exception(
+            "Error processing Login Data",
+            error=str(e),
+            object_id=file_enriched.object_id,
+            file_path=file_enriched.path,
+        )
+        raise

libs/chromium/chromium/retry.py

@@ -0,0 +1,426 @@
+"""Retry decryption logic for Chromium cookies and logins."""
+
+from uuid import UUID
+
+import psycopg
+from common.logger import get_logger
+from nemesis_dpapi import Blob, DpapiManager, MasterKeyNotDecryptedError, MasterKeyNotFoundError
+
+from .helpers import decrypt_chrome_string, get_postgres_connection_str, get_state_key_bytes
+
+logger = get_logger(__name__)
+
+
+async def retry_decrypt_chromium_data(
+    masterkey_guid: UUID, dpapi_manager: DpapiManager, masterkey_type: str | None = None
+) -> dict:
+    """Retry decrypting cookies and logins that failed to decrypt previously.
+
+    When a new masterkey becomes available, this function:
+    1. Retries DPAPI-encrypted cookies/logins that use this masterkey
+    2. Retries key/abe-encrypted cookies/logins (state keys may now be decrypted)
+
+    Args:
+        masterkey_guid: The GUID of the newly available masterkey
+        dpapi_manager: DpapiManager instance for decryption
+        masterkey_type: Optional masterkey type ('system', 'user', 'unknown')
+
+    Returns:
+        Dict with statistics: {
+            "cookies_attempted": int,
+            "cookies_decrypted": int,
+            "logins_attempted": int,
+            "logins_decrypted": int,
+            "errors": list
+        }
+    """
+    result = {
+        "cookies_attempted": 0,
+        "cookies_decrypted": 0,
+        "logins_attempted": 0,
+        "logins_decrypted": 0,
+        "errors": [],
+    }
+
+    conn_str = get_postgres_connection_str()
+
+    try:
+        with psycopg.connect(conn_str) as pg_conn:
+            # Retry cookies
+            cookies_result = await _retry_decrypt_cookies(masterkey_guid, dpapi_manager, pg_conn)
+            result["cookies_attempted"] = cookies_result["attempted"]
+            result["cookies_decrypted"] = cookies_result["decrypted"]
+            result["errors"].extend(cookies_result["errors"])
+
+            # Retry logins
+            logins_result = await _retry_decrypt_logins(masterkey_guid, dpapi_manager, pg_conn)
+            result["logins_attempted"] = logins_result["attempted"]
+            result["logins_decrypted"] = logins_result["decrypted"]
+            result["errors"].extend(logins_result["errors"])
+
+            logger.warning(
+                "Completed retroactive chromium data decryption for masterkey",
+                masterkey_guid=masterkey_guid,
+                cookies_attempted=result["cookies_attempted"],
+                cookies_decrypted=result["cookies_decrypted"],
+                logins_attempted=result["logins_attempted"],
+                logins_decrypted=result["logins_decrypted"],
+                errors=len(result["errors"]),
+            )
+
+    except Exception as e:
+        error_msg = f"Database error during retroactive chromium data decryption: {str(e)}"
+        logger.exception("Error in retry_decrypt_chromium_data", masterkey_guid=masterkey_guid, error=str(e))
+        result["errors"].append(error_msg)
+
+    return result
+
+
+async def _retry_decrypt_cookies(masterkey_guid: UUID, dpapi_manager: DpapiManager, pg_conn) -> dict:
+    """Retry decrypting cookies that previously failed.
+
+    Args:
+        masterkey_guid: The GUID of the newly available masterkey
+        dpapi_manager: DpapiManager instance for decryption
+        pg_conn: PostgreSQL connection
+
+    Returns:
+        Dict with {"attempted": int, "decrypted": int, "errors": list}
+    """
+    result = {"attempted": 0, "decrypted": 0, "errors": []}
+
+    # Find all undecrypted cookies
+    # For DPAPI: only those matching this masterkey
+    # For key/abe: all undecrypted (state keys may now be available)
+    with pg_conn.cursor() as cur:
+        cur.execute(
+            """
+            SELECT id, encryption_type, masterkey_guid, state_key_id,
+                   value_enc, source, username, browser
+            FROM chromium.cookies
+            WHERE is_decrypted = FALSE
+            AND (
+                (encryption_type = 'dpapi' AND masterkey_guid = %s)
+                OR encryption_type IN ('key', 'abe')
+            )
+            """,
+            (masterkey_guid,),
+        )
+        cookies = cur.fetchall()
+
+    logger.debug(
+        "Found undecrypted cookies to retry",
+        masterkey_guid=masterkey_guid,
+        count=len(cookies),
+    )
+
+    for cookie in cookies:
+        result["attempted"] += 1
+        cookie_id, encryption_type, cookie_masterkey_guid, state_key_id, value_enc, source, username, browser = cookie
+
+        try:
+            value_dec = None
+            decrypted = False
+
+            # Try DPAPI decryption
+            if encryption_type == "dpapi" and cookie_masterkey_guid == masterkey_guid:
+                try:
+                    value_dec_bytes = await dpapi_manager.decrypt_blob(Blob.from_bytes(value_enc))
+                    if value_dec_bytes:
+                        value_dec = value_dec_bytes.decode("utf-8", errors="replace")
+                        decrypted = True
+                        logger.debug("Successfully decrypted cookie with DPAPI", cookie_id=cookie_id)
+                except (MasterKeyNotFoundError, MasterKeyNotDecryptedError):
+                    # Still not available
+                    pass
+                except Exception as e:
+                    logger.warning("Error decrypting cookie with DPAPI", cookie_id=cookie_id, error=str(e))
+
+            # Try state key decryption (key/abe)
+            elif encryption_type in ["key", "abe"]:
+                # Try with existing state_key_id if available
+                if state_key_id:
+                    state_key_bytes = get_state_key_bytes(state_key_id, encryption_type, pg_conn)
+                    if state_key_bytes:
+                        try:
+                            value_dec_bytes = decrypt_chrome_string(value_enc, state_key_bytes, encryption_type)
+                            if value_dec_bytes:
+                                # Apply offset handling for cookies
+                                if encryption_type == "abe" and len(value_dec_bytes) > 32:
+                                    value_dec_bytes = value_dec_bytes[32:]
+                                elif encryption_type == "key" and len(value_dec_bytes) > 48:
+                                    value_dec_bytes = value_dec_bytes[32:-16]
+                                elif encryption_type == "key" and len(value_dec_bytes) > 16:
+                                    value_dec_bytes = value_dec_bytes[:-16]
+
+                                value_dec = value_dec_bytes.decode("utf-8", errors="replace")
+                                decrypted = True
+                                logger.debug(
+                                    "Successfully decrypted cookie with state key",
+                                    cookie_id=cookie_id,
+                                    state_key_id=state_key_id,
+                                )
+                        except Exception as e:
+                            logger.debug(
+                                "Failed to decrypt cookie with state key",
+                                cookie_id=cookie_id,
+                                state_key_id=state_key_id,
+                                error=str(e),
+                            )
+
+                # If no state_key_id or decryption failed, try to find matching state key
+                if not decrypted and username and browser:
+                    try:
+                        with pg_conn.cursor() as cur2:
+                            # Try to find a decrypted state key for this source/username/browser
+                            cur2.execute(
+                                """
+                                SELECT id FROM chromium.state_keys
+                                WHERE source = %s AND username = %s AND browser = %s
+                                AND (
+                                    (key_is_decrypted = TRUE AND %s = 'key')
+                                    OR (app_bound_key_is_decrypted = TRUE AND %s = 'abe')
+                                )
+                                """,
+                                (source, username, browser, encryption_type, encryption_type),
+                            )
+                            state_key_row = cur2.fetchone()
+                            if state_key_row:
+                                new_state_key_id = state_key_row[0]
+                                state_key_bytes = get_state_key_bytes(new_state_key_id, encryption_type, pg_conn)
+                                if state_key_bytes:
+                                    try:
+                                        value_dec_bytes = decrypt_chrome_string(
+                                            value_enc, state_key_bytes, encryption_type
+                                        )
+                                        if value_dec_bytes:
+                                            # Apply offset handling
+                                            if encryption_type == "abe" and len(value_dec_bytes) > 32:
+                                                value_dec_bytes = value_dec_bytes[32:]
+                                            elif encryption_type == "key" and len(value_dec_bytes) > 48:
+                                                value_dec_bytes = value_dec_bytes[32:-16]
+                                            elif encryption_type == "key" and len(value_dec_bytes) > 16:
+                                                value_dec_bytes = value_dec_bytes[:-16]
+
+                                            value_dec = value_dec_bytes.decode("utf-8", errors="replace")
+                                            decrypted = True
+                                            state_key_id = new_state_key_id
+                                            logger.debug(
+                                                "Successfully decrypted cookie with newly found state key",
+                                                cookie_id=cookie_id,
+                                                state_key_id=new_state_key_id,
+                                            )
+                                    except Exception as e:
+                                        logger.debug(
+                                            "Failed to decrypt cookie with newly found state key",
+                                            cookie_id=cookie_id,
+                                            error=str(e),
+                                        )
+                    except Exception as e:
+                        logger.debug("Error looking up state key for cookie", cookie_id=cookie_id, error=str(e))
+
+            # Update database if decrypted
+            if decrypted and value_dec:
+                with pg_conn.cursor() as cur:
+                    cur.execute(
+                        """
+                        UPDATE chromium.cookies
+                        SET value_dec = %s, is_decrypted = TRUE, state_key_id = %s
+                        WHERE id = %s
+                        """,
+                        (value_dec, state_key_id, cookie_id),
+                    )
+                pg_conn.commit()
+                result["decrypted"] += 1
+
+        except Exception as e:
+            error_msg = f"Error processing cookie {cookie_id}: {str(e)}"
+            logger.warning("Failed to retry decrypt cookie", cookie_id=cookie_id, error=str(e))
+            result["errors"].append(error_msg)
+
+    return result
+
+
+async def _retry_decrypt_logins(masterkey_guid: UUID, dpapi_manager: DpapiManager, pg_conn) -> dict:
+    """Retry decrypting logins that previously failed.
+
+    Args:
+        masterkey_guid: The GUID of the newly available masterkey
+        dpapi_manager: DpapiManager instance for decryption
+        pg_conn: PostgreSQL connection
+
+    Returns:
+        Dict with {"attempted": int, "decrypted": int, "errors": list}
+    """
+    result = {"attempted": 0, "decrypted": 0, "errors": []}
+
+    # Find all undecrypted logins
+    with pg_conn.cursor() as cur:
+        cur.execute(
+            """
+            SELECT id, encryption_type, masterkey_guid, state_key_id,
+                   password_value_enc, source, username, browser
+            FROM chromium.logins
+            WHERE is_decrypted = FALSE
+            AND (
+                (encryption_type = 'dpapi' AND masterkey_guid = %s)
+                OR encryption_type IN ('key', 'abe')
+            )
+            """,
+            (masterkey_guid,),
+        )
+        logins = cur.fetchall()
+
+    logger.debug(
+        "Found undecrypted logins to retry",
+        masterkey_guid=masterkey_guid,
+        count=len(logins),
+    )
+
+    for login in logins:
+        result["attempted"] += 1
+        (
+            login_id,
+            encryption_type,
+            login_masterkey_guid,
+            state_key_id,
+            password_value_enc,
+            source,
+            username,
+            browser,
+        ) = login
+
+        try:
+            password_dec = None
+            decrypted = False
+
+            # Try DPAPI decryption
+            if encryption_type == "dpapi" and login_masterkey_guid == masterkey_guid:
+                try:
+                    password_dec_bytes = await dpapi_manager.decrypt_blob(Blob.from_bytes(password_value_enc))
+                    if password_dec_bytes:
+                        password_dec = password_dec_bytes.decode("utf-8", errors="replace")
+                        decrypted = True
+                        logger.debug("Successfully decrypted login with DPAPI", login_id=login_id)
+                except (MasterKeyNotFoundError, MasterKeyNotDecryptedError):
+                    # Still not available
+                    pass
+                except Exception as e:
+                    logger.warning("Error decrypting login with DPAPI", login_id=login_id, error=str(e))
+
+            # Try state key decryption (key/abe)
+            elif encryption_type in ["key", "abe"]:
+                # Try with existing state_key_id if available
+                if state_key_id:
+                    state_key_bytes = get_state_key_bytes(state_key_id, encryption_type, pg_conn)
+                    if state_key_bytes:
+                        try:
+                            password_dec_bytes = decrypt_chrome_string(
+                                password_value_enc, state_key_bytes, encryption_type
+                            )
+                            if password_dec_bytes:
+                                # Apply offset handling for passwords
+                                if encryption_type == "key" and len(password_dec_bytes) > 32:
+                                    password_dec_bytes = password_dec_bytes[32:-16]
+                                elif encryption_type == "key" and len(password_dec_bytes) > 16:
+                                    password_dec_bytes = password_dec_bytes[:-16]
+                                # v20 passwords typically don't have offset like cookies
+
+                                password_dec = password_dec_bytes.decode("utf-8", errors="replace")
+                                decrypted = True
+                                logger.debug(
+                                    "Successfully decrypted login with state key",
+                                    login_id=login_id,
+                                    state_key_id=state_key_id,
+                                )
+                        except Exception as e:
+                            logger.debug(
+                                "Failed to decrypt login with state key",
+                                login_id=login_id,
+                                state_key_id=state_key_id,
+                                error=str(e),
+                            )
+
+                # If no state_key_id or decryption failed, try to find matching state key
+                if not decrypted and source:
+                    try:
+                        with pg_conn.cursor() as cur2:
+                            if username and browser:
+                                # Try to find a decrypted state key for this source/username/browser
+                                cur2.execute(
+                                    """
+                                    SELECT id FROM chromium.state_keys
+                                    WHERE source = %s AND username = %s AND browser = %s
+                                    AND (
+                                        (key_is_decrypted = TRUE AND %s = 'key')
+                                        OR (app_bound_key_is_decrypted = TRUE AND %s = 'abe')
+                                    )
+                                    """,
+                                    (source, username, browser, encryption_type, encryption_type),
+                                )
+                            else:
+                                # if no username/password, just restrict to SOURCE
+                                cur2.execute(
+                                    """
+                                    SELECT id FROM chromium.state_keys
+                                    WHERE source = %s
+                                    AND (
+                                        (key_is_decrypted = TRUE AND %s = 'key')
+                                        OR (app_bound_key_is_decrypted = TRUE AND %s = 'abe')
+                                    )
+                                    """,
+                                    (source, encryption_type, encryption_type),
+                                )
+                            state_key_row = cur2.fetchone()
+                            if state_key_row:
+                                new_state_key_id = state_key_row[0]
+                                state_key_bytes = get_state_key_bytes(new_state_key_id, encryption_type, pg_conn)
+                                if state_key_bytes:
+                                    try:
+                                        password_dec_bytes = decrypt_chrome_string(
+                                            password_value_enc, state_key_bytes, encryption_type
+                                        )
+                                        if password_dec_bytes:
+                                            # Apply offset handling
+                                            if encryption_type == "key" and len(password_dec_bytes) > 32:
+                                                password_dec_bytes = password_dec_bytes[32:-16]
+                                            elif encryption_type == "key" and len(password_dec_bytes) > 16:
+                                                password_dec_bytes = password_dec_bytes[:-16]
+
+                                            password_dec = password_dec_bytes.decode("utf-8", errors="replace")
+                                            decrypted = True
+                                            state_key_id = new_state_key_id
+                                            logger.debug(
+                                                "Successfully decrypted login with newly found state key",
+                                                login_id=login_id,
+                                                state_key_id=new_state_key_id,
+                                            )
+                                    except Exception as e:
+                                        logger.debug(
+                                            "Failed to decrypt login with newly found state key",
+                                            login_id=login_id,
+                                            error=str(e),
+                                        )
+                    except Exception as e:
+                        logger.debug("Error looking up state key for login", login_id=login_id, error=str(e))
+
+            # Update database if decrypted
+            if decrypted and password_dec:
+                with pg_conn.cursor() as cur:
+                    cur.execute(
+                        """
+                        UPDATE chromium.logins
+                        SET password_value_dec = %s, is_decrypted = TRUE, state_key_id = %s
+                        WHERE id = %s
+                        """,
+                        (password_dec, state_key_id, login_id),
+                    )
+                pg_conn.commit()
+                result["decrypted"] += 1
+
+        except Exception as e:
+            error_msg = f"Error processing login {login_id}: {str(e)}"
+            logger.warning("Failed to retry decrypt login", login_id=login_id, error=str(e))
+            result["errors"].append(error_msg)
+
+    return result

libs/chromium/pyproject.toml

@@ -0,0 +1,25 @@
+[tool.poetry]
+name = "chromium"
+version = "0.1.0"
+description = "Modules Nemesis uses to handle Chromium files"
+authors = ["SpecterOps"]
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = ">=3.12,<4.0"
+impacket = ">=0.12.0,<0.13.0"
+psycopg = {extras = ["binary"], version = ">=3.0.0,<4.0.0"}
+dapr = "1.16.0"
+structlog = ">=20.0.0,<30.0.0"
+nemesis_dpapi = { path = "../nemesis_dpapi", develop = true }
+file_linking = { path = "../file_linking", develop = true }
+common = { path = "../../libs/common", develop = true }
+
+[tool.poetry.group.dev.dependencies]
+ruff = "^0.9.2"
+pytest = "^8.4.2"
+pytest-asyncio = "^1.2.0"
+
+[build-system]
+requires = ["poetry-core>=2.0.0,<3.0.0"]
+build-backend = "poetry.core.masonry.api"

libs/chromium/tests/test_example.py

@@ -0,0 +1,3 @@
+def test_example():
+    """Simple example test to verify pytest is working."""
+    assert True

libs/common/.vscode/settings.json

@@ -8,13 +8,12 @@
     "[python]": {
         "editor.formatOnSave": true,
         "editor.codeActionsOnSave": {
-            "source.fixAll": "explicit",
-            "source.organizeImports": "explicit"
+            "source.fixAll.ruff": "explicit",
+            "source.organizeImports.ruff": "explicit"
         },
         "editor.defaultFormatter": "charliermarsh.ruff"
     },
     "autoDocstring.docstringFormat": "google",
-    "editor.formatOnSave": true,
     "files.exclude": {
         "**/.DS_Store": true,
         "**/.git": true,
@@ -62,4 +61,5 @@
     "python.testing.unittestEnabled": false,
     "python.testing.pytestEnabled": true,
     "python.analysis.typeCheckingMode": "basic",
+    "python.defaultInterpreterPath": "${workspaceFolder}/.venv/bin/python"
 }

libs/common/common/db.py

@@ -0,0 +1,47 @@
+from functools import lru_cache
+from urllib.parse import quote_plus
+
+from dapr.clients import DaprClient
+
+_DAPR_SECRET_STORE_NAME = "nemesis-secret-store"
+
+
+@lru_cache(maxsize=1)
+def get_postgres_connection_str(dapr_client: DaprClient | None = None) -> str:
+    """Get PostgreSQL connection string from Dapr secrets by building it from individual parameters."""
+
+    def fetch_secrets(client: DaprClient) -> dict:
+        """Fetch all required PostgreSQL secrets."""
+        secrets = {}
+        secret_keys = ["POSTGRES_USER", "POSTGRES_PASSWORD", "POSTGRES_HOST", "POSTGRES_PORT", "POSTGRES_DB", "POSTGRES_PARAMETERS"]
+
+        for key in secret_keys:
+            try:
+                secret = client.get_secret(store_name=_DAPR_SECRET_STORE_NAME, key=key)
+                secrets[key] = secret.secret[key]
+            except Exception as e:
+                raise ValueError(f"Failed to fetch {key} from Dapr secret store: {e}") from e
+
+        return secrets
+
+    if dapr_client:
+        secrets = fetch_secrets(dapr_client)
+    else:
+        with DaprClient() as client:
+            secrets = fetch_secrets(client)
+
+    # Build the connection string from individual parameters
+    # URL-encode user and password to handle special characters like @, :, /, etc.
+    user = quote_plus(secrets["POSTGRES_USER"])
+    password = quote_plus(secrets["POSTGRES_PASSWORD"])
+    host = secrets["POSTGRES_HOST"]
+    port = secrets["POSTGRES_PORT"]
+    db = secrets["POSTGRES_DB"]
+    parameters = secrets["POSTGRES_PARAMETERS"]
+
+    output = f"postgresql://{user}:{password}@{host}:{port}/{db}?{parameters}"
+
+    if not output.startswith("postgresql://"):
+        raise ValueError("Constructed POSTGRES connection string must start with 'postgresql://'")
+
+    return output

libs/common/common/dependency_checks.py

@@ -3,7 +3,6 @@ import os
 import shutil
 from collections.abc import Sequence
 from pathlib import Path
-from typing import Optional
 
 logger = logging.getLogger(__name__)
 logger.addHandler(logging.NullHandler())
@@ -22,7 +21,7 @@ def find_missing_path_dependencies(
     commands: Sequence[str],
     *,
     raise_error: bool = True,
-    search_paths: Optional[Sequence[Path]] = None,
+    search_paths: Sequence[Path] | None = None,
 ) -> list[str]:
     """Checks if specified commands are available in system PATH.
 

libs/common/common/helpers.py

@@ -1,10 +1,9 @@
 import hashlib
+import io
+import posixpath
 import re
 import subprocess
-
-import structlog
-
-logger = structlog.get_logger(module=__name__)
+from typing import BinaryIO
 
 
 def calculate_file_hash(file_path: str, hash_type: str) -> str:
@@ -250,6 +249,100 @@ def sanitize_file_path(file_path: str, num_chars=4):
     return f"{sanitized_base}.{extension[0]}" if extension else sanitized_base
 
 
+def get_file_extension(filepath):
+    # Get just the final filename component of the path
+    base_name = posixpath.basename(filepath)
+
+    # Split on the last dot, but only if the dot isn't the first character
+    if base_name.startswith(".") or "." not in base_name:
+        return ""
+
+    name_parts = base_name.split(".")
+    if len(name_parts) > 1:
+        return "." + name_parts[-1]
+    return ""
+
+def get_drive_from_path(path: str) -> str | None:
+    """
+    Extract Windows drive letter from a file path.
+
+    Supports two path formats:
+    1. POSIX-style with leading slash: "/C:/Users/..." or "/D:/Data/..."
+    2. Windows-style without leading slash: "C:/Users/..." or "D:/Data/..."
+
+    Args:
+        path: File path string to parse
+
+    Returns:
+        str | None: Drive letter with colon, or None if no valid drive found
+            - For POSIX-style paths: Returns with leading slash (e.g., "/C:", "/D:")
+            - For Windows-style paths: Returns without leading slash (e.g., "C:", "D:")
+
+    Examples:
+        >>> get_drive_from_path("/C:/Users/john/file.txt")
+        '/C:'
+        >>> get_drive_from_path("C:/Users/john/file.txt")
+        'C:'
+        >>> get_drive_from_path("/D:/Data/files")
+        '/D:'
+        >>> get_drive_from_path("invalid/path")
+        None
+
+    Supported drive letters: A-Z (case-insensitive)
+    """
+    parts = path.split("/")
+
+    # Handle paths without leading slash (e.g., "C:/Users/...")
+    if len(parts) >= 1 and parts[0]:
+        drive_part = parts[0]
+
+        # Validate drive part
+        if not drive_part:
+            return None
+
+        # Must be exactly 2 characters: letter + colon
+        if len(drive_part) != 2:
+            return None
+
+        # Second character must be ':'
+        if drive_part[1] != ":":
+            return None
+
+        # First character must be a letter (A-Z or a-z)
+        if not drive_part[0].isalpha():
+            return None
+
+        # Return drive without trailing slash for paths without leading slash
+        return drive_part
+
+    # For paths like "/C:/Users/...", parts[0] is empty and parts[1] contains the drive
+    if len(parts) >= 2:
+        drive_part = parts[1]
+
+        # Validate drive part
+        if not drive_part:
+            return None
+
+        # Must be exactly 2 characters: letter + colon
+        if len(drive_part) != 2:
+            return None
+
+        # Second character must be ':'
+        if drive_part[1] != ":":
+            return None
+
+        # First character must be a letter (A-Z or a-z)
+        if not drive_part[0].isalpha():
+            return None
+
+        # Return "/" + drive letter with colon
+        # e.g., "C:" -> "/C:"
+        return f"/{drive_part}"
+
+    return None
+
+
+
 def extract_all_strings(filename: str, min_len: int = 5):
     """
     Returns a combined list of all single-byte ASCII strings
@@ -274,6 +367,22 @@ def extract_all_strings(filename: str, min_len: int = 5):
     return all_strings
 
 
+def create_text_reader(binary_file: BinaryIO) -> io.TextIOWrapper:
+    """Creates a text reader that handles BOMs and mixed content"""
+
+    bom_check = binary_file.read(4)
+    binary_file.seek(0)  # Reset to start
+
+    if bom_check.startswith(b"\xff\xfe"):
+        return io.TextIOWrapper(binary_file, encoding="utf-16le")
+    elif bom_check.startswith(b"\xfe\xff"):
+        return io.TextIOWrapper(binary_file, encoding="utf-16be")
+    elif bom_check.startswith(b"\xef\xbb\xbf"):
+        return io.TextIOWrapper(binary_file, encoding="utf-8-sig")
+    else:
+        return io.TextIOWrapper(binary_file, encoding="utf-8", errors="replace")
+
+
 def escape_markdown(text):
     """
     Escapes markdown control characters in text by adding backslashes.

libs/common/common/logger.py

@@ -0,0 +1,67 @@
+import logging
+import os
+import sys
+
+import structlog
+from structlog.stdlib import ProcessorFormatter
+
+LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO").upper()
+NUMERIC_LEVEL = getattr(logging, LOG_LEVEL, logging.INFO)
+
+WORKFLOW_RUNTIME_LOG_LEVEL = os.getenv("WORKFLOW_RUNTIME_LOG_LEVEL", "WARNING")
+WORKFLOW_CLIENT_LOG_LEVEL = os.getenv("WORKFLOW_CLIENT_LOG_LEVEL", "WARNING")
+
+
+def add_callsite_from_record(_logger: logging.Logger, _method_name: str, event_dict: dict) -> dict:
+    record = event_dict.get("_record")
+    if record is not None:
+        event_dict.setdefault("logger", record.name)
+        event_dict.setdefault("module", record.module)
+        event_dict.setdefault("func", record.funcName)
+        event_dict.setdefault("line", record.lineno)
+    return event_dict
+
+
+foreign_pre_chain = [
+    add_callsite_from_record,
+    structlog.stdlib.add_log_level,
+    structlog.processors.StackInfoRenderer(),
+    structlog.processors.format_exc_info,
+]
+
+formatter = ProcessorFormatter(
+    processor=structlog.dev.ConsoleRenderer(colors=True),
+    foreign_pre_chain=foreign_pre_chain,
+)
+
+handler = logging.StreamHandler(stream=sys.stdout)
+handler.setFormatter(formatter)
+
+root = logging.getLogger()
+root.handlers[:] = [handler]
+root.setLevel("WARNING")
+logging.captureWarnings(True)
+
+# structlog -> hand off to ProcessorFormatter (no direct rendering here)
+structlog.configure(
+    processors=[
+        structlog.stdlib.filter_by_level,
+        structlog.stdlib.add_logger_name,  # adds "logger" for your own logs
+        structlog.stdlib.add_log_level,
+        structlog.stdlib.PositionalArgumentsFormatter(),
+        structlog.processors.TimeStamper(fmt="iso"),
+        structlog.processors.StackInfoRenderer(),
+        structlog.processors.format_exc_info,
+        structlog.processors.UnicodeDecoder(),
+        ProcessorFormatter.wrap_for_formatter,  # defer final render
+    ],
+    context_class=dict,
+    logger_factory=structlog.stdlib.LoggerFactory(),
+    wrapper_class=structlog.stdlib.BoundLogger,
+    cache_logger_on_first_use=True,
+)
+
+
+def get_logger(name: str | None = None) -> structlog.stdlib.BoundLogger:
+    logging.getLogger(name).setLevel(LOG_LEVEL)
+    return structlog.get_logger(name)

libs/common/common/models.py

@@ -1,12 +1,16 @@
 # src/common/models.py
 from datetime import datetime
 from enum import Enum
-from typing import Any, Generic, TypeVar
+from typing import TYPE_CHECKING, Any
 
-import structlog
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, ConfigDict, Field, field_serializer, field_validator
 
-logger = structlog.get_logger(module=__name__)
+from .logger import get_logger
+
+if TYPE_CHECKING:
+    from .models2.api import FileMetadata
+
+logger = get_logger(__name__)
 
 
 ##########################################
@@ -28,6 +32,7 @@ class FindingCategory(str, Enum):
     YARA_MATCH = "yara_match"
     PII = "pii"
     MISC = "misc"
+    INFORMATIONAL = "informational"
 
 
 class FindingOrigin(str, Enum):
@@ -59,6 +64,59 @@ class Alert(BaseModel):
     service: str | None = None  # service that sent the message (optional)
 
 
+##########################################
+#
+# .NET
+#
+##########################################
+
+
+class DotNetInput(BaseModel):
+    object_id: str
+
+
+class DotNetMethodInfo(BaseModel):
+    MethodName: str
+    FilterLevel: str | None = None
+
+
+class DotNetAssemblyAnalysis(BaseModel):
+    AssemblyName: str
+    Error: str | None = None
+    RemotingChannels: list[str] = []
+    IsWCFServer: bool = False
+    IsWCFClient: bool = False
+    SerializationGadgetCalls: dict[str, list[DotNetMethodInfo]] = {}
+    WcfServerCalls: dict[str, list[DotNetMethodInfo]] = {}
+    ClientCalls: dict[str, list[DotNetMethodInfo]] = {}
+    RemotingCalls: dict[str, list[DotNetMethodInfo]] = {}
+    ExecutionCalls: dict[str, list[DotNetMethodInfo]] = {}
+
+
+class DotNetOutput(BaseModel):
+    model_config = ConfigDict(populate_by_name=True)
+
+    object_id: str = Field(alias="objectId")
+    decompilation: str | None = None
+    analysis: DotNetAssemblyAnalysis | None = None
+
+    @field_validator("analysis", mode="before")
+    @classmethod
+    def parse_analysis_json(cls, v):
+        """Parse analysis from JSON string if needed"""
+        if v is None:
+            return None
+        if isinstance(v, str):
+            import json
+
+            try:
+                return json.loads(v)
+            except Exception as e:
+                logger.warning(f"Failed to parse DotNet analysis JSON: {e}")
+                return None
+        return v
+
+
 ##########################################
 #
 # Special case for NoseyParker
@@ -70,6 +128,14 @@ class NoseyParkerInput(BaseModel):
     object_id: str
 
 
+class GitCommitInfo(BaseModel):
+    commit_id: str
+    author: str
+    author_email: str
+    commit_date: str
+    message: str
+
+
 class MatchLocation(BaseModel):
     line: int
     column: int
@@ -81,6 +147,8 @@ class MatchInfo(BaseModel):
     matched_content: str
     location: MatchLocation
     snippet: str
+    file_path: str | None = None
+    git_commit: GitCommitInfo | None = None
 
 
 class ScanStats(BaseModel):
@@ -90,31 +158,21 @@ class ScanStats(BaseModel):
     bytes_scanned: int
     matches_found: int
 
-    # Allow aliases for field names
-    class Config:
-        populate_by_name = True
-        extra = "ignore"  # Ignore extra fields
-
 
 class ScanResults(BaseModel):
     scan_duration_ms: int
     bytes_scanned: int
-    matches: list[MatchInfo] = []  # Default to empty list
+    matches: list[MatchInfo]
     stats: ScanStats
-
-    class Config:
-        populate_by_name = True
-        extra = "ignore"  # Ignore extra fields
+    scan_type: str = "regular"  # "regular", "zip", "git_repo"
 
 
 class NoseyParkerOutput(BaseModel):
+    model_config = ConfigDict(populate_by_name=True, extra="ignore")
+
     object_id: str
     scan_result: ScanResults
 
-    class Config:
-        populate_by_name = True
-        extra = "ignore"  # Ignore extra fields
-
     # Add a factory method to handle flexible parsing
     @classmethod
     def from_dict(cls, data: dict):
@@ -137,32 +195,80 @@ class NoseyParkerOutput(BaseModel):
                         bytes_scanned=data.get("scan_result", {}).get("stats", {}).get("bytes_scanned", 0),
                         matches_found=data.get("scan_result", {}).get("stats", {}).get("matches_found", 0),
                     ),
+                    scan_type=data.get("scan_result", {}).get("scan_type", "regular"),
                 ),
             )
 
 
+##########################################
+#
+# Bulk Enrichment
+#
+##########################################
+
+
+class BulkEnrichmentEvent(BaseModel):
+    enrichment_name: str
+    object_id: str
+
+
+class SingleEnrichmentWorkflowInput(BaseModel):
+    """Input model for single enrichment workflows (bulk operations)."""
+
+    enrichment_name: str
+    object_id: str
+
+
 ##########################################
 #
 # Model for files submitted to the API
 #
 ##########################################
 class File(BaseModel):
+    model_config = ConfigDict(
+        exclude_none=True,
+        exclude_unset=True,
+    )
+
     object_id: str
     agent_id: str
+    source: str | None = None
     project: str
     timestamp: datetime
     expiration: datetime
-    path: str | None = None
+    path: str  # | None = None
     originating_object_id: str | None = None
+    originating_container_id: str | None = None
     nesting_level: int | None = None
     creation_time: str | None = None
     access_time: str | None = None
     modification_time: str | None = None
 
-    class Config:
-        exclude_none = True
-        exclude_unset = True
-        json_encoders = {datetime: lambda dt: dt.isoformat()}
+    @field_serializer("timestamp", "expiration")
+    def serialize_datetime(self, dt: datetime, _info):
+        return dt.isoformat()
+
+    @classmethod
+    def from_file_metadata(cls, metadata: "FileMetadata", object_id: str) -> "File":
+        """
+        Create a File instance from FileMetadata and object_id.
+
+        Args:
+            metadata: FileMetadata object containing upload metadata
+            object_id: The object ID of the uploaded file
+
+        Returns:
+            File instance ready for submission
+        """
+        return cls(
+            object_id=object_id,
+            agent_id=metadata.agent_id,
+            source=metadata.source,
+            project=metadata.project,
+            timestamp=metadata.timestamp,
+            expiration=metadata.expiration,
+            path=metadata.path,
+        )
 
 
 ##########################################
@@ -199,9 +305,6 @@ class FileEnriched(File):
     is_plaintext: bool
     is_container: bool
 
-    class Config:
-        json_encoders = {datetime: lambda dt: dt.isoformat()}
-
 
 ##########################################
 #
@@ -218,10 +321,7 @@ class WorkflowStatus(BaseModel):
     result: dict | None = None
 
 
-T = TypeVar("T")
-
-
-class CloudEvent(BaseModel, Generic[T]):
+class CloudEvent[T](BaseModel):
     """Cloud event schema used in Dapr pub/sub"""
 
     data: T

libs/common/common/models2/api.py

@@ -1,9 +1,10 @@
 import logging
+import re
 from datetime import UTC, datetime
-from typing import Annotated, Union
+from typing import Annotated, Literal
 from uuid import UUID
 
-from pydantic import BaseModel, BeforeValidator, Field
+from pydantic import BaseModel, BeforeValidator, Field, field_serializer, field_validator
 
 logger = logging.getLogger(__name__)
 
@@ -12,14 +13,6 @@ class ErrorResponse(BaseModel):
     detail: str = Field(..., description="Error message details")
 
 
-class ValidationError(BaseModel):
-    """Model representing a validation error"""
-
-    loc: list[Union[str, int]]
-    msg: str
-    type: str
-
-
 class FileWithMetadataResponse(BaseModel):
     """Response for combined file and metadata uploads"""
 
@@ -67,28 +60,107 @@ def ensure_utc_datetime(value) -> datetime:
 UTCDatetime = Annotated[datetime, BeforeValidator(ensure_utc_datetime)]
 
 
+class FileFilters(BaseModel):
+    """File filtering configuration for container extraction"""
+
+    include: list[str] | None = Field(
+        default=None, description="Patterns for files to include. If empty/None, all files are included by default."
+    )
+    exclude: list[str] | None = Field(
+        default=None, description="Patterns for files to exclude. Takes precedence over include patterns."
+    )
+    pattern_type: Literal["glob", "regex"] = Field(
+        default="glob",
+        description="Type of patterns to use: 'glob' for shell-style wildcards, 'regex' for regular expressions",
+    )
+
+    @field_validator("include", "exclude")
+    @classmethod
+    def validate_patterns(cls, v, info):
+        """Validate that patterns can be compiled if regex type"""
+        if v is None:
+            return v
+
+        # Access other field values through info.data
+        pattern_type = info.data.get("pattern_type", "glob")
+
+        if pattern_type == "regex":
+            # Validate that all regex patterns compile
+            for pattern in v:
+                try:
+                    re.compile(pattern, re.IGNORECASE)
+                except re.error as e:
+                    raise ValueError(f"Invalid regex pattern '{pattern}': {e}") from e
+
+        return v
+
+
 class FileMetadata(BaseModel):
     """Metadata model for file uploads"""
 
     agent_id: str
+    source: str | None = None
     project: str
-    timestamp: datetime = Field(description="ISO 8601 formatted timestamp")
-    expiration: datetime = Field(description="ISO 8601 formatted expiration date")
+    timestamp: datetime | None = Field(
+        default=None, description="ISO 8601 formatted timestamp of when the data was collected"
+    )
+    expiration: datetime | None = Field(
+        default=None, description="ISO 8601 formatted expiration date (when the data should be deleted)"
+    )
     path: str
+    file_filters: FileFilters | None = Field(
+        default=None, description="Optional file filtering configuration for container extraction"
+    )
 
     model_config = {
         "json_schema_extra": {
             "example": {
                 "agent_id": "beacon123",
+                "source": "host://192.168.1.1",
                 "project": "assess-test",
                 "timestamp": "2025-01-06T23:48:46.925656Z",
                 "expiration": "2026-01-06T23:48:46.925656Z",
                 "path": "/path/to/file",
+                "file_filters": {
+                    "include": ["*.exe", "*/Users/**/*"],
+                    "exclude": ["*/Windows/**/*", "*.tmp"],
+                    "pattern_type": "glob",
+                },
             }
         }
     }
 
-    model_config = {"json_encoders": {datetime: lambda dt: dt.isoformat()}}
+    @field_serializer("timestamp", "expiration", when_used="unless-none")
+    def serialize_datetime(self, dt: datetime, _info):
+        return dt.isoformat()
+
+
+class ContainerFromMountRequest(BaseModel):
+    """Request model for processing container files from mounted folder"""
+
+    filename: str = Field(description="Name of the container file in the mounted folder")
+    metadata: FileMetadata = Field(description="File metadata for processing")
+
+    model_config = {
+        "json_schema_extra": {
+            "example": {
+                "filename": "large_archive.zip",
+                "metadata": {
+                    "agent_id": "beacon123",
+                    "source": "host://192.168.1.1",
+                    "project": "assess-test",
+                    "timestamp": "2025-01-06T23:48:46.925656Z",
+                    "expiration": "2026-01-06T23:48:46.925656Z",
+                    "path": "/mounted/large_archive.zip",
+                    "file_filters": {
+                        "include": ["*.exe", "*/Users/**/*"],
+                        "exclude": ["*/Windows/**/*", "*.tmp"],
+                        "pattern_type": "glob",
+                    },
+                },
+            }
+        }
+    }
 
 
 ####################

libs/common/common/models2/dpapi.py

@@ -0,0 +1,171 @@
+"""DPAPI credential models for API requests."""
+
+import re
+from typing import Annotated, Literal, Union
+from uuid import UUID
+
+from common.logger import get_logger
+from nemesis_dpapi.types import Sid
+from pydantic import BaseModel, Discriminator, Field, Tag, field_serializer, field_validator
+
+logger = get_logger(__name__)
+
+
+class PasswordCredentialKey(BaseModel):
+    """Use a plaintext password to derive a DPAPI credential key."""
+
+    model_config = {"frozen": True, "extra": "forbid"}
+
+    type: Literal["password"]
+    value: str  # Plain text password
+    user_sid: Sid  # Required for to derive MK encryption keys
+
+
+class NtlmHashCredentialKey(BaseModel):
+    """Use an NTLM hash to derive a DPAPI credential key."""
+
+    model_config = {"frozen": True, "extra": "forbid"}
+
+    type: Literal["cred_key_ntlm"]
+    value: str  # Hex string representation of NTLM hash (16 bytes)
+    user_sid: Sid  # Required for to derive MK encryption keys
+
+    @field_validator("value")
+    @classmethod
+    def validate_ntlm_hash_length(cls, v):
+        """Validate that value is exactly 32 hex characters (16 bytes)."""
+        if len(v) != 32:
+            raise ValueError(f"NTLM hash value must be exactly 32 hex characters (16 bytes), got {len(v)} characters")
+        if not re.match(r"^[0-9a-fA-F]+$", v):
+            raise ValueError("NTLM hash value must contain only hex characters (0-9, a-f, A-F)")
+        return v
+
+
+class Sha1CredentialKey(BaseModel):
+    """Use a SHA1 hash to derive a DPAPI credential key."""
+
+    model_config = {"frozen": True, "extra": "forbid"}
+
+    type: Literal["cred_key_sha1"]
+    value: str  # Hex string representation of credential key (20 bytes)
+    user_sid: Sid  # Required for to derive MK encryption keys
+
+    @field_validator("value")
+    @classmethod
+    def validate_cred_key_length(cls, v):
+        """Validate that value is either 40 hex characters (20 bytes)."""
+        if len(v) != 40:
+            raise ValueError(f"SHA1 credential key value must be 40 hex characters (20 bytes), got {len(v)} characters")
+        if not re.match(r"^[0-9a-fA-F]+$", v):
+            raise ValueError("SHA1 Credential key value must contain only hex characters (0-9, a-f, A-F)")
+        return v
+
+
+class Pbkdf2StrongCredentialKey(BaseModel):
+    """Credential key object."""
+
+    model_config = {"frozen": True, "extra": "forbid"}
+
+    type: Literal["cred_key_pbkdf2"]
+    value: str  # Hex string representation of credential key (16 bytes)
+    user_sid: Sid  # Required to derive MK encryption keys
+
+    @field_validator("value")
+    @classmethod
+    def validate_cred_key_length(cls, v):
+        """Validate that value is either 32 hex characters (16 bytes)."""
+        if len(v) != 32:
+            raise ValueError(
+                f"Secure credential key (PBKDF2) value must be exactly 32 hex characters (16 bytes), got {len(v)} characters"
+            )
+        if not re.match(r"^[0-9a-fA-F]+$", v):
+            raise ValueError("Secure credential key (PBKDF2) value must contain only hex characters (0-9, a-f, A-F)")
+        return v
+
+
+class DomainBackupKeyCredential(BaseModel):
+    """Domain backup key credential (PVK format)."""
+
+    model_config = {"frozen": True, "extra": "forbid"}
+
+    type: Literal["domain_backup_key"]
+    value: str  # Base64 encoded PVK data
+    guid: str  # Domain backup key GUID (UUID format)
+    domain_controller: str | None = None  # Optional domain controller
+
+    @field_validator("guid")
+    @classmethod
+    def validate_guid_format(cls, v):
+        """Validate that guid is a valid UUID format."""
+        try:
+            UUID(v)  # This will raise ValueError if not valid UUID format
+            return v
+        except ValueError as e:
+            raise ValueError(f"guid must be a valid UUID format, got: {v}") from e
+
+
+class MasterKeyGuidPair(BaseModel):
+    """Strongly typed master key data."""
+
+    model_config = {"frozen": True, "extra": "forbid"}
+
+    guid: UUID = Field(description="Master key GUID")
+    key_hex: str = Field(description="Hex-encoded master key bytes", pattern=r"^[0-9a-fA-F]+$")
+
+    @field_serializer("guid")
+    def serialize_guid(self, value: UUID) -> str:
+        return str(value)
+
+
+class MasterKeyGuidPairList(BaseModel):
+    """Decrypted master key/GUID pairs."""
+
+    model_config = {"frozen": True, "extra": "forbid"}
+
+    type: Literal["master_key_guid_pair"]
+    value: list[MasterKeyGuidPair]
+
+
+class DpapiSystemCredentialRequest(BaseModel):
+    """DPAPI_SYSTEM LSA Secret credential sent in an API request."""
+
+    model_config = {"frozen": True, "extra": "forbid"}
+
+    type: Literal["dpapi_system"]
+    value: str = Field(
+        description="Hex-encoded DPAPI_SYSTEM LSA secret (40 bytes)",
+        pattern=r"^[0-9a-fA-F]{80}$",
+    )
+
+    @field_validator("value")
+    @classmethod
+    def validate_hex_length(cls, v):
+        """Validate that value is exactly 80 hex characters (40 bytes)."""
+        if len(v) != 80:
+            raise ValueError(
+                f"DPAPI_SYSTEM value must be exactly 80 hex characters (40 bytes), got {len(v)} characters"
+            )
+        if not re.match(r"^[0-9a-fA-F]+$", v):
+            raise ValueError("DPAPI_SYSTEM value must contain only hex characters (0-9, a-f, A-F)")
+        return v
+
+
+def get_credential_type(v):
+    """Discriminator function to determine credential type from 'type' field."""
+    if isinstance(v, dict):
+        return v.get("type")
+    return getattr(v, "type", None)
+
+
+type DpapiCredentialRequest = Annotated[
+    Union[
+        Annotated[PasswordCredentialKey, Tag("password")],
+        Annotated[NtlmHashCredentialKey, Tag("cred_key_ntlm")],
+        Annotated[Sha1CredentialKey, Tag("cred_key_sha1")],
+        Annotated[Pbkdf2StrongCredentialKey, Tag("cred_key_pbkdf2")],
+        Annotated[DomainBackupKeyCredential, Tag("domain_backup_key")],
+        Annotated[MasterKeyGuidPairList, Tag("master_key_guid_pair")],
+        Annotated[DpapiSystemCredentialRequest, Tag("dpapi_system")],
+    ],
+    Field(discriminator=Discriminator(get_credential_type)),
+]

libs/common/common/models2/enrichments.py

@@ -0,0 +1,18 @@
+"""API models for the /enrichments route."""
+
+from pydantic import BaseModel
+
+
+class EnrichmentRequest(BaseModel):
+    object_id: str
+
+
+class EnrichmentResponse(BaseModel):
+    status: str
+    message: str
+    object_id: str
+    instance_id: str
+
+
+class ModulesListResponse(BaseModel):
+    modules: list[str]

libs/common/common/state_helpers.py

@@ -1,72 +1,113 @@
 import json
 
+import asyncpg
 import psycopg
-import structlog
-from dapr.clients import DaprClient
-
+from common.db import get_postgres_connection_str
 from common.models import FileEnriched
 
-logger = structlog.get_logger(module=__name__)
+from .logger import get_logger
+
+logger = get_logger(__name__)
+
+# Single source of truth for file_enriched query (using psycopg style with %s placeholders)
+_FILE_ENRICHED_SELECT_QUERY = """
+    SELECT
+        object_id, agent_id, source, project, timestamp, expiration,
+        path, file_name, extension, size, magic_type, mime_type,
+        is_plaintext, is_container, originating_object_id,
+        nesting_level, file_creation_time, file_access_time,
+        file_modification_time, security_info, hashes
+    FROM files_enriched
+"""
+
+_FILE_ENRICHED_SELECT_QUERY_PSYCHOPG = f"""
+{_FILE_ENRICHED_SELECT_QUERY}
+WHERE object_id = %s
+"""
+
+_FILE_ENRICHED_SELECT_QUERY_ASYNCPG = f"""
+{_FILE_ENRICHED_SELECT_QUERY}
+WHERE object_id = $1
+"""
 
 
-with DaprClient() as client:
-    secret = client.get_secret(store_name="nemesis-secret-store", key="POSTGRES_CONNECTION_STRING")
-    postgres_connection_string = secret.secret["POSTGRES_CONNECTION_STRING"]
+def _transform_file_enriched_data(file_data: dict) -> dict:
+    """
+    Transform raw database data into format suitable for FileEnriched model.
+
+    Handles:
+    - UUID to string conversion
+    - Datetime to ISO format conversion
+    - JSON field parsing
+    - None value removal
+
+    Args:
+        file_data: Dictionary of raw database values
+
+    Returns:
+        Transformed dictionary ready for FileEnriched.model_validate()
+    """
+    # Convert UUID to string
+    if "object_id" in file_data and file_data["object_id"]:
+        file_data["object_id"] = str(file_data["object_id"])
+    if "originating_object_id" in file_data and file_data["originating_object_id"]:
+        file_data["originating_object_id"] = str(file_data["originating_object_id"])
+
+    # Convert datetime objects to ISO format strings
+    datetime_fields = [
+        "timestamp",
+        "expiration",
+        "file_creation_time",
+        "file_access_time",
+        "file_modification_time",
+    ]
+    for field in datetime_fields:
+        if field in file_data and file_data[field]:
+            file_data[field] = file_data[field].isoformat()
+
+    # Handle JSON fields (parse if string, keep as-is if dict)
+    if "security_info" in file_data and isinstance(file_data["security_info"], str):
+        file_data["security_info"] = json.loads(file_data["security_info"])
+    if "hashes" in file_data and isinstance(file_data["hashes"], str):
+        file_data["hashes"] = json.loads(file_data["hashes"])
+
+    # Remove None values
+    file_data = {k: v for k, v in file_data.items() if v is not None}
+
+    return file_data
 
 
 def get_file_enriched(object_id: str) -> FileEnriched:
-    """Retrieve a file_enriched record from PostgreSQL and parse it into a FileEnriched object."""
+    """
+    Retrieve a file_enriched record from PostgreSQL (synchronous).
+
+    Args:
+        object_id: The object_id to query
+
+    Returns:
+        FileEnriched model instance
+
+    Raises:
+        ValueError: If no record found for object_id
+        Exception: For database or parsing errors
+    """
     try:
-        with psycopg.connect(postgres_connection_string) as conn:
+        with psycopg.connect(get_postgres_connection_str()) as conn:
             with conn.cursor() as cur:
-                # Query remains the same
-                cur.execute(
-                    """
-                    SELECT
-                        object_id, agent_id, project, timestamp, expiration, path,
-                        file_name, extension, size, magic_type, mime_type,
-                        is_plaintext, is_container, originating_object_id,
-                        nesting_level, file_creation_time, file_access_time,
-                        file_modification_time, security_info, hashes
-                    FROM files_enriched
-                    WHERE object_id = %s
-                """,
-                    (object_id,),
-                )
+                cur.execute(_FILE_ENRICHED_SELECT_QUERY_PSYCHOPG, (object_id,))
 
                 result = cur.fetchone()
                 if not result:
                     raise ValueError(f"No file_enriched record found for object_id {object_id}")
 
+                if not cur.description:
+                    raise RuntimeError("Query returned no column descriptions")
+
                 columns = [desc[0] for desc in cur.description]
                 file_data = dict(zip(columns, result))
 
-                # Convert UUID to string
-                if "object_id" in file_data and file_data["object_id"]:
-                    file_data["object_id"] = str(file_data["object_id"])
-                if "originating_object_id" in file_data and file_data["originating_object_id"]:
-                    file_data["originating_object_id"] = str(file_data["originating_object_id"])
-
-                # Convert datetime objects to ISO format strings
-                datetime_fields = [
-                    "timestamp",
-                    "expiration",
-                    "file_creation_time",
-                    "file_access_time",
-                    "file_modification_time",
-                ]
-                for field in datetime_fields:
-                    if field in file_data and file_data[field]:
-                        file_data[field] = file_data[field].isoformat()
-
-                # Handle JSON fields
-                if "security_info" in file_data and isinstance(file_data["security_info"], str):
-                    file_data["security_info"] = json.loads(file_data["security_info"])
-                if "hashes" in file_data and isinstance(file_data["hashes"], str):
-                    file_data["hashes"] = json.loads(file_data["hashes"])
-
-                # Remove None values
-                file_data = {k: v for k, v in file_data.items() if v is not None}
+                # Transform data using shared helper
+                file_data = _transform_file_enriched_data(file_data)
 
                 return FileEnriched.model_validate(file_data)
 
@@ -76,3 +117,51 @@ def get_file_enriched(object_id: str) -> FileEnriched:
     except Exception as e:
         logger.exception(e, message="Error retrieving file_enriched from PostgreSQL")
         raise
+
+
+async def get_file_enriched_async(object_id: str, connection: str | asyncpg.Pool | None = None) -> FileEnriched:
+    """
+    Retrieve a file_enriched record from PostgreSQL (asynchronous using asyncpg).
+
+    Args:
+        object_id: The object_id to query
+        connection: Optional connection string or asyncpg.Pool. If not provided, uses get_postgres_connection_str()
+
+    Returns:
+        FileEnriched model instance
+
+    Raises:
+        ValueError: If no record found for object_id
+        Exception: For database or parsing errors
+    """
+    try:
+        # Determine if we're using a pool or creating a new connection
+        if isinstance(connection, asyncpg.Pool):
+            # Use the provided pool
+            row = await connection.fetchrow(_FILE_ENRICHED_SELECT_QUERY_ASYNCPG, object_id)
+        else:
+            # Use connection string (provided or default)
+            connection_string = connection if connection is not None else get_postgres_connection_str()
+            conn = await asyncpg.connect(connection_string)
+            try:
+                row = await conn.fetchrow(_FILE_ENRICHED_SELECT_QUERY_ASYNCPG, object_id)
+            finally:
+                await conn.close()
+
+        if not row:
+            raise ValueError(f"No file_enriched record found for object_id {object_id}")
+
+        # Convert asyncpg.Record to dict
+        file_data = dict(row)
+
+        # Transform data using shared helper
+        file_data = _transform_file_enriched_data(file_data)
+
+        return FileEnriched.model_validate(file_data)
+
+    except ValueError as e:
+        logger.error(f"File not found: {str(e)}")
+        raise
+    except Exception as e:
+        logger.exception(e, message="Error retrieving file_enriched from PostgreSQL (async)")
+        raise

libs/common/common/storage.py

@@ -2,14 +2,15 @@ import tempfile
 import uuid
 from io import BytesIO
 
-import structlog
 from dapr.clients import DaprClient
 from fastapi import UploadFile
 from minio import Minio
 from minio.error import S3Error
 from urllib3 import PoolManager, Retry
 
-logger = structlog.get_logger(module=__name__)
+from .logger import get_logger
+
+logger = get_logger(__name__)
 
 
 class StorageMinio:
@@ -58,11 +59,12 @@ class StorageMinio:
                 logger.exception(e, message="Failed to download file")
                 raise
             finally:
-                logger.info("Downloaded file", file_uuid=file_uuid)
+                logger.debug("Downloaded file", file_uuid=file_uuid)
 
             return temp_file
         except Exception as e:
             logger.exception(e, file_uuid=file_uuid, bucket_name=self.bucket_name)
+            raise
 
     def download_bytes(self, file_uuid: str, offset: int = 0, length: int = 0) -> bytes:
         try:
@@ -75,7 +77,7 @@ class StorageMinio:
                 file_data = response.read()
                 response.close()
 
-                logger.info("Successfully downloaded file", file_uuid=file_uuid)
+                logger.debug("Successfully downloaded file", file_uuid=file_uuid)
                 return file_data
 
             except BaseException as e:
@@ -115,7 +117,7 @@ class StorageMinio:
                     yield chunk
 
                 response.close()
-                logger.info("Successfully streamed file", file_uuid=file_uuid)
+                logger.debug("Successfully streamed file", file_uuid=file_uuid)
 
             except BaseException as e:
                 logger.exception(e, message="Failed to stream file")
@@ -131,10 +133,7 @@ class StorageMinio:
             return self.minio_client.stat_object(self.bucket_name, object_name)
         except Exception as e:
             logger.exception(e, "Error pulling object stats", object_name=object_name)
-
-    def check_bucket_exists(self):
-        """Returns True if the bucket exists, false if it doesn't."""
-        return self.minio_client.bucket_exists(self.bucket_name)
+            raise
 
     def check_file_exists(self, object_name):
         """Check if a file exists."""
@@ -149,41 +148,10 @@ class StorageMinio:
             # For other errors, raise the exception
             raise
 
-    def ensure_bucket_exists(self):
-        """Checks if the bucket exists and creates it w/ the LifecycleConfig if not."""
-        try:
-            if not self.check_bucket_exists():
-                logger.info("Creating Minio bucket", bucket=self.bucket_name)
-                self.minio_client.make_bucket(f"{self.bucket_name}")
-
-                # # since this is the only place that creates the bucket, we can set
-                # #   the auto-expiration policy here
-                # # # NOTE: this is now handled by the Housekeeping service
-
-                # config = LifecycleConfig(
-                #     [
-                #         Rule(
-                #             ENABLED,
-                #             rule_filter=Filter(prefix=""),
-                #             rule_id=f"expire-{self.storage_expiration_days}-days",
-                #             expiration=Expiration(days=self.storage_expiration_days),
-                #         ),
-                #     ],
-                # )
-                # logger.info(
-                #     f"Setting Minio bucket files to expire in {self.storage_expiration_days} days",
-                #     bucket=self.bucket_name,
-                # )
-                # self.minio_client.set_bucket_lifecycle(self.bucket_name, config)
-        except Exception as e:
-            logger.exception(e, bucket_name=self.bucket_name)
-
     def upload_uploadfile(self, file: UploadFile) -> uuid.UUID:
         # uploads an UploadFile post directly from FastAPI
         try:
-            self.ensure_bucket_exists()
-
-            logger.info(f"Uploading UploadFile {file.filename} to storage")
+            logger.debug(f"Uploading UploadFile {file.filename} to storage")
 
             # Get file size using SpooledTemporaryFile instead of async read
             file_size = 0
@@ -204,16 +172,16 @@ class StorageMinio:
                 file.file,  # Use file.file directly
                 length=file_size,  # Provide the calculated size
             )
-            logger.debug("Object upload compelted", file_name=file.filename)
+            logger.debug("Object upload completed", file_name=file.filename)
 
             return file_uuid
 
         except Exception as e:
             logger.exception(e, bucket_name=self.bucket_name)
+            raise
 
     def upload_file(self, file_path: str) -> uuid.UUID:
         try:
-            self.ensure_bucket_exists()
             logger.debug("Uploading file to storage", file_path=file_path)
             file_uuid = f"{uuid.uuid4()}"
             self.minio_client.fput_object(
@@ -224,10 +192,10 @@ class StorageMinio:
             return file_uuid
         except Exception as e:
             logger.exception(e, file_path=file_path, bucket_name=self.bucket_name)
+            raise
 
     def upload(self, data: bytes) -> uuid.UUID:
         try:
-            self.ensure_bucket_exists()
             logger.debug(f"Uploading {len(data)} bytes to storage")
             file_uuid = f"{uuid.uuid4()}"
             self.minio_client.put_object(
@@ -239,6 +207,7 @@ class StorageMinio:
             return file_uuid
         except Exception as e:
             logger.exception(e, bucket_name=self.bucket_name)
+            raise
 
     def delete_object(self, object_id: str) -> bool:
         """Delete a single object from Minio storage.
@@ -281,7 +250,7 @@ class StorageMinio:
         upload creates everything correctly.
         """
 
-        logger.info("Deleting all files from bucket", bucket_name=self.bucket_name)
+        logger.debug("Deleting all files from bucket", bucket_name=self.bucket_name)
 
         try:
             files = self.minio_client.list_objects(self.bucket_name, recursive=True)

libs/common/common/workflows/setup.py

@@ -0,0 +1,63 @@
+import asyncio
+from collections.abc import Callable
+from functools import wraps
+from typing import Any
+
+import dapr.ext.workflow as wf
+from common.logger import WORKFLOW_RUNTIME_LOG_LEVEL
+from dapr.ext.workflow.logger.options import LoggerOptions
+
+wf_runtime: wf.WorkflowRuntime = wf.WorkflowRuntime(
+    logger_options=LoggerOptions(
+        log_level=WORKFLOW_RUNTIME_LOG_LEVEL,
+    )
+)
+
+
+def set_fastapi_loop(loop: asyncio.AbstractEventLoop) -> None:
+    global fastapi_loop
+    fastapi_loop = loop
+
+
+def workflow_activity(fn: Callable | None = None, *, name: str | None = None) -> Callable:
+    """
+    Decorator to mark an async function as a workflow activity.
+    The default @wf_runtime.activity decorator does not support async functions.
+
+    Can be used with or without parentheses:
+        @workflow_activity
+        async def my_activity(): ...
+
+        @workflow_activity()
+        async def my_activity(): ...
+
+        @workflow_activity(name="custom_name")
+        async def my_activity(): ...
+    """
+
+    def decorator(func: Callable) -> Callable:
+        # To facilitate unit-testing, avoid using the @wf_runtime.activity decorator
+        # and simply return the function as is.
+        # if settings.pytest_running:
+        #     return func
+
+        @wf_runtime.activity(name=name)
+        @wraps(func)
+        def wrapped_fn(*args, **kwargs) -> Any:  # type: ignore
+            result = func(*args, **kwargs)
+            if not asyncio.iscoroutine(result):
+                # If the result is not a coroutine, just return it as is.
+                return result
+
+            if fastapi_loop is None:
+                raise RuntimeError("FastAPI event loop is not set.")
+            return asyncio.run_coroutine_threadsafe(result, fastapi_loop).result()
+
+        return wrapped_fn
+
+    # If called without parentheses, fn will be the function
+    if fn is not None:
+        return decorator(fn)
+
+    # If called with parentheses (with or without arguments), return the decorator
+    return decorator

libs/common/pyproject.toml

@@ -9,84 +9,19 @@ readme = "README.md"
 python = ">=3.12,<4.0"
 pydantic = "^2.10.5"
 structlog = "^25.1.0"
-dapr = "^1.14.0"
+dapr = "1.16.0"
 minio = "^7.2.14"
 fastapi = "^0.115.6"
+dapr-ext-workflow = "^1.16.0"
+psycopg = {extras = ["pool"], version = "^3.2.9"}
+asyncpg = "^0.30.0"
 
 [tool.poetry.group.dev.dependencies]
 ruff = "^0.9.2"
+pytest = "^8.4.1"
+pytest-asyncio = "^1.1.0"
 
 [build-system]
 requires = ["poetry-core>=2.0.0,<3.0.0"]
 build-backend = "poetry.core.masonry.api"
 
-[tool.ruff]
-select = [
-    "E",   # pycodestyle errors
-    "W",   # pycodestyle warnings
-    "F",   # pyflakes
-    "I",   # isort
-    "C",   # flake8-comprehensions
-    "B",   # flake8-bugbear
-    "UP",  # pyupgrade
-    "NPY", # numpydoc
-    "A",   # flake8-annotations
-    "TCH001",   # Move application-only imports into TYPE_CHECKING block
-    "TCH002",   # Move third-party imports into TYPE_CHECKING block
-    "TCH003",   # Move standard library imports into TYPE_CHECKING block
-]
-ignore = [
-    "E501",  # line too long, handled by black
-    "B008",  # do not perform function calls in argument defaults
-    "C901",  # too complex
-    "W191",  # indentation contains tabs
-    "F722",  # syntax error in forward annotation
-    "UP007", # X | Y syntax while we're still supporting 3.9
-    "UP038", # isinstance() X | Y instance ^
-    "B905",  # zip() without strict (isn't supported in 3.9)
-]
-
-exclude = [
-    ".bzr",
-    ".direnv",
-    ".eggs",
-    ".git",
-    ".git-rewrite",
-    ".hg",
-    ".mypy_cache",
-    ".nox",
-    ".pants.d",
-    ".pytype",
-    ".ruff_cache",
-    ".svn",
-    ".tox",
-    ".venv",
-    "__pypackages__",
-    "_build",
-    "buck-out",
-    "build",
-    "dist",
-    "node_modules",
-    "venv",
-]
-
-line-length = 120
-indent-width = 4
-target-version = "py312"
-
-[tool.ruff.lint]
-fixable = ["ALL"]
-unfixable = ["B"]
-
-dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
-extend-select = [
-    "I", # isort, added per instructions in the VS code extension (https://marketplace.visualstudio.com/items?itemName=charliermarsh.ruff)
-]
-
-[tool.ruff.format]
-quote-style = "double"
-indent-style = "space"
-skip-magic-trailing-comma = false
-line-ending = "auto"
-
-

libs/common/tests/test_example.py

@@ -0,0 +1,3 @@
+def test_example():
+    """Simple example test to verify pytest is working."""
+    assert True

libs/common/tests/test_helpers.py

@@ -0,0 +1,118 @@
+"""Tests for chromium.local_state module."""
+
+from common.helpers import get_drive_from_path
+
+
+class TestGetDriveFromPath:
+    """Test suite for get_drive_from_path function."""
+
+    def test_valid_path_with_colon(self):
+        """Test valid path with colon in drive letter."""
+        assert get_drive_from_path("/C:/Users/test") == "/C:"
+        assert get_drive_from_path("/D:/Program Files") == "/D:"
+        assert get_drive_from_path("/E:/temp") == "/E:"
+
+    def test_invalid_path_without_colon(self):
+        """Test invalid path without colon in drive letter - should fail."""
+        assert get_drive_from_path("/C/Users/test") is None
+        assert get_drive_from_path("/D/Program Files") is None
+        assert get_drive_from_path("/E/temp") is None
+
+    def test_lowercase_drive_letters(self):
+        """Test lowercase drive letters are preserved."""
+        assert get_drive_from_path("/c:/Users/test") == "/c:"
+        assert get_drive_from_path("/d:/temp") == "/d:"
+
+    def test_mixed_case_drive_letters(self):
+        """Test mixed case scenarios."""
+        assert get_drive_from_path("/C:/Users/test") == "/C:"
+        assert get_drive_from_path("/c:/Users/test") == "/c:"
+
+    def test_all_valid_drive_letters(self):
+        """Test all valid drive letters A-Z with colon."""
+        for letter in "ABCDEFGHIJKLMNOPQRSTUVWXYZ":
+            assert get_drive_from_path(f"/{letter}:/Users") == f"/{letter}:"
+            # Without colon should fail
+            assert get_drive_from_path(f"/{letter}/Users") is None
+
+    def test_invalid_numeric_drive(self):
+        """Test invalid numeric drive letters."""
+        assert get_drive_from_path("/1:/Users/test") is None
+        assert get_drive_from_path("/123/Users/test") is None
+        assert get_drive_from_path("/0:/temp") is None
+
+    def test_invalid_special_characters(self):
+        """Test invalid special characters as drive."""
+        assert get_drive_from_path("/$:/Users/test") is None
+        assert get_drive_from_path("/@:/Users/test") is None
+        assert get_drive_from_path("/#/Users/test") is None
+        assert get_drive_from_path("/*/Users/test") is None
+
+    def test_invalid_multiple_letters(self):
+        """Test invalid multiple letters without colon."""
+        assert get_drive_from_path("/AB/Users/test") is None
+        assert get_drive_from_path("/CD:/Users/test") is None
+        assert get_drive_from_path("/ABC/Users/test") is None
+
+    def test_invalid_colon_usage(self):
+        """Test invalid colon usage."""
+        assert get_drive_from_path("/C;/Users/test") is None  # semicolon instead
+        assert get_drive_from_path("/C::/Users/test") is None  # double colon
+        assert get_drive_from_path("/:C/Users/test") is None  # colon before letter
+
+    def test_path_without_leading_slash(self):
+        """Test paths without leading slash."""
+        assert get_drive_from_path("C:/Users/test") == "C:"
+        assert get_drive_from_path("C/Users/test") is None
+
+    def test_path_with_only_drive(self):
+        """Test paths with only drive letter."""
+        assert get_drive_from_path("/C:") == "/C:"
+        assert get_drive_from_path("/C") is None  # Without colon should fail
+        assert get_drive_from_path("/D:") == "/D:"
+
+    def test_path_with_trailing_elements(self):
+        """Test various path structures."""
+        assert get_drive_from_path("/C:/Users/test/Documents/file.txt") == "/C:"
+        assert get_drive_from_path("/C/Users/test/Documents/file.txt") is None  # Without colon should fail
+        assert get_drive_from_path("/D:/Program Files/App/config.ini") == "/D:"
+
+    def test_edge_case_single_slash(self):
+        """Test edge case of single slash."""
+        assert get_drive_from_path("/") is None
+
+    def test_edge_case_double_slash(self):
+        """Test edge case of double slash."""
+        assert get_drive_from_path("//") is None
+        assert get_drive_from_path("//C/Users") is None
+
+    def test_path_with_spaces(self):
+        """Test paths with spaces in directories."""
+        assert get_drive_from_path("/C:/Program Files/Test") == "/C:"
+        assert get_drive_from_path("/C/Program Files/Test") is None  # Without colon should fail
+
+    def test_mixed_slashes_in_path(self):
+        """Test that function handles forward slashes correctly."""
+        # The function expects POSIX-style paths with forward slashes
+        assert get_drive_from_path("/C:/Users/test") == "/C:"
+        assert get_drive_from_path("/C/Users/test") is None  # Without colon should fail
+
+    def test_unicode_characters(self):
+        """Test paths with unicode characters in later parts."""
+        assert get_drive_from_path("/C:/Users/тест") == "/C:"
+        assert get_drive_from_path("/C/Users/测试") is None  # Without colon should fail
+
+    def test_empty_drive_after_slash(self):
+        """Test path with empty drive section."""
+        assert get_drive_from_path("//Users/test") is None
+
+    def test_real_world_chrome_paths(self):
+        """Test with real-world Chrome Local State paths."""
+        assert get_drive_from_path("/C:/Users/itadmin/AppData/Local/Google/Chrome/User Data/Local State") == "/C:"
+        assert get_drive_from_path("/C/Users/itadmin/AppData/Local/Google/Chrome/User Data/Local State") is None  # Without colon should fail
+        assert get_drive_from_path("/C/DPAPIUser/AppData/Local/Google/Chrome/User Data/Local State") is None  # Without colon should fail
+
+    def test_alternate_browser_paths(self):
+        """Test with other Chromium-based browser paths."""
+        assert get_drive_from_path("/D:/Users/test/AppData/Local/Microsoft/Edge/User Data/Local State") == "/D:"
+        assert get_drive_from_path("/E/Users/test/AppData/Local/BraveSoftware/Brave-Browser/User Data/Local State") is None  # Without colon should fail

libs/file_enrichment_modules/.vscode/settings.json

@@ -6,15 +6,13 @@
         "editor.formatOnSave": false
     },
     "[python]": {
-        "editor.formatOnSave": true,
         "editor.codeActionsOnSave": {
-            "source.fixAll": "explicit",
-            "source.organizeImports": "explicit",
+            "source.fixAll.ruff": "explicit",
+            "source.organizeImports.ruff": "explicit"
         },
-        "editor.defaultFormatter": "charliermarsh.ruff",
+        "editor.defaultFormatter": "charliermarsh.ruff"
     },
     "autoDocstring.docstringFormat": "google",
-    "editor.formatOnSave": true,
     "files.exclude": {
         "**/.DS_Store": true,
         "**/.git": true,
@@ -46,16 +44,6 @@
         "reportMissingModuleSource": "none",
     },
     "python.analysis.useLibraryCodeForTypes": true, // Pyright
-    // "python.formatting.provider": "none",
-    "python.languageServer": "Pylance",
-    "python.linting.enabled": true,
-    "python.linting.flake8Args": [
-        "--max-line-length=240",
-        "--ignore=E203,E722,W503",
-    ],
-    "python.linting.flake8Enabled": true,
-    "python.linting.lintOnSave": true,
-    "python.linting.pylintEnabled": false,
     "python.testing.pytestArgs": [
         "tests"
     ],

libs/file_enrichment_modules/file_enrichment_modules/base64_decoder/analyzer.py

@@ -5,16 +5,15 @@ import re
 import tempfile
 from typing import Any
 
-import structlog
 from common.helpers import is_plaintext
+from common.logger import get_logger
 from common.models import EnrichmentResult, File, Transform
 from common.state_helpers import get_file_enriched
 from common.storage import StorageMinio
 from dapr.clients import DaprClient
-
 from file_enrichment_modules.module_loader import EnrichmentModule
 
-logger = structlog.get_logger(module=__name__)
+logger = get_logger(__name__)
 
 
 class Base64DecoderAnalyzer(EnrichmentModule):
@@ -35,8 +34,13 @@ class Base64DecoderAnalyzer(EnrichmentModule):
         # Allow whitespace/newlines within long sequences
         self.long_base64_pattern = re.compile(r"([A-Za-z0-9+/\s]{200,}={0,2})")
 
-    def should_process(self, object_id: str) -> bool:
-        """Determine if this module should run on plaintext files."""
+    def should_process(self, object_id: str, file_path: str | None = None) -> bool:
+        """Determine if this module should run on plaintext files.
+
+        Args:
+            object_id: The object ID of the file
+            file_path: Optional path to already downloaded file
+        """
 
         # there are some performance issues, so we're disabling this one for now
         return False
@@ -56,7 +60,15 @@ class Base64DecoderAnalyzer(EnrichmentModule):
 
         try:
             num_bytes = file_enriched.size if file_enriched.size < self.size_limit else self.size_limit
-            file_bytes = self.storage.download_bytes(file_enriched.object_id, length=num_bytes)
+
+            if file_path:
+                # Use provided file path
+                with open(file_path, "rb") as f:
+                    file_bytes = f.read(num_bytes)
+            else:
+                # Fallback to original method
+                file_bytes = self.storage.download_bytes(file_enriched.object_id, length=num_bytes)
+
             file_content = file_bytes.decode("utf-8", errors="ignore")
 
             # Quick check using efficient patterns
@@ -214,15 +226,28 @@ class Base64DecoderAnalyzer(EnrichmentModule):
         )
         return candidates
 
-    def process(self, object_id: str) -> EnrichmentResult | None:
-        """Process file to find and decode base64 content."""
+    def process(self, object_id: str, file_path: str | None = None) -> EnrichmentResult | None:
+        """Process file to find and decode base64 content.
+
+        Args:
+            object_id: The object ID of the file
+            file_path: Optional path to already downloaded file
+        """
         try:
             file_enriched = get_file_enriched(object_id)
             enrichment_result = EnrichmentResult(module_name=self.name)
 
             # Download and read the file content (respect size limit)
             num_bytes = file_enriched.size if file_enriched.size < self.size_limit else self.size_limit
-            file_bytes = self.storage.download_bytes(file_enriched.object_id, length=num_bytes)
+
+            if file_path:
+                # Use provided file path
+                with open(file_path, "rb") as f:
+                    file_bytes = f.read(num_bytes)
+            else:
+                # Fallback to original method
+                file_bytes = self.storage.download_bytes(file_enriched.object_id, length=num_bytes)
+
             file_content = file_bytes.decode("utf-8", errors="ignore")
 
             # Extract potential base64 candidates with efficient filtering
@@ -296,6 +321,7 @@ class Base64DecoderAnalyzer(EnrichmentModule):
                         file_message = File(
                             object_id=decoded_object_id,
                             agent_id=file_enriched.agent_id,
+                            source=file_enriched.source,
                             project=file_enriched.project,
                             timestamp=file_enriched.timestamp,
                             expiration=file_enriched.expiration,

libs/file_enrichment_modules/file_enrichment_modules/chromium_cookies/analyzer.py

@@ -0,0 +1,257 @@
+# enrichment_modules/chromium_cookies/analyzer.py
+import csv
+import sqlite3
+import tempfile
+from datetime import UTC, datetime
+from typing import TYPE_CHECKING
+
+import yara_x
+from chromium import convert_chromium_timestamp, process_chromium_cookies
+from common.logger import get_logger
+from common.models import EnrichmentResult, Transform
+from common.state_helpers import get_file_enriched, get_file_enriched_async
+from common.storage import StorageMinio
+from file_enrichment_modules.module_loader import EnrichmentModule
+
+if TYPE_CHECKING:
+    import asyncio
+
+    from nemesis_dpapi import DpapiManager
+
+logger = get_logger(__name__)
+
+
+class ChromeCookiesParser(EnrichmentModule):
+    def __init__(self):
+        super().__init__("chrome_cookies_parser")
+        self.storage = StorageMinio()
+
+        # the workflows this module should automatically run in
+        self.workflows = ["default"]
+
+        self.dpapi_manager: DpapiManager = None  # type: ignore
+        self.loop: asyncio.AbstractEventLoop = None  # type: ignore
+
+        # Yara rule to check for Chrome Cookies tables
+        self.yara_rule = yara_x.compile("""
+rule Chrome_Cookies_Tables
+{
+    meta:
+        description = "Detects Chrome/Chromium cookies database tables"
+
+    strings:
+        $cookies_table = "CREATE TABLE cookies"
+        $cookies_index = "CREATE UNIQUE INDEX cookies_unique_index"
+
+    condition:
+        all of them
+}
+        """)
+
+    def should_process(self, object_id: str, file_path: str | None = None) -> bool:
+        """Determine if this module should run.
+
+        Args:
+            object_id: The object ID of the file
+            file_path: Optional path to already downloaded file
+        """
+
+        file_enriched = get_file_enriched(object_id)
+
+        if "sqlite 3.x database" not in file_enriched.magic_type.lower():
+            return False
+
+        if file_path:
+            # Use provided file path
+            with open(file_path, "rb") as f:
+                file_bytes = f.read()
+        else:
+            # Fallback to downloading the file itself
+            file_bytes = self.storage.download_bytes(file_enriched.object_id)
+
+        if file_enriched.is_plaintext:
+            return False
+
+        # Verify Chrome cookies tables using Yara
+        should_run = len(self.yara_rule.scan(file_bytes).matching_rules) > 0
+
+        return should_run
+
+    async def process(self, object_id: str, file_path: str | None = None) -> EnrichmentResult | None:
+        """Do the file enrichment.
+
+        Args:
+            object_id: The object ID of the file
+            file_path: Optional path to already downloaded file
+
+        Returns:
+            EnrichmentResult or None if processing fails
+        """
+        return await self._process_async(object_id, file_path)
+
+    async def _process_async(self, object_id: str, file_path: str | None = None) -> EnrichmentResult | None:
+        """Process Chrome Cookies database.
+
+        Args:
+            object_id: The object ID of the file
+            file_path: Optional path to already downloaded file
+        """
+        try:
+            file_enriched = await get_file_enriched_async(object_id)
+            enrichment_result = EnrichmentResult(module_name=self.name, dependencies=self.dependencies)
+            transforms = []
+
+            # Use the chromium library to process and insert into database
+            process_chromium_cookies(object_id, file_path)
+
+            # Configure SQLite to handle non-UTF8 data for report generation
+            def adapt_bytes(b):
+                return b.hex() if b is not None else None
+
+            def convert_bytes(hex_str):
+                return bytes.fromhex(hex_str) if hex_str is not None else None
+
+            sqlite3.register_adapter(bytes, adapt_bytes)
+            sqlite3.register_converter("BLOB", convert_bytes)
+
+            if file_path:
+                # Use provided file path
+                conn = sqlite3.connect(file_path, detect_types=sqlite3.PARSE_DECLTYPES)
+            else:
+                # Fallback to downloading the file itself
+                with self.storage.download(file_enriched.object_id) as temp_file:
+                    conn = sqlite3.connect(temp_file.name, detect_types=sqlite3.PARSE_DECLTYPES)
+
+            # Set text factory to handle non-UTF8 strings
+            conn.text_factory = lambda x: x.decode("utf-8", errors="replace")
+            cursor = conn.cursor()
+
+            # Generate summary report
+            report_lines = []
+
+            # Cookies summary
+            cursor.execute("SELECT COUNT(*) FROM cookies")
+            cookie_count = cursor.fetchone()[0]
+            report_lines.append("# Chrome Cookies Summary")
+            report_lines.append(f"\nTotal cookies: {cookie_count}")
+
+            # Count of non-expired cookies at time of processing
+            cursor.execute(
+                """
+                SELECT COUNT(*) FROM cookies
+                WHERE expires_utc IS NULL OR expires_utc > ?
+            """,
+                (int((datetime.now(UTC).timestamp() - 11644473600) * 1000000),),
+            )
+            non_expired_count = cursor.fetchone()[0]
+            report_lines.append(f"Non-expired cookies (at time of processing): {non_expired_count}")
+
+            # Most recently accessed cookies
+            cursor.execute("""
+                SELECT host_key, name, last_access_utc
+                FROM cookies
+                ORDER BY last_access_utc DESC
+                LIMIT 10
+            """)
+            report_lines.append("\n## Most Recently Accessed Cookies")
+            report_lines.append("\n| Last Access Time | Host | Cookie Name |")
+            report_lines.append("| ---------------- | ---- | ----------- |")
+            for host_key, name, last_access_utc in cursor.fetchall():
+                last_access_iso = convert_chromium_timestamp(last_access_utc, True)
+                # Escape pipe characters to prevent table formatting issues
+                safe_host = host_key.replace("|", "\\|") if host_key else ""
+                safe_name = name.replace("|", "\\|") if name else ""
+                report_lines.append(f"| {last_access_iso} | {safe_host} | {safe_name} |")
+
+            # Create summary report transform
+            with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8") as tmp_report:
+                tmp_report.write("\n".join(report_lines))
+                tmp_report.flush()
+                report_object_id = self.storage.upload_file(tmp_report.name)
+
+                transforms.append(
+                    Transform(
+                        type="finding_summary",
+                        object_id=f"{report_object_id}",
+                        metadata={
+                            "file_name": f"{file_enriched.file_name}.md",
+                            "display_type_in_dashboard": "markdown",
+                            "default_display": True,
+                        },
+                    )
+                )
+
+            # Export cookies table
+            cursor.execute("""
+                SELECT creation_utc, host_key, source_port, path, name, expires_utc,
+                       last_access_utc, last_update_utc, is_secure, is_httponly,
+                       is_persistent, samesite
+                FROM cookies
+            """)
+            with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", newline="") as tmp_cookies:
+                writer = csv.writer(tmp_cookies)
+                writer.writerow(
+                    [
+                        "creation_utc",
+                        "host_key",
+                        "source_port",
+                        "path",
+                        "name",
+                        "expires_utc",
+                        "last_access_utc",
+                        "last_update_utc",
+                        "is_secure",
+                        "is_httponly",
+                        "is_persistent",
+                        "samesite",
+                    ]
+                )
+                for row in cursor:
+                    # Convert Chromium timestamps to ISO format
+                    creation_utc = convert_chromium_timestamp(row[0], True) if row[0] else None
+                    expires_utc = convert_chromium_timestamp(row[5], True) if row[5] else None
+                    last_access_utc = convert_chromium_timestamp(row[6], True) if row[6] else None
+                    last_update_utc = convert_chromium_timestamp(row[7], True) if row[7] else None
+
+                    writer.writerow(
+                        [
+                            creation_utc,
+                            row[1],
+                            row[2],
+                            row[3],
+                            row[4],
+                            expires_utc,
+                            last_access_utc,
+                            last_update_utc,
+                            row[8],
+                            row[9],
+                            row[10],
+                            row[11],
+                        ]
+                    )
+                tmp_cookies.flush()
+                cookies_object_id = self.storage.upload_file(tmp_cookies.name)
+
+                transforms.append(
+                    Transform(
+                        type="chromium_cookies",
+                        object_id=f"{cookies_object_id}",
+                        metadata={
+                            "file_name": f"{file_enriched.file_name}_chromium_cookies.csv",
+                            "offer_as_download": True,
+                        },
+                    )
+                )
+
+            conn.close()
+            enrichment_result.transforms = transforms
+            return enrichment_result
+
+        except Exception as e:
+            logger.exception(
+                e, message="Error processing Chrome Cookies database", object_id=object_id, file_path=file_enriched.path
+            )
+
+
+def create_enrichment_module() -> EnrichmentModule:
+    return ChromeCookiesParser()

libs/file_enrichment_modules/file_enrichment_modules/chromium_history/analyzer.py

@@ -2,17 +2,16 @@
 import csv
 import sqlite3
 import tempfile
-from datetime import UTC, datetime, timedelta
 
-import structlog
 import yara_x
+from chromium import convert_chromium_timestamp, process_chromium_history
+from common.logger import get_logger
 from common.models import EnrichmentResult, Transform
 from common.state_helpers import get_file_enriched
 from common.storage import StorageMinio
-
 from file_enrichment_modules.module_loader import EnrichmentModule
 
-logger = structlog.get_logger(module=__name__)
+logger = get_logger(__name__)
 
 
 class ChromeHistoryParser(EnrichmentModule):
@@ -39,166 +38,186 @@ rule Chrome_Downloads_Tables
 }
         """)
 
-    def should_process(self, object_id: str) -> bool:
-        """Determine if this module should run."""
+    def should_process(self, object_id: str, file_path: str | None = None) -> bool:
+        """Determine if this module should run.
+
+        Args:
+            object_id: The object ID of the file
+            file_path: Optional path to already downloaded file
+        """
+
         file_enriched = get_file_enriched(object_id)
 
         # Check if filename is exactly "History" and SQLite magic type
         if not (file_enriched.file_name == "History" and "sqlite 3.x database" in file_enriched.magic_type.lower()):
             return False
 
+        if file_enriched.is_plaintext:
+            return False
+
+        if file_path:
+            # Use provided file path
+            with open(file_path, "rb") as f:
+                file_bytes = f.read()
+        else:
+            # Fallback to downloading the file itself
+            file_bytes = self.storage.download_bytes(file_enriched.object_id)
+
         # Verify Chrome history tables using Yara
-        file_bytes = self.storage.download_bytes(file_enriched.object_id)
         should_run = len(self.yara_rule.scan(file_bytes).matching_rules) > 0
 
-        logger.debug(f"ChromeHistoryParser should_run: {should_run}")
         return should_run
 
-    def _chrome_time_to_iso(self, chrome_time: int) -> str:
-        """Convert Chrome timestamp to ISO 8601."""
-        if not chrome_time:
-            return ""
-        # Chrome stores timestamps as microseconds since 1601-01-01 UTC
-        epoch = datetime(1601, 1, 1, tzinfo=UTC)
-        dt = epoch + timedelta(microseconds=chrome_time)
-        return dt.isoformat()
+    def process(self, object_id: str, file_path: str | None = None) -> EnrichmentResult | None:
+        """Process Chrome History database.
 
-    def process(self, object_id: str) -> EnrichmentResult | None:
-        """Process Chrome History database."""
+        Args:
+            object_id: The object ID of the file
+            file_path: Optional path to already downloaded file
+        """
         try:
             file_enriched = get_file_enriched(object_id)
             enrichment_result = EnrichmentResult(module_name=self.name, dependencies=self.dependencies)
             transforms = []
 
-            with self.storage.download(file_enriched.object_id) as temp_file:
-                # Configure SQLite to handle non-UTF8 data
-                def adapt_bytes(b):
-                    return b.hex() if b is not None else None
+            # Use the chromium library to process and insert into the database
+            process_chromium_history(object_id, file_path)
 
-                def convert_bytes(hex_str):
-                    return bytes.fromhex(hex_str) if hex_str is not None else None
+            # Configure SQLite to handle non-UTF8 data for report generation
+            def adapt_bytes(b):
+                return b.hex() if b is not None else None
 
-                sqlite3.register_adapter(bytes, adapt_bytes)
-                sqlite3.register_converter("BLOB", convert_bytes)
+            def convert_bytes(hex_str):
+                return bytes.fromhex(hex_str) if hex_str is not None else None
 
-                conn = sqlite3.connect(temp_file.name, detect_types=sqlite3.PARSE_DECLTYPES)
-                # Set text factory to handle non-UTF8 strings
-                conn.text_factory = lambda x: x.decode("utf-8", errors="replace")
-                cursor = conn.cursor()
+            sqlite3.register_adapter(bytes, adapt_bytes)
+            sqlite3.register_converter("BLOB", convert_bytes)
 
-                # Generate summary report
-                report_lines = []
+            if file_path:
+                # Use provided file path
+                conn = sqlite3.connect(file_path, detect_types=sqlite3.PARSE_DECLTYPES)
+            else:
+                # Fallback to downloading the file itself
+                with self.storage.download(file_enriched.object_id) as temp_file:
+                    conn = sqlite3.connect(temp_file.name, detect_types=sqlite3.PARSE_DECLTYPES)
 
-                # URLs summary
-                cursor.execute("SELECT COUNT(*) FROM urls")
-                url_count = cursor.fetchone()[0]
-                report_lines.append("# Chrome History Summary")
-                report_lines.append(f"\nTotal URLs visited: {url_count}")
+            # Set text factory to handle non-UTF8 strings
+            conn.text_factory = lambda x: x.decode("utf-8", errors="replace")
+            cursor = conn.cursor()
 
-                # Top 10 visited URLs
-                cursor.execute("""
-                    SELECT url, visit_count
-                    FROM urls
-                    ORDER BY visit_count DESC
-                    LIMIT 10
-                """)
-                report_lines.append("\n## Top 10 Most Visited URLs")
-                report_lines.append("\n| URL | Visit Count |")
-                report_lines.append("| --- | ----------- |")
-                for url, count in cursor.fetchall():
-                    report_lines.append(f"| {url} | {count} |")
+            # Generate summary report
+            report_lines = []
 
-                # Downloads summary
-                cursor.execute("SELECT COUNT(*) FROM downloads")
-                download_count = cursor.fetchone()[0]
-                report_lines.append(f"\nTotal Downloads: {download_count}")
+            # URLs summary
+            cursor.execute("SELECT COUNT(*) FROM urls")
+            url_count = cursor.fetchone()[0]
+            report_lines.append("# Chrome History Summary")
+            report_lines.append(f"\nTotal URLs visited: {url_count}")
 
-                # Recent downloads
-                cursor.execute("""
-                    SELECT target_path, tab_url, end_time
-                    FROM downloads
-                    ORDER BY end_time DESC
-                    LIMIT 10
-                """)
-                report_lines.append("\n## Most Recent Downloads")
-                report_lines.append("\n| Time | Path | Source URL |")
-                report_lines.append("| ---- | ---- | ---------- |")
-                for path, url, end_time in cursor.fetchall():
-                    end_time_iso = self._chrome_time_to_iso(end_time)
-                    # Escape pipe characters in paths and URLs to prevent table formatting issues
-                    safe_path = path.replace("|", "\\|") if path else ""
-                    safe_url = url.replace("|", "\\|") if url else ""
-                    report_lines.append(f"| {end_time_iso} | {safe_path} | {safe_url} |")
+            # Top 10 visited URLs
+            cursor.execute("""
+                SELECT url, visit_count
+                FROM urls
+                ORDER BY visit_count DESC
+                LIMIT 10
+            """)
+            report_lines.append("\n## Top 10 Most Visited URLs")
+            report_lines.append("\n| URL | Visit Count |")
+            report_lines.append("| --- | ----------- |")
+            for url, count in cursor.fetchall():
+                report_lines.append(f"| {url} | {count} |")
 
-                # Create summary report transform
-                with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8") as tmp_report:
-                    tmp_report.write("\n".join(report_lines))
-                    tmp_report.flush()
-                    object_id = self.storage.upload_file(tmp_report.name)
+            # Downloads summary
+            cursor.execute("SELECT COUNT(*) FROM downloads")
+            download_count = cursor.fetchone()[0]
+            report_lines.append(f"\nTotal Downloads: {download_count}")
 
-                    transforms.append(
-                        Transform(
-                            type="finding_summary",
-                            object_id=f"{object_id}",
-                            metadata={
-                                "file_name": f"{file_enriched.file_name}.md",
-                                "display_type_in_dashboard": "markdown",
-                                "default_display": True,
-                            },
-                        )
+            # Recent downloads
+            cursor.execute("""
+                SELECT target_path, tab_url, end_time
+                FROM downloads
+                ORDER BY end_time DESC
+                LIMIT 10
+            """)
+            report_lines.append("\n## Most Recent Downloads")
+            report_lines.append("\n| Time | Path | Source URL |")
+            report_lines.append("| ---- | ---- | ---------- |")
+            for path, url, end_time in cursor.fetchall():
+                end_time_iso = convert_chromium_timestamp(end_time, True)
+                # Escape pipe characters in paths and URLs to prevent table formatting issues
+                safe_path = path.replace("|", "\\|") if path else ""
+                safe_url = url.replace("|", "\\|") if url else ""
+                report_lines.append(f"| {end_time_iso} | {safe_path} | {safe_url} |")
+
+            # Create summary report transform
+            with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8") as tmp_report:
+                tmp_report.write("\n".join(report_lines))
+                tmp_report.flush()
+                report_object_id = self.storage.upload_file(tmp_report.name)
+
+                transforms.append(
+                    Transform(
+                        type="finding_summary",
+                        object_id=f"{report_object_id}",
+                        metadata={
+                            "file_name": f"{file_enriched.file_name}.md",
+                            "display_type_in_dashboard": "markdown",
+                            "default_display": True,
+                        },
                     )
+                )
 
-                # Export URLs table
-                cursor.execute("""
-                    SELECT url, title, visit_count, last_visit_time
-                    FROM urls
-                """)
-                with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", newline="") as tmp_urls:
-                    writer = csv.writer(tmp_urls)
-                    writer.writerow(["url", "title", "visit_count", "last_visit_time"])
-                    for row in cursor:
-                        writer.writerow([row[0], row[1], row[2], self._chrome_time_to_iso(row[3])])
-                    tmp_urls.flush()
-                    object_id = self.storage.upload_file(tmp_urls.name)
+            # Export URLs table
+            cursor.execute("""
+                SELECT url, title, visit_count, last_visit_time
+                FROM urls
+            """)
+            with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", newline="") as tmp_urls:
+                writer = csv.writer(tmp_urls)
+                writer.writerow(["url", "title", "visit_count", "last_visit_time"])
+                for row in cursor:
+                    writer.writerow([row[0], row[1], row[2], convert_chromium_timestamp(row[3], True)])
+                tmp_urls.flush()
+                urls_object_id = self.storage.upload_file(tmp_urls.name)
 
-                    transforms.append(
-                        Transform(
-                            type="chromium_urls",
-                            object_id=f"{object_id}",
-                            metadata={
-                                "file_name": f"{file_enriched.file_name}_chromium_urls.csv",
-                                "offer_as_download": True,
-                            },
-                        )
+                transforms.append(
+                    Transform(
+                        type="chromium_urls",
+                        object_id=f"{urls_object_id}",
+                        metadata={
+                            "file_name": f"{file_enriched.file_name}_chromium_urls.csv",
+                            "offer_as_download": True,
+                        },
                     )
+                )
 
-                # Export downloads table
-                cursor.execute("""
-                    SELECT target_path, total_bytes, end_time, tab_url, mime_type
-                    FROM downloads
-                """)
-                with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", newline="") as tmp_downloads:
-                    writer = csv.writer(tmp_downloads)
-                    writer.writerow(["target_path", "total_bytes", "end_time", "tab_url", "mime_type"])
-                    for row in cursor:
-                        writer.writerow([row[0], row[1], self._chrome_time_to_iso(row[2]), row[3], row[4]])
-                    tmp_downloads.flush()
-                    object_id = self.storage.upload_file(tmp_downloads.name)
+            # Export downloads table
+            cursor.execute("""
+                SELECT target_path, total_bytes, end_time, tab_url, mime_type
+                FROM downloads
+            """)
+            with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", newline="") as tmp_downloads:
+                writer = csv.writer(tmp_downloads)
+                writer.writerow(["target_path", "total_bytes", "end_time", "tab_url", "mime_type"])
+                for row in cursor:
+                    writer.writerow([row[0], row[1], convert_chromium_timestamp(row[2], True), row[3], row[4]])
+                tmp_downloads.flush()
+                downloads_object_id = self.storage.upload_file(tmp_downloads.name)
 
-                    transforms.append(
-                        Transform(
-                            type="chromium_downloads",
-                            object_id=f"{object_id}",
-                            metadata={
-                                "file_name": f"{file_enriched.file_name}_chromium_downloads.csv",
-                                "offer_as_download": True,
-                            },
-                        )
+                transforms.append(
+                    Transform(
+                        type="chromium_downloads",
+                        object_id=f"{downloads_object_id}",
+                        metadata={
+                            "file_name": f"{file_enriched.file_name}_chromium_downloads.csv",
+                            "offer_as_download": True,
+                        },
                     )
+                )
 
-                conn.close()
-                enrichment_result.transforms = transforms
-                return enrichment_result
+            conn.close()
+            enrichment_result.transforms = transforms
+            return enrichment_result
 
         except Exception as e:
             logger.exception(e, message="Error processing Chrome History database")

libs/file_enrichment_modules/file_enrichment_modules/chromium_localstate/analyzer.py

@@ -0,0 +1,110 @@
+# enrichment_modules/chromium_logins/analyzer.py
+
+import asyncio
+from typing import TYPE_CHECKING
+
+import yara_x
+from chromium import process_chromium_local_state
+from common.logger import get_logger
+from common.models import EnrichmentResult
+from common.state_helpers import get_file_enriched
+from common.storage import StorageMinio
+from file_enrichment_modules.module_loader import EnrichmentModule
+from nemesis_dpapi import DpapiManager
+
+if TYPE_CHECKING:
+    from nemesis_dpapi import DpapiManager
+
+logger = get_logger(__name__)
+
+
+class ChromeLocalStateParser(EnrichmentModule):
+    def __init__(self):
+        super().__init__("chrome_local_state_parser")
+        self.storage = StorageMinio()
+
+        # the workflows this module should automatically run in
+        self.workflows = ["default"]
+
+        self.dpapi_manager: DpapiManager = None  # type: ignore
+        self.loop: asyncio.AbstractEventLoop = None  # type: ignore
+
+        # Yara rule to check for Chrome Login Data tables
+        self.yara_rule = yara_x.compile("""
+rule Chrome_Local_State
+{
+    meta:
+        description = "Detects Chrome/Chromium Local State json"
+
+    strings:
+        $local_state_1 = "\\"os_crypt\\""
+        $local_state_2 = "\\"encrypted_key\\""
+        $local_state_3 = "\\"user_experience_metrics\\""
+
+    condition:
+        all of them
+}
+        """)
+
+    def should_process(self, object_id: str, file_path: str | None = None) -> bool:
+        """Determine if this module should run.
+
+        Args:
+            object_id: The object ID of the file
+            file_path: Optional path to already downloaded file
+        """
+
+        file_enriched = get_file_enriched(object_id)
+
+        # Check if file is < 5 megs and JSON magic type
+        if not ((file_enriched.size < 5000000) and ("json" in file_enriched.magic_type.lower())):
+            return False
+
+        if file_path:
+            # Use provided file path
+            with open(file_path, "rb") as f:
+                file_bytes = f.read()
+        else:
+            # Fallback to downloading the file itself
+            file_bytes = self.storage.download_bytes(file_enriched.object_id)
+
+        # Verify Chrome Local State using Yara
+        should_run = len(self.yara_rule.scan(file_bytes).matching_rules) > 0
+
+        return should_run
+
+    async def process(self, object_id: str, file_path: str | None = None) -> EnrichmentResult | None:
+        """Process Chrome Local State files.
+
+        Args:
+            object_id: The object ID of the file
+            file_path: Optional path to already downloaded file
+        """
+        return await self._process_async(object_id, file_path)
+
+    async def _process_async(self, object_id: str, file_path: str | None = None) -> EnrichmentResult | None:
+        """Async helper for process method."""
+
+        try:
+            # Use the chromium library to process and insert into database
+            state_key_data = await process_chromium_local_state(self.dpapi_manager, object_id, file_path)
+
+            if state_key_data:
+                # Debug: Check for coroutines in state_key_data
+                import inspect
+
+                for key, value in state_key_data.items():
+                    if inspect.iscoroutine(value):
+                        logger.error(f"FOUND COROUTINE IN state_key_data['{key}']!")
+
+                enrichment = EnrichmentResult(module_name=self.name)
+                enrichment.results = {"parsed": state_key_data}
+
+                return enrichment
+
+        except Exception as e:
+            logger.exception(e, message="Error processing Chrome Local State file")
+
+
+def create_enrichment_module() -> EnrichmentModule:
+    return ChromeLocalStateParser()

libs/file_enrichment_modules/file_enrichment_modules/chromium_logins/analyzer.py

@@ -0,0 +1,232 @@
+# enrichment_modules/chromium_logins/analyzer.py
+import asyncio
+import csv
+import sqlite3
+import tempfile
+from typing import TYPE_CHECKING
+
+import yara_x
+from chromium import convert_chromium_timestamp, process_chromium_logins
+from common.logger import get_logger
+from common.models import EnrichmentResult, Transform
+from common.state_helpers import get_file_enriched, get_file_enriched_async
+from common.storage import StorageMinio
+from file_enrichment_modules.module_loader import EnrichmentModule
+
+if TYPE_CHECKING:
+    from nemesis_dpapi import DpapiManager
+
+logger = get_logger(__name__)
+
+
+class ChromeLoginsParser(EnrichmentModule):
+    def __init__(self):
+        super().__init__("chrome_logins_parser")
+        self.storage = StorageMinio()
+
+        # the workflows this module should automatically run in
+        self.workflows = ["default"]
+
+        self.dpapi_manager: DpapiManager = None  # type: ignore
+        self.loop: asyncio.AbstractEventLoop = None  # type: ignore
+
+        # Yara rule to check for Chrome Login Data tables
+        self.yara_rule = yara_x.compile("""
+rule Chrome_Logins_Tables
+{
+    meta:
+        description = "Detects Chrome/Chromium logins database tables"
+
+    strings:
+        $logins_table = "CREATE TABLE logins "
+        $logins_table2 = "CREATE TABLE insecure_credentials"
+
+    condition:
+        all of them
+}
+        """)
+
+    def should_process(self, object_id: str, file_path: str | None = None) -> bool:
+        """Determine if this module should run.
+
+        Args:
+            object_id: The object ID of the file
+            file_path: Optional path to already downloaded file
+        """
+
+        file_enriched = get_file_enriched(object_id)
+
+        if "sqlite 3.x database" not in file_enriched.magic_type.lower():
+            return False
+
+        if file_enriched.is_plaintext:
+            return False
+
+        if file_path:
+            # Use provided file path
+            with open(file_path, "rb") as f:
+                file_bytes = f.read()
+        else:
+            # Fallback to downloading the file itself
+            file_bytes = self.storage.download_bytes(file_enriched.object_id)
+
+        # Verify Chrome Login Data tables using Yara
+        should_run = len(self.yara_rule.scan(file_bytes).matching_rules) > 0
+
+        return should_run
+
+    async def process(self, object_id: str, file_path: str | None = None) -> EnrichmentResult | None:
+        """Do the file enrichment.
+
+        Args:
+            object_id: The object ID of the file
+            file_path: Optional path to already downloaded file
+
+        Returns:
+            EnrichmentResult or None if processing fails
+        """
+        return await self._process_async(object_id, file_path)
+
+    async def _process_async(self, object_id: str, file_path: str | None = None) -> EnrichmentResult | None:
+        """Process Chrome Login Data database.
+
+        Args:
+            object_id: The object ID of the file
+            file_path: Optional path to already downloaded file
+        """
+        try:
+            file_enriched = await get_file_enriched_async(object_id)
+            enrichment_result = EnrichmentResult(module_name=self.name, dependencies=self.dependencies)
+            transforms = []
+
+            # Use the chromium library to process and insert into database
+            process_chromium_logins(object_id, file_path)
+
+            # Configure SQLite to handle non-UTF8 data for report generation
+            def adapt_bytes(b):
+                return b.hex() if b is not None else None
+
+            def convert_bytes(hex_str):
+                return bytes.fromhex(hex_str) if hex_str is not None else None
+
+            sqlite3.register_adapter(bytes, adapt_bytes)
+            sqlite3.register_converter("BLOB", convert_bytes)
+
+            if file_path:
+                # Use provided file path
+                conn = sqlite3.connect(file_path, detect_types=sqlite3.PARSE_DECLTYPES)
+            else:
+                # Fallback to downloading the file itself
+                with self.storage.download(file_enriched.object_id) as temp_file:
+                    conn = sqlite3.connect(temp_file.name, detect_types=sqlite3.PARSE_DECLTYPES)
+
+            # Set text factory to handle non-UTF8 strings
+            conn.text_factory = lambda x: x.decode("utf-8", errors="replace")
+            cursor = conn.cursor()
+
+            # Generate summary report
+            report_lines = []
+
+            # Logins summary
+            cursor.execute("SELECT COUNT(*) FROM logins")
+            login_count = cursor.fetchone()[0]
+            report_lines.append("# Chrome Logins Summary")
+            report_lines.append(f"\nTotal logins: {login_count}")
+
+            # Count of logins with non-empty password_value
+            cursor.execute("SELECT COUNT(*) FROM logins WHERE password_value IS NOT NULL")
+            password_count = cursor.fetchone()[0]
+            report_lines.append(f"\nLogins with saved passwords: {password_count}")
+
+            # Most recently used logins
+            cursor.execute("""
+                SELECT origin_url, username_value, date_last_used
+                FROM logins
+                ORDER BY date_last_used DESC
+                LIMIT 10
+            """)
+            report_lines.append("\n## Most Recently Used Logins")
+            report_lines.append("\n| Last Used Time | Origin URL | Username |")
+            report_lines.append("| -------------- | ---------- | -------- |")
+            for origin_url, username_value, date_last_used in cursor.fetchall():
+                last_used_iso = convert_chromium_timestamp(date_last_used, True)
+                # Escape pipe characters to prevent table formatting issues
+                safe_origin = origin_url.replace("|", "\\|") if origin_url else ""
+                safe_username = username_value.replace("|", "\\|") if username_value else ""
+                report_lines.append(f"| {last_used_iso} | {safe_origin} | {safe_username} |")
+
+            # Create summary report transform
+            with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8") as tmp_report:
+                tmp_report.write("\n".join(report_lines))
+                tmp_report.flush()
+                report_object_id = self.storage.upload_file(tmp_report.name)
+
+                transforms.append(
+                    Transform(
+                        type="finding_summary",
+                        object_id=f"{report_object_id}",
+                        metadata={
+                            "file_name": f"{file_enriched.file_name}.md",
+                            "display_type_in_dashboard": "markdown",
+                            "default_display": True,
+                        },
+                    )
+                )
+
+            # Export logins table
+            cursor.execute("""
+                SELECT origin_url, username_value, signon_realm, date_created,
+                       date_last_used, date_password_modified, times_used
+                FROM logins
+            """)
+            with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", newline="") as tmp_logins:
+                writer = csv.writer(tmp_logins)
+                writer.writerow(
+                    [
+                        "origin_url",
+                        "username_value",
+                        "signon_realm",
+                        "date_created",
+                        "date_last_used",
+                        "date_password_modified",
+                        "times_used",
+                    ]
+                )
+                for row in cursor:
+                    # Convert Chromium timestamps to ISO format
+                    date_created = convert_chromium_timestamp(row[3], True) if row[3] else None
+                    date_last_used = convert_chromium_timestamp(row[4], True) if row[4] else None
+                    date_password_modified = convert_chromium_timestamp(row[5], True) if row[5] else None
+
+                    writer.writerow(
+                        [row[0], row[1], row[2], date_created, date_last_used, date_password_modified, row[6]]
+                    )
+                tmp_logins.flush()
+                logins_object_id = self.storage.upload_file(tmp_logins.name)
+
+                transforms.append(
+                    Transform(
+                        type="chromium_logins",
+                        object_id=f"{logins_object_id}",
+                        metadata={
+                            "file_name": f"{file_enriched.file_name}_chromium_logins.csv",
+                            "offer_as_download": True,
+                        },
+                    )
+                )
+
+            conn.close()
+            enrichment_result.transforms = transforms
+            return enrichment_result
+
+        except Exception as e:
+            logger.exception(
+                e,
+                message="Error processing Chrome Login Data database",
+                object_id=object_id,
+                file_path=file_enriched.path,
+            )
+
+
+def create_enrichment_module() -> EnrichmentModule:
+    return ChromeLoginsParser()